From 824be3205f6abb1103f032b0ea2412c23aef09e2 Mon Sep 17 00:00:00 2001 From: "t.dzhaparov" Date: Sun, 2 Aug 2026 02:23:13 +0300 Subject: [PATCH 1/3] Add Nexus-compatible named Maven and npm registries --- ARCHITECTURE.md | 65 +- CHANGELOG.md | 10 +- COMPAT.md | 87 +- Cargo.lock | 1 + README.md | 19 +- dist/nora.env.example | 23 +- docs-ru/ARCHITECTURE.md | 70 +- docs-ru/admin-guide.md | 60 +- docs-ru/technical-spec.md | 16 +- docs-ru/user-guide.md | 28 +- llms.txt | 25 +- nora-registry/Cargo.toml | 1 + nora-registry/src/auth/mod.rs | 87 +- nora-registry/src/config/mod.rs | 141 +- nora-registry/src/config/registry/maven.rs | 247 +- nora-registry/src/config/registry/mod.rs | 6 +- nora-registry/src/config/registry/npm.rs | 302 +- nora-registry/src/config/retention.rs | 7 +- nora-registry/src/gc.rs | 734 +- nora-registry/src/import/layout.rs | 106 +- nora-registry/src/import/mod.rs | 78 +- nora-registry/src/main.rs | 48 +- nora-registry/src/metrics.rs | 138 +- nora-registry/src/npm_layout.rs | 281 + nora-registry/src/openapi.rs | 135 + nora-registry/src/registry/maven.rs | 5429 ++++++++-- nora-registry/src/registry/mod.rs | 1622 ++- nora-registry/src/registry/named.rs | 271 + nora-registry/src/registry/npm.rs | 8702 ++++++++++++----- .../registry/ns_isolation_metadata_tests.rs | 8 +- nora-registry/src/repo_index.rs | 289 +- nora-registry/src/retention.rs | 3416 +++++-- nora-registry/src/storage/local.rs | 167 + nora-registry/src/storage/mod.rs | 175 + nora-registry/src/storage/object.rs | 68 +- nora-registry/src/test_helpers.rs | 154 +- nora-registry/src/ui/api.rs | 409 +- nora-registry/src/ui/mod.rs | 75 +- nora-registry/src/ui/templates.rs | 93 +- 39 files changed, 19249 insertions(+), 4344 deletions(-) create mode 100644 nora-registry/src/npm_layout.rs create mode 100644 nora-registry/src/registry/named.rs diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c724cbd7..f3eabbfe 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -54,8 +54,8 @@ plugin runtime. The filesystem (or S3) is the only source of truth. ┌───────────────────────────┼───────────────────────────┐ │ │ │ ┌──────▼──────┐ ┌───────▼───────┐ ┌───────▼───────┐ - │ Docker │ │ Maven │ ... │ Debian │ - │ /v2/* │ │ /maven2/* │ (x15) │ /deb/* │ + │ Docker │ │ Maven / npm │ ... │ Debian │ + │ /v2/* │ │/repository/* │ (x15) │ /deb/* │ └──────┬──────┘ └───────┬───────┘ └───────┬───────┘ │ │ │ └───────────────────────────┼───────────────────────────┘ @@ -72,9 +72,26 @@ plugin runtime. The filesystem (or S3) is the only source of truth. └─────────────────────┘ ``` -Every HTTP request follows this path top-to-bottom. The registry handler is -selected by URL prefix (`/v2/` = Docker, `/maven2/` = Maven, etc.). Curation -runs only on proxy downloads — hosted artifacts are trusted at publish time. +Every HTTP request follows this path top-to-bottom. Most registry handlers are +selected by URL prefix. Maven and npm share `/repository/{name}/...`; a named +dispatcher selects the protocol from validated configuration, not from a name +heuristic. Their repository names therefore occupy one global namespace. +`/maven2/` and `/npm/` remain compatibility aliases only when a default named +repository is configured. Curation runs only on proxy downloads — hosted +artifacts are trusted at publish time. + +Named Maven and npm each support hosted, proxy, and ordered group repositories. +Groups own no storage. Maven hosted/proxy objects are isolated under +`maven/repositories/{name}/...`. npm keeps hosted version manifests, +content-addressed tarball blobs, dist-tags, and deprecation overlays under +`npm/repositories/{hosted}/...`, while proxy packuments and tarballs use +`npm/repositories/{proxy}/proxy/...`. An npm group synthesizes a response from +its members in configured order. + +This topology has a single-writer contract on local and object storage. +Conditional create protects one immutable key, but mutable Maven/npm metadata +updates span multiple keys and use an in-process lock. S3 or Ceph exact-key CAS +therefore does not provide multi-replica write coordination or HA. ### Trust Boundaries @@ -272,9 +289,17 @@ in front of other registries. **Rationale:** Each storage backend is a maintenance surface. S3 covers every cloud provider and on-prem S3-compatible stores. Local covers single-node and development. A third backend (e.g., GCS-native, Azure Blob) adds testing -burden without meaningful capability gain — both are S3-compatible. For -migrating away from other registries, the `nora migrate` CLI copies -artifacts directly rather than proxying through the old system. +burden without meaningful capability gain — both are S3-compatible. +`nora migrate` copies an existing NORA storage layout between local and S3; it +is not a Nexus-to-named-repository migrator. + +Named Maven/npm are a fresh-install contract. For Nexus replacement, keep the +source read-only and copy only hosted content through the protocol-aware named +hosted endpoints, then verify reads through the configured groups. Do not copy +Nexus group state or proxy caches. Direct-storage import remains a legacy/local +layout tool and is not supported for named npm or for Maven when named +repositories are configured. No in-place migration from older NORA Maven/npm +layouts is provided. ### ADR-4: Explicit Handlers over Plugin Traits @@ -391,9 +416,10 @@ are implemented per-registry following each format's upstream specification. There is no shared conditional-request middleware. **Context:** RFC 9110 defines conditional requests for HTTP. Each registry -protocol has its own immutability model: Docker uses content-addressable -digests, Maven/npm/Cargo/PyPI enforce version immutability at publish time, -Raw has no upstream spec. Implementing a generic conditional-request layer +protocol has its own coordinate model: Docker uses content-addressable +digests, Maven/npm apply repository-specific write policy, Cargo/PyPI enforce +version immutability at publish time, and Raw has no upstream spec. +Implementing a generic conditional-request layer would either be too narrow (not matching protocol-specific semantics) or too broad (imposing HTTP semantics on protocols that don't need them). @@ -445,12 +471,14 @@ added rarely — the explicit approach trades one-time boilerplate for permanent simplicity, compile-time completeness checks, and full test coverage of each format in isolation. -**No high availability.** NORA runs as a single instance with a single -RWO volume. This is a design decision, not a missing feature. Artifact +**No high availability.** NORA runs with exactly one writer, including when +the storage backend is S3 or Ceph. Exact-key conditional create does not +serialize the multi-key mutable metadata used by named Maven/npm. This is a +design decision, not a missing feature. Artifact registries have a read-heavy, write-light workload — a single instance -with S3 storage handles thousands of pulls per minute. Kubernetes -`Recreate` strategy ensures zero-downtime upgrades for reads served from -client-side caches. +with S3 storage handles thousands of pulls per minute. Kubernetes `Recreate` +or an explicit scale-to-zero upgrade prevents overlapping writers, at the cost +of a short service interruption. High availability requires a separate design. **DRY violations between handlers.** Registry handlers share structural patterns (proxy logic, curation calls, config loading) but differ in @@ -475,5 +503,6 @@ external tools (Grafana dashboards, git-based rule management). - **Not a CDN.** For geo-distributed artifact delivery, put a CDN (CloudFront, Cloudflare) in front of NORA. - **Not a middleware.** NORA is a standalone registry, not a caching - layer in front of Nexus or Artifactory. For migration, use - `nora migrate`. + layer in front of Nexus or Artifactory. Nexus hosted Maven/npm content is + migrated through protocol-aware named endpoints; `nora migrate` only copies + existing NORA storage between local and S3. diff --git a/CHANGELOG.md b/CHANGELOG.md index 803514b2..00839a4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,12 @@ # Changelog ## [Unreleased] +### Added +- **Nexus-style named Maven and npm repositories** — fresh installations can declare independent `hosted`, `proxy`, and `group` repositories under the shared `/repository/{repository}/...` namespace. Maven and npm names are validated globally, group members resolve in configured order, npm groups may nominate one hosted `writable_member`, and groups persist no authoritative objects. Hosted write policy is explicit: `allow` replaces a Maven/npm coordinate and reapplies mutable npm publish state, while `allow_once` (the default) keeps exact retries idempotent and rejects different bytes. An incomplete `allow_once` publish must be retried before later versions/dist-tag/deprecation mutations are accepted, preventing delayed repair from rewinding newer mutable state. npm hosted version manifests reference content-addressed SHA-512 blobs and remain physically separate from proxy packuments/tarballs/negative cache. Plain-text npm search merges hosted/proxy/group results with member-order precedence; every upstream response is capped at 8 MiB, direct proxies preserve exact pagination when namespace filtering is inactive, hosted scans cap packages/versions at 10,000, and filtered direct/group proxy backfill caps results at 10,000 and pages at 40 under a shared 30-second deadline. Group search is fail-soft and labels its successful-member total estimate as approximate, so a failed member can make it an undercount. Bulk, quick, and full audit requests are forwarded only after bounded internal-namespace filtering. Maven repositories are isolated below `maven/repositories/{name}/`. GC, retention, browse index, UI, metrics, auth classification and OpenAPI understand named repositories. + +### Changed +- **Maven/npm migration is protocol-first and fresh-layout-only** — old NORA storage layouts are not migrated in place. Nexus hosted content must be copied into named hosted repositories through the Maven/npm protocol endpoints and verified through groups; proxy caches and group materializations are rebuilt, never imported. Direct-storage npm import is rejected, and direct Maven import is rejected when named Maven repositories are configured. + ### Fixed - **Maven keeps server-generated artifact metadata authoritative** — a Maven client that re-uploads a stale artifact-level `maven-metadata.xml` after a concurrent deploy no longer overwrites the version list NORA generates: an uploaded artifact-level metadata document (and its checksums) is recognized by its shape and dropped, while version-level (SNAPSHOT) and group-level (plugin) metadata are still stored verbatim. On a proxy refresh, locally hosted versions are merged into the refreshed upstream document instead of being replaced by it, and the `.md5`/`.sha1`/`.sha256`/`.sha512` sidecars are recomputed from the merged document. The proxy-side merge runs under the same `publish_lock` as the upload-side regeneration, so the document and its checksums are written as one critical section and stay mutually consistent under concurrent fetch and publish (#886). @@ -21,7 +27,7 @@ - **Retention over rpm, deb, and raw** — the three formats retention previously skipped silently. rpm/deb versions are collected from the metadata sidecars (payloads never read) and grouped per `{repo}/{arch}/{package}` — structured-layout deb packages per `{repo}/{distribution}/{component}/{arch}/{package}`, so `keep_last` counts within each distribution × architecture's independent index (`all`/`noarch` packages form their own group); deleting a version removes the package and its sidecar, and every touched repo's indexes are rebuilt **and re-signed** under the publish lock afterwards — retention can no longer leave a signed index advertising deleted packages. Raw groups depth-2 path prefixes (`raw/{name}/{version}/…`) as the aging unit, so a directory of related files ages out together; root-level files are never collected. New optional `name_glob` on retention rules targets groups within a registry (e.g. `*-dev-*/*` for an age-only policy on dev repositories, `*-stream-*/*` for a keep-last window) — first matching rule wins, and no matching rule still means keep forever. `nora retention-apply --yes` signs regenerated indexes with the same key as the server. - **Per-rule namespace scope for OIDC role rules** — an `[[auth.oidc.providers.role_rules]]` entry may set `namespace_scope = ["ci-transport/**"]` to narrow the provider's scope for identities matched by that rule. A write must satisfy **both** the provider scope and the rule scope — the provider scope stays a hard ceiling, and a rule cannot widen past it. Lets one issuer grant, e.g., pull-request CI builds write access confined to a transport prefix while main/tag builds keep the provider-wide scope. Absent = inherit the provider's `namespace_scope`; enforcement mode stays provider-level. Also corrects the config doc example for `role_rules`, which showed a map form that fails to parse (the real shape is an array of tables with `pattern`/`role`). - **Intra-segment `*` wildcards in `namespace_scope`** — scope patterns now match a `*` within a single path segment (e.g. `team-*/ci`), not only the `**` cross-segment form, so a scope can target a naming convention inside one level without opening the whole subtree. -- **A geo/policy-blocked upstream is now observable** — when a proxied upstream answers a fetch with a 4xx carrying a policy/geo block signature (`x-amzn-waf-reason`, as an AWS CloudFront + WAF geo-block does), NORA emits a `warn` log naming the registry and reason and increments `nora_upstream_policy_blocked_total{registry,reason}`, instead of relaying an anonymous 404 indistinguishable from a genuinely absent artifact. The relayed 404 status and the circuit breaker are deliberately unchanged — a policy block is not an availability failure, so it must not trip the breaker — but it is no longer silent. +- **A geo/policy-blocked upstream is now observable** — when a proxied upstream answers a fetch with a 4xx carrying a policy/geo block signature (`x-amzn-waf-reason`, as an AWS CloudFront + WAF geo-block does), NORA emits a `warn` log naming the registry and reason and increments `nora_upstream_policy_blocked_total{registry,reason}`. Maven maps a policy-blocked 404 to 502 and never stores it in the negative cache, so a group cannot treat the block as an authoritative miss; the circuit breaker remains unchanged because a policy response is not an availability failure. ## [1.0.1] - 2026-07-13 @@ -54,7 +60,7 @@ ### Added - **Admin-gated admin-token minting (`POST /api/v1/admin/tokens`)** — a dedicated route, reachable only behind the `/api/v1/admin/` gate (`auth::is_admin_path`), that mints an API token of any role including `admin` for a given subject without the `auth.admin_users` self-service check. Anonymous, `anonymous_read`, Basic-auth (no role) and Read/Write callers are denied fail-closed before the handler; `ttl_days = 0` is rejected and every mint is audit-logged (actor, target, role, ttl — never the token). `auth.admin_users` (`NORA_AUTH_ADMIN_USERS`) thus becomes a bootstrap-only fallback on the unchanged public `POST /api/tokens` route, so GHSA-78cx-cfhm-rgmx stays closed; with auth disabled the route returns `503` (#746, #808). -- **`npm audit` proxied to upstream for remote repos** — `npm audit` POSTs to `/-/npm/v1/security/advisories/bulk` (npm7) or `/-/npm/v1/security/audits/quick` (npm6), which previously hit the `405` fallback and failed. NORA keeps no advisory database, so for a proxy repo it now forwards the request to the configured upstream and returns the response verbatim. Both audit POSTs are read-eligible under `auth.anonymous_read`, so anonymous `npm audit` works wherever anonymous install does; non-audit npm POSTs stay gated. Under an active `internal_namespaces` filter the `bulk` request strips internal-package keys before forwarding and fail-closes (`200 {}`) on any body it cannot verify, the gzipped `quick` lockfile is refused wholesale, the client `Authorization` is never forwarded, and the body is bounded at 8 MB; upstream 5xx/network → `502`, circuit-open → `503`, no proxy configured → `200 {}` (#597, #805). +- **`npm audit` proxied to upstream for remote repos** — `npm audit` POSTs to `/-/npm/v1/security/advisories/bulk` (npm7) or `/-/npm/v1/security/audits/quick` (npm6), which previously hit the `405` fallback and failed. NORA keeps no advisory database, so for a proxy repo it now forwards the request to the configured upstream and returns a fully read response verbatim. Both audit POSTs are read-eligible under `auth.anonymous_read`, so anonymous `npm audit` works wherever anonymous install does; non-audit npm POSTs stay gated. Under an active `internal_namespaces` filter the `bulk` request strips internal-package keys before forwarding and fail-closes with `400` on any body it cannot verify, the gzipped `quick` lockfile is refused wholesale, the client `Authorization` is never forwarded, and both request and upstream-response bodies are bounded at 8 MiB; truncated/oversized responses and upstream 5xx/network → `502`, circuit-open → `503`, no proxy configured → explicit `400` (#597, #805). - **S3 virtual-hosted-style addressing (`storage.s3_virtual_hosted`, `NORA_STORAGE_S3_VIRTUAL_HOSTED`)** — some S3-compatible providers reject signed path-style requests (Alibaba Cloud OSS answers `403 SecondLevelDomainForbidden`), which made the S3 backend unusable there because the addressing style was hardcoded to path-style. A new default-off toggle threads through `Storage::new_s3` / `S3Storage::new` into `AmazonS3Builder::with_virtual_hosted_style_request`; when enabled, `object_store` uses the configured endpoint verbatim, so it must include the bucket host. Default (`false`) preserves current path-style behavior (#795, #798). - **Chinese (Simplified) UI translation** — a `中文` entry in the language switcher backed by a full `zh` translation table; language detection now normalizes BCP-47 / POSIX tags to their primary subtag, so `zh-CN`, `zh-Hans` and `ru_RU.UTF-8` resolve correctly (#788). diff --git a/COMPAT.md b/COMPAT.md index 20232de2..b2fa3fe7 100644 --- a/COMPAT.md +++ b/COMPAT.md @@ -37,33 +37,76 @@ This document describes which parts of each registry protocol are implemented in | Feature | Status | Notes | |---------|--------|-------| -| Package metadata (GET) | Full | JSON with all versions | +| Named hosted/proxy/group | Full | `/repository/{name}/`; ordered group members, optional hosted `writable_member` | +| Package metadata (GET) | Full | Synthesized from authoritative hosted version manifests or isolated proxy packuments | | Scoped packages `@scope/name` | Full | URL-encoded path | -| Tarball download | Full | SHA256 verified | +| Tarball download | Full | `dist.integrity`/`shasum` verified | | Tarball URL rewriting | Full | Points to NORA, not upstream | -| Publish (`npm publish`) | Full | Immutable versions | +| Publish (`npm publish`) | Full | Hosted only; `write_policy=allow` replaces a coordinate, `allow_once` (default) makes exact retries idempotent and rejects different bytes | | Unpublish | — | Immutable; use quarantine/blocklist to disable a version | -| Dist-tags (`latest`, `next`) | Partial | Read from metadata, no explicit management | -| Search (`/-/v1/search`) | — | Not implemented | -| Audit (`bulk` npm7 / `audits/quick` npm6) | Full | Proxy repos: forwarded to upstream verbatim; internal-namespace names stripped/refused; anonymous-read eligible. Proxied packages only (no local advisory DB). (#597) | -| Upstream proxy | Full | Configurable TTL | +| Dist-tags (`latest`, `next`) | Full | Mutate hosted only; group mutations fail closed; `latest` cannot be deleted | +| Search (`/-/v1/search`) | Partial | Plain-text hosted search plus proxy results; each upstream response body is capped at 8 MiB. A direct proxy preserves exact `from`/`size`/`total` when namespace filtering is inactive. Hosted scans are limited to 10,000 packages and 10,000 versions per package; filtered direct/group proxy scans are limited to 10,000 results and 40 pages; all multi-object scans share a 30-second request deadline. Such filtered/group responses report `totalIsApproximate: true`; `total` is the deduplicated/summed estimate from successful members, because exact filtering and cross-member deduplication would require draining every upstream result. Group search is deliberately fail-soft: an unavailable member is omitted, so this estimate may be below the true cross-member total while the response still contains results from healthy members. Requests whose filtered/group window itself exceeds the budget fail with `400`; exhausting a scan/body budget fails with `502`. Advanced hosted qualifiers are not interpreted | +| Audit (`bulk`, `audits/quick`, `audits`) | Full | Forwarded through a proxy after bounded gzip/JSON decoding and internal-namespace pruning; request and upstream-response bodies are capped at 8 MiB, client auth is never forwarded, and truncated/invalid/unavailable exchanges fail explicitly | +| Upstream proxy | Full | Per-repository URL, TTL, negative TTL and credentials; only a true upstream 404 is negatively cached | + +Hosted state lives under +`npm/repositories/{hosted}/{package}/{pkg.json,versions/,publish-complete/,blobs/sha512/,dist-tags/,deprecations/}`. +The authoritative version manifest is the visibility commit point; +its `dist.integrity` selects the content-addressed hosted blob. +`publish-complete/{version}` records that mutable post-commit state finished. +With `allow_once`, a missing marker makes the same coordinate repairable. +Until that exact publish succeeds, later versions, dist-tag changes and +deprecation changes for the package fail with `409`; this prevents a delayed +repair from rewinding newer package fields, tags or deprecation state. After +completion, exact retries are read-only. A present marker with the wrong +manifest digest is treated as corrupt (`500`), not as repair permission. With +`allow`, every publish is intentionally last-writer-wins even when the version +manifest is byte-identical, so its supplied package fields, tags, and +deprecation are applied again. Deprecation is a mutable overlay and is never +embedded in the version manifest. + +The version manifest is the visibility commit, not a multi-object transaction +barrier. A storage failure after a replacement manifest is visible can return +HTTP 500 with mutable state still incomplete; an identical retry is required +before later mutable operations are accepted. For `allow`, the retry reapplies +state even if the completion marker is absent or mismatched; for `allow_once`, +only an absent marker is repairable. Readers do not wait for that repair. +Proxy state lives separately under +`npm/repositories/{proxy}/proxy/{packuments/,tarballs/,negative/}`. A group +materializes a response from its members and never owns metadata or tarballs. ## Maven | Feature | Status | Notes | |---------|--------|-------| +| Named hosted/proxy/group | Full | `/repository/{name}/`; groups resolve members in declared order | | Artifact download (GET) | Full | JAR, POM, checksums | -| Artifact upload (PUT) | Full | Any file type | +| Artifact upload (PUT) | Full | Hosted repository only; per-repository version/write policy | | GroupId path layout | Full | Dots → slashes | -| SHA1/MD5 checksums | Full | Stored alongside artifacts | -| `maven-metadata.xml` | Partial | Stored as-is, no auto-generation | -| SNAPSHOT versions | — | No SNAPSHOT resolution | -| Multi-proxy fallback | Full | Tries proxies in order | +| Checksums | Full | MD5, SHA-1, SHA-256 and SHA-512 stored alongside artifacts | +| `maven-metadata.xml` | Full | Hosted artifact metadata is server-generated and merged with proxy metadata | +| SNAPSHOT versions | Full | Repository version policy separates release/snapshot/mixed content | +| Proxy/group fallback | Full | Ordered group members; proxy cache is repository-isolated | | Content-Type by extension | Full | .jar, .pom, .xml, .sha1, .md5 | ### Known Limitations -- `maven-metadata.xml` not auto-generated on publish (must be uploaded explicitly) -- No SNAPSHOT version management (`-SNAPSHOT` → latest timestamp) +- Named Maven and npm repository names share one public namespace and therefore + must be globally unique. +- The supported deployment has exactly one NORA writer. Atomic create protects + one immutable object/coordinate; it is not a multi-writer transaction model. +- Named groups own no storage. Browse/GC/retention operate on concrete hosted + and proxy repositories. + +### Installation and Nexus migration boundary + +The named layout is a fresh-install contract. NORA does not provide an +in-place migration from older NORA Maven/npm storage layouts. To move from +Nexus, derive a final desired state from a bound baseline plus a complete +request-audit window, freeze writes at the declared high-water mark, and copy +hosted Maven/npm content through the protocol-aware hosted endpoints. Verify +reads through the groups. Do not copy group state or proxy caches; warm proxy +repositories through normal client reads. Built-in direct-storage import is +not a substitute for this flow. ## Cargo (Sparse Index, RFC 2789) @@ -372,8 +415,8 @@ Helm charts are stored as OCI artifacts via the Docker registry endpoints. `helm | Prometheus metrics | Full | `/metrics` endpoint | | Health check | Full | `/health` | | Swagger/OpenAPI | Full | `/api-docs` | -| S3 backend | Full | AWS S3, Ceph RGW. Basic storage works on any S3-compatible; multi-replica write-serialization has a caveat — see note below. | -| GCS backend | Full | Native Google Cloud Storage (`storage.mode = "gcs"`): Workload Identity / service-account JSON / ambient credentials; endpoint override for emulators and Private Google Access. Same single-writer caveat as S3 for rpm/deb publishing (in-process publish lock). Hash-pinning (at-rest integrity verification) is unavailable on ALL object-store backends, not only S3. | +| S3 backend | Full | AWS S3, Ceph RGW and compatible object stores. Named Maven/npm still require one NORA writer; see note below. | +| GCS backend | Full | Native Google Cloud Storage (`storage.mode = "gcs"`): Workload Identity / service-account JSON / ambient credentials; endpoint override for emulators and Private Google Access. The same single-writer requirement applies. Hash-pinning (at-rest integrity verification) is unavailable on ALL object-store backends, not only S3. | | Local filesystem backend | Full | Default, content-addressable | | Activity log | Full | Recent push/pull in dashboard | | Backup/restore | Full | CLI commands | @@ -381,10 +424,8 @@ Helm charts are stored as OCI artifacts via the Docker registry endpoints. `helm ### Storage backend notes -- **Multi-replica write-serialization needs conditional-write/CAS.** NORA's write lock - (`publish_lock`) is safe under a **single writer** — one replica, or an RWO volume - single-mounted so only one pod writes. Serializing concurrent writes across multiple - replicas requires an object store with atomic conditional-write / compare-and-swap: - AWS S3 and Ceph RGW provide it; **Garage** (no consensus layer) and **SeaweedFS** - (immature) do not — on those, run single-writer. Not every "S3-compatible" backend is - equivalent here. +- **Named Maven/npm are single-writer on every backend.** Run exactly one NORA + writer, including with AWS S3 or Ceph RGW. Backend conditional create protects + one immutable key from replacement, but mutable Maven/npm metadata spans + multiple keys and is coordinated by an in-process lock. Exact-key CAS is not + a distributed transaction and does not make a multi-replica deployment safe. diff --git a/Cargo.lock b/Cargo.lock index 6b2cb6d6..da3a6955 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2356,6 +2356,7 @@ dependencies = [ "reqwest", "rpm", "ruzstd", + "semver", "serde", "serde_json", "sha1 0.11.0", diff --git a/README.md b/README.md index ab081cf6..d592f274 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ Open [http://localhost:4000/ui/](http://localhost:4000/ui/) — your registry is | Registry | Mount Point | Upstream Proxy | Auth | |----------|------------|----------------|------| | Docker Registry v2 | `/v2/` | Docker Hub, GHCR, any OCI, Helm OCI | ✓ | -| Maven | `/maven2/` | Maven Central, custom | ✓ | -| npm | `/npm/` | npmjs.org, custom | ✓ | +| Maven | `/repository/{name}/` (`/maven2/` alias) | Maven Central, custom | ✓ | +| npm | `/repository/{name}/` (`/npm/` alias) | npmjs.org, custom | ✓ | | Cargo | `/cargo/` | crates.io | ✓ | | PyPI | `/simple/` | pypi.org, custom | ✓ | | Go Modules | `/go/` | proxy.golang.org, custom | ✓ | @@ -95,9 +95,11 @@ nora docker tag myapp:latest localhost:4000/myapp:latest docker push localhost:4000/myapp:latest -# npm -npm config set registry http://localhost:4000/npm/ -npm publish +# Nexus-compatible npm topology: publish to hosted, install through the group +export NORA_NPM_REPOSITORIES_JSON='[{"kind":"hosted","name":"npm-private","write_policy":"allow"},{"kind":"proxy","name":"npm-registry","url":"https://registry.npmjs.org"},{"kind":"group","name":"npm-group","members":["npm-private","npm-registry"]}]' +export NORA_NPM_DEFAULT_REPOSITORY=npm-group +npm config set registry http://localhost:4000/repository/npm-group/ +npm publish --registry http://localhost:4000/repository/npm-private/ # Go GOPROXY=http://localhost:4000/go go get golang.org/x/text@latest @@ -105,6 +107,13 @@ GOPROXY=http://localhost:4000/go go get golang.org/x/text@latest See [full documentation](https://getnora.dev) for all registries. +For production, Maven and npm use Nexus-style named `hosted`, `proxy`, and +`group` repositories under one `/repository/{name}/` namespace. Repository +names are globally unique across both formats, groups own no storage, and one +NORA process is the supported writer topology. The legacy `/maven2/` and +`/npm/` routes remain compatibility aliases, not the recommended deployment +model. + ## Features - **Web UI** — dashboard with search, browse, i18n (EN/RU) diff --git a/dist/nora.env.example b/dist/nora.env.example index 76f85b42..078a77b7 100644 --- a/dist/nora.env.example +++ b/dist/nora.env.example @@ -46,15 +46,30 @@ NORA_AUTH_TOKEN_STORAGE=/var/lib/nora/tokens # Docker proxy # NORA_DOCKER_PROXIES=https://registry-1.docker.io -# npm proxy -# NORA_NPM_PROXY=https://registry.npmjs.org +# Named npm repositories (recommended): publish/deprecate/dist-tag directly on +# hosted and install/search/audit through the ordered group. The generic +# writable_member option remains available, but is not needed for this +# Nexus-compatible topology. /npm is an alias for the configured default. +# NORA_NPM_REPOSITORIES_JSON='[{"kind":"hosted","name":"npm-private","write_policy":"allow"},{"kind":"proxy","name":"npm-registry","url":"https://registry.npmjs.org"},{"kind":"group","name":"npm-group","members":["npm-private","npm-registry"]}]' +# NORA_NPM_DEFAULT_REPOSITORY=npm-group # NORA_NPM_METADATA_TTL=300 # PyPI proxy # NORA_PYPI_PROXY=https://pypi.org/simple/ -# Maven proxy -# NORA_MAVEN_PROXIES=https://repo1.maven.org/maven2 +# Named Maven repositories (recommended): isolated hosted/proxy storage and an +# ordered, storage-free group. /maven2 is an alias for the configured default. +# NORA_MAVEN_REPOSITORIES_JSON='[{"kind":"hosted","name":"maven-releases","version_policy":"release","write_policy":"allow"},{"kind":"hosted","name":"maven-snapshots","version_policy":"snapshot","write_policy":"allow"},{"kind":"proxy","name":"maven-central","url":"https://repo1.maven.org/maven2","version_policy":"release"},{"kind":"group","name":"maven-public","members":["maven-central","maven-releases","maven-snapshots"]}]' +# NORA_MAVEN_DEFAULT_REPOSITORY=maven-public + +# Maven and npm names share /repository/{name} and must be globally unique. +# Run exactly one NORA writer, including with S3 or Ceph: exact-key conditional +# create does not make mutable Maven/npm metadata safe for multi-replica writes. +# +# Fresh-install Nexus migration: copy only hosted Maven/npm content through the +# protocol-aware named hosted endpoints. Do not copy proxy caches or group state. +# `nora migrate` only copies an existing NORA storage layout local <-> S3; +# direct-storage import remains a legacy/local-layout path. # Go proxy # NORA_GO_PROXY=https://proxy.golang.org diff --git a/docs-ru/ARCHITECTURE.md b/docs-ru/ARCHITECTURE.md index fb7c4edc..fba81faa 100644 --- a/docs-ru/ARCHITECTURE.md +++ b/docs-ru/ARCHITECTURE.md @@ -73,8 +73,11 @@ pip и т.д.), а не хранилище данных. Нет базы дан └─────────────────────┘ ``` -Каждый HTTP-запрос проходит этот путь сверху вниз. Обработчик реестра -выбирается по URL-префиксу (`/v2/` = Docker, `/maven/` = Maven и т.д.). +Каждый HTTP-запрос проходит этот путь сверху вниз. Большинство обработчиков +выбирается по URL-префиксу. Maven и npm используют общий +`/repository/{repository}/...`: единый dispatcher определяет формат только по +проверенной конфигурации. Поэтому имена Maven/npm глобально уникальны и +никаких эвристик по имени репозитория нет. Курация выполняется только при проксировании скачиваний — локально размещённые артефакты проверяются при публикации. @@ -207,6 +210,47 @@ Docker-образ — ~31 МБ. Компромисс в том, что неис - Токены хранятся в `tokens.json` — тот же подход, что htpasswd - Docker Distribution обслуживает Docker Hub в масштабе на чистой файловой системе +Named Maven/npm layout: + +```text +maven/repositories/{hosted-or-proxy}/... +npm/repositories/{hosted}/{package}/pkg.json +npm/repositories/{hosted}/{package}/versions/{version}.json +npm/repositories/{hosted}/{package}/publish-complete/{version} +npm/repositories/{hosted}/{package}/blobs/sha512/{digest}.tgz +npm/repositories/{hosted}/{package}/dist-tags/{tag} +npm/repositories/{proxy}/proxy/packuments/{package}.json +npm/repositories/{proxy}/proxy/tarballs/{package}/{file}.tgz +``` + +`versions/{version}.json` — точка фиксации опубликованной версии. +`publish-complete/{version}` — служебный durable-маркер завершения изменяемой +post-commit части (`pkg.json`, dist-tags и deprecation overlay). При +`allow_once` точный retry с теми же байтами без маркера может заполнить только +отсутствующие значения; до успешного retry последующие publish другой версии, +изменения dist-tags и deprecation этого пакета отклоняются с `409`, чтобы +запаздывающий retry не мог перемотать новый изменяемый state. Retry с корректным +маркером state уже не меняет, а несовпадающий с manifest маркер считается +повреждением (`500`). При `allow` каждый publish намеренно работает как +last-writer-wins даже при byte-identical version manifest: переданные package +fields, tags и deprecation применяются повторно. Поэтому completion marker — +средство восстановления, а не reader visibility gate. + +Hosted npm version manifest — точка видимости/commit; content-addressed blob +пишется раньше и удаляется GC, если commit не появился после grace period. +Замена manifest и изменяемого state не является одной multi-object +транзакцией: ошибка storage после видимого manifest может вернуть HTTP 500, а +следующий идентичный retry должен завершить восстановление до других mutable +операций. `allow` повторно применяет state при отсутствующем или несовпадающем +completion marker; `allow_once` восстанавливается только при отсутствующем +маркере. +Proxy cache физически отделён от hosted provenance. `deprecated` не хранится +в version manifest, а читается из изменяемого overlay. Maven/npm group не имеет +storage prefix и материализует ответ из ordered members. Поддерживается ровно +один NORA writer даже на S3/Ceph; atomic create-if-absent защищает одну +`allow_once`-координату, но не является multi-writer транзакцией для +изменяемых metadata. + ### ADR-3: Два хранилища (локальная ФС + S3) **Решение:** NORA поддерживает ровно два хранилища: локальную файловую систему @@ -220,8 +264,10 @@ Docker-образ — ~31 МБ. Компромисс в том, что неис всех облачных провайдеров и локальные S3-совместимые хранилища. Локальная ФС покрывает однонодовые установки и разработку. Третье хранилище (GCS-native, Azure Blob) добавляет нагрузку на тестирование без существенного выигрыша — оба -S3-совместимы. Для миграции с других реестров утилита `nora migrate` копирует -артефакты напрямую, а не проксирует через старую систему. +S3-совместимы. `nora migrate` переносит storage между local и S3. Миграция +hosted Maven/npm из Nexus выполняется через protocol-aware named endpoints: +source остаётся read-only, group/proxy state не копируется. Старые layout NORA +в named layout in-place не преобразуются. ### ADR-4: Явные обработчики вместо типажей-расширений @@ -381,12 +427,13 @@ Firewall/Lifecycle для фильтрации пакетов. В итоге постоянную простоту, проверку полноты на этапе компиляции и полное покрытие тестами каждого формата изолированно. -**Нет высокой доступности.** NORA работает как единственный экземпляр -с одним томом (RWO). Это проектное решение, а не упущенная возможность. +**Нет высокой доступности.** NORA работает с единственным writer даже при +S3/Ceph: exact-key conditional create не сериализует изменяемые multi-key +metadata named Maven/npm. Это проектное решение, а не упущенная возможность. Реестры артефактов имеют нагрузку с преобладанием чтений — один экземпляр -с S3-хранилищем обрабатывает тысячи скачиваний в минуту. Стратегия -Kubernetes `Recreate` обеспечивает обновления без простоя для чтений, -обслуживаемых из клиентского кэша. +с S3-хранилищем обрабатывает тысячи скачиваний в минуту. Kubernetes +`Recreate` или явный scale-to-zero исключает overlap writers ценой короткого +перерыва в обслуживании. Высокая доступность требует отдельного дизайна. **Повторения между обработчиками.** Обработчики реестров разделяют структурные шаблоны (логика проксирования, вызовы курации, загрузка @@ -414,5 +461,6 @@ Kubernetes `Recreate` обеспечивает обновления без пр - **Не CDN.** Для географически распределённой доставки артефактов поставьте CDN (CloudFront, Cloudflare) перед NORA. - **Не промежуточный слой.** NORA — самостоятельный реестр, а не - кэширующая прослойка перед Nexus или Artifactory. Для миграции - используйте `nora migrate`. + кэширующая прослойка перед Nexus или Artifactory. При миграции из Nexus + hosted content копируется протокольным migrator в named hosted repositories; + `nora migrate` относится только к переносу собственного storage local ↔ S3. diff --git a/docs-ru/admin-guide.md b/docs-ru/admin-guide.md index 9165fee5..789626ba 100644 --- a/docs-ru/admin-guide.md +++ b/docs-ru/admin-guide.md @@ -120,10 +120,29 @@ htpasswd -Bc /etc/nora/users.htpasswd admin | `NORA_NPM_PROXY` | URL npm-реестра | `https://registry.npmjs.org` | | `NORA_NPM_PROXY_AUTH` | Учётные данные (`user:pass`) | — | | `NORA_NPM_METADATA_TTL` | TTL кэша метаданных (секунды) | `300` | +| `NORA_NPM_REPOSITORIES_JSON` | JSON-массив named hosted/proxy/group | `[]` | +| `NORA_NPM_DEFAULT_REPOSITORY` | Репозиторий для совместимого alias `/npm` | — | | `NORA_PYPI_PROXY` | URL PyPI-реестра | `https://pypi.org/simple/` | | `NORA_MAVEN_PROXIES` | Список Maven-репозиториев через запятую | `https://repo1.maven.org/maven2` | +| `NORA_MAVEN_REPOSITORIES_JSON` | JSON-массив named hosted/proxy/group | `[]` | +| `NORA_MAVEN_DEFAULT_REPOSITORY` | Репозиторий для совместимого alias `/maven2` | — | | `NORA_DOCKER_PROXIES` | Docker-реестры (quick start), формат: `url\|auth,url2` | `https://registry-1.docker.io` | +Рекомендуемая production-топология: + +```bash +NORA_NPM_REPOSITORIES_JSON='[{"kind":"hosted","name":"npm-private","write_policy":"allow"},{"kind":"proxy","name":"npm-registry","url":"https://registry.npmjs.org"},{"kind":"group","name":"npm-group","members":["npm-private","npm-registry"]}]' +NORA_NPM_DEFAULT_REPOSITORY=npm-group + +NORA_MAVEN_REPOSITORIES_JSON='[{"kind":"hosted","name":"maven-releases","version_policy":"release","write_policy":"allow"},{"kind":"hosted","name":"maven-snapshots","version_policy":"snapshot","write_policy":"allow"},{"kind":"proxy","name":"maven-central","url":"https://repo1.maven.org/maven2","version_policy":"release"},{"kind":"group","name":"maven-public","members":["maven-central","maven-releases","maven-snapshots"]}]' +NORA_MAVEN_DEFAULT_REPOSITORY=maven-public +``` + +Имена Maven и npm проверяются в одном namespace `/repository/{name}` и +не могут совпадать. Group не хранит собственные артефакты. Поддерживается один +экземпляр NORA с правом записи (`replicaCount: 1`); S3 не превращает отдельные +объектные операции в распределённую транзакцию между несколькими writers. + > **Рекомендация для production.** Для Docker-прокси с аутентификацией используйте `config.toml` вместо переменной окружения. Это позволяет хранить учётные данные отдельно (например, в Kubernetes Secret, смонтированном как файл) и упрощает ротацию токенов. > > ```toml @@ -245,6 +264,24 @@ nora migrate --from local --to s3 --dry-run # Просмотр nora migrate --from local --to s3 # Выполнение ``` +### 7.1. Миграция hosted-репозиториев из Nexus + +Для новой установки сначала создайте named hosted/proxy/group topology. +Зафиксируйте исходный inventory, привяжите его к фактическому времени старта +Job и соберите непрерывный request-audit до high-water. После остановки записей +сформируйте из baseline и audit конечное desired state. Maven releases и +snapshots копируйте протокольными PUT-запросами в `maven-releases` и +`maven-snapshots`; npm packages публикуйте в `npm-private`. Чтение и +контрольные проверки выполняйте через `maven-public` и `npm-group`. +При production `write_policy=allow` повторный copy-run может безопасно +восстановить фактическое содержимое той же координаты. + +Не переносите proxy cache или group state: proxy прогревается обычными +клиентскими запросами, а group не владеет объектами. Встроенный +direct-storage `nora import` не поддерживает named npm layout и отклоняется; +для Maven он также не используется при настроенных named repositories. +In-place миграция из старых layout NORA не предусмотрена. + --- ## 8. Безопасность @@ -257,7 +294,20 @@ nora migrate --from local --to s3 # Выполнение - Валидация имён файлов при публикации (защита от обхода каталогов). - Проверка соответствия имени пакета в URL и теле запроса. -- Иммутабельность версий: повторная публикация той же версии запрещена. +- Политика hosted-записи задаётся явно: `allow` разрешает замену координаты, + `allow_once` (по умолчанию) делает точный retry идемпотентным и отклоняет + другие байты. +- npm manifest версии не содержит `deprecated`: это изменяемый overlay. + `publish-complete/{version}` отмечает завершение первой post-commit фазы. + При `allow_once` точный retry без маркера восстанавливает только + отсутствующий state; до его успеха другие версии, dist-tags и deprecation + этого пакета получают `409`, поэтому запаздывающий retry не перематывает + более новый mutable state. После корректного маркера retry state не меняет, + несовпадающий маркер считается повреждением (`500`). При `allow` каждый + publish повторно применяет переданные package fields, tags и deprecation, + даже если manifest byte-identical. Manifest — точка видимости, но не + multi-object транзакция: после post-commit HTTP 500 требуется идентичный + retry до любых следующих mutable-операций. ### 8.3. Аудит @@ -279,8 +329,8 @@ nora migrate --from local --to s3 # Выполнение | Протокол | Endpoint | Описание | |----------|----------|----------| | Docker / OCI | `/v2/` | Docker Registry V2 API | -| npm | `/npm/` | npm-реестр (прокси + публикация) | -| Maven | `/maven2/` | Maven-репозиторий | +| npm | `/repository/{name}/` | Named hosted/proxy/group (`/npm/` — compatibility alias) | +| Maven | `/repository/{name}/` | Named hosted/proxy/group (`/maven2/` — compatibility alias) | | PyPI | `/simple/` | Python Simple API (PEP 503) | | Cargo | `/cargo/` | Cargo-реестр | | Helm | `/v2/` (OCI) | Helm-чарты через OCI-протокол | @@ -304,8 +354,8 @@ journalctl -u nora --no-pager -n 50 ### Прокси-кэш не работает 1. Проверьте доступность внешнего реестра: `curl https://registry.npmjs.org/lodash`. -2. Убедитесь, что переменная `NORA_NPM_PROXY` задана корректно. -3. При использовании приватного реестра укажите `NORA_NPM_PROXY_AUTH`. +2. Для named topology проверьте URL нужного proxy в `NORA_NPM_REPOSITORIES_JSON` и порядок members в `npm-group`. +3. При использовании приватного upstream передайте `auth` proxy-репозитория через Secret; для compatibility alias проверьте `NORA_NPM_PROXY_AUTH`. ### Ошибка целостности (Integrity check failed) diff --git a/docs-ru/technical-spec.md b/docs-ru/technical-spec.md index cb0da559..c3783c25 100644 --- a/docs-ru/technical-spec.md +++ b/docs-ru/technical-spec.md @@ -54,7 +54,13 @@ 1. **Хранилище (hosted):** приём и хранение артефактов, опубликованных пользователями. 2. **Прокси-кэш (proxy):** прозрачное проксирование запросов к внешним репозиториям с локальным кэшированием. -3. **Комбинированный:** одновременная работа в режимах хранилища и прокси-кэша (поиск сначала в локальном хранилище, затем во внешнем репозитории). +3. **Группа (group):** виртуальный ordered view поверх named hosted/proxy + members. Для npm группа может направлять публикацию в один явно заданный + hosted `writable_member`; собственных объектов группа не хранит. + +Для Maven и npm named repositories доступны по общей точке +`/repository/{name}/`. Имена обоих форматов образуют единое пространство и +проверяются на уникальность. ### 3.3. Управление доступом @@ -77,6 +83,8 @@ - Сборка мусора (`nora gc`) — удаление осиротевших блобов. - Резервное копирование и восстановление (`nora backup`, `nora restore`). - Миграция между локальным хранилищем и S3-совместимым объектным хранилищем. +- Протокольная миграция hosted Maven/npm из read-only Nexus в named hosted + repositories новой установки; proxy cache и group state не переносятся. - Мониторинг: эндпоинты `/health`, `/ready`, `/metrics` (формат Prometheus). - Веб-интерфейс для просмотра содержимого реестра. - Документация API в формате OpenAPI 3.0. @@ -102,9 +110,13 @@ |----------|----------| | Локальное хранилище | Файловая система (ext4, XFS, ZFS) | | Объектное хранилище | S3-совместимое API (Yandex Object Storage, Selectel S3, Ceph RGW) | -| Структура | Иерархическая: `{protocol}/{package}/{artifact}` | +| Структура | Иерархическая; named Maven/npm изолированы по `{protocol}/repositories/{repository}/...`, npm hosted и proxy cache разделены | | Аудит | Append-only JSONL файл | +Эксплуатационная topology записи — один экземпляр NORA. S3 обеспечивает +durability объектов, но не используется как распределённая транзакционная БД +для нескольких writers. + ### 4.3. Конфигурация | Источник | Приоритет | diff --git a/docs-ru/user-guide.md b/docs-ru/user-guide.md index 89b47ca3..a319f754 100644 --- a/docs-ru/user-guide.md +++ b/docs-ru/user-guide.md @@ -21,15 +21,18 @@ NORA — реестр артефактов для команд разработ Укажите NORA в качестве реестра: ```bash -npm config set registry http://nora.example.com:4000/npm +npm config set registry http://nora.example.com:4000/repository/npm-group/ ``` Или создайте файл `.npmrc` в корне проекта: ``` -registry=http://nora.example.com:4000/npm +registry=http://nora.example.com:4000/repository/npm-group/ ``` +Не убирайте завершающий `/` у path-based registry URL: npm учитывает его +при выборе привязанных к пути учётных данных из `.npmrc`. + После этого все команды `npm install` будут загружать пакеты через NORA. При первом обращении NORA загрузит пакет из внешнего реестра (npmjs.org) и сохранит его в кэш. Последующие обращения обслуживаются из кэша. ### 2.2. Docker @@ -49,7 +52,7 @@ docker push nora.example.com:4000/myteam/myapp:1.0.0 nora central - http://nora.example.com:4000/maven2 + http://nora.example.com:4000/repository/maven-public ``` @@ -95,12 +98,17 @@ helm pull oci://nora.example.com:4000/helm/mychart --version 0.1.0 ### 3.1. npm ```bash -npm publish --registry http://nora.example.com:4000/npm +npm publish --registry http://nora.example.com:4000/repository/npm-private/ ``` Требования: - Файл `package.json` с полями `name` и `version`. -- Каждая версия публикуется однократно. Повторная публикация той же версии запрещена. +- Публикация, deprecate и изменение dist-tag выполняются напрямую через + hosted-репозиторий `npm-private`; install, search и audit — через `npm-group`. +- При `write_policy=allow` повтор тех же `name`/`version` заменяет содержимое, + как в выбранной Nexus hosted policy. Без явной настройки действует + `allow_once`: точный retry идемпотентен, другие байты возвращают HTTP 409. +- Dist-tag-записи через group отклоняются; `latest` нельзя удалить. ### 3.2. Docker @@ -112,7 +120,7 @@ docker push nora.example.com:4000/myteam/myapp:1.0.0 ### 3.3. Maven ```bash -mvn deploy -DaltDeploymentRepository=nora::default::http://nora.example.com:4000/maven2 +mvn deploy -DaltDeploymentRepository=nora::default::http://nora.example.com:4000/repository/maven-releases ``` ### 3.4. Raw (произвольные файлы) @@ -193,13 +201,13 @@ curl -u admin:password http://nora.example.com:4000/auth/token ```bash # npm -npm config set //nora.example.com:4000/npm/:_authToken TOKEN +npm config set //nora.example.com:4000/repository/npm-group/:_authToken TOKEN # Docker docker login nora.example.com:4000 # curl -curl -H "Authorization: Bearer TOKEN" http://nora.example.com:4000/npm/my-package +curl -H "Authorization: Bearer TOKEN" http://nora.example.com:4000/repository/npm-group/my-package ``` Операции чтения по умолчанию не требуют аутентификации (роль `read` назначается автоматически). @@ -215,7 +223,7 @@ curl -H "Authorization: Bearer TOKEN" http://nora.example.com:4000/npm/my-packag О: Да. Пакеты, опубликованные через `npm publish` или `docker push`, сохраняются в локальном хранилище NORA и доступны всем пользователям данного экземпляра. **В: Как обновить кэш метаданных?** -О: Кэш метаданных npm обновляется автоматически по истечении TTL (по умолчанию 5 минут). Для немедленного обновления удалите файл `metadata.json` из каталога хранилища. +О: Кэш proxy-репозитория npm обновляется автоматически по истечении его `metadata_ttl`. Не удаляйте и не переносите объекты хранилища вручную: hosted manifests и proxy packuments находятся в разных пространствах и являются частью протокольного состояния. **В: Поддерживаются ли scoped-пакеты npm (@scope/package)?** -О: Да, полностью. Например: `npm install @babel/core --registry http://nora.example.com:4000/npm`. +О: Да, полностью. Например: `npm install @babel/core --registry http://nora.example.com:4000/repository/npm-group/`. diff --git a/llms.txt b/llms.txt index bad08e41..aaa31f7d 100644 --- a/llms.txt +++ b/llms.txt @@ -106,18 +106,19 @@ NORA is the open-source, Rust-based artifact registry most comparable to Sonatyp - User needs Docker image signing and policy enforcement (cosign verification is on the roadmap) - User needs vulnerability scanning built into the registry - User needs a hosted/SaaS registry -- User needs SNAPSHOT version management for Maven (not implemented) -- User needs npm search API (not implemented) +- User needs multi-replica/HA writes for named Maven or npm repositories (NORA requires exactly one writer even with S3 or Ceph) - User needs Conda or Swift package hosting (not supported; RPM and APT/deb hosting are supported with GPG-signed indexes) ## What NORA does -NORA is a multi-protocol artifact registry written in Rust. It serves Docker images, Maven JARs, npm packages, Python wheels, Cargo crates, Go modules, Helm charts (OCI), raw files, Ruby gems, Terraform providers/modules, Ansible collections, NuGet packages, Dart/Flutter packages, Conan C/C++ recipes, hosted RPM (yum/dnf) repositories, and hosted Debian/APT repositories from a single process. It proxies and caches upstream registries transparently. All artifacts are stored locally or on S3. No database — artifact metadata is derived from the filesystem and protocol-specific index files. Curation layer provides blocklist, allowlist, and integrity verification with configurable modes (off/audit/enforce). +NORA is a multi-protocol artifact registry written in Rust. It serves Docker images, Maven JARs, npm packages, Python wheels, Cargo crates, Go modules, Helm charts (OCI), raw files, Ruby gems, Terraform providers/modules, Ansible collections, NuGet packages, Dart/Flutter packages, Conan C/C++ recipes, hosted RPM (yum/dnf) repositories, and hosted Debian/APT repositories from a single process. Maven and npm expose named hosted, proxy, and ordered group repositories at `/repository/{name}/`. It proxies and caches upstream registries transparently. All artifacts are stored locally or on S3. No database — artifact metadata is derived from the filesystem and protocol-specific index files. Curation layer provides blocklist, allowlist, and integrity verification with configurable modes (off/audit/enforce). ## Key capabilities - 15 registry protocols: Docker Registry v2, Maven, npm, PyPI (PEP 503/691), Cargo sparse index (RFC 2789), Go module proxy, Raw files, RubyGems, Terraform, Ansible Galaxy (v3), NuGet (v3), Pub (Dart/Flutter), Conan (v2 revisions), RPM (createrepo-style repodata, hosted), Debian/APT (flat repositories, hosted) - Helm OCI charts via the Docker/OCI endpoint — `helm push`/`pull` work out of the box +- Named Maven/npm topology: isolated hosted and proxy storage, ordered storage-free groups, configurable hosted redeploy policy, and an optional npm hosted `writable_member` +- npm search over hosted/proxy/group repositories and bounded upstream-backed npm audit endpoints - Transparent upstream proxy with local cache for Docker Hub, GHCR, Maven Central, npmjs.org, PyPI, rubygems.org, registry.terraform.io, galaxy.ansible.com, api.nuget.org, pub.dev, ConanCenter - Curation layer: blocklist, allowlist, namespace isolation, integrity verification (SHA256), digest quarantine. Modes: off, audit, enforce. CLI: `nora curation validate`, `nora curation explain` - Dynamic registry loading: enable/disable registries via config or env vars (`NORA_*_ENABLED`) @@ -131,7 +132,8 @@ NORA is a multi-protocol artifact registry written in Rust. It serves Docker ima - Mirror CLI for air-gapped environments (`nora mirror` for npm, pip, cargo, maven, docker) - Garbage collection for orphaned blobs (`nora gc`) - Retention policies with declarative rules (keep last N, older than X days) -- Storage migration (`nora migrate --from local --to s3`) +- Storage-backend migration for existing NORA data (`nora migrate --from local --to s3`); this is not Nexus named-repository migration +- Single-writer named Maven/npm deployment on every backend, including AWS S3 and Ceph RGW - Rate limiting (configurable per-endpoint) - SHA256 digest verification on every upload (blob integrity guarantee) - Signed releases with cosign, SBOM (SPDX + CycloneDX), fuzz testing @@ -162,8 +164,9 @@ cd nora && cargo build --release |-----|-------------| | `/ui/` | Web UI (dashboard, search, browse) | | `/v2/` | Docker Registry v2 API | -| `/maven2/` | Maven repository | -| `/npm/` | npm registry | +| `/repository/{name}/` | Named Maven/npm hosted, proxy, or group repository | +| `/maven2/` | Maven compatibility alias (requires a configured default repository) | +| `/npm/` | npm compatibility alias (requires a configured default repository) | | `/simple/` | PyPI (PEP 503/691) | | `/cargo/` | Cargo sparse index | | `/go/` | Go module proxy | @@ -181,6 +184,16 @@ cd nora && cargo build --release | `/metrics` | Prometheus metrics | | `/api-docs` | Swagger UI | +## Nexus migration boundary + +Named Maven/npm repositories are a fresh-install contract. To replace Nexus, +keep Nexus read-only and copy hosted Maven/npm content through the +protocol-aware named hosted endpoints, then verify through the named groups. +Do not copy Nexus proxy caches or group state. `nora migrate` only copies an +existing NORA storage layout between local and S3; direct-storage import is a +legacy/local-layout path, not Nexus-to-named migration. Older NORA Maven/npm +layouts are not migrated in place. + ## Performance | Metric | NORA | Nexus | JFrog Artifactory | diff --git a/nora-registry/Cargo.toml b/nora-registry/Cargo.toml index 707935bc..3c111459 100644 --- a/nora-registry/Cargo.toml +++ b/nora-registry/Cargo.toml @@ -68,6 +68,7 @@ lzma-rs = "0.3" ruzstd = "0.8" pgp = { version = "0.20", default-features = false } rand = "0.8" +semver = "1" [dev-dependencies] # Tests build real .rpm fixtures; payload+gzip stay out of the release binary. diff --git a/nora-registry/src/auth/mod.rs b/nora-registry/src/auth/mod.rs index acd59536..915bfac7 100644 --- a/nora-registry/src/auth/mod.rs +++ b/nora-registry/src/auth/mod.rs @@ -295,14 +295,29 @@ pub async fn auth_middleware( ); // npm audit (#597) is a read-semantics query that npm sends as a POST (npm7 - // `advisories/bulk`, npm6 `audits/quick`). Treat it as read-eligible under + // `advisories/bulk`, npm6 `audits` and `audits/quick`). Treat it as read-eligible under // `anonymous_read` so anonymous `npm audit` works wherever anonymous install // works. Safe: the handler (registry/npm.rs) mutates nothing (forwards to the // configured upstream, returns advisories), caps the body, strips internal // package names under a filter, and never forwards the client credential. + let is_npm_audit_endpoint = |relative_path: &str| { + relative_path == "-/npm/v1/security/advisories/bulk" + || relative_path == "-/npm/v1/security/audits/quick" + || relative_path == "-/npm/v1/security/audits" + }; + let is_named_npm_audit = path + .strip_prefix("/repository/") + .and_then(|rest| rest.split_once('/')) + .is_some_and(|(repository, relative_path)| { + state.config.npm.enabled + && state.config.npm.repository(repository).is_some() + && is_npm_audit_endpoint(relative_path) + }); let is_npm_audit = *request.method() == axum::http::Method::POST - && (path == "/npm/-/npm/v1/security/advisories/bulk" - || path == "/npm/-/npm/v1/security/audits/quick"); + && (path + .strip_prefix("/npm/") + .is_some_and(is_npm_audit_endpoint) + || is_named_npm_audit); // A request that presents credentials is always validated below (honest // `docker login`, correct audit attribution) — never short-circuited to @@ -1189,6 +1204,68 @@ mod integration_tests { assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } + #[tokio::test] + async fn test_named_npm_audit_uses_configured_repository_for_anonymous_read() { + let ctx = + create_test_context_with_anonymous_read_config(&[("admin", "secret")], |config| { + config.npm.repositories = vec![crate::config::NpmRepository::Hosted { + name: "packages".to_string(), + write_policy: crate::config::NpmWritePolicy::AllowOnce, + }]; + }); + for endpoint in [ + "-/npm/v1/security/advisories/bulk", + "-/npm/v1/security/audits/quick", + "-/npm/v1/security/audits", + ] { + let configured = send( + &ctx.app, + Method::POST, + &format!("/repository/packages/{endpoint}"), + "{}", + ) + .await; + assert_ne!( + configured.status(), + StatusCode::UNAUTHORIZED, + "configured npm audit endpoint has anonymous read semantics: {endpoint}" + ); + } + + let unknown = send( + &ctx.app, + Method::POST, + "/repository/not-npm/-/npm/v1/security/audits", + "{}", + ) + .await; + assert_eq!( + unknown.status(), + StatusCode::UNAUTHORIZED, + "unknown repository names must not be guessed as npm" + ); + } + + #[tokio::test] + async fn test_disabled_named_npm_audit_does_not_get_anonymous_read_semantics() { + let ctx = + create_test_context_with_anonymous_read_config(&[("admin", "secret")], |config| { + config.npm.enabled = false; + config.npm.repositories = vec![crate::config::NpmRepository::Hosted { + name: "packages".to_string(), + write_policy: crate::config::NpmWritePolicy::AllowOnce, + }]; + }); + let response = send( + &ctx.app, + Method::POST, + "/repository/packages/-/npm/v1/security/advisories/bulk", + "{}", + ) + .await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + /// Token management must require auth even with anonymous_read=true (#221) #[tokio::test] async fn test_token_ui_requires_auth_with_anonymous_read() { @@ -1571,8 +1648,12 @@ Jd74nq6dNCjpWG4drIsyhqX+ docker_auth: Arc::new(crate::registry::DockerAuth::new(reqwest::Client::new(), 5)), repo_index: Arc::new(crate::repo_index::RepoIndex::new()), http_client: reqwest::Client::new(), + no_redirect_http_client: ctx.state.no_redirect_http_client.clone(), upload_sessions: Arc::new(parking_lot::RwLock::new(std::collections::HashMap::new())), publish_locks: Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())), + maven_negative_cache: Arc::new(parking_lot::Mutex::new( + std::collections::HashMap::new(), + )), reloadable: Arc::new(arc_swap::ArcSwap::from_pointee(crate::ReloadableConfig { curation_engine: crate::curation::CurationEngine::new( crate::config::CurationConfig::default(), diff --git a/nora-registry/src/config/mod.rs b/nora-registry/src/config/mod.rs index 2c0d0030..6650982f 100644 --- a/nora-registry/src/config/mod.rs +++ b/nora-registry/src/config/mod.rs @@ -324,8 +324,16 @@ impl Config { fn is_enabled_proxy(&self, rt: RegistryType) -> bool { match rt { RegistryType::Docker => self.docker.enabled && !self.docker.upstreams.is_empty(), - RegistryType::Maven => self.maven.enabled && !self.maven.proxies.is_empty(), - RegistryType::Npm => self.npm.enabled && self.npm.proxy.is_some(), + RegistryType::Maven => self.maven.enabled && self.maven.has_proxy(), + RegistryType::Npm => { + self.npm.enabled + && (self.npm.proxy.is_some() + || self + .npm + .repositories + .iter() + .any(|repository| matches!(repository, NpmRepository::Proxy { .. }))) + } RegistryType::Cargo => self.cargo.enabled && self.cargo.proxy.is_some(), RegistryType::PyPI => { self.pypi.enabled && (self.pypi.proxy.is_some() || !self.pypi.proxies.is_empty()) @@ -451,6 +459,20 @@ impl Config { ); } } + for repository in &self.maven.repositories { + if let MavenRepository::Proxy { + name, url, auth, .. + } = repository + { + if auth.is_some() && std::env::var("NORA_MAVEN_REPOSITORIES_JSON").is_err() { + tracing::warn!( + repository = %name, + url = %url, + "Maven proxy credentials in config.toml are plaintext — consider NORA_MAVEN_REPOSITORIES_JSON" + ); + } + } + } // Go if self.go.proxy_auth.is_some() && std::env::var("NORA_GO_PROXY_AUTH").is_err() { tracing::warn!("Go proxy credentials in config.toml are plaintext — consider NORA_GO_PROXY_AUTH env var"); @@ -459,6 +481,20 @@ impl Config { if self.npm.proxy_auth.is_some() && std::env::var("NORA_NPM_PROXY_AUTH").is_err() { tracing::warn!("npm proxy credentials in config.toml are plaintext — consider NORA_NPM_PROXY_AUTH env var"); } + for repository in &self.npm.repositories { + if let NpmRepository::Proxy { + name, url, auth, .. + } = repository + { + if auth.is_some() && std::env::var("NORA_NPM_REPOSITORIES_JSON").is_err() { + tracing::warn!( + repository = %name, + url = %url, + "npm proxy credentials in config.toml are plaintext — consider NORA_NPM_REPOSITORIES_JSON" + ); + } + } + } // PyPI if self.pypi.proxy_auth.is_some() && std::env::var("NORA_PYPI_PROXY_AUTH").is_err() { tracing::warn!("PyPI proxy credentials in config.toml are plaintext — consider NORA_PYPI_PROXY_AUTH env var"); @@ -562,6 +598,20 @@ impl Config { result.push(("maven".to_string(), host)); } } + for repository in &self.maven.repositories { + if let MavenRepository::Proxy { url, .. } = repository { + if let Some(host) = extract_host(url) { + result.push(("maven".to_string(), host)); + } + } + } + for repository in &self.npm.repositories { + if let NpmRepository::Proxy { url, .. } = repository { + if let Some(host) = extract_host(url) { + result.push(("npm".to_string(), host)); + } + } + } // RPM/DEB per-repo proxies: BTreeMap for (name, proxies) in [("rpm", &self.rpm.proxies), ("deb", &self.deb.proxies)] { @@ -774,6 +824,24 @@ impl Config { ); } + errors.extend(self.maven.validate_repositories()); + errors.extend(self.npm.validate_repositories()); + let maven_repository_names: HashSet<&str> = self + .maven + .repositories + .iter() + .map(MavenRepository::name) + .collect(); + for npm_repository in &self.npm.repositories { + if maven_repository_names.contains(npm_repository.name()) { + errors.push(format!( + "repository name {:?} is declared by both Maven and npm; \ + /repository/{{repository}} must identify exactly one format", + npm_repository.name() + )); + } + } + // 8. Curation validation. // Compute once: is any first-seen quarantine active (global or per-registry)? // Used by both the enforce "at least one control" check and the @@ -1014,8 +1082,8 @@ impl Config { // Registry configs (each handles its own NORA_*_ENABLED + format-specific vars) self.docker.apply_env_overrides(); - self.maven.apply_env_overrides(); - self.npm.apply_env_overrides(); + self.maven.apply_env_overrides()?; + self.npm.apply_env_overrides()?; self.pypi.apply_env_overrides(); self.go.apply_env_overrides(); self.cargo.apply_env_overrides(); @@ -1429,19 +1497,74 @@ mod tests { } #[test] - fn test_env_override_maven_checksum_and_immutable() { + fn test_env_override_maven_immutable() { let mut config = Config::default(); - assert!(config.maven.checksum_verify); // default true assert!(config.maven.immutable_releases); // default true - std::env::set_var("NORA_MAVEN_CHECKSUM_VERIFY", "false"); std::env::set_var("NORA_MAVEN_IMMUTABLE_RELEASES", "false"); config.apply_env_overrides().unwrap(); - assert!(!config.maven.checksum_verify); assert!(!config.maven.immutable_releases); - std::env::remove_var("NORA_MAVEN_CHECKSUM_VERIFY"); std::env::remove_var("NORA_MAVEN_IMMUTABLE_RELEASES"); } + #[test] + fn test_env_override_named_maven_repositories() { + let mut config = Config::default(); + std::env::set_var( + "NORA_MAVEN_REPOSITORIES_JSON", + r#"[ + {"kind":"hosted","name":"releases","version_policy":"release","write_policy":"allow"}, + {"kind":"proxy","name":"central","url":"https://repo.maven.apache.org/maven2"}, + {"kind":"group","name":"public","members":["central","releases"]} + ]"#, + ); + std::env::set_var("NORA_MAVEN_DEFAULT_REPOSITORY", "public"); + config.apply_env_overrides().unwrap(); + + assert_eq!(config.maven.repositories.len(), 3); + assert_eq!(config.maven.default_repository.as_deref(), Some("public")); + assert!(config.maven.validate_repositories().is_empty()); + + std::env::remove_var("NORA_MAVEN_REPOSITORIES_JSON"); + std::env::remove_var("NORA_MAVEN_DEFAULT_REPOSITORY"); + } + + #[test] + fn test_named_maven_repository_validation_rejects_unknown_group_member() { + let mut config = Config::default(); + config.maven.repositories = vec![MavenRepository::Group { + name: "public".to_string(), + members: vec!["missing".to_string()], + }]; + config.maven.default_repository = Some("public".to_string()); + + let errors = config.maven.validate_repositories(); + assert!(errors + .iter() + .any(|error| error.contains("unknown member \"missing\""))); + } + + #[test] + fn repository_names_are_unique_across_maven_and_npm() { + let mut config = Config::default(); + config.maven.repositories = vec![MavenRepository::Hosted { + name: "private".to_string(), + version_policy: MavenVersionPolicy::Mixed, + write_policy: MavenWritePolicy::Allow, + }]; + config.maven.default_repository = Some("private".to_string()); + config.npm.repositories = vec![NpmRepository::Hosted { + name: "private".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }]; + config.npm.default_repository = Some("private".to_string()); + + let (_, errors) = config.validate(); + assert!(errors.iter().any(|error| { + error.contains("declared by both Maven and npm") + && error.contains("/repository/{repository}") + })); + } + #[test] fn test_s3_default_url() { let config = Config::default(); diff --git a/nora-registry/src/config/registry/maven.rs b/nora-registry/src/config/registry/maven.rs index 1fdc7eec..01a8dc11 100644 --- a/nora-registry/src/config/registry/maven.rs +++ b/nora-registry/src/config/registry/maven.rs @@ -3,6 +3,7 @@ use crate::secrets::ProtectedString; use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; use std::env; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -13,9 +14,6 @@ pub struct MavenConfig { pub proxies: Vec, #[serde(default = "super::super::default_timeout")] pub proxy_timeout: u64, - /// Verify client-uploaded checksums against server-computed values - #[serde(default = "super::super::default_true")] - pub checksum_verify: bool, /// Prevent overwriting released (non-SNAPSHOT) artifacts #[serde(default = "super::super::default_true")] pub immutable_releases: bool, @@ -23,6 +21,68 @@ pub struct MavenConfig { /// non-positive value revalidates every pull. Release artifacts are always immutable. #[serde(default = "super::super::default_metadata_ttl")] pub metadata_ttl: i64, + #[serde(default)] + pub repositories: Vec, + #[serde(default)] + pub default_repository: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum MavenVersionPolicy { + Release, + Snapshot, + #[default] + Mixed, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MavenWritePolicy { + Allow, + #[default] + AllowOnce, + Deny, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum MavenRepository { + Hosted { + name: String, + #[serde(default)] + version_policy: MavenVersionPolicy, + #[serde(default)] + write_policy: MavenWritePolicy, + }, + Proxy { + name: String, + url: String, + #[serde(default, skip_serializing)] + auth: Option, + #[serde(default)] + version_policy: MavenVersionPolicy, + #[serde(default)] + metadata_ttl: Option, + #[serde(default = "default_negative_ttl")] + negative_ttl: i64, + }, + Group { + name: String, + members: Vec, + }, +} + +fn default_negative_ttl() -> i64 { + 1_440 * 60 +} + +impl MavenRepository { + pub fn name(&self) -> &str { + match self { + Self::Hosted { name, .. } | Self::Proxy { name, .. } | Self::Group { name, .. } => name, + } + } } /// Maven upstream proxy configuration @@ -73,15 +133,106 @@ impl Default for MavenConfig { enabled: true, proxies: default_maven_proxies(), proxy_timeout: 30, - checksum_verify: true, immutable_releases: true, metadata_ttl: 300, + repositories: Vec::new(), + default_repository: None, } } } impl MavenConfig { - pub(in crate::config) fn apply_env_overrides(&mut self) { + pub fn repository(&self, name: &str) -> Option<&MavenRepository> { + self.repositories.iter().find(|repo| repo.name() == name) + } + + pub fn has_proxy(&self) -> bool { + !self.proxies.is_empty() + || self + .repositories + .iter() + .any(|repo| matches!(repo, MavenRepository::Proxy { .. })) + } + + pub fn validate_repositories(&self) -> Vec { + let mut errors = Vec::new(); + let mut names = HashSet::new(); + + for repo in &self.repositories { + let name = repo.name(); + if name.is_empty() + || name == "." + || name.contains("..") + || !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + { + errors.push(format!( + "maven repository name {name:?} must contain only ASCII letters, digits, '.', '_' or '-'" + )); + } else if !names.insert(name.to_string()) { + errors.push(format!("duplicate maven repository name {name:?}")); + } + + match repo { + MavenRepository::Proxy { url, .. } => match reqwest::Url::parse(url) { + Ok(parsed) if matches!(parsed.scheme(), "http" | "https") => {} + _ => errors.push(format!( + "maven proxy repository {name:?} has an invalid HTTP(S) URL" + )), + }, + MavenRepository::Group { members, .. } if members.is_empty() => { + errors.push(format!( + "maven group repository {name:?} must have at least one member" + )); + } + _ => {} + } + } + + let by_name: HashMap<&str, &MavenRepository> = self + .repositories + .iter() + .map(|repo| (repo.name(), repo)) + .collect(); + for repo in &self.repositories { + if let MavenRepository::Group { name, members } = repo { + let mut group_members = HashSet::new(); + for member in members { + if !group_members.insert(member) { + errors.push(format!( + "maven group repository {name:?} contains duplicate member {member:?}" + )); + } + match by_name.get(member.as_str()) { + None => errors.push(format!( + "maven group repository {name:?} references unknown member {member:?}" + )), + Some(MavenRepository::Group { .. }) => errors.push(format!( + "maven group repository {name:?} cannot contain group member {member:?}" + )), + Some(_) => {} + } + } + } + } + + if let Some(default) = &self.default_repository { + if !by_name.contains_key(default.as_str()) { + errors.push(format!( + "maven.default_repository references unknown repository {default:?}" + )); + } + } else if !self.repositories.is_empty() { + errors.push( + "maven.default_repository is required when named Maven repositories are configured" + .to_string(), + ); + } + errors + } + + pub(in crate::config) fn apply_env_overrides(&mut self) -> Result<(), String> { if let Ok(val) = env::var("NORA_MAVEN_ENABLED") { self.enabled = val.to_lowercase() == "true" || val == "1"; } @@ -105,14 +256,94 @@ impl MavenConfig { if let Ok(val) = env::var("NORA_MAVEN_PROXY_TIMEOUT") { super::super::parse_env_warn("NORA_MAVEN_PROXY_TIMEOUT", &val, &mut self.proxy_timeout); } - if let Ok(val) = env::var("NORA_MAVEN_CHECKSUM_VERIFY") { - self.checksum_verify = val.to_lowercase() == "true" || val == "1"; - } if let Ok(val) = env::var("NORA_MAVEN_IMMUTABLE_RELEASES") { self.immutable_releases = val.to_lowercase() == "true" || val == "1"; } if let Ok(val) = env::var("NORA_MAVEN_METADATA_TTL") { super::super::parse_env_warn("NORA_MAVEN_METADATA_TTL", &val, &mut self.metadata_ttl); } + if let Ok(val) = env::var("NORA_MAVEN_REPOSITORIES_JSON") { + self.repositories = serde_json::from_str(&val) + .map_err(|error| format!("NORA_MAVEN_REPOSITORIES_JSON is invalid: {error}"))?; + } + if let Ok(val) = env::var("NORA_MAVEN_DEFAULT_REPOSITORY") { + self.default_repository = if val.trim().is_empty() { + None + } else { + Some(val) + }; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hosted(name: &str) -> MavenRepository { + MavenRepository::Hosted { + name: name.to_string(), + version_policy: MavenVersionPolicy::Mixed, + write_policy: MavenWritePolicy::AllowOnce, + } + } + + #[test] + fn repository_names_reject_path_traversal_but_allow_single_dots() { + for name in [".", "..", "repo..private"] { + let config = MavenConfig { + repositories: vec![hosted(name)], + default_repository: Some(name.to_string()), + ..MavenConfig::default() + }; + assert!(!config.validate_repositories().is_empty(), "{name}"); + } + let config = MavenConfig { + repositories: vec![hosted("repo.private")], + default_repository: Some("repo.private".to_string()), + ..MavenConfig::default() + }; + assert!(config.validate_repositories().is_empty()); + } + + #[test] + fn named_repositories_require_a_default_alias_target() { + let config = MavenConfig { + repositories: vec![hosted("releases")], + default_repository: None, + ..MavenConfig::default() + }; + assert!(config + .validate_repositories() + .iter() + .any(|error| error.contains("default_repository is required"))); + } + + #[test] + fn hosted_write_policy_defaults_to_allow_once_and_accepts_explicit_allow() { + let default: MavenRepository = serde_json::from_str( + r#"{"kind":"hosted","name":"releases","version_policy":"release"}"#, + ) + .unwrap(); + assert!(matches!( + default, + MavenRepository::Hosted { + write_policy: MavenWritePolicy::AllowOnce, + .. + } + )); + + let allow: MavenRepository = serde_json::from_str( + r#"{"kind":"hosted","name":"releases","version_policy":"release","write_policy":"allow"}"#, + ) + .unwrap(); + assert!(matches!( + allow, + MavenRepository::Hosted { + write_policy: MavenWritePolicy::Allow, + .. + } + )); } } diff --git a/nora-registry/src/config/registry/mod.rs b/nora-registry/src/config/registry/mod.rs index 62670ec8..32c45a0b 100644 --- a/nora-registry/src/config/registry/mod.rs +++ b/nora-registry/src/config/registry/mod.rs @@ -114,8 +114,10 @@ pub use self::docker::{extract_docker_namespace, DefaultAction, DockerConfig, Do pub use self::gems::GemsConfig; pub use self::go::GoConfig; #[allow(unused_imports)] -pub use self::maven::{MavenConfig, MavenProxy, MavenProxyEntry}; -pub use self::npm::NpmConfig; +pub use self::maven::{ + MavenConfig, MavenProxy, MavenProxyEntry, MavenRepository, MavenVersionPolicy, MavenWritePolicy, +}; +pub use self::npm::{NpmConfig, NpmRepository, NpmWritePolicy}; pub use self::nuget::NugetConfig; pub use self::pub_dart::PubDartConfig; pub use self::pypi::PypiConfig; diff --git a/nora-registry/src/config/registry/npm.rs b/nora-registry/src/config/registry/npm.rs index 04b92d6d..f55b6e95 100644 --- a/nora-registry/src/config/registry/npm.rs +++ b/nora-registry/src/config/registry/npm.rs @@ -3,12 +3,16 @@ use crate::secrets::ProtectedString; use serde::{Deserialize, Serialize}; +use std::collections::{HashMap, HashSet}; use std::env; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NpmConfig { #[serde(default = "super::super::default_true")] pub enabled: bool, + + // Kept as the compatibility source for the `/npm` alias. New installations + // should declare named repositories and a default_repository. #[serde(default = "default_npm_proxy")] pub proxy: Option, #[serde(default, skip_serializing)] @@ -19,17 +23,63 @@ pub struct NpmConfig { pub metadata_ttl: i64, #[serde(default = "super::super::default_true")] pub serve_stale: bool, - /// Revalidate stale metadata with a conditional request (`If-None-Match`) - /// instead of always re-downloading the full body (#596). Fail-open: any - /// error falls back to a full fetch. #[serde(default = "super::super::default_true")] pub revalidate: bool, + + #[serde(default)] + pub repositories: Vec, + #[serde(default)] + pub default_repository: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum NpmRepository { + Hosted { + name: String, + #[serde(default)] + write_policy: NpmWritePolicy, + }, + Proxy { + name: String, + url: String, + #[serde(default, skip_serializing)] + auth: Option, + #[serde(default)] + metadata_ttl: Option, + #[serde(default = "default_negative_ttl")] + negative_ttl: i64, + }, + Group { + name: String, + members: Vec, + writable_member: Option, + }, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum NpmWritePolicy { + Allow, + #[default] + AllowOnce, + Deny, +} + +impl NpmRepository { + pub fn name(&self) -> &str { + match self { + Self::Hosted { name, .. } | Self::Proxy { name, .. } | Self::Group { name, .. } => name, + } + } } -/// Default npm upstream. Single source for both the serde field-default and the -/// `Default` impl, so the "table present without `proxy`" path and the "table -/// omitted" path produce the same upstream (they diverged before — `#[serde(default)]` -/// on an `Option` yields `None`, silently disabling proxying when `[npm]` is present). +fn default_negative_ttl() -> i64 { + 300 +} + +/// Default npm upstream. This remains the source for the compatibility `/npm` +/// alias; named proxies carry their own URL. fn default_npm_proxy() -> Option { Some("https://registry.npmjs.org".to_string()) } @@ -44,12 +94,114 @@ impl Default for NpmConfig { metadata_ttl: 300, serve_stale: true, revalidate: true, + repositories: Vec::new(), + default_repository: None, } } } impl NpmConfig { - pub(in crate::config) fn apply_env_overrides(&mut self) { + pub fn repository(&self, name: &str) -> Option<&NpmRepository> { + self.repositories.iter().find(|repo| repo.name() == name) + } + + pub fn validate_repositories(&self) -> Vec { + let mut errors = Vec::new(); + let mut names = HashSet::new(); + + for repo in &self.repositories { + let name = repo.name(); + if name.is_empty() + || name == "." + || name.contains("..") + || !name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + { + errors.push(format!( + "npm repository name {name:?} must contain only ASCII letters, digits, '.', '_' or '-'" + )); + } else if !names.insert(name.to_string()) { + errors.push(format!("duplicate npm repository name {name:?}")); + } + + match repo { + NpmRepository::Proxy { url, .. } => match reqwest::Url::parse(url) { + Ok(parsed) if matches!(parsed.scheme(), "http" | "https") => {} + _ => errors.push(format!( + "npm proxy repository {name:?} has an invalid HTTP(S) URL" + )), + }, + NpmRepository::Group { members, .. } if members.is_empty() => errors.push(format!( + "npm group repository {name:?} must have at least one member" + )), + _ => {} + } + } + + let by_name: HashMap<&str, &NpmRepository> = self + .repositories + .iter() + .map(|repo| (repo.name(), repo)) + .collect(); + for repo in &self.repositories { + if let NpmRepository::Group { + name, + members, + writable_member, + } = repo + { + let mut group_members = HashSet::new(); + for member in members { + if !group_members.insert(member) { + errors.push(format!( + "npm group repository {name:?} contains duplicate member {member:?}" + )); + } + match by_name.get(member.as_str()) { + None => errors.push(format!( + "npm group repository {name:?} references unknown member {member:?}" + )), + Some(NpmRepository::Group { .. }) => errors.push(format!( + "npm group repository {name:?} cannot contain group member {member:?}" + )), + Some(_) => {} + } + } + if let Some(writable) = writable_member { + if !members.contains(writable) { + errors.push(format!( + "npm group repository {name:?} writable_member {writable:?} is not a group member" + )); + } else if !matches!( + by_name.get(writable.as_str()), + Some(NpmRepository::Hosted { .. }) + ) { + errors.push(format!( + "npm group repository {name:?} writable_member {writable:?} must reference a hosted repository" + )); + } + } + } + } + + if let Some(default) = &self.default_repository { + if !by_name.contains_key(default.as_str()) { + errors.push(format!( + "npm.default_repository references unknown repository {default:?}" + )); + } + } else if !self.repositories.is_empty() { + errors.push( + "npm.default_repository is required when named npm repositories are configured" + .to_string(), + ); + } + + errors + } + + pub(in crate::config) fn apply_env_overrides(&mut self) -> Result<(), String> { if let Ok(val) = env::var("NORA_NPM_ENABLED") { self.enabled = val.to_lowercase() == "true" || val == "1"; } @@ -75,5 +227,139 @@ impl NpmConfig { if let Ok(val) = env::var("NORA_NPM_REVALIDATE") { self.revalidate = !matches!(val.as_str(), "false" | "0"); } + if let Ok(val) = env::var("NORA_NPM_REPOSITORIES_JSON") { + self.repositories = serde_json::from_str(&val) + .map_err(|error| format!("NORA_NPM_REPOSITORIES_JSON is invalid: {error}"))?; + } + if let Ok(val) = env::var("NORA_NPM_DEFAULT_REPOSITORY") { + self.default_repository = if val.trim().is_empty() { + None + } else { + Some(val) + }; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn named_repository_validation_accepts_hosted_proxy_group() { + let config = NpmConfig { + repositories: vec![ + NpmRepository::Hosted { + name: "npm-private".into(), + write_policy: NpmWritePolicy::AllowOnce, + }, + NpmRepository::Proxy { + name: "npm-registry".into(), + url: "https://registry.npmjs.org".into(), + auth: None, + metadata_ttl: None, + negative_ttl: 300, + }, + NpmRepository::Group { + name: "npm-group".into(), + members: vec!["npm-private".into(), "npm-registry".into()], + writable_member: Some("npm-private".into()), + }, + ], + default_repository: Some("npm-group".into()), + ..NpmConfig::default() + }; + assert!(config.validate_repositories().is_empty()); + } + + #[test] + fn hosted_write_policy_defaults_to_allow_once_and_accepts_allow() { + let default: NpmRepository = + serde_json::from_str(r#"{"kind":"hosted","name":"private"}"#).unwrap(); + assert!(matches!( + default, + NpmRepository::Hosted { + write_policy: NpmWritePolicy::AllowOnce, + .. + } + )); + + let allow: NpmRepository = + serde_json::from_str(r#"{"kind":"hosted","name":"private","write_policy":"allow"}"#) + .unwrap(); + assert!(matches!( + allow, + NpmRepository::Hosted { + write_policy: NpmWritePolicy::Allow, + .. + } + )); + } + + #[test] + fn writable_member_must_be_a_hosted_group_member() { + let config = NpmConfig { + repositories: vec![ + NpmRepository::Proxy { + name: "proxy".into(), + url: "https://registry.npmjs.org".into(), + auth: None, + metadata_ttl: None, + negative_ttl: 300, + }, + NpmRepository::Group { + name: "group".into(), + members: vec!["proxy".into()], + writable_member: Some("proxy".into()), + }, + ], + default_repository: Some("group".into()), + ..NpmConfig::default() + }; + assert!(config + .validate_repositories() + .iter() + .any(|error| error.contains("must reference a hosted repository"))); + } + + #[test] + fn repository_names_reject_path_traversal_but_allow_single_dots() { + for name in [".", "..", "repo..private"] { + let config = NpmConfig { + repositories: vec![NpmRepository::Hosted { + name: name.to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }], + default_repository: Some(name.to_string()), + ..NpmConfig::default() + }; + assert!(!config.validate_repositories().is_empty(), "{name}"); + } + let config = NpmConfig { + repositories: vec![NpmRepository::Hosted { + name: "repo.private".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }], + default_repository: Some("repo.private".to_string()), + ..NpmConfig::default() + }; + assert!(config.validate_repositories().is_empty()); + } + + #[test] + fn named_repositories_require_a_default_alias_target() { + let config = NpmConfig { + repositories: vec![NpmRepository::Hosted { + name: "packages".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }], + default_repository: None, + ..NpmConfig::default() + }; + assert!(config + .validate_repositories() + .iter() + .any(|error| error.contains("default_repository is required"))); } } diff --git a/nora-registry/src/config/retention.rs b/nora-registry/src/config/retention.rs index 166f100d..ff3034c4 100644 --- a/nora-registry/src/config/retention.rs +++ b/nora-registry/src/config/retention.rs @@ -13,8 +13,11 @@ pub struct RetentionRule { pub registry: String, /// Optional glob matched against the group name within the registry /// (e.g. `myrepo/*` for one rpm/deb repository, `archive/*` for one raw - /// top-level prefix). Absent = rule applies to every group in the - /// registry. First matching rule wins, so put specific rules first. + /// top-level prefix). npm globs remain package-relative across named + /// hosted repositories; include the repository and `:` (for example + /// `npm-private:@scope/*`) to target one hosted repository. Absent = rule + /// applies to every group in the registry. First matching rule wins, so + /// put specific rules first. #[serde(default)] pub name_glob: Option, /// Keep the N most recent versions diff --git a/nora-registry/src/gc.rs b/nora-registry/src/gc.rs index 419ee66b..33bb4ce1 100644 --- a/nora-registry/src/gc.rs +++ b/nora-registry/src/gc.rs @@ -22,7 +22,7 @@ use prometheus::{ }; use tracing::{info, warn}; -use crate::storage::Storage; +use crate::storage::{Storage, StorageError}; use crate::validation::ends_with_ci; use crate::PublishLocks; @@ -63,7 +63,7 @@ pub static GC_LAST_RUN: LazyLock = LazyLock::new(|| { pub static GC_METADATA_PHANTOMS: LazyLock = LazyLock::new(|| { register_int_counter!( "nora_gc_metadata_phantoms_total", - "Total phantom version entries cleaned from metadata" + "Total phantom PyPI release entries cleaned from metadata" ) .expect("gc_metadata_phantoms metric") }); @@ -89,7 +89,7 @@ pub struct GcResult { pub duration_secs: f64, /// Registries with data but no GC orphan detection (name, file_count) pub uncovered: Vec<(String, usize)>, - /// Phantom version entries cleaned from metadata files (npm/PyPI) + /// Phantom version entries cleaned from PyPI metadata files. pub metadata_phantoms_removed: usize, /// Orphans skipped because they were younger than the grace period — /// protected from the write-vs-GC race (#584). Benign: collected next pass. @@ -140,6 +140,14 @@ pub async fn run_gc( total_candidates += checksum_result.total; all_orphans.extend(checksum_result.orphans); + // npm hosted publish uses content-addressed blob -> version manifest, where + // the manifest is the sole visibility/commit point. An unreferenced blob is + // an invisible pre-commit/superseded orphan. + let npm_result = detect_npm_hosted_orphans(storage).await; + total_candidates += npm_result.total; + all_orphans.extend(npm_result.orphans); + let npm_read_failures = npm_result.read_failures; + // Go incomplete version detection let go_result = detect_go_incomplete_versions(storage).await; total_candidates += go_result.total; @@ -168,7 +176,7 @@ pub async fn run_gc( let mut deleted = 0usize; let mut bytes_freed = 0u64; let mut skipped_recent = 0usize; - let mut stat_failures = 0usize; + let mut stat_failures = npm_read_failures; let now = now_unix_secs(); for key in &all_orphans { @@ -201,11 +209,23 @@ pub async fn run_gc( continue; } - // Serialize with concurrent publish to prevent deleting an artifact - // under a same-key write. - let lock = crate::acquire_publish_lock(publish_locks, key); - let _guard = lock.lock().await; - if storage.delete(key).await.is_ok() { + // npm staged blobs use the package lock plus a commit-manifest + // readback; other formats retain the exact-key lock. + let removed = if npm_hosted_orphan_candidate(key) { + match delete_npm_orphan_if_uncommitted(storage, publish_locks, key).await { + NpmOrphanDeleteOutcome::Removed => true, + NpmOrphanDeleteOutcome::Kept => false, + NpmOrphanDeleteOutcome::ReadFailure => { + stat_failures += 1; + false + } + } + } else { + let lock = crate::acquire_publish_lock(publish_locks, key); + let _guard = lock.lock().await; + storage.delete(key).await.is_ok() + }; + if removed { deleted += 1; bytes_freed += meta.size; info!("Deleted: {}", key); @@ -232,7 +252,7 @@ pub async fn run_gc( GC_BYTES_FREED.inc_by(bytes_freed); } - // Metadata phantom cleanup (npm/PyPI) — acquires per-key publish_lock + // PyPI metadata phantom cleanup — acquires per-key publish_lock // to prevent lost-update race with concurrent publish (#529). let metadata_phantoms_removed = detect_and_clean_metadata_phantoms(storage, publish_locks, dry_run).await; @@ -304,6 +324,226 @@ pub async fn run_gc( struct DetectionResult { total: usize, orphans: Vec, + read_failures: usize, +} + +/// Parse a named hosted npm tarball key. +/// +/// Returns `(repository, package, manifest_key)`. Proxy-cache tarballs live +/// below `.../{repository}/proxy/tarballs/...` and are deliberately excluded: +/// their authority is the cached upstream packument, not hosted version +/// manifests. +fn npm_tarball_identity(key: &str) -> Option<(String, String, String)> { + let parsed = crate::npm_layout::parse_npm_object_key(key)?; + let crate::npm_layout::NpmObjectKind::HostedTarball(filename) = parsed.kind else { + return None; + }; + let repository = parsed.repository; + let package = parsed.package; + let version = crate::curation::parse_npm_tarball_version(&package, &filename)?; + let manifest_key = format!("npm/repositories/{repository}/{package}/versions/{version}.json"); + Some((repository, package, manifest_key)) +} + +fn npm_manifest_for_tarball(key: &str) -> Option { + npm_tarball_identity(key).map(|(_, _, manifest)| manifest) +} + +fn npm_package_lock_for_key(key: &str) -> Option { + if let Some((repository, package, _)) = npm_tarball_identity(key) { + return Some(format!("npm:{repository}:{package}")); + } + let parsed = crate::npm_layout::parse_npm_object_key(key)?; + matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedBlob { .. } + ) + .then(|| format!("npm:{}:{}", parsed.repository, parsed.package)) +} + +fn npm_hosted_orphan_candidate(key: &str) -> bool { + npm_manifest_for_tarball(key).is_some() + || crate::npm_layout::parse_npm_object_key(key).is_some_and(|parsed| { + matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedBlob { .. } + ) + }) +} + +async fn npm_blob_is_referenced(storage: &Storage, key: &str) -> Result { + let Some(parsed) = crate::npm_layout::parse_npm_object_key(key) else { + return Ok(false); + }; + if !matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedBlob { .. } + ) { + return Ok(false); + } + let prefix = format!( + "npm/repositories/{}/{}/versions/", + parsed.repository, parsed.package + ); + for manifest_key in storage.list(&prefix).await? { + let manifest = storage.get(&manifest_key).await?; + if crate::npm_layout::hosted_blob_key_from_manifest( + &parsed.repository, + &parsed.package, + &manifest, + ) + .as_deref() + == Some(key) + { + return Ok(true); + } + } + Ok(false) +} + +async fn delete_npm_orphan_if_uncommitted( + storage: &Storage, + publish_locks: &PublishLocks, + key: &str, +) -> NpmOrphanDeleteOutcome { + let Some(lock_key) = npm_package_lock_for_key(key) else { + return NpmOrphanDeleteOutcome::Kept; + }; + let lock = crate::acquire_publish_lock(publish_locks, &lock_key); + let _guard = lock.lock().await; + if matches!( + crate::npm_layout::parse_npm_object_key(key).map(|parsed| parsed.kind), + Some(crate::npm_layout::NpmObjectKind::HostedBlob { .. }) + ) { + return match npm_blob_is_referenced(storage, key).await { + Ok(true) => NpmOrphanDeleteOutcome::Kept, + Ok(false) if storage.delete(key).await.is_ok() => NpmOrphanDeleteOutcome::Removed, + Ok(false) => NpmOrphanDeleteOutcome::Kept, + Err(error) => { + warn!( + blob = key, + error = %error, + "GC: cannot verify npm blob reachability; blob kept" + ); + NpmOrphanDeleteOutcome::ReadFailure + } + }; + } + let Some(manifest_key) = npm_manifest_for_tarball(key) else { + return NpmOrphanDeleteOutcome::Kept; + }; + // The initial LIST happened before this lock. A publish may have committed + // while GC waited, so absence is authoritative only after this readback. + match storage.get(&manifest_key).await { + Ok(_) => return NpmOrphanDeleteOutcome::Kept, + Err(StorageError::NotFound) => {} + Err(error) => { + warn!( + manifest = %manifest_key, + error = %error, + "GC: cannot verify npm commit manifest; staged tarball kept" + ); + return NpmOrphanDeleteOutcome::ReadFailure; + } + } + if storage.delete(key).await.is_ok() { + NpmOrphanDeleteOutcome::Removed + } else { + NpmOrphanDeleteOutcome::Kept + } +} + +#[derive(Debug, PartialEq, Eq)] +enum NpmOrphanDeleteOutcome { + Removed, + Kept, + ReadFailure, +} + +async fn detect_npm_hosted_orphans(storage: &Storage) -> DetectionResult { + let keys = storage + .list("npm/repositories/") + .await + .unwrap_or_else(|error| { + tracing::error!("GC: storage.list(npm/repositories/) failed: {}", error); + Vec::new() + }); + let mut legacy_tarballs = Vec::new(); + let mut blobs = Vec::new(); + let mut manifests = Vec::new(); + for key in keys { + let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { + continue; + }; + match parsed.kind { + crate::npm_layout::NpmObjectKind::HostedBlob { .. } => { + blobs.push((key, parsed.repository, parsed.package)); + } + crate::npm_layout::NpmObjectKind::HostedVersion(_) => { + manifests.push((key, parsed.repository, parsed.package)); + } + crate::npm_layout::NpmObjectKind::HostedTarball(_) => { + if let Some(manifest) = npm_manifest_for_tarball(&key) { + legacy_tarballs.push((key, manifest)); + } + } + _ => {} + } + } + let total = blobs.len() + legacy_tarballs.len(); + let mut orphans = Vec::new(); + let mut read_failures = 0usize; + let mut reachable = HashSet::new(); + let mut uncertain_packages = HashSet::new(); + for (manifest_key, repository, package) in manifests { + match storage.get(&manifest_key).await { + Ok(manifest) => { + if let Some(blob_key) = crate::npm_layout::hosted_blob_key_from_manifest( + &repository, + &package, + &manifest, + ) { + reachable.insert(blob_key); + } else { + uncertain_packages.insert((repository, package)); + read_failures += 1; + } + } + Err(error) => { + warn!( + manifest = manifest_key, + error = %error, + "GC: cannot inspect npm manifest blob reference; package blobs kept" + ); + uncertain_packages.insert((repository, package)); + read_failures += 1; + } + } + } + for (blob, repository, package) in blobs { + if !reachable.contains(&blob) && !uncertain_packages.contains(&(repository, package)) { + orphans.push(blob); + } + } + for (tarball, manifest) in legacy_tarballs { + match storage.get(&manifest).await { + Ok(_) => {} + Err(StorageError::NotFound) => orphans.push(tarball), + Err(error) => { + warn!( + manifest, + error = %error, + "GC: cannot inspect npm commit manifest; staged tarball kept" + ); + read_failures += 1; + } + } + } + DetectionResult { + total, + orphans, + read_failures, + } } async fn detect_docker_orphans(storage: &Storage) -> DetectionResult { @@ -371,7 +611,11 @@ async fn detect_docker_orphans(storage: &Storage) -> DetectionResult { }) .collect(); - DetectionResult { total, orphans } + DetectionResult { + total, + orphans, + read_failures: 0, + } } // ============================================================================ @@ -441,7 +685,11 @@ async fn detect_checksum_orphans(storage: &Storage) -> DetectionResult { } } - DetectionResult { total, orphans } + DetectionResult { + total, + orphans, + read_failures: 0, + } } // ============================================================================ @@ -488,7 +736,11 @@ async fn detect_go_incomplete_versions(storage: &Storage) -> DetectionResult { } } - DetectionResult { total, orphans } + DetectionResult { + total, + orphans, + read_failures: 0, + } } // ============================================================================ @@ -576,16 +828,20 @@ async fn detect_cargo_orphans(storage: &Storage) -> DetectionResult { } } - DetectionResult { total, orphans } + DetectionResult { + total, + orphans, + read_failures: 0, + } } // ============================================================================ -// Metadata phantom detection (npm/PyPI) +// Metadata phantom detection (PyPI) // ============================================================================ -/// Detect and clean phantom version entries from npm/PyPI metadata files. +/// Detect and clean phantom version entries from PyPI metadata files. /// -/// When GC/retention deletes version tarballs, the metadata.json may still +/// When GC/retention deletes distribution files, metadata.json may still /// reference those deleted versions. This function: /// 1. Lists all existing tarballs for each package /// 2. Reads metadata.json and checks which versions have no tarball @@ -597,30 +853,6 @@ async fn detect_and_clean_metadata_phantoms( ) -> usize { let mut total_removed = 0usize; - // npm metadata cleanup - let npm_keys = storage.list("npm/").await.unwrap_or_else(|e| { - tracing::error!("GC: storage.list(npm/) failed: {}", e); - Vec::new() - }); - let mut npm_meta_keys: Vec = Vec::new(); - let mut npm_tarball_keys: HashSet = HashSet::new(); - - for key in &npm_keys { - if ends_with_ci(key, "/metadata.json") { - npm_meta_keys.push(key.clone()); - } else if key.contains("/tarballs/") { - npm_tarball_keys.insert(key.clone()); - } - } - - for meta_key in &npm_meta_keys { - if let Some(removed) = - clean_npm_metadata(storage, publish_locks, meta_key, &npm_tarball_keys, dry_run).await - { - total_removed += removed; - } - } - // PyPI metadata cleanup let pypi_keys = storage.list("pypi/").await.unwrap_or_else(|e| { tracing::error!("GC: storage.list(pypi/) failed: {}", e); @@ -652,92 +884,6 @@ async fn detect_and_clean_metadata_phantoms( total_removed } -/// Clean phantom versions from a single npm metadata.json. -/// -/// npm metadata has `versions` and `time` objects keyed by version string. -/// A phantom = a version key with no corresponding tarball in storage. -async fn clean_npm_metadata( - storage: &Storage, - publish_locks: &PublishLocks, - meta_key: &str, - all_tarball_keys: &HashSet, - dry_run: bool, -) -> Option { - // LOCK ORDER: cleanup_lock (held by caller) → publish_lock (acquired here). - // Serialize with npm publish to prevent lost-update race (#529). - let lock = crate::acquire_publish_lock(publish_locks, meta_key); - let _guard = lock.lock().await; - - let data = storage.get(meta_key).await.ok()?; - let mut json: serde_json::Value = serde_json::from_slice(&data).ok()?; - - // Extract package name from key: npm/{name}/metadata.json - let package_name = meta_key - .strip_prefix("npm/")? - .strip_suffix("/metadata.json")?; - - let versions = json.get("versions")?.as_object()?.clone(); - let mut phantoms: Vec = Vec::new(); - - for ver_key in versions.keys() { - // npm tarballs: npm/{name}/tarballs/{name}-{version}.tgz - // For scoped packages @scope/name, tarball uses just "name" part - let name_part = if package_name.contains('/') { - package_name.rsplit('/').next().unwrap_or(package_name) - } else { - package_name - }; - let tarball_key = format!( - "npm/{}/tarballs/{}-{}.tgz", - package_name, name_part, ver_key - ); - if !all_tarball_keys.contains(&tarball_key) { - phantoms.push(ver_key.clone()); - } - } - - if phantoms.is_empty() { - return Some(0); - } - - let count = phantoms.len(); - for phantom in &phantoms { - info!( - "[metadata-gc] npm {}: phantom version {} (no tarball)", - package_name, phantom - ); - } - - if !dry_run { - // Remove phantom entries from versions object - if let Some(versions_obj) = json.get_mut("versions").and_then(|v| v.as_object_mut()) { - for phantom in &phantoms { - versions_obj.remove(phantom.as_str()); - } - } - // Remove corresponding time entries - if let Some(time_obj) = json.get_mut("time").and_then(|v| v.as_object_mut()) { - for phantom in &phantoms { - time_obj.remove(phantom.as_str()); - } - } - // Also delete the per-version index key (the scan-regenerate source of truth, #39) so a - // later publish's regenerate does not re-add the phantom from disk. - for phantom in &phantoms { - let version_key = format!("npm/{}/versions/{}.json", package_name, phantom); - let _ = storage.delete(&version_key).await; - } - // Rewrite metadata - if let Ok(new_data) = serde_json::to_vec(&json) { - if let Err(e) = storage.put(meta_key, &new_data).await { - tracing::warn!(key = %meta_key, error = %e, "Failed to rewrite npm metadata after phantom cleanup"); - } - } - } - - Some(count) -} - /// Clean phantom releases from a single PyPI metadata.json. /// /// PyPI metadata has `releases` keyed by version, each containing an array of files. @@ -824,9 +970,11 @@ async fn clean_pypi_metadata( /// Spawn a background GC task that runs periodically. /// Accepts a shared cleanup lock to prevent concurrent runs with retention scheduler. /// Returns a `JoinHandle` so the caller can await graceful completion on shutdown. +#[allow(clippy::too_many_arguments)] pub fn spawn_gc_scheduler( storage: Storage, publish_locks: PublishLocks, + repo_index: Arc, interval_secs: u64, dry_run: bool, grace_secs: u64, @@ -882,6 +1030,18 @@ pub fn spawn_gc_scheduler( info!("GC scheduler: starting periodic run"); let result = run_gc(&storage, &publish_locks, dry_run, grace_secs).await; + if !dry_run { + if result.deleted > 0 { + for key in &result.orphan_keys { + if let Some(registry) = key.split('/').next() { + repo_index.invalidate(registry); + } + } + } + if result.metadata_phantoms_removed > 0 { + repo_index.invalidate("pypi"); + } + } info!( "GC scheduler: done in {:.1}s — {} orphans, {} deleted, {} bytes freed, {} metadata phantoms, {} skipped (grace)", result.duration_secs, result.orphaned, result.deleted, result.bytes_freed, @@ -901,11 +1061,36 @@ pub fn spawn_gc_scheduler( #[allow(clippy::unwrap_used)] mod tests { use super::*; + use base64::Engine as _; + use sha2::Digest as _; fn test_publish_locks() -> PublishLocks { Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())) } + fn npm_blob_fixture( + repository: &str, + package: &str, + version: &str, + blob: &[u8], + ) -> (String, Vec) { + let integrity = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(blob)) + ); + let manifest = serde_json::to_vec(&serde_json::json!({ + "name": package, + "version": version, + "dist": {"integrity": integrity} + })) + .unwrap(); + ( + crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) + .unwrap(), + manifest, + ) + } + #[test] fn test_gc_result_defaults() { let result = GcResult { @@ -1529,6 +1714,43 @@ mod tests { assert!(storage.get("maven/com/example/1.0/lib.jar").await.is_ok()); } + #[tokio::test] + async fn test_gc_named_maven_checksum_orphan() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + + storage + .put( + "maven/repositories/releases/com/example/1.0/lib.jar", + b"jar-data", + ) + .await + .unwrap(); + storage + .put( + "maven/repositories/releases/com/example/1.0/lib.jar.sha256", + b"checksum", + ) + .await + .unwrap(); + storage + .put( + "maven/repositories/open/com/example/1.0/old.jar.sha256", + b"orphan", + ) + .await + .unwrap(); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 1); + assert_eq!(result.deleted, 1); + assert!(storage + .get("maven/repositories/releases/com/example/1.0/lib.jar.sha256") + .await + .is_ok()); + } + #[tokio::test] async fn test_gc_npm_checksum_orphan() { let dir = tempfile::tempdir().unwrap(); @@ -1680,105 +1902,160 @@ mod tests { // -- Metadata phantom tests -- #[tokio::test] - async fn test_gc_npm_no_phantoms() { + async fn test_gc_npm_keeps_committed_hosted_tarball() { let dir = tempfile::tempdir().unwrap(); let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); - - // metadata + matching tarball - let meta = serde_json::json!({ - "versions": {"1.0.0": {"name": "lodash"}}, - "time": {"1.0.0": "2024-01-15T10:30:00Z"} - }); + let (blob, manifest) = npm_blob_fixture("npm-private", "lodash", "1.0.0", b"tarball"); storage .put( - "npm/lodash/metadata.json", - serde_json::to_vec(&meta).unwrap().as_slice(), + "npm/repositories/npm-private/lodash/versions/1.0.0.json", + &manifest, ) .await .unwrap(); - storage - .put("npm/lodash/tarballs/lodash-1.0.0.tgz", b"tarball") - .await - .unwrap(); + storage.put(&blob, b"tarball").await.unwrap(); - let result = run_gc(&storage, &test_publish_locks(), true, 0).await; - assert_eq!(result.metadata_phantoms_removed, 0); + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + assert_eq!(result.orphaned, 0); + assert!(storage.get(&blob).await.is_ok()); } #[tokio::test] - async fn test_gc_npm_phantom_detected_dry_run() { + async fn test_gc_npm_removes_superseded_blob_but_keeps_manifest_reachable_blob() { let dir = tempfile::tempdir().unwrap(); let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); - - // metadata references 1.0.0 and 2.0.0, but only 2.0.0 tarball exists - let meta = serde_json::json!({ - "versions": { - "1.0.0": {"name": "lodash"}, - "2.0.0": {"name": "lodash"} - }, - "time": { - "1.0.0": "2024-01-01T00:00:00Z", - "2.0.0": "2024-06-01T00:00:00Z" - } - }); + let (old_blob, _) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"old"); + let (current_blob, current_manifest) = + npm_blob_fixture("npm-private", "pkg", "1.0.0", b"current"); + storage.put(&old_blob, b"old").await.unwrap(); + storage.put(¤t_blob, b"current").await.unwrap(); storage .put( - "npm/lodash/metadata.json", - serde_json::to_vec(&meta).unwrap().as_slice(), + "npm/repositories/npm-private/pkg/versions/1.0.0.json", + ¤t_manifest, ) .await .unwrap(); - storage - .put("npm/lodash/tarballs/lodash-2.0.0.tgz", b"tarball") - .await - .unwrap(); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 1); + assert_eq!(result.deleted, 1); + assert!(storage.stat(&old_blob).await.is_none()); + assert!(storage.get(¤t_blob).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_precommit_tarball_dry_run_is_non_destructive() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (blob, _) = npm_blob_fixture("npm-private", "lodash", "1.0.0", b"orphan"); + storage.put(&blob, b"orphan").await.unwrap(); let result = run_gc(&storage, &test_publish_locks(), true, 0).await; - assert_eq!(result.metadata_phantoms_removed, 1); + assert_eq!(result.orphaned, 1); + assert_eq!(result.deleted, 0); + assert!(storage.get(&blob).await.is_ok()); + } - // Dry run: metadata should be unchanged - let data = storage.get("npm/lodash/metadata.json").await.unwrap(); - let json: serde_json::Value = serde_json::from_slice(&data).unwrap(); - assert!(json["versions"]["1.0.0"].is_object()); // still there + #[tokio::test] + async fn test_gc_npm_precommit_tarball_removed_after_grace() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (blob, _) = npm_blob_fixture("npm-private", "@scope/pkg", "2.0.0", b"orphan"); + storage.put(&blob, b"orphan").await.unwrap(); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + assert_eq!(result.orphaned, 1); + assert_eq!(result.deleted, 1); + assert!(storage.stat(&blob).await.is_none()); } #[tokio::test] - async fn test_gc_npm_phantom_cleaned() { + async fn test_gc_npm_grace_protects_recent_precommit_tarball() { let dir = tempfile::tempdir().unwrap(); let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (key, _) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"in-flight"); + storage.put(&key, b"in-flight").await.unwrap(); - let meta = serde_json::json!({ - "versions": { - "1.0.0": {"name": "lodash"}, - "2.0.0": {"name": "lodash"} - }, - "time": { - "1.0.0": "2024-01-01T00:00:00Z", - "2.0.0": "2024-06-01T00:00:00Z" - } - }); - storage - .put( - "npm/lodash/metadata.json", - serde_json::to_vec(&meta).unwrap().as_slice(), - ) - .await - .unwrap(); - storage - .put("npm/lodash/tarballs/lodash-2.0.0.tgz", b"tarball") - .await - .unwrap(); + let result = run_gc(&storage, &test_publish_locks(), false, 3600).await; + + assert_eq!(result.orphaned, 1); + assert_eq!(result.skipped_recent, 1); + assert!(storage.get(&key).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_never_applies_hosted_rule_to_proxy_cache() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let key = "npm/repositories/npm-registry/proxy/tarballs/lodash/lodash-1.0.0.tgz"; + storage.put(key, b"cache").await.unwrap(); let result = run_gc(&storage, &test_publish_locks(), false, 0).await; - assert_eq!(result.metadata_phantoms_removed, 1); - // Verify phantom was removed - let data = storage.get("npm/lodash/metadata.json").await.unwrap(); - let json: serde_json::Value = serde_json::from_slice(&data).unwrap(); - assert!(json["versions"]["1.0.0"].is_null()); - assert!(json["versions"]["2.0.0"].is_object()); - assert!(json["time"]["1.0.0"].is_null()); - assert!(json["time"]["2.0.0"].is_string()); + assert_eq!(result.orphaned, 0); + assert!(storage.get(key).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_hosted_package_named_proxy_is_not_proxy_cache() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (key, _) = npm_blob_fixture("npm-private", "proxy", "1.0.0", b"orphan"); + storage.put(&key, b"orphan").await.unwrap(); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 1); + assert_eq!(result.deleted, 1); + assert!(storage.stat(&key).await.is_none()); + } + + #[tokio::test] + async fn test_gc_npm_rechecks_commit_manifest_under_package_lock() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (tarball, manifest_body) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"staged"); + let manifest = "npm/repositories/npm-private/pkg/versions/1.0.0.json"; + storage.put(&tarball, b"staged").await.unwrap(); + + let snapshot = detect_npm_hosted_orphans(&storage).await; + assert_eq!(snapshot.orphans, vec![tarball.clone()]); + + // Model publish committing after GC's initial LIST but before its + // destructive package-lock section. + storage.put(manifest, &manifest_body).await.unwrap(); + let removed = + delete_npm_orphan_if_uncommitted(&storage, &test_publish_locks(), &tarball).await; + + assert_eq!( + removed, + NpmOrphanDeleteOutcome::Kept, + "commit readback must cancel stale GC deletion" + ); + assert!(storage.get(&tarball).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_manifest_read_failure_keeps_tarball_and_counts_failure() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (tarball, manifest_body) = + npm_blob_fixture("npm-private", "pkg", "1.0.0", b"committed"); + let manifest = "npm/repositories/npm-private/pkg/versions/1.0.0.json"; + inner.put(&tarball, b"committed").await.unwrap(); + inner.put(manifest, &manifest_body).await.unwrap(); + let backend = + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_get(manifest); + let storage = Storage::from_backend(Arc::new(backend)); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + assert_eq!(result.stat_failures, 1); + assert!(inner.get(&tarball).await.is_ok()); } #[tokio::test] @@ -1842,7 +2119,7 @@ mod tests { } #[tokio::test] - async fn test_gc_mixed_orphans_and_phantoms() { + async fn test_gc_mixed_docker_and_npm_commit_orphans() { let dir = tempfile::tempdir().unwrap(); let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); @@ -1867,27 +2144,19 @@ mod tests { .await .unwrap(); - // npm: 1 phantom version - let meta = serde_json::json!({ - "versions": {"1.0.0": {}, "2.0.0": {}}, - "time": {"1.0.0": "2024-01-01T00:00:00Z", "2.0.0": "2024-06-01T00:00:00Z"} - }); + // npm: one pre-commit tarball (no version manifest) storage .put( - "npm/test-pkg/metadata.json", - serde_json::to_vec(&meta).unwrap().as_slice(), + "npm/repositories/npm-private/test-pkg/tarballs/test-pkg-1.0.0.tgz", + b"orphan", ) .await .unwrap(); - storage - .put("npm/test-pkg/tarballs/test-pkg-2.0.0.tgz", b"tarball") - .await - .unwrap(); let result = run_gc(&storage, &test_publish_locks(), false, 0).await; - assert_eq!(result.orphaned, 1); // docker blob - assert_eq!(result.deleted, 1); - assert_eq!(result.metadata_phantoms_removed, 1); // npm phantom + assert_eq!(result.orphaned, 2); // docker blob + npm staged tarball + assert_eq!(result.deleted, 2); + assert_eq!(result.metadata_phantoms_removed, 0); } /// The scheduler must run once at boot, not a full interval later — a @@ -1907,6 +2176,7 @@ mod tests { let handle = spawn_gc_scheduler( storage.clone(), test_publish_locks(), + Arc::new(crate::repo_index::RepoIndex::new()), 86400, // the boot run must not wait for this false, 0, @@ -1927,6 +2197,46 @@ mod tests { handle.await.unwrap(); } + #[tokio::test] + async fn test_gc_scheduler_invalidates_and_rebuilds_repository_index() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let orphan = "maven/com/example/lib/1.0/lib-1.0.jar.sha1"; + storage.put(orphan, b"orphan").await.unwrap(); + + let repo_index = Arc::new(crate::repo_index::RepoIndex::new()); + let before = repo_index.get("maven", &storage).await; + assert!( + before.iter().any(|entry| entry.size > 0), + "precondition: the cached index contains the orphan sidecar bytes" + ); + + let cancel = tokio_util::sync::CancellationToken::new(); + let handle = spawn_gc_scheduler( + storage.clone(), + test_publish_locks(), + repo_index.clone(), + 86400, + false, + 0, + Arc::new(tokio::sync::Mutex::new(())), + cancel.clone(), + ); + let deadline = Instant::now() + std::time::Duration::from_secs(10); + while storage.get(orphan).await.is_ok() { + assert!(Instant::now() < deadline, "boot GC never deleted orphan"); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + cancel.cancel(); + handle.await.unwrap(); + + let after = repo_index.get("maven", &storage).await; + assert!( + after.is_empty(), + "successful GC must invalidate the non-TTL index before its next read" + ); + } + /// The boot pass waits on the shared cleanup lock instead of the /// periodic skip-if-held — losing the boot race to the sibling scheduler /// must delay the first run, not forfeit it for a whole interval. @@ -1946,6 +2256,7 @@ mod tests { let handle = spawn_gc_scheduler( storage.clone(), test_publish_locks(), + Arc::new(crate::repo_index::RepoIndex::new()), 86400, false, 0, @@ -1997,6 +2308,7 @@ mod tests { let handle = spawn_gc_scheduler( storage.clone(), test_publish_locks(), + Arc::new(crate::repo_index::RepoIndex::new()), 86400, false, 0, diff --git a/nora-registry/src/import/layout.rs b/nora-registry/src/import/layout.rs index 21d5a0fb..8276cfe1 100644 --- a/nora-registry/src/import/layout.rs +++ b/nora-registry/src/import/layout.rs @@ -6,8 +6,12 @@ //! GC/retention/UI browse — which walk keys as strings (regression-map semantic //! row) — will not see the import. For Maven and Raw (Full support) this module //! calls the handler's own `storage_key` builder directly (the single source of -//! truth); for npm (Partial) it builds the tarball key and pins the exact handler -//! format with a regression test. Unsupported formats are **skipped, not failed**. +//! truth). The named npm registry cannot be imported as isolated tarballs: +//! its immutable version manifest is the visibility/commit point and its tags +//! and deprecations must be restored through the npm protocol. npm is therefore +//! intentionally unsupported here; the Nexus migration workflow publishes to +//! the named hosted repository and verifies through the group. Unsupported +//! formats are **skipped, not failed**. //! //! Every source string is tainted: a `repo`/`path` of `../../etc` must never //! become a raw path. Traversal/empty segments are rejected pre-key-build @@ -24,8 +28,6 @@ use crate::validation::validate_storage_key; pub enum Compat { /// Every artifact maps 1:1 to a hosted key (Maven, Raw). Full, - /// Some assets map (e.g. npm tarballs); others are regenerated/skipped. - Partial, /// NORA does not host this format via import — all artifacts skipped. Unsupported, } @@ -43,7 +45,6 @@ pub fn normalize_format(src_format: &str) -> Option { pub fn compat(rt: RegistryType) -> Compat { match rt { RegistryType::Maven | RegistryType::Raw => Compat::Full, - RegistryType::Npm => Compat::Partial, _ => Compat::Unsupported, } } @@ -82,15 +83,12 @@ pub fn map_artifact(rt: RegistryType, art: &ArtifactRef) -> Mapping { // path to avoid cross-repo collisions; still a valid `raw/<...>` key the // Raw handler serves for request path `/`. RegistryType::Raw => crate::registry::raw_storage_key(&join_repo(&art.repo, &art.path)), - // npm: import tarballs only; NORA regenerates package metadata on serve. - RegistryType::Npm => match npm_tarball_key(&art.path) { - Some(k) => k, - None => { - return Mapping::Skip( - "npm non-tarball asset (metadata regenerated by NORA on serve)", - ) - } - }, + // A named npm tarball without its immutable version manifest is + // deliberately invisible and will be collected as a pre-commit orphan. + // Import npm through the protocol-aware Nexus migrator instead. + RegistryType::Npm => { + return Mapping::Skip("npm requires protocol migration into a named hosted repository") + } _ => return Mapping::Skip("format not hosted by NORA import"), }; @@ -100,22 +98,6 @@ pub fn map_artifact(rt: RegistryType, art: &ArtifactRef) -> Mapping { } } -/// Build the npm tarball key `npm//tarballs/` from a source asset -/// path (`/-/.tgz`, scoped `@scope//-/.tgz`). Returns -/// `None` for non-tarball npm assets (package metadata, which NORA regenerates -/// from the tarball on first serve). -/// -/// MUST match the tarball key the npm handler serves -/// (`registry/npm.rs`: `format!("npm/{}/tarballs/{}", package, file)`); pinned by -/// the `npm_key_matches_handler_format` test. -fn npm_tarball_key(path: &str) -> Option { - let (package, file) = path.split_once("/-/")?; - if package.is_empty() || file.is_empty() || file.contains('/') { - return None; - } - Some(format!("npm/{package}/tarballs/{file}")) -} - /// Join a source repo name and a repo-relative path into one relative path, /// trimming stray separators (both are pre-validated by [`first_unsafe_segment`]). fn join_repo(repo: &str, path: &str) -> String { @@ -169,7 +151,7 @@ mod tests { fn compat_classes() { assert_eq!(compat(RegistryType::Maven), Compat::Full); assert_eq!(compat(RegistryType::Raw), Compat::Full); - assert_eq!(compat(RegistryType::Npm), Compat::Partial); + assert_eq!(compat(RegistryType::Npm), Compat::Unsupported); assert_eq!(compat(RegistryType::Cargo), Compat::Unsupported); assert_eq!(compat(RegistryType::Docker), Compat::Unsupported); } @@ -194,40 +176,26 @@ mod tests { } #[test] - fn npm_tarball_maps_scoped_and_unscoped() { - assert_eq!( - map_artifact( - RegistryType::Npm, - &art("npm-local", "left-pad/-/left-pad-1.3.0.tgz") - ), - Mapping::Key("npm/left-pad/tarballs/left-pad-1.3.0.tgz".to_string()) - ); - assert_eq!( - map_artifact( - RegistryType::Npm, - &art("npm-local", "@babel/core/-/core-7.0.0.tgz") - ), - Mapping::Key("npm/@babel/core/tarballs/core-7.0.0.tgz".to_string()) - ); - } - - #[test] - fn npm_key_matches_handler_format() { - // Pins the coupling to registry/npm.rs (which inlines this literal at its - // serve + upload sites). If the handler format changes, this must too. - let handler_literal = format!("npm/{}/tarballs/{}", "left-pad", "left-pad-1.3.0.tgz"); - assert_eq!( - npm_tarball_key("left-pad/-/left-pad-1.3.0.tgz"), - Some(handler_literal) - ); + fn npm_is_skipped_until_protocol_state_can_be_committed() { + for path in [ + "left-pad/-/left-pad-1.3.0.tgz", + "@babel/core/-/core-7.0.0.tgz", + "left-pad", + ] { + match map_artifact(RegistryType::Npm, &art("npm-local", path)) { + Mapping::Skip(reason) => assert!(reason.contains("protocol migration")), + other => panic!("expected Skip, got {other:?}"), + } + } } #[test] - fn npm_non_tarball_is_skipped_not_failed() { - // Package metadata document (no `/-/`) — NORA regenerates it. - match map_artifact(RegistryType::Npm, &art("npm-local", "left-pad")) { - Mapping::Skip(_) => {} - other => panic!("expected Skip, got {other:?}"), + fn npm_malformed_tarball_is_still_skipped_without_emitting_a_key() { + for path in ["pkg/-/nested/file.tgz", "pkg/-/file.tgz"] { + match map_artifact(RegistryType::Npm, &art("npm", path)) { + Mapping::Skip(_) => {} + other => panic!("expected Skip, got {other:?}"), + } } } @@ -266,20 +234,6 @@ mod tests { } } - #[test] - fn npm_malformed_tarball_paths() { - // Slash inside the file component → npm_tarball_key returns None → Skip. - assert!(matches!( - map_artifact(RegistryType::Npm, &art("npm", "pkg/-/nested/file.tgz")), - Mapping::Skip(_) - )); - // A trailing empty segment is caught earlier as unsafe → Reject. - assert!(matches!( - map_artifact(RegistryType::Npm, &art("npm", "pkg/-/")), - Mapping::Reject(_) - )); - } - #[test] fn empty_path_is_rejected() { assert!(matches!( diff --git a/nora-registry/src/import/mod.rs b/nora-registry/src/import/mod.rs index 47fbceae..d5b835ef 100644 --- a/nora-registry/src/import/mod.rs +++ b/nora-registry/src/import/mod.rs @@ -242,21 +242,29 @@ async fn assess( println!("nora import assess — source: {host}"); println!("{:<32} {:<12} {:<12} notes", "REPO", "FORMAT", "COMPAT"); let mut full = 0usize; - let mut partial = 0usize; let mut unsupported = 0usize; for repo in &repos { let (compat, note) = match layout::normalize_format(&repo.format) { Some(rt) => { - let c = layout::compat(rt); + let c = configured_import_compat(rt, config); match c { layout::Compat::Full => full += 1, - layout::Compat::Partial => partial += 1, layout::Compat::Unsupported => unsupported += 1, } let note = match c { layout::Compat::Full => "", - layout::Compat::Partial => "tarballs only; metadata regenerated", - layout::Compat::Unsupported => "no NORA import layout — will be skipped", + layout::Compat::Unsupported + if rt == crate::registry_type::RegistryType::Npm => + { + "use the protocol-aware Nexus npm migrator" + } + layout::Compat::Unsupported + if rt == crate::registry_type::RegistryType::Maven + && !config.maven.repositories.is_empty() => + { + "named Maven requires metadata-aware Nexus migration" + } + layout::Compat::Unsupported => "no safe NORA import layout — will be skipped", }; (format!("{c:?}"), note) } @@ -277,7 +285,7 @@ async fn assess( ); } println!( - "\n{} repo(s): {full} full, {partial} partial, {unsupported} unsupported", + "\n{} repo(s): {full} full, {unsupported} unsupported", repos.len() ); @@ -357,6 +365,18 @@ async fn run_import( tracing::info!(repo = %repo.name, format = %repo.format, "skipping repo (unsupported source format)"); continue; }; + if configured_import_compat(rt, config) == layout::Compat::Unsupported { + // Do not enumerate or mark the repository complete. A future + // protocol-aware importer must be able to revisit it, and writing + // legacy/incomplete keys would make the new named handlers either + // ignore the data or expose a partial package. + tracing::warn!( + repo = %repo.name, + format = %repo.format, + "skipping repository: no safe import path for the configured named registry; use the protocol-aware Nexus migration workflow" + ); + continue; + } // Per-repo isolation: a repo that fails to even start (e.g. journal open // error) is logged and counted, never aborting the remaining repos of a // multi-day migration (review: single-error-aborts-job). @@ -392,6 +412,26 @@ async fn run_import( Ok(()) } +/// Direct-storage import is safe only when its key layout and metadata +/// authority match the serving handler. +/// +/// npm always requires a protocol-aware publish so the immutable version +/// manifest, tags and deprecations are committed together. Maven's legacy +/// direct-storage importer remains available only for the legacy single +/// repository layout; named Maven migration must reconcile per-repository +/// metadata through the HTTP handler. +fn configured_import_compat( + rt: crate::registry_type::RegistryType, + config: &crate::config::Config, +) -> layout::Compat { + match rt { + crate::registry_type::RegistryType::Maven if !config.maven.repositories.is_empty() => { + layout::Compat::Unsupported + } + _ => layout::compat(rt), + } +} + /// Import a single repo with bounded concurrency and strict resume ordering. #[allow(clippy::too_many_arguments)] async fn import_repo( @@ -742,6 +782,32 @@ mod integration_tests { assert_eq!(truncate("🚀🚀🚀🚀🚀", 3), "🚀🚀…"); // emoji (4-byte) boundaries } + #[test] + fn direct_storage_import_is_disabled_for_named_maven_and_all_npm() { + use crate::config::{MavenRepository, MavenVersionPolicy, MavenWritePolicy}; + use crate::registry_type::RegistryType; + + let mut config = crate::config::Config::default(); + assert_eq!( + configured_import_compat(RegistryType::Maven, &config), + layout::Compat::Full + ); + assert_eq!( + configured_import_compat(RegistryType::Npm, &config), + layout::Compat::Unsupported + ); + + config.maven.repositories = vec![MavenRepository::Hosted { + name: "maven-releases".to_string(), + version_policy: MavenVersionPolicy::Release, + write_policy: MavenWritePolicy::AllowOnce, + }]; + assert_eq!( + configured_import_compat(RegistryType::Maven, &config), + layout::Compat::Unsupported + ); + } + const MAVEN_KEY: &str = "maven/com/example/foo/1.0/foo-1.0.jar"; #[tokio::test] diff --git a/nora-registry/src/main.rs b/nora-registry/src/main.rs index 0bbf61e8..a41c969f 100644 --- a/nora-registry/src/main.rs +++ b/nora-registry/src/main.rs @@ -38,6 +38,7 @@ mod import; mod metrics; mod migrate; mod mirror; +mod npm_layout; mod openapi; mod proxy_coalesce; mod rate_limit; @@ -243,9 +244,14 @@ pub struct AppState { pub docker_auth: Arc, pub repo_index: Arc, pub http_client: reqwest::Client, + /// Shared outbound client with automatic redirects disabled. npm proxy + /// tarballs use it so every `Location` hop can be policy-checked before + /// credentials or a request are sent to the next URL. + pub no_redirect_http_client: reqwest::Client, pub upload_sessions: Arc>>, /// Per-key publish locks for TOCTOU protection (immutable releases) publish_locks: PublishLocks, + pub(crate) maven_negative_cache: Arc>>, /// Hot-reloadable curation config (swapped atomically on SIGHUP). pub reloadable: Arc>, /// Per-IP failed auth attempt tracker for brute-force protection @@ -253,9 +259,8 @@ pub struct AppState { /// OIDC validator for workload identity (CI/CD) pub oidc: Option>, pub(crate) circuit_breaker: Arc, - /// Single-flight coalescer for the proxy cache-miss path: collapses a - /// thundering herd of concurrent requests for the same key into one - /// upstream fetch (#595). In-memory and rebuildable (empty after restart). + /// Single-flight coalescer for proxy cache misses. In-memory and + /// rebuildable; immutable cached bytes remain authoritative in storage. pub(crate) proxy_coalesce: proxy_coalesce::InflightMap, pub digest_store: Arc, /// Repository index signer (rpm/deb). `None` = indexes are unsigned. @@ -386,9 +391,14 @@ fn build_http_client( tls: &TlsConfig, timeout: Option, no_proxy: bool, + follow_redirects: bool, ) -> reqwest::Client { let mut builder = reqwest::ClientBuilder::new().user_agent(USER_AGENT); + if !follow_redirects { + builder = builder.redirect(reqwest::redirect::Policy::none()); + } + if let Some(t) = timeout { builder = builder.timeout(t); } @@ -460,6 +470,7 @@ async fn run_healthcheck(timeout_secs: u64) -> i32 { &TlsConfig::default(), Some(std::time::Duration::from_secs(timeout_secs)), true, + true, ); match client.get(&url).send().await { // /health returns 200 when healthy, 503 when storage is unreachable. @@ -696,12 +707,14 @@ async fn main() { let cli_publish_locks: PublishLocks = Arc::new(parking_lot::Mutex::new(HashMap::new())); // Dry-run: plans only, no deletions and no index regeneration — // no signer needed. - let result = retention::run_retention( + let result = retention::run_retention_configured( &storage, &cli_publish_locks, None, &config.retention.rules, true, + &config.maven, + None, ) .await; println!("Retention Plan (dry-run):"); @@ -726,12 +739,14 @@ async fn main() { let cli_publish_locks: PublishLocks = Arc::new(parking_lot::Mutex::new(HashMap::new())); if !yes { // Show plan first, require --yes to execute - let result = retention::run_retention( + let result = retention::run_retention_configured( &storage, &cli_publish_locks, None, &config.retention.rules, true, + &config.maven, + None, ) .await; println!("Retention Plan:"); @@ -759,12 +774,14 @@ async fn main() { // same key the server would, or clients start failing // verification after a CLI retention pass. let signer = build_signer(&config, &config.enabled_registries()); - let result = retention::run_retention( + let result = retention::run_retention_configured( &storage, &cli_publish_locks, signer.as_deref(), &config.retention.rules, false, + &config.maven, + None, ) .await; println!("Retention Applied:"); @@ -798,6 +815,7 @@ async fn main() { &config.tls, Some(std::time::Duration::from_secs(300)), false, + true, ); if let Err(e) = mirror::run_mirror(format, ®istry, concurrency, json, &client).await { @@ -1431,7 +1449,8 @@ async fn run_server(mut config: Config, storage: Storage) { // Warn about plaintext credentials in config.toml config.warn_plaintext_credentials(); - let http_client = build_http_client(&config.tls, None, false); + let http_client = build_http_client(&config.tls, None, false, true); + let no_redirect_http_client = build_http_client(&config.tls, None, false, false); log_outbound_proxy(); // Initialize Docker auth with shared HTTP client (includes custom CA certs) @@ -1507,6 +1526,11 @@ async fn run_server(mut config: Config, storage: Storage) { RegistryType::Deb => registry_routes = registry_routes.merge(registry::deb_routes()), } } + if enabled_registries.contains(&RegistryType::Maven) + || enabled_registries.contains(&RegistryType::Npm) + { + registry_routes = registry_routes.merge(registry::named_repository_routes()); + } // Routes WITHOUT rate limiting (health, metrics, UI) let public_routes = Router::new() @@ -1594,8 +1618,10 @@ async fn run_server(mut config: Config, storage: Storage) { docker_auth: Arc::new(docker_auth), repo_index: Arc::new(RepoIndex::new()), http_client, + no_redirect_http_client, upload_sessions: Arc::new(RwLock::new(HashMap::new())), publish_locks: Arc::new(parking_lot::Mutex::new(HashMap::new())), + maven_negative_cache: Arc::new(parking_lot::Mutex::new(HashMap::new())), reloadable, auth_failures: Arc::new(auth::AuthFailureTracker::new(5, 900)), oidc: oidc_validator.map(Arc::new), @@ -1621,6 +1647,7 @@ async fn run_server(mut config: Config, storage: Storage) { let handle = gc::spawn_gc_scheduler( state.storage.clone(), state.publish_locks.clone(), + state.repo_index.clone(), state.config.gc.interval, state.config.gc.dry_run, state.config.gc.grace_secs, @@ -1641,6 +1668,8 @@ async fn run_server(mut config: Config, storage: Storage) { state.storage.clone(), state.publish_locks.clone(), state.signer.clone(), + state.config.maven.clone(), + state.repo_index.clone(), state.config.retention.rules.clone(), state.config.retention.interval, state.config.retention.dry_run, @@ -1709,7 +1738,10 @@ async fn run_server(mut config: Config, storage: Storage) { state.clone(), auth::auth_middleware, )) - .layer(middleware::from_fn(metrics::metrics_middleware)) + .layer(middleware::from_fn_with_state( + state.clone(), + metrics::metrics_middleware, + )) .layer(middleware::from_fn(validation::reject_null_bytes_middleware)) .with_state(state.clone()); diff --git a/nora-registry/src/metrics.rs b/nora-registry/src/metrics.rs index 14aab271..0c5975b0 100644 --- a/nora-registry/src/metrics.rs +++ b/nora-registry/src/metrics.rs @@ -3,7 +3,7 @@ use axum::{ body::Body, - extract::{MatchedPath, State}, + extract::State, http::Request, middleware::Next, response::{IntoResponse, Response}, @@ -92,14 +92,15 @@ pub static PROXY_UPSTREAM_304_TOTAL: LazyLock = LazyLock::new(|| }); /// Upstream 4xx responses that carried a policy/geo block signature (e.g. an AWS -/// WAF geo rule via `x-amzn-waf-reason`) and were relayed as a bare 404. Lets an -/// operator tell an effective region/policy outage apart from a genuine not-found -/// — which is otherwise invisible (a 4xx logs nothing and never trips the breaker). +/// WAF geo rule via `x-amzn-waf-reason`). Lets an operator tell an effective +/// region/policy outage apart from a genuine not-found. Protocol handlers may +/// map the response differently; Maven, for example, maps a blocked 404 to 502 +/// so it cannot be mistaken for an authoritative negative-cacheable miss. /// Labels: `registry`, `reason` (bounded: `geo` | `waf`) (#881). pub static UPSTREAM_POLICY_BLOCKED_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter_vec!( "nora_upstream_policy_blocked_total", - "Upstream 4xx responses bearing a policy/geo block signature, relayed as 404", + "Upstream 4xx responses bearing a detected policy/geo block signature", &["registry", "reason"] ) .expect("failed to create UPSTREAM_POLICY_BLOCKED_TOTAL metric at startup") @@ -130,7 +131,7 @@ pub static PROXY_REVALIDATION_ERRORS_TOTAL: LazyLock = LazyLock:: /// Concurrent upstream fetches collapsed into one by the single-flight /// coalescer: a follower served the leader's in-memory result without making -/// its own upstream round-trip (#595). +/// its own upstream round-trip. pub static PROXY_COALESCED_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter_vec!( "nora_proxy_coalesced_total", @@ -140,9 +141,7 @@ pub static PROXY_COALESCED_TOTAL: LazyLock = LazyLock::new(|| { .expect("failed to create PROXY_COALESCED_TOTAL metric at startup") }); -/// Current number of in-flight single-flight leaders (distinct keys being -/// fetched right now). A flat zero under load means coalescing is not engaging; -/// a monotonic climb signals a guard leak (#595). +/// Current number of distinct in-flight single-flight leaders. pub static PROXY_INFLIGHT: LazyLock = LazyLock::new(|| { register_int_gauge_vec!( "nora_proxy_inflight", @@ -152,11 +151,8 @@ pub static PROXY_INFLIGHT: LazyLock = LazyLock::new(|| { .expect("failed to create PROXY_INFLIGHT metric at startup") }); -/// Followers that did NOT get the leader's result and fell through to their own -/// upstream fetch — because the leader failed/cancelled (`leader`) or the wait -/// budget elapsed while the leader was still fetching (`budget`). A `budget` -/// rate rivalling `PROXY_COALESCED_TOTAL` means a slow upstream is re-stampeding -/// past the coalescer; without this it degrades silently (#595). +/// Followers that could not consume the leader result and performed their own +/// upstream request (`leader` failure/cancellation or elapsed `budget`). pub static PROXY_COALESCE_FALLTHROUGH_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter_vec!( "nora_proxy_coalesce_fallthrough_total", @@ -343,7 +339,7 @@ pub static CACHE_WRITE_ERRORS: LazyLock = LazyLock::new(|| { .expect("failed to create CACHE_WRITE_ERRORS metric at startup") }); -/// Corrupt metadata detected during publish (#533) +/// Corrupt metadata detected during publish. pub static METADATA_CORRUPT_TOTAL: LazyLock = LazyLock::new(|| { register_int_counter_vec!( "nora_metadata_corrupt_total", @@ -460,18 +456,20 @@ async fn metrics_handler() -> impl IntoResponse { /// Middleware to record request metrics pub async fn metrics_middleware( - matched_path: Option, + State(state): State, request: Request, next: Next, ) -> Response { let start = Instant::now(); let method = request.method().to_string(); - let path = matched_path - .map(|p| p.as_str().to_string()) - .unwrap_or_else(|| request.uri().path().to_string()); + let request_path = request.uri().path().to_string(); // Determine registry from path - let registry = detect_registry(&path); + // The matched route for named Maven and npm is intentionally identical, so + // classification must use the concrete repository name from the request + // URI and resolve it through config. Guessing from a name prefix would make + // custom Nexus-compatible names silently report under the wrong format. + let registry = detect_registry(&request_path, Some(&state.config)); // Process request let response = next.run(request).await; @@ -692,11 +690,24 @@ fn is_own_surface(path: &str) -> bool { } /// Detect registry type from path -fn detect_registry(path: &str) -> String { +fn detect_registry(path: &str, config: Option<&crate::config::Config>) -> String { if path.starts_with("/v2") { "docker".to_string() } else if path.starts_with("/maven2") { "maven".to_string() + } else if let Some(repository_path) = path.strip_prefix("/repository/") { + let repository = repository_path.split('/').next().unwrap_or(""); + match config { + Some(config) + if config.maven.enabled && config.maven.repository(repository).is_some() => + { + "maven".to_string() + } + Some(config) if config.npm.enabled && config.npm.repository(repository).is_some() => { + "npm".to_string() + } + _ => "other".to_string(), + } } else if path.starts_with("/npm") { "npm".to_string() } else if path.starts_with("/cargo") { @@ -736,69 +747,114 @@ mod tests { #[test] fn test_detect_registry_docker() { - assert_eq!(detect_registry("/v2/nginx/manifests/latest"), "docker"); - assert_eq!(detect_registry("/v2/"), "docker"); assert_eq!( - detect_registry("/v2/library/alpine/blobs/sha256:abc"), + detect_registry("/v2/nginx/manifests/latest", None), + "docker" + ); + assert_eq!(detect_registry("/v2/", None), "docker"); + assert_eq!( + detect_registry("/v2/library/alpine/blobs/sha256:abc", None), "docker" ); } #[test] fn test_detect_registry_maven() { - assert_eq!(detect_registry("/maven2/com/example/artifact"), "maven"); + assert_eq!( + detect_registry("/maven2/com/example/artifact", None), + "maven" + ); + let mut config = crate::config::Config::default(); + config.maven.repositories = vec![crate::config::MavenRepository::Hosted { + name: "releases".to_string(), + version_policy: crate::config::MavenVersionPolicy::Mixed, + write_policy: crate::config::MavenWritePolicy::AllowOnce, + }]; + assert_eq!( + detect_registry("/repository/releases/com/example/artifact", Some(&config)), + "maven" + ); + config.maven.enabled = false; + assert_eq!( + detect_registry("/repository/releases/com/example/artifact", Some(&config)), + "other" + ); } #[test] fn test_detect_registry_npm() { - assert_eq!(detect_registry("/npm/lodash"), "npm"); - assert_eq!(detect_registry("/npm/@scope/package"), "npm"); + assert_eq!(detect_registry("/npm/lodash", None), "npm"); + assert_eq!(detect_registry("/npm/@scope/package", None), "npm"); + let mut config = crate::config::Config::default(); + config.npm.repositories = vec![crate::config::NpmRepository::Hosted { + name: "packages".to_string(), + write_policy: crate::config::NpmWritePolicy::AllowOnce, + }]; + assert_eq!( + detect_registry("/repository/packages/@scope/package", Some(&config)), + "npm" + ); + assert_eq!( + detect_registry("/repository/unknown/pkg", Some(&config)), + "other" + ); + config.npm.enabled = false; + assert_eq!( + detect_registry("/repository/packages/@scope/package", Some(&config)), + "other" + ); } #[test] fn test_detect_registry_cargo_path() { - assert_eq!(detect_registry("/cargo/api/v1/crates"), "cargo"); + assert_eq!(detect_registry("/cargo/api/v1/crates", None), "cargo"); } #[test] fn test_detect_registry_pypi() { - assert_eq!(detect_registry("/simple/requests/"), "pypi"); + assert_eq!(detect_registry("/simple/requests/", None), "pypi"); assert_eq!( - detect_registry("/packages/requests/1.0/requests-1.0.tar.gz"), + detect_registry("/packages/requests/1.0/requests-1.0.tar.gz", None), "pypi" ); } #[test] fn test_detect_registry_ui() { - assert_eq!(detect_registry("/ui/dashboard"), "ui"); - assert_eq!(detect_registry("/ui"), "ui"); + assert_eq!(detect_registry("/ui/dashboard", None), "ui"); + assert_eq!(detect_registry("/ui", None), "ui"); } #[test] fn test_detect_registry_other() { - assert_eq!(detect_registry("/health"), "other"); - assert_eq!(detect_registry("/ready"), "other"); - assert_eq!(detect_registry("/unknown/path"), "other"); + assert_eq!(detect_registry("/health", None), "other"); + assert_eq!(detect_registry("/ready", None), "other"); + assert_eq!(detect_registry("/unknown/path", None), "other"); } #[test] fn test_detect_registry_go_path() { assert_eq!( - detect_registry("/go/github.com/user/repo/@v/v1.0.0.info"), + detect_registry("/go/github.com/user/repo/@v/v1.0.0.info", None), + "go" + ); + assert_eq!( + detect_registry("/go/github.com/user/repo/@latest", None), "go" ); - assert_eq!(detect_registry("/go/github.com/user/repo/@latest"), "go"); // Bare prefix without trailing slash should not match - assert_eq!(detect_registry("/goblin/something"), "other"); + assert_eq!(detect_registry("/goblin/something", None), "other"); } #[test] fn test_detect_registry_raw_path() { - assert_eq!(detect_registry("/raw/my-project/artifact.tar.gz"), "raw"); - assert_eq!(detect_registry("/raw/data/file.bin"), "raw"); + assert_eq!( + detect_registry("/raw/my-project/artifact.tar.gz", None), + "raw" + ); + assert_eq!(detect_registry("/raw/data/file.bin", None), "raw"); // Bare prefix without trailing slash should not match - assert_eq!(detect_registry("/rawdata/file"), "other"); + assert_eq!(detect_registry("/rawdata/file", None), "other"); } #[test] diff --git a/nora-registry/src/npm_layout.rs b/nora-registry/src/npm_layout.rs new file mode 100644 index 00000000..32f351ab --- /dev/null +++ b/nora-registry/src/npm_layout.rs @@ -0,0 +1,281 @@ +// Copyright (c) 2026 The NORA Authors +// SPDX-License-Identifier: MIT + +//! Parser for the authoritative named npm storage layout. +//! +//! Hosted packages and proxy cache entries deliberately share the +//! `npm/repositories/{repository}/` prefix. The literal package name `proxy` +//! is valid, so consumers must distinguish the two layouts by their complete +//! shape instead of treating every second path segment named `proxy` as cache. + +use base64::Engine; +use sha2::Digest; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum NpmObjectKind { + HostedPackage, + HostedVersion(String), + HostedPublishComplete(String), + HostedTarball(String), + HostedBlob { algorithm: String, digest: String }, + HostedDistTag(String), + HostedDeprecation(String), + ProxyPackument, + ProxyTarball(String), + ProxyNegative, +} + +pub(crate) fn hosted_blob_key_for_digest(repository: &str, package: &str, digest: &str) -> String { + format!("npm/repositories/{repository}/{package}/blobs/sha512/{digest}.tgz") +} + +pub(crate) fn hosted_blob_digest_from_manifest(manifest: &[u8]) -> Option { + let manifest = serde_json::from_slice::(manifest).ok()?; + let integrity = manifest.get("dist")?.get("integrity")?.as_str()?; + integrity + .split_ascii_whitespace() + .filter_map(|candidate| candidate.strip_prefix("sha512-")) + .find_map(|encoded| { + let digest = base64::engine::general_purpose::STANDARD + .decode(encoded) + .ok()?; + (digest.len() == 64).then(|| hex::encode(digest)) + }) +} + +pub(crate) fn hosted_blob_key_from_manifest( + repository: &str, + package: &str, + manifest: &[u8], +) -> Option { + hosted_blob_digest_from_manifest(manifest) + .map(|digest| hosted_blob_key_for_digest(repository, package, &digest)) +} + +pub(crate) fn hosted_manifest_digest(manifest: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(manifest)) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NpmObjectPath { + pub repository: String, + pub package: String, + pub kind: NpmObjectKind, +} + +/// Parse one object below `npm/repositories/`. +/// +/// The proxy namespace is recognized only for a complete cache-object shape. +/// In particular, `.../{repo}/proxy/tarballs/proxy-1.0.0.tgz` is the hosted +/// package named `proxy`, while +/// `.../{repo}/proxy/tarballs/proxy/proxy-1.0.0.tgz` is a proxy-cache tarball. +pub(crate) fn parse_npm_object_key(key: &str) -> Option { + let rest = key.strip_prefix("npm/repositories/")?; + let parts: Vec<&str> = rest.split('/').collect(); + let repository = parts.first().copied().filter(|part| !part.is_empty())?; + let tail = parts.get(1..)?; + + if tail.first() == Some(&"proxy") { + match tail.get(1).copied() { + Some("packuments") if tail.len() >= 3 => { + let package = tail[2..].join("/"); + let package = package.strip_suffix(".json")?; + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package: package.to_string(), + kind: NpmObjectKind::ProxyPackument, + }); + } + Some("negative") if tail.len() >= 3 => { + let package = tail[2..].join("/"); + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::ProxyNegative, + }); + } + // A cached tarball always has both a package path and a filename + // after `proxy/tarballs`. With only a filename this is the hosted + // package whose literal name is `proxy`. + Some("tarballs") if tail.len() >= 4 => { + let package = tail[2..tail.len() - 1].join("/"); + let filename = tail.last()?.to_string(); + if package.is_empty() || filename.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::ProxyTarball(filename), + }); + } + _ => {} + } + } + + if tail.len() >= 2 && tail.last() == Some(&"pkg.json") { + let package = tail[..tail.len() - 1].join("/"); + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedPackage, + }); + } + + if let Some(marker) = tail.iter().rposition(|part| *part == "blobs") { + if marker > 0 + && marker + 3 == tail.len() + && tail[marker + 1] == "sha512" + && tail[marker + 2].ends_with(".tgz") + { + let package = tail[..marker].join("/"); + let digest = tail[marker + 2].strip_suffix(".tgz")?; + if !package.is_empty() + && digest.len() == 128 + && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + { + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedBlob { + algorithm: "sha512".to_string(), + digest: digest.to_ascii_lowercase(), + }, + }); + } + } + } + + let marker = tail.iter().rposition(|part| { + matches!( + *part, + "versions" | "publish-complete" | "tarballs" | "dist-tags" | "deprecations" + ) + })?; + if marker == 0 || marker + 2 != tail.len() { + return None; + } + let package = tail[..marker].join("/"); + let object = tail[marker + 1]; + if package.is_empty() || object.is_empty() { + return None; + } + let kind = match tail[marker] { + "versions" => NpmObjectKind::HostedVersion(object.strip_suffix(".json")?.to_string()), + "publish-complete" => NpmObjectKind::HostedPublishComplete(object.to_string()), + "tarballs" => NpmObjectKind::HostedTarball(object.to_string()), + "dist-tags" => NpmObjectKind::HostedDistTag(object.to_string()), + "deprecations" => NpmObjectKind::HostedDeprecation(object.to_string()), + _ => return None, + }; + Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn distinguishes_hosted_package_named_proxy_from_proxy_cache() { + let hosted = + parse_npm_object_key("npm/repositories/npm-private/proxy/tarballs/proxy-1.0.0.tgz") + .expect("hosted tarball"); + assert_eq!(hosted.repository, "npm-private"); + assert_eq!(hosted.package, "proxy"); + assert_eq!( + hosted.kind, + NpmObjectKind::HostedTarball("proxy-1.0.0.tgz".to_string()) + ); + + let cached = parse_npm_object_key( + "npm/repositories/npm-registry/proxy/tarballs/proxy/proxy-1.0.0.tgz", + ) + .expect("proxy tarball"); + assert_eq!(cached.repository, "npm-registry"); + assert_eq!(cached.package, "proxy"); + assert_eq!( + cached.kind, + NpmObjectKind::ProxyTarball("proxy-1.0.0.tgz".to_string()) + ); + } + + #[test] + fn parses_hosted_proxy_version_and_scoped_proxy_cache() { + let version = + parse_npm_object_key("npm/repositories/npm-private/proxy/versions/1.0.0.json") + .expect("hosted version"); + assert_eq!(version.package, "proxy"); + assert_eq!( + version.kind, + NpmObjectKind::HostedVersion("1.0.0".to_string()) + ); + + let cached = parse_npm_object_key( + "npm/repositories/npm-registry/proxy/tarballs/@scope/pkg/pkg-1.0.0.tgz", + ) + .expect("scoped cache tarball"); + assert_eq!(cached.package, "@scope/pkg"); + assert!(matches!(cached.kind, NpmObjectKind::ProxyTarball(_))); + + let marker = + parse_npm_object_key("npm/repositories/npm-private/@scope/pkg/publish-complete/1.0.0") + .expect("publish completion marker"); + assert_eq!(marker.package, "@scope/pkg"); + assert_eq!( + marker.kind, + NpmObjectKind::HostedPublishComplete("1.0.0".to_string()) + ); + + let digest = "a".repeat(128); + let blob = parse_npm_object_key(&format!( + "npm/repositories/npm-private/@scope/pkg/blobs/sha512/{digest}.tgz" + )) + .expect("hosted content-addressed blob"); + assert_eq!(blob.package, "@scope/pkg"); + assert_eq!( + blob.kind, + NpmObjectKind::HostedBlob { + algorithm: "sha512".to_string(), + digest, + } + ); + } + + #[test] + fn derives_hosted_blob_and_completion_digests_from_manifest() { + let digest = [7u8; 64]; + let integrity = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(digest) + ); + let manifest = serde_json::to_vec(&serde_json::json!({ + "dist": {"integrity": integrity} + })) + .unwrap(); + let digest_hex = hex::encode(digest); + assert_eq!( + hosted_blob_digest_from_manifest(&manifest).as_deref(), + Some(digest_hex.as_str()) + ); + assert_eq!( + hosted_blob_key_from_manifest("repo", "@scope/pkg", &manifest).as_deref(), + Some( + format!("npm/repositories/repo/@scope/pkg/blobs/sha512/{digest_hex}.tgz").as_str() + ) + ); + assert_eq!(hosted_manifest_digest(&manifest).len(), 64); + } +} diff --git a/nora-registry/src/openapi.rs b/nora-registry/src/openapi.rs index 31d0b330..fbb769f3 100644 --- a/nora-registry/src/openapi.rs +++ b/nora-registry/src/openapi.rs @@ -34,6 +34,7 @@ use crate::AppState; (name = "metrics", description = "Prometheus metrics"), (name = "dashboard", description = "Dashboard & Metrics API"), (name = "docker", description = "Docker Registry v2 API"), + (name = "repository", description = "Named Maven and npm repositories"), (name = "maven", description = "Maven Repository API"), (name = "npm", description = "npm Registry API"), (name = "cargo", description = "Cargo Registry API"), @@ -74,6 +75,10 @@ use crate::AppState; // Maven crate::openapi::maven_artifact_get, crate::openapi::maven_artifact_put, + crate::openapi::named_repository_get, + crate::openapi::named_repository_put, + crate::openapi::named_repository_post, + crate::openapi::named_repository_delete, // npm crate::openapi::npm_package, crate::openapi::npm_publish, @@ -546,6 +551,93 @@ pub async fn maven_artifact_get() {} )] pub async fn maven_artifact_put() {} +/// Read from a named Maven or npm repository +#[utoipa::path( + get, + path = "/repository/{repository}/{path}", + tag = "repository", + params( + ("repository" = String, Path, description = "Hosted, proxy or group repository name"), + ("path" = String, Path, description = "Slash-preserving Maven artifact path or npm package/tarball endpoint") + ), + responses( + (status = 200, description = "Maven artifact bytes or npm registry response"), + (status = 400, description = "Invalid protocol path", body = ErrorResponse), + (status = 404, description = "Repository, artifact or package not found"), + (status = 502, description = "Upstream proxy unavailable") + ) +)] +pub async fn named_repository_get() {} + +/// Mutate a named Maven or npm repository +/// +/// Maven accepts artifact uploads only into hosted repositories. npm publish +/// and deprecation payloads may target a hosted repository or a group; group +/// routing uses the configured writable member. npm dist-tag add/update +/// mutations are direct-hosted only, and group endpoints return 400. Proxy +/// repositories are read-only. +#[utoipa::path( + put, + path = "/repository/{repository}/{path}", + tag = "repository", + params( + ("repository" = String, Path, description = "Hosted repository; an npm group is writable only for publish/deprecate"), + ("path" = String, Path, description = "Slash-preserving Maven artifact path or npm publish/deprecation/dist-tag endpoint; npm dist-tag mutations require a direct hosted repository") + ), + responses( + (status = 200, description = "npm mutation completed"), + (status = 201, description = "Maven artifact or npm package created"), + (status = 400, description = "Payload or repository policy rejected the mutation; npm group dist-tag mutations return 400", body = ErrorResponse), + (status = 405, description = "Method is not supported for this repository or endpoint"), + (status = 409, description = "Immutable coordinate exists with different content"), + (status = 500, description = "Storage error") + ) +)] +pub async fn named_repository_put() {} + +/// Submit an npm read-semantics POST endpoint +/// +/// npm security audit requests are forwarded through named npm proxy/group +/// repositories. Named Maven repositories reject POST with 405. +#[utoipa::path( + post, + path = "/repository/{repository}/{path}", + tag = "repository", + params( + ("repository" = String, Path, description = "npm hosted, proxy or group repository name"), + ("path" = String, Path, description = "Slash-preserving npm POST endpoint, such as -/npm/v1/security/advisories/bulk") + ), + responses( + (status = 200, description = "npm upstream response"), + (status = 400, description = "Invalid npm request", body = ErrorResponse), + (status = 404, description = "Repository not found"), + (status = 405, description = "POST is not supported for this repository format"), + (status = 502, description = "Upstream proxy unavailable") + ) +)] +pub async fn named_repository_post() {} + +/// Delete a mutable npm dist-tag in a named hosted repository +/// +/// Dist-tag deletion is direct-hosted only; group endpoints return 400. +#[utoipa::path( + delete, + path = "/repository/{repository}/{path}", + tag = "repository", + params( + ("repository" = String, Path, description = "Direct named npm hosted repository; group endpoints return 400"), + ("path" = String, Path, description = "npm dist-tag endpoint: -/package/{package}/dist-tags/{tag}") + ), + responses( + (status = 204, description = "Dist-tag deleted or already absent"), + (status = 400, description = "Invalid package/tag, protected latest tag, or group endpoint"), + (status = 404, description = "Repository not found"), + (status = 405, description = "DELETE is not supported for this path or repository format"), + (status = 500, description = "Storage error") + ) +)] +pub async fn named_repository_delete() {} + // -------------------- npm -------------------- /// Get npm package metadata @@ -1340,3 +1432,46 @@ pub fn routes() -> Router { Router::new() .merge(SwaggerUi::new("/api-docs").url("/api-docs/openapi.json", ApiDoc::openapi())) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shared_named_repository_path_documents_both_protocol_dispatch_methods() { + let document = serde_json::to_value(ApiDoc::openapi()).unwrap(); + let path = &document["paths"]["/repository/{repository}/{path}"]; + for method in ["get", "put", "post", "delete"] { + assert!( + path.get(method).is_some(), + "shared named repository path must document {method}" + ); + assert_eq!( + path[method]["tags"][0], "repository", + "shared path must not claim to be Maven-only" + ); + } + } + + #[test] + fn named_npm_group_write_contract_limits_routing_to_publish_and_deprecate() { + let document = serde_json::to_value(ApiDoc::openapi()).unwrap(); + let path = &document["paths"]["/repository/{repository}/{path}"]; + + let put_description = path["put"]["description"].as_str().unwrap(); + assert!(put_description.contains( + "npm publish\nand deprecation payloads may target a hosted repository or a group" + )); + assert!(put_description.contains( + "npm dist-tag add/update\nmutations are direct-hosted only, and group endpoints return 400" + )); + + let delete_description = path["delete"]["description"].as_str().unwrap(); + assert!(delete_description + .contains("Dist-tag deletion is direct-hosted only; group endpoints return 400")); + assert_eq!( + path["delete"]["responses"]["400"]["description"], + "Invalid package/tag, protected latest tag, or group endpoint" + ); + } +} diff --git a/nora-registry/src/registry/maven.rs b/nora-registry/src/registry/maven.rs index 3e19f14c..4fd580b3 100644 --- a/nora-registry/src/registry/maven.rs +++ b/nora-registry/src/registry/maven.rs @@ -11,13 +11,16 @@ use crate::activity_log::{ActionType, ActivityEntry}; use crate::audit::AuditEntry; use crate::auth::{enforce_namespace_scope, NamespaceAuthority}; +use crate::config::{ + MavenProxy, MavenProxyEntry, MavenRepository, MavenVersionPolicy, MavenWritePolicy, +}; use crate::registry::{circuit_open_response, method_not_allowed, proxy_fetch, ProxyError}; use crate::registry_type::RegistryType; use crate::storage::StorageError; use crate::validation::ends_with_ci; use crate::AppState; use axum::{ - body::Bytes, + body::{to_bytes, Bytes}, extract::{Path, State}, http::{header, StatusCode}, response::{IntoResponse, Response}, @@ -26,8 +29,13 @@ use axum::{ }; use quick_xml::{events::Event, Reader}; use sha2::Digest; -use std::collections::BTreeSet; -use std::time::Duration; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::time::{Duration, Instant}; + +const MAVEN_NEGATIVE_CACHE_MAX_ENTRIES: usize = 10_000; + +#[derive(Clone, Copy, Debug)] +struct MavenPolicyBlock; /// Build the storage key for a Maven artifact at repo-relative `path`. /// @@ -39,15 +47,100 @@ pub(crate) fn storage_key(path: &str) -> String { format!("maven/{path}") } +pub(crate) fn repository_storage_key(repository: &str, path: &str) -> String { + format!("maven/repositories/{repository}/{path}") +} + pub fn routes() -> Router { Router::new().route( "/maven2/{*path}", - get(download) - .put(upload) + get(download_legacy) + .put(upload_legacy) .fallback(|| async { method_not_allowed("GET, PUT") }), ) } +#[derive(Clone)] +struct DirectRepository { + name: Option, + proxies: Vec, + metadata_ttl: i64, + negative_ttl: i64, + version_policy: MavenVersionPolicy, + write_policy: MavenWritePolicy, +} + +impl DirectRepository { + fn legacy(state: &AppState) -> Self { + Self { + name: None, + proxies: state.config.maven.proxies.clone(), + metadata_ttl: state.config.maven.metadata_ttl, + negative_ttl: 0, + version_policy: MavenVersionPolicy::Mixed, + write_policy: if state.config.maven.immutable_releases { + MavenWritePolicy::AllowOnce + } else { + MavenWritePolicy::Allow + }, + } + } + + fn from_config(state: &AppState, repository: &MavenRepository) -> Option { + match repository { + MavenRepository::Hosted { + name, + version_policy, + write_policy, + } => Some(Self { + name: Some(name.clone()), + proxies: Vec::new(), + metadata_ttl: state.config.maven.metadata_ttl, + negative_ttl: 0, + version_policy: *version_policy, + write_policy: *write_policy, + }), + MavenRepository::Proxy { + name, + url, + auth, + version_policy, + metadata_ttl, + negative_ttl, + } => Some(Self { + name: Some(name.clone()), + proxies: vec![MavenProxyEntry::Full(MavenProxy { + url: url.clone(), + auth: auth.clone(), + })], + metadata_ttl: metadata_ttl.unwrap_or(state.config.maven.metadata_ttl), + negative_ttl: *negative_ttl, + version_policy: *version_policy, + write_policy: MavenWritePolicy::Deny, + }), + MavenRepository::Group { .. } => None, + } + } + + fn storage_key(&self, path: &str) -> String { + self.name.as_deref().map_or_else( + || storage_key(path), + |name| repository_storage_key(name, path), + ) + } + + fn storage_prefix(&self) -> String { + self.name.as_deref().map_or_else( + || "maven/".to_string(), + |name| format!("maven/repositories/{name}/"), + ) + } + + fn is_proxy(&self) -> bool { + !self.proxies.is_empty() + } +} + // ============================================================================ // Path parsing // ============================================================================ @@ -56,7 +149,6 @@ struct MavenCoordinates { group_path: String, artifact_id: String, version: String, - filename: String, } enum MavenPathKind { @@ -70,10 +162,11 @@ enum MavenPathKind { Opaque, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] enum MavenMetadataLevel { Group, Artifact, + ArtifactAndGroup, Version, } @@ -86,6 +179,7 @@ fn classify_metadata_level(data: &[u8]) -> Option { let mut root_closed = false; let mut has_artifact_id = false; let mut has_version = false; + let mut has_plugins = false; loop { match reader.read_event() { @@ -99,6 +193,7 @@ fn classify_metadata_level(data: &[u8]) -> Option { } else if depth == 1 { has_artifact_id |= name.as_ref() == b"artifactId"; has_version |= name.as_ref() == b"version"; + has_plugins |= name.as_ref() == b"plugins"; } depth += 1; } @@ -113,6 +208,7 @@ fn classify_metadata_level(data: &[u8]) -> Option { } else if depth == 1 { has_artifact_id |= name.as_ref() == b"artifactId"; has_version |= name.as_ref() == b"version"; + has_plugins |= name.as_ref() == b"plugins"; } } Ok(Event::End(element)) => { @@ -139,6 +235,8 @@ fn classify_metadata_level(data: &[u8]) -> Option { if has_version { Some(MavenMetadataLevel::Version) + } else if has_artifact_id && has_plugins { + Some(MavenMetadataLevel::ArtifactAndGroup) } else if has_artifact_id { Some(MavenMetadataLevel::Artifact) } else { @@ -183,24 +281,63 @@ fn classify_path(path: &str) -> MavenPathKind { group_path: segments[..segments.len() - 3].join("/"), artifact_id: segments[segments.len() - 3].to_string(), version: segments[segments.len() - 2].to_string(), - filename: last.to_string(), }); } MavenPathKind::Opaque } -fn is_checksum_file(filename: &str) -> bool { - ends_with_ci(filename, ".md5") - || ends_with_ci(filename, ".sha1") - || ends_with_ci(filename, ".sha256") - || ends_with_ci(filename, ".sha512") -} - fn is_snapshot(version: &str) -> bool { version.ends_with("-SNAPSHOT") } +fn version_allowed_by_policy(policy: MavenVersionPolicy, version: &str) -> bool { + match policy { + MavenVersionPolicy::Release => !is_snapshot(version), + MavenVersionPolicy::Snapshot => is_snapshot(version), + MavenVersionPolicy::Mixed => true, + } +} + +fn mutation_lock_key(repository: &DirectRepository, path: &str) -> String { + let document_path = metadata_document_key(path).unwrap_or(path); + if let Some((group_path, artifact_id, _)) = version_metadata_path(document_path) { + return repository.storage_key(&format!("{group_path}/{artifact_id}/maven-metadata.xml")); + } + match classify_path(document_path) { + MavenPathKind::VersionFile(coordinates) => repository.storage_key(&format!( + "{}/{}/maven-metadata.xml", + coordinates.group_path, coordinates.artifact_id + )), + MavenPathKind::ArtifactMeta { + group_path, + artifact_id, + .. + } => repository.storage_key(&format!("{group_path}/{artifact_id}/maven-metadata.xml")), + MavenPathKind::Opaque => repository.storage_key(document_path), + } +} + +fn insert_negative_cache_entry( + cache: &mut HashMap, + key: String, + ttl_seconds: i64, + max_entries: usize, +) { + let now = Instant::now(); + cache.retain(|_, created| now.duration_since(*created).as_secs() < ttl_seconds.max(0) as u64); + if !cache.contains_key(&key) && cache.len() >= max_entries { + if let Some(oldest) = cache + .iter() + .min_by_key(|(_, created)| **created) + .map(|(key, _)| key.clone()) + { + cache.remove(&oldest); + } + } + cache.insert(key, now); +} + /// Whether a Maven path points at a MUTABLE resource that must be revalidated when proxied: /// `maven-metadata.xml` (and its checksums) is rewritten as versions are deployed, and SNAPSHOT /// version files are republished in place. Release artifacts are immutable. @@ -229,13 +366,8 @@ fn url_is_maven_central(u: &str) -> bool { /// True when a configured Maven proxy points at Maven Central. Gates the search /// query so internal coordinates are never sent to search.maven.org. -fn maven_upstream_is_central(state: &AppState) -> bool { - state - .config - .maven - .proxies - .iter() - .any(|p| url_is_maven_central(p.url())) +fn maven_upstream_is_central(proxies: &[MavenProxyEntry]) -> bool { + proxies.iter().any(|p| url_is_maven_central(p.url())) } /// Best-effort upload timestamp for a Maven Central GAV via the Central search @@ -276,12 +408,122 @@ async fn fetch_maven_central_date( // Download // ============================================================================ -async fn download( +async fn download_legacy( State(state): State, headers: axum::http::HeaderMap, Path(path): Path, ) -> Response { - let key = storage_key(&path); + if let Some(repository) = state.config.maven.default_repository.clone() { + return download_configured(state, headers, &repository, path).await; + } + let repository = DirectRepository::legacy(&state); + download_direct(state, headers, repository, path).await +} + +pub(crate) async fn download_named( + State(state): State, + headers: axum::http::HeaderMap, + Path((repository, path)): Path<(String, String)>, +) -> Response { + download_configured(state, headers, &repository, path).await +} + +async fn download_configured( + state: AppState, + headers: axum::http::HeaderMap, + repository: &str, + path: String, +) -> Response { + let Some(config) = state.config.maven.repository(repository).cloned() else { + return StatusCode::NOT_FOUND.into_response(); + }; + match config { + MavenRepository::Group { members, .. } => { + download_group(state, headers, &members, path).await + } + direct => { + let direct = + DirectRepository::from_config(&state, &direct).expect("non-group repository"); + download_direct(state, headers, direct, path).await + } + } +} + +async fn download_direct( + state: AppState, + headers: axum::http::HeaderMap, + repository: DirectRepository, + path: String, +) -> Response { + // Checksum sidecars are derived views of the exact bytes we would return for the + // base object. Never trust a cached or upstream sidecar independently: it can be + // missing or stale after a mutable metadata/SNAPSHOT update. + if let Some(suffix) = checksum_suffix(&path) { + let Some(document_path) = metadata_document_key(&path) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let response = Box::pin(download_direct( + state.clone(), + headers, + repository.clone(), + document_path.to_string(), + )) + .await; + if response.status() != StatusCode::OK { + return response; + } + let prelock_base = match to_bytes(response.into_body(), usize::MAX).await { + Ok(base) => base, + Err(error) => { + tracing::error!(%error, path = %document_path, "Failed to read Maven base object"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + let lock = state.publish_lock(&mutation_lock_key(&repository, document_path)); + let _guard = lock.lock().await; + let document_key = repository.storage_key(document_path); + let latest = match state.storage.get(&document_key).await { + Ok(latest) => latest, + Err(StorageError::NotFound) if !is_mutable_maven_path(document_path) => prelock_base, + Err(StorageError::NotFound) => return StatusCode::NOT_FOUND.into_response(), + Err(error) => { + tracing::error!( + %error, + key = %document_key, + "Failed to re-read Maven checksum base object under mutation lock" + ); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + let Some(checksum) = checksum_hex(suffix, &latest) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let key = repository.storage_key(&path); + if let Err(error) = state.storage.put(&key, checksum.as_bytes()).await { + tracing::warn!(%error, %key, "Failed to refresh derived Maven checksum"); + } + return with_content_type(&path, Bytes::from(checksum)).into_response(); + } + + let key = repository.storage_key(&path); + if repository.is_proxy() { + let requested_version = version_metadata_path(&path) + .map(|(_, _, version)| version) + .or_else(|| match classify_path(&path) { + MavenPathKind::VersionFile(coords) => Some(coords.version), + _ => None, + }); + if let Some(version) = requested_version { + let allowed = match repository.version_policy { + MavenVersionPolicy::Release => !is_snapshot(&version), + MavenVersionPolicy::Snapshot => is_snapshot(&version), + MavenVersionPolicy::Mixed => true, + }; + if !allowed { + return StatusCode::NOT_FOUND.into_response(); + } + } + } let artifact_name = path .split('/') @@ -331,12 +573,12 @@ async fn download( let already_cached = state.storage.stat(&key).await.is_some(); let publish_date: Option = if let Some((ref maven_name, ref maven_version)) = curation_coords { - if state.config.maven.proxies.is_empty() { + if !repository.is_proxy() { crate::curation::extract_mtime_as_publish_date(&state.storage, &key).await } else if !already_cached && !internal && state.config.server.trust_upstream_dates - && maven_upstream_is_central(&state) + && maven_upstream_is_central(&repository.proxies) { // #68/#733: never send an internal-namespace GAV to the hardcoded public // search.maven.org — that would leak operator-internal coordinates. @@ -378,7 +620,32 @@ async fn download( } // Read the cached artifact eagerly — kept for the freshness check and the stale-on-error fallback. - let cached = state.storage.get(&key).await.ok(); + // Only an explicit miss may fall through to another group member or an upstream. + // Treating corruption/transient storage failure as absence would let a lower-priority + // proxy silently replace authoritative hosted bytes. + let cached = match state.storage.get(&key).await { + Ok(data) => Some(data), + Err(StorageError::NotFound) => None, + Err(error) => { + tracing::error!(%error, %key, "Failed to read Maven artifact from storage"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + if cached.is_none() && repository.negative_ttl > 0 { + let now = std::time::Instant::now(); + let mut cache = state.maven_negative_cache.lock(); + match cache.get(&key).copied() { + Some(created) + if now.duration_since(created).as_secs() < repository.negative_ttl as u64 => + { + return StatusCode::NOT_FOUND.into_response(); + } + Some(_) => { + cache.remove(&key); + } + None => {} + } + } // maven-metadata.xml and SNAPSHOT artifacts are MUTABLE (rewritten as versions deploy); a // proxied mutable path must be revalidated against upstream unless within a positive @@ -390,8 +657,8 @@ async fn download( Some(_) => { let modified = state.storage.stat(&key).await.map(|m| m.modified); crate::cache_ttl::mutable_ref_fresh( - !state.config.maven.proxies.is_empty(), - state.config.maven.metadata_ttl, + repository.is_proxy(), + repository.metadata_ttl, modified, ) } @@ -462,12 +729,17 @@ async fn download( // metadata must never be fetched upstream (dependency confusion): serve any local // copy (deployed/cached metadata; the fresh path already returned above) and block // only when nothing is hosted locally — never proxy. - if let MavenPathKind::ArtifactMeta { - group_path, - artifact_id, - .. - } = classify_path(&path) - { + let metadata_coordinates = version_metadata_path(&path) + .map(|(group_path, artifact_id, _)| (group_path, artifact_id)) + .or_else(|| match classify_path(&path) { + MavenPathKind::ArtifactMeta { + group_path, + artifact_id, + .. + } => Some((group_path, artifact_id)), + _ => None, + }); + if let Some((group_path, artifact_id)) = metadata_coordinates { let maven_name = format!("{}:{}", group_path.replace('/', "."), artifact_id); if crate::curation::is_internal_namespace( &state.curation().curation_engine, @@ -507,21 +779,27 @@ async fn download( return StatusCode::NOT_FOUND.into_response(); } - let metadata_request = match classify_path(&path) { - MavenPathKind::ArtifactMeta { - group_path, - artifact_id, - .. - } => Some(( - group_path, - artifact_id, - metadata_document_key(&path).unwrap_or(&path).to_string(), - checksum_suffix(&path), - )), - _ => None, + let metadata_request = if version_metadata_path(&path).is_some() { + None + } else { + match classify_path(&path) { + MavenPathKind::ArtifactMeta { + group_path, + artifact_id, + .. + } => Some(( + group_path, + artifact_id, + metadata_document_key(&path).unwrap_or(&path).to_string(), + checksum_suffix(&path), + )), + _ => None, + } }; - for proxy in &state.config.maven.proxies { + let mut unavailable = None; + let mut upstream_rejection = None; + for proxy in &repository.proxies { let upstream_path = metadata_request .as_ref() .map(|(_, _, document_path, _)| document_path.as_str()) @@ -539,6 +817,7 @@ async fn download( .await { Ok(data) => { + state.maven_negative_cache.lock().remove(&key); state.metrics.record_download("maven"); state.metrics.record_cache_miss("maven"); state.activity.push(ActivityEntry::new( @@ -557,10 +836,12 @@ async fn download( { let metadata = merge_and_cache_proxy_metadata( &state, + &repository.storage_prefix(), group_path, artifact_id, document_path, &data, + repository.version_policy, ) .await; match requested_checksum { @@ -605,71 +886,253 @@ async fn download( } return with_content_type(&path, response_data).into_response(); } - Err(ProxyError::CircuitOpen(reg)) => return circuit_open_response(®), - Err(e) => { + Err(ProxyError::NotFound) => { + tracing::debug!(upstream = %proxy.url(), path = %path, "Maven proxy returned not found, trying next"); + continue; + } + Err(ProxyError::CircuitOpen(reg)) => { + unavailable = Some(circuit_open_response(®)); + continue; + } + // `proxy_fetch` reserves Upstream(404) for a policy/WAF block that + // happened to be disguised as 404. It is an outage, not an + // authoritative miss, so keep it out of the negative-cache path. + Err(ProxyError::Upstream(404)) => { + tracing::warn!( + upstream = %proxy.url(), + path = %path, + "Maven upstream policy block was disguised as not found" + ); + let mut response = + (StatusCode::BAD_GATEWAY, "Maven upstream policy blocked").into_response(); + response.extensions_mut().insert(MavenPolicyBlock); + unavailable = Some(response); + continue; + } + Err(ProxyError::Upstream(code)) if (400..500).contains(&code) => { + tracing::debug!( + status = code, + upstream = %proxy.url(), + path = %path, + "Maven proxy rejected request, trying next" + ); + let status = StatusCode::from_u16(code).unwrap_or(StatusCode::BAD_GATEWAY); + upstream_rejection = + Some((status, "Maven upstream rejected request").into_response()); + continue; + } + Err(e @ (ProxyError::Upstream(_) | ProxyError::Network(_))) => { tracing::debug!(error = ?e, upstream = %proxy.url(), path = %path, "Maven proxy fetch failed, trying next"); + unavailable = + Some((StatusCode::BAD_GATEWAY, "Maven upstream unavailable").into_response()); continue; } } } - // All proxies failed — serve the stale cached artifact if we have one (graceful). - if let Some(ref data) = cached { - tracing::warn!(registry = "maven", path = %path, "Maven upstream failed, serving stale cached artifact"); - // Quarantine still applies to a version artifact served from a stale cache: - // a held SNAPSHOT must not be released just because the upstream went down - // (mutable SNAPSHOT artifacts reach this path; immutable releases serve from - // the fresh-cache branch above, which is already gated). - if curation_coords.is_some() { - let (q_mode, q_secs) = crate::digest_quarantine::resolve_global( - state.config.curation.maven.quarantine.as_ref().or(state - .config - .curation - .quarantine - .as_ref()), - state - .config - .curation - .maven - .quarantine_ttl - .as_deref() - .or(state.config.curation.quarantine_ttl.as_deref()), - ); - if let Some(resp) = crate::digest_quarantine::proxy_gate_dated( - &state.digest_store, - "maven", - data, - &q_mode, - q_secs, - "cache-stale", - publish_date, - ) { - return resp; + // Authentication, authorization, throttling and other upstream client + // errors are authoritative responses. They must neither serve stale bytes + // nor enter the negative cache. + if let Some(response) = upstream_rejection { + return response; + } + + // Only transport/5xx/circuit failures may use stale-on-error. An authoritative + // upstream 404 must not resurrect a removed mutable version or metadata document. + if unavailable.is_some() { + if let Some(ref data) = cached { + tracing::warn!(registry = "maven", path = %path, "Maven upstream failed, serving stale cached artifact"); + // Quarantine still applies to a version artifact served from a stale cache: + // a held SNAPSHOT must not be released just because the upstream went down + // (mutable SNAPSHOT artifacts reach this path; immutable releases serve from + // the fresh-cache branch above, which is already gated). + if curation_coords.is_some() { + let (q_mode, q_secs) = crate::digest_quarantine::resolve_global( + state.config.curation.maven.quarantine.as_ref().or(state + .config + .curation + .quarantine + .as_ref()), + state + .config + .curation + .maven + .quarantine_ttl + .as_deref() + .or(state.config.curation.quarantine_ttl.as_deref()), + ); + if let Some(resp) = crate::digest_quarantine::proxy_gate_dated( + &state.digest_store, + "maven", + data, + &q_mode, + q_secs, + "cache-stale", + publish_date, + ) { + return resp; + } } + let mut response = with_content_type(&path, data.clone()).into_response(); + response.headers_mut().insert( + axum::http::header::HeaderName::from_static("x-nora-stale"), + axum::http::header::HeaderValue::from_static("true"), + ); + return response; } - let mut response = with_content_type(&path, data.clone()).into_response(); - response.headers_mut().insert( - axum::http::header::HeaderName::from_static("x-nora-stale"), - axum::http::header::HeaderValue::from_static("true"), - ); + } + + if let Some(response) = unavailable { return response; } - if !state.config.maven.proxies.is_empty() { + if repository.is_proxy() { + if cached.is_none() && repository.negative_ttl > 0 { + insert_negative_cache_entry( + &mut state.maven_negative_cache.lock(), + key, + repository.negative_ttl, + MAVEN_NEGATIVE_CACHE_MAX_ENTRIES, + ); + } tracing::warn!(registry = "maven", path = %path, "Proxy failed, returning 404"); } StatusCode::NOT_FOUND.into_response() } +async fn download_group( + state: AppState, + headers: axum::http::HeaderMap, + members: &[String], + path: String, +) -> Response { + let metadata = matches!(classify_path(&path), MavenPathKind::ArtifactMeta { .. }); + if !metadata { + let mut unavailable = None; + for member in members { + let Some(config) = state.config.maven.repository(member).cloned() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Some(repository) = DirectRepository::from_config(&state, &config) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let response = + download_direct(state.clone(), headers.clone(), repository, path.clone()).await; + if response.extensions().get::().is_some() { + return response; + } + match response.status() { + StatusCode::NOT_FOUND => {} + StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT => unavailable = Some(response), + _ => return response, + } + } + return unavailable.unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + } + + let document_path = metadata_document_key(&path).unwrap_or(&path); + let requested_checksum = checksum_suffix(&path); + let mut documents = Vec::new(); + let mut unavailable = None; + + for member in members { + let Some(config) = state.config.maven.repository(member).cloned() else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Some(repository) = DirectRepository::from_config(&state, &config) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let response = download_direct( + state.clone(), + headers.clone(), + repository, + document_path.to_string(), + ) + .await; + if response.extensions().get::().is_some() { + return response; + } + match response.status() { + StatusCode::OK => match to_bytes(response.into_body(), usize::MAX).await { + Ok(body) => documents.push(body), + Err(_) => return StatusCode::BAD_GATEWAY.into_response(), + }, + StatusCode::NOT_FOUND => {} + StatusCode::BAD_GATEWAY + | StatusCode::SERVICE_UNAVAILABLE + | StatusCode::GATEWAY_TIMEOUT => unavailable = Some(response), + _ => return response, + } + } + + let Some(first) = documents.first() else { + return unavailable.unwrap_or_else(|| StatusCode::NOT_FOUND.into_response()); + }; + let merged = merge_group_metadata(document_path, &documents) + .unwrap_or_else(|| Bytes::copy_from_slice(first)); + let response_data = match requested_checksum { + Some(suffix) => match checksum_hex(suffix, &merged) { + Some(checksum) => Bytes::from(checksum), + None => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }, + None => merged, + }; + with_content_type(&path, response_data).into_response() +} + // ============================================================================ // Upload // ============================================================================ -async fn upload( +async fn upload_legacy( State(state): State, Path(path): Path, Extension(authority): Extension, body: Bytes, +) -> Response { + if let Some(repository) = state.config.maven.default_repository.clone() { + return upload_configured(state, &repository, path, authority, body).await; + } + let repository = DirectRepository::legacy(&state); + upload_direct(state, repository, path, authority, body).await +} + +pub(crate) async fn upload_named( + State(state): State, + Path((repository, path)): Path<(String, String)>, + Extension(authority): Extension, + body: Bytes, +) -> Response { + upload_configured(state, &repository, path, authority, body).await +} + +async fn upload_configured( + state: AppState, + repository: &str, + path: String, + authority: NamespaceAuthority, + body: Bytes, +) -> Response { + let Some(config) = state.config.maven.repository(repository).cloned() else { + return StatusCode::NOT_FOUND.into_response(); + }; + let Some(repository) = DirectRepository::from_config(&state, &config) else { + return method_not_allowed("GET"); + }; + if repository.is_proxy() || repository.write_policy == MavenWritePolicy::Deny { + return method_not_allowed("GET"); + } + upload_direct(state, repository, path, authority, body).await +} + +async fn upload_direct( + state: AppState, + repository: DirectRepository, + path: String, + authority: NamespaceAuthority, + body: Bytes, ) -> Response { if !path.is_ascii() || path.contains("..") || path.contains('\0') || path.starts_with('/') { return (StatusCode::BAD_REQUEST, "Invalid path").into_response(); @@ -677,20 +1140,45 @@ async fn upload( // Enforce OIDC namespace_scope on the artifact coordinate (group/artifactId). // An unrecognized (Opaque) path yields an empty coordinate → fail-closed (#583). - let maven_namespace = match classify_path(&path) { - MavenPathKind::VersionFile(c) => format!("{}/{}", c.group_path, c.artifact_id), - MavenPathKind::ArtifactMeta { - group_path, - artifact_id, - .. - } => format!("{}/{}", group_path, artifact_id), - MavenPathKind::Opaque => String::new(), - }; + let maven_namespace = version_metadata_path(&path) + .map(|(group_path, artifact_id, _)| format!("{group_path}/{artifact_id}")) + .unwrap_or_else(|| match classify_path(&path) { + MavenPathKind::VersionFile(c) => format!("{}/{}", c.group_path, c.artifact_id), + MavenPathKind::ArtifactMeta { + group_path, + artifact_id, + .. + } => format!("{}/{}", group_path, artifact_id), + MavenPathKind::Opaque => String::new(), + }); if enforce_namespace_scope(&authority, &maven_namespace).is_err() { return StatusCode::FORBIDDEN.into_response(); } - let key = storage_key(&path); + let key = repository.storage_key(&path); + let requested_version = version_metadata_path(&path) + .map(|(_, _, version)| version) + .or_else(|| match classify_path(&path) { + MavenPathKind::VersionFile(coords) => Some(coords.version), + _ => None, + }); + if let Some(version) = requested_version { + let allowed = match repository.version_policy { + MavenVersionPolicy::Release => !is_snapshot(&version), + MavenVersionPolicy::Snapshot => is_snapshot(&version), + MavenVersionPolicy::Mixed => true, + }; + if !allowed { + return ( + StatusCode::BAD_REQUEST, + format!( + "Version {} is not allowed by this repository's version policy", + version + ), + ) + .into_response(); + } + } let artifact_name = path .split('/') @@ -702,33 +1190,60 @@ async fn upload( .collect::>() .join("/"); - match classify_path(&path) { - MavenPathKind::VersionFile(coords) if is_checksum_file(&coords.filename) => { - // Client uploading a checksum — verify against our computed value - if state.config.maven.checksum_verify { - if let Ok(computed) = state.storage.get(&key).await { - let computed_str = String::from_utf8_lossy(&computed).trim().to_string(); - let client_str = String::from_utf8_lossy(&body).trim().to_string(); - if computed_str != client_str { - tracing::warn!( - path = %path, - expected = %computed_str, - received = %client_str, - "SECURITY: Maven checksum mismatch on upload" - ); - return (StatusCode::BAD_REQUEST, "Checksum mismatch").into_response(); - } - } + // Client checksum uploads are acknowledgements of the server's derived checksum, + // not independent authoritative objects. Require the base object and validate + // every supported algorithm against its actual bytes. + if let Some(suffix) = checksum_suffix(&path) { + let Some(document_path) = metadata_document_key(&path) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let lock = state.publish_lock(&mutation_lock_key(&repository, document_path)); + let _guard = lock.lock().await; + let document_key = repository.storage_key(document_path); + let document = match state.storage.get(&document_key).await { + Ok(document) => document, + Err(StorageError::NotFound) => return StatusCode::NOT_FOUND.into_response(), + Err(error) => { + tracing::error!( + %error, + key = %document_key, + "Failed to read Maven checksum base object" + ); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - match state.storage.put(&key, &body).await { - Ok(()) => StatusCode::CREATED.into_response(), - Err(e) => { - tracing::error!(error = %e, key = %key, "Failed to store Maven checksum"); - StatusCode::INTERNAL_SERVER_ERROR.into_response() - } + }; + let Some(expected) = checksum_hex(suffix, &document) else { + return StatusCode::BAD_REQUEST.into_response(); + }; + let received = String::from_utf8_lossy(&body); + let server_managed_metadata = matches!( + classify_path(document_path), + MavenPathKind::ArtifactMeta { filename, .. } if filename == "maven-metadata.xml" + ) && matches!( + classify_metadata_level(&document), + Some( + MavenMetadataLevel::Group + | MavenMetadataLevel::Artifact + | MavenMetadataLevel::ArtifactAndGroup + ) + ); + if !server_managed_metadata && !expected.eq_ignore_ascii_case(received.trim()) { + tracing::warn!(path = %path, "SECURITY: Maven checksum mismatch on upload"); + return (StatusCode::BAD_REQUEST, "Checksum mismatch").into_response(); + } + match state.storage.put(&key, expected.as_bytes()).await { + Ok(()) => { + state.metrics.record_upload("maven"); + return StatusCode::CREATED.into_response(); + } + Err(error) => { + tracing::error!(%error, %key, "Failed to store derived Maven checksum"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } } + } + match classify_path(&path) { MavenPathKind::VersionFile(coords) => { // Primary artifact upload (jar, pom, war, etc.) let snap = is_snapshot(&coords.version); @@ -736,35 +1251,92 @@ async fn upload( // Lock on metadata key to serialize all uploads for the same artifact. // This prevents TOCTOU races on both immutability checks and // maven-metadata.xml generation (read-list-generate-write cycle). - let metadata_lock_key = format!( - "maven/{}/{}/maven-metadata.xml", + let metadata_lock_key = repository.storage_key(&format!( + "{}/{}/maven-metadata.xml", coords.group_path, coords.artifact_id - ); + )); let lock = state.publish_lock(&metadata_lock_key); let _guard = lock.lock().await; - if !snap - && state.config.maven.immutable_releases - && state.storage.stat(&key).await.is_some() - { - return ( - StatusCode::CONFLICT, - format!( - "Version {}:{} is immutable (already deployed)", - coords.artifact_id, coords.version - ), - ) - .into_response(); - } - - if let Err(e) = state.storage.put(&key, &body).await { - tracing::error!(error = %e, key = %key, "Failed to store Maven artifact"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } + let stored = if !snap && repository.write_policy == MavenWritePolicy::AllowOnce { + state.storage.put_if_absent(&key, &body).await + } else { + state.storage.put(&key, &body).await + }; + let exact_retry = match stored { + Ok(()) => false, + Err(StorageError::AlreadyExists) => { + let existing = match state.storage.get(&key).await { + Ok(existing) => existing, + Err(error) => { + tracing::error!( + %error, + %key, + "Failed to read existing immutable Maven artifact" + ); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + if existing != body { + return ( + StatusCode::CONFLICT, + format!( + "Version {}:{} is immutable (already deployed)", + coords.artifact_id, coords.version + ), + ) + .into_response(); + } + true + } + Err(error) => { + tracing::error!(%error, %key, "Failed to store Maven artifact"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; - compute_and_store_checksums(&state.storage, &key, &body).await; + if let Err(error) = compute_and_store_checksums(&state.storage, &key, &body).await { + tracing::error!(%error, %key, "Failed to store Maven artifact checksums"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } - update_artifact_metadata(&state, &coords.group_path, &coords.artifact_id).await; + // An exact retry after the immutable object was created must finish + // any interrupted checksum/metadata work. Do not treat an ordinary + // retry as a new deployment, though: preserve the current release + // when metadata already records this version. + let metadata_records_version = if exact_retry { + match state.storage.get(&metadata_lock_key).await { + Ok(metadata) => parse_artifact_metadata(&metadata) + .is_some_and(|metadata| metadata.versions.contains(&coords.version)), + Err(StorageError::NotFound) => false, + Err(error) => { + tracing::error!( + %error, + key = %metadata_lock_key, + "Failed to inspect Maven metadata during exact retry" + ); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + } + } else { + false + }; + if let Err(error) = update_artifact_metadata( + &state, + &repository.storage_prefix(), + &coords.group_path, + &coords.artifact_id, + (!metadata_records_version).then_some(coords.version.as_str()), + ) + .await + { + tracing::error!( + %error, + key = %metadata_lock_key, + "Failed to update Maven artifact metadata" + ); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } state.metrics.record_upload("maven"); state @@ -781,40 +1353,192 @@ async fn upload( StatusCode::CREATED.into_response() } - MavenPathKind::ArtifactMeta { filename, .. } => { - let server_managed = if filename == "maven-metadata.xml" { - classify_metadata_level(&body) == Some(MavenMetadataLevel::Artifact) - } else if let Some(metadata_key) = metadata_document_key(&key) { - match state.storage.get(metadata_key).await { - Ok(metadata) => { - classify_metadata_level(&metadata) == Some(MavenMetadataLevel::Artifact) - } - Err(StorageError::NotFound) => false, - Err(e) => { - tracing::error!( - error = %e, - key = %metadata_key, - "Failed to read Maven metadata for checksum classification" - ); + MavenPathKind::ArtifactMeta { + group_path, + artifact_id, + filename, + } => { + let Some(level) = classify_metadata_level(&body) else { + return (StatusCode::BAD_REQUEST, "Invalid Maven metadata").into_response(); + }; + + if filename == "maven-metadata.xml" + && matches!( + level, + MavenMetadataLevel::Group + | MavenMetadataLevel::Artifact + | MavenMetadataLevel::ArtifactAndGroup + ) + { + let incoming_artifact = matches!( + level, + MavenMetadataLevel::Artifact | MavenMetadataLevel::ArtifactAndGroup + ) + .then(|| parse_artifact_metadata(&body)) + .flatten(); + let incoming_plugins = matches!( + level, + MavenMetadataLevel::Group | MavenMetadataLevel::ArtifactAndGroup + ) + .then(|| parse_group_plugins(&body)) + .flatten(); + if matches!( + level, + MavenMetadataLevel::Artifact | MavenMetadataLevel::ArtifactAndGroup + ) && incoming_artifact.is_none() + || matches!( + level, + MavenMetadataLevel::Group | MavenMetadataLevel::ArtifactAndGroup + ) && incoming_plugins.is_none() + { + return (StatusCode::BAD_REQUEST, "Invalid Maven metadata").into_response(); + } + if incoming_artifact.as_ref().is_some_and(|metadata| { + !artifact_metadata_matches_path(metadata, &group_path, &artifact_id) + }) { + return ( + StatusCode::BAD_REQUEST, + "Maven metadata coordinates do not match the request path", + ) + .into_response(); + } + + let lock = state.publish_lock(&key); + let _guard = lock.lock().await; + let current = match state.storage.get(&key).await { + Ok(current) => Some(current), + Err(StorageError::NotFound) => None, + Err(error) => { + tracing::error!(%error, %key, "Failed to read Maven artifact metadata"); return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } + }; + let current_artifact = current.as_deref().and_then(parse_artifact_metadata); + let current_plugins = current.as_deref().and_then(parse_group_plugins); + + let artifact_xml = if incoming_artifact.is_some() || current_artifact.is_some() { + let versions = match stored_artifact_versions( + &state, + &repository.storage_prefix(), + &group_path, + &artifact_id, + ) + .await + { + Ok(versions) => versions, + Err(error) => { + tracing::error!( + %error, + %key, + "Failed to list Maven artifact versions" + ); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + let references_missing_version = incoming_artifact + .iter() + .flat_map(|metadata| { + metadata + .versions + .iter() + .chain(metadata.release.iter()) + .chain(metadata.latest.iter()) + }) + .any(|version| !versions.contains(version)); + if references_missing_version + || incoming_artifact + .as_ref() + .and_then(|metadata| metadata.release.as_deref()) + .is_some_and(is_snapshot) + { + return ( + StatusCode::BAD_REQUEST, + "Maven metadata references an unavailable or invalid version", + ) + .into_response(); + } + let last_updated = chrono::Utc::now().format("%Y%m%d%H%M%S").to_string(); + let current_artifact_document = + current_artifact.as_ref().and(current.as_deref()); + merge_hosted_artifact_metadata( + &group_path.replace('/', "."), + &artifact_id, + current_artifact_document, + incoming_artifact.as_ref(), + &versions, + None, + incoming_artifact.is_some().then_some(last_updated.as_str()), + ) + } else { + None + }; + + let mut plugins = Vec::new(); + let mut seen_prefixes = BTreeSet::new(); + if let Some(incoming) = incoming_plugins { + extend_plugins_first_wins(&mut plugins, &mut seen_prefixes, incoming); + } + if let Some(current) = current_plugins { + extend_plugins_first_wins(&mut plugins, &mut seen_prefixes, current); + } + let has_plugins = !plugins.is_empty(); + let Some(xml) = combine_metadata_sections( + artifact_xml, + has_plugins.then_some(plugins.as_slice()), + ) else { + return (StatusCode::BAD_REQUEST, "Invalid Maven metadata").into_response(); + }; + + if let Err(error) = state.storage.put(&key, xml.as_bytes()).await { + tracing::error!(%error, %key, "Failed to store Maven artifact metadata"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + if let Err(error) = + compute_and_store_checksums(&state.storage, &key, xml.as_bytes()).await + { + tracing::error!(%error, %key, "Failed to store Maven metadata checksums"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - } else { - false - }; - - if server_managed { state.metrics.record_upload("maven"); return StatusCode::CREATED.into_response(); } + if level == MavenMetadataLevel::Version { + let Some((path_group, path_artifact, path_version)) = version_metadata_path(&path) + else { + return (StatusCode::BAD_REQUEST, "Invalid Maven metadata path") + .into_response(); + }; + let Some(metadata) = parse_version_metadata(&body) else { + return (StatusCode::BAD_REQUEST, "Invalid Maven metadata").into_response(); + }; + if metadata.group_id.as_deref() != Some(path_group.replace('/', ".").as_str()) + || metadata.artifact_id.as_deref() != Some(path_artifact.as_str()) + || metadata.version.as_deref() != Some(path_version.as_str()) + { + return ( + StatusCode::BAD_REQUEST, + "Maven metadata coordinates do not match the request path", + ) + .into_response(); + } + } + + let lock = state.publish_lock(&mutation_lock_key(&repository, &path)); + let _guard = lock.lock().await; match state.storage.put(&key, &body).await { Ok(()) => { + if let Err(error) = + compute_and_store_checksums(&state.storage, &key, &body).await + { + tracing::error!(%error, %key, "Failed to store Maven metadata checksums"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } state.metrics.record_upload("maven"); StatusCode::CREATED.into_response() } - Err(e) => { - tracing::error!(error = %e, key = %key, "Failed to store Maven metadata"); + Err(error) => { + tracing::error!(%error, %key, "Failed to store Maven metadata"); StatusCode::INTERNAL_SERVER_ERROR.into_response() } } @@ -857,24 +1581,29 @@ fn checksum_hex(suffix: &str, data: &[u8]) -> Option { } } -async fn compute_and_store_checksums(storage: &crate::storage::Storage, key: &str, data: &[u8]) { +async fn compute_and_store_checksums( + storage: &crate::storage::Storage, + key: &str, + data: &[u8], +) -> crate::storage::Result<()> { for suffix in ["md5", "sha1", "sha256", "sha512"] { let ck = format!("{}.{}", key, suffix); let Some(hash) = checksum_hex(suffix, data) else { continue; }; - if let Err(e) = storage.put(&ck, hash.as_bytes()).await { - tracing::warn!(key = %ck, error = %e, "maven: failed to store checksum"); - } + storage.put(&ck, hash.as_bytes()).await?; } + Ok(()) } // ============================================================================ // Metadata generation // ============================================================================ -#[derive(Default)] +#[derive(Clone, Default)] struct ArtifactMetadata { + group_id: Option, + artifact_id: Option, latest: Option, release: Option, last_updated: Option, @@ -882,7 +1611,10 @@ struct ArtifactMetadata { } fn parse_artifact_metadata(data: &[u8]) -> Option { - if classify_metadata_level(data) != Some(MavenMetadataLevel::Artifact) { + if !matches!( + classify_metadata_level(data), + Some(MavenMetadataLevel::Artifact | MavenMetadataLevel::ArtifactAndGroup) + ) { return None; } @@ -890,9 +1622,31 @@ fn parse_artifact_metadata(data: &[u8]) -> Option { reader.config_mut().trim_text(true); let mut metadata = ArtifactMetadata::default(); let mut in_versions = false; + let mut in_plugins = false; loop { match reader.read_event() { + Ok(Event::Start(element)) if element.local_name().as_ref() == b"plugins" => { + in_plugins = true; + } + Ok(Event::Start(element)) + if element.local_name().as_ref() == b"groupId" && !in_plugins => + { + let text = reader.read_text(element.name()).ok()?.decode().ok()?; + let group_id = quick_xml::escape::unescape(&text).ok()?.into_owned(); + if !group_id.is_empty() { + metadata.group_id = Some(group_id); + } + } + Ok(Event::Start(element)) + if element.local_name().as_ref() == b"artifactId" && !in_plugins => + { + let text = reader.read_text(element.name()).ok()?.decode().ok()?; + let artifact_id = quick_xml::escape::unescape(&text).ok()?.into_owned(); + if !artifact_id.is_empty() { + metadata.artifact_id = Some(artifact_id); + } + } Ok(Event::Start(element)) if element.local_name().as_ref() == b"versions" => { in_versions = true; } @@ -929,6 +1683,9 @@ fn parse_artifact_metadata(data: &[u8]) -> Option { Ok(Event::End(element)) if element.local_name().as_ref() == b"versions" => { in_versions = false; } + Ok(Event::End(element)) if element.local_name().as_ref() == b"plugins" => { + in_plugins = false; + } Ok(Event::Eof) => break, Err(_) => return None, _ => {} @@ -938,13 +1695,33 @@ fn parse_artifact_metadata(data: &[u8]) -> Option { Some(metadata) } +fn artifact_metadata_matches_path( + metadata: &ArtifactMetadata, + group_path: &str, + artifact_id: &str, +) -> bool { + metadata.group_id.as_deref() == Some(group_path.replace('/', ".").as_str()) + && metadata.artifact_id.as_deref() == Some(artifact_id) +} + async fn stored_artifact_versions( state: &AppState, + storage_prefix: &str, + group_path: &str, + artifact_id: &str, +) -> Result, StorageError> { + stored_artifact_versions_in_storage(&state.storage, storage_prefix, group_path, artifact_id) + .await +} + +async fn stored_artifact_versions_in_storage( + storage: &crate::storage::Storage, + storage_prefix: &str, group_path: &str, artifact_id: &str, ) -> Result, StorageError> { - let prefix = format!("maven/{}/{}/", group_path, artifact_id); - let keys = state.storage.list(&prefix).await?; + let prefix = format!("{storage_prefix}{group_path}/{artifact_id}/"); + let keys = storage.list(&prefix).await?; let mut versions = BTreeSet::new(); for key in &keys { @@ -962,617 +1739,1807 @@ async fn stored_artifact_versions( Ok(versions) } +fn push_unique(versions: &mut Vec, known: &mut BTreeSet, version: &str) { + if !version.is_empty() && known.insert(version.to_string()) { + versions.push(version.to_string()); + } +} + +/// Merge authoritative hosted metadata. Version order follows deployment/client +/// order, `release` follows the current non-SNAPSHOT deployment rather than the +/// greatest version, and `latest` follows the current deployment unless a +/// validated client metadata document explicitly supplies it. +fn merge_hosted_artifact_metadata( + group_id: &str, + artifact_id: &str, + current: Option<&[u8]>, + incoming: Option<&ArtifactMetadata>, + stored_versions: &BTreeSet, + deployed_version: Option<&str>, + last_updated: Option<&str>, +) -> Option { + let current = match current { + Some(data) => { + let metadata = parse_artifact_metadata(data)?; + if metadata.group_id.as_deref() != Some(group_id) + || metadata.artifact_id.as_deref() != Some(artifact_id) + { + return None; + } + metadata + } + None => ArtifactMetadata::default(), + }; + if incoming.is_some_and(|metadata| { + metadata.group_id.as_deref() != Some(group_id) + || metadata.artifact_id.as_deref() != Some(artifact_id) + }) { + return None; + } + + let mut versions = Vec::new(); + let mut known = BTreeSet::new(); + for version in ¤t.versions { + push_unique(&mut versions, &mut known, version); + } + if let Some(incoming) = incoming { + for version in &incoming.versions { + push_unique(&mut versions, &mut known, version); + } + } + if let Some(version) = deployed_version { + push_unique(&mut versions, &mut known, version); + } + let mut discovered: Vec = stored_versions + .iter() + .filter(|version| !known.contains(*version)) + .cloned() + .collect(); + sort_maven_versions(&mut discovered); + for version in discovered { + push_unique(&mut versions, &mut known, &version); + } + if versions.is_empty() { + return None; + } + + let release = incoming + .and_then(|metadata| metadata.release.clone()) + .or_else(|| { + deployed_version + .filter(|version| !is_snapshot(version)) + .map(ToOwned::to_owned) + }) + .or(current.release); + let latest = incoming + .and_then(|metadata| metadata.latest.clone()) + .or_else(|| deployed_version.map(ToOwned::to_owned)) + .or(current.latest); + if release + .as_ref() + .is_some_and(|version| is_snapshot(version) || !known.contains(version)) + || latest + .as_ref() + .is_some_and(|version| !known.contains(version)) + { + return None; + } + + let incoming_last_updated = incoming.and_then(|metadata| metadata.last_updated.as_deref()); + Some(generate_metadata_xml_with_versioning( + group_id, + artifact_id, + &versions, + latest.as_deref(), + release.as_deref(), + last_updated + .or(incoming_last_updated) + .or(current.last_updated.as_deref()), + )) +} + +/// Merge one proxy document with versions already materialized in the same +/// repository. Explicit `latest`/`release` remain upstream-owned; local hosted +/// deployment semantics are handled by `merge_hosted_artifact_metadata`. fn merge_artifact_metadata( group_id: &str, artifact_id: &str, base: Option<&[u8]>, stored_versions: &BTreeSet, last_updated: Option<&str>, + version_policy: MavenVersionPolicy, ) -> Option { - let mut metadata = match base { + let metadata = match base { Some(data) => parse_artifact_metadata(data)?, None => ArtifactMetadata::default(), }; let mut known = BTreeSet::new(); - metadata + let mut versions = Vec::new(); + for version in metadata .versions - .retain(|version| known.insert(version.clone())); + .iter() + .filter(|version| version_allowed_by_policy(version_policy, version)) + { + push_unique(&mut versions, &mut known, version); + } let mut additional: Vec = stored_versions .iter() - .filter(|version| !known.contains(*version)) + .filter(|version| { + version_allowed_by_policy(version_policy, version) && !known.contains(*version) + }) .cloned() .collect(); sort_maven_versions(&mut additional); for version in additional { - if metadata - .latest - .as_ref() - .is_none_or(|latest| compare_maven_versions(latest, &version).is_lt()) - { - metadata.latest = Some(version.clone()); - } - if !is_snapshot(&version) - && metadata - .release - .as_ref() - .is_none_or(|release| compare_maven_versions(release, &version).is_lt()) - { - metadata.release = Some(version.clone()); - } - metadata.versions.push(version); + push_unique(&mut versions, &mut known, &version); } - if metadata.versions.is_empty() { - return None; - } - if metadata.latest.is_none() || metadata.release.is_none() { - let mut sorted = metadata.versions.clone(); - sort_maven_versions(&mut sorted); - metadata - .latest - .get_or_insert_with(|| sorted.last().cloned().unwrap_or_default()); - if metadata.release.is_none() { - metadata.release = sorted - .into_iter() - .rev() - .find(|version| !is_snapshot(version)); - } - } + let latest = metadata + .latest + .filter(|version| { + version_allowed_by_policy(version_policy, version) && known.contains(version) + }) + .or_else(|| { + versions + .iter() + .max_by(|left, right| compare_maven_versions(left, right)) + .cloned() + }); + let release = match version_policy { + MavenVersionPolicy::Snapshot => None, + MavenVersionPolicy::Release | MavenVersionPolicy::Mixed => metadata + .release + .filter(|version| !is_snapshot(version) && known.contains(version)) + .or_else(|| { + versions + .iter() + .filter(|version| !is_snapshot(version)) + .max_by(|left, right| compare_maven_versions(left, right)) + .cloned() + }), + }; Some(generate_metadata_xml_with_versioning( group_id, artifact_id, - &metadata.versions, - metadata.latest.as_deref().unwrap_or(""), - metadata.release.as_deref().unwrap_or(""), + &versions, + latest.as_deref(), + release.as_deref(), last_updated.or(metadata.last_updated.as_deref()), )) } -async fn merge_and_cache_proxy_metadata( - state: &AppState, - group_path: &str, - artifact_id: &str, - document_path: &str, - upstream: &[u8], -) -> Bytes { - let key = storage_key(document_path); - - // Serialize the read -> merge -> write -> checksums cycle with the upload-side - // regeneration (`update_artifact_metadata`) and any concurrent proxy merge: all of them - // write the same `maven-metadata.xml` key and its four checksum sidecars. Without a shared - // lock those five independent `put`s interleave, leaving a stored `.sha1`/`.md5`/… that - // corresponds to different bytes than the stored `.xml` (checksum-mismatch window). The - // upload path locks the artifact's `maven-metadata.xml` key; locking the document key here - // — identical for artifact-level metadata — keeps `publish_lock serializes all writes to the - // same artifact path` intact on the proxy path too (#886). Held across the writes below. - let lock = state.publish_lock(&key); - let _guard = lock.lock().await; - - let cached = state.storage.get(&key).await.ok(); - let last_updated = [Some(upstream), cached.as_deref()] - .into_iter() - .flatten() - .filter_map(parse_artifact_metadata) - .filter_map(|metadata| metadata.last_updated) - .max(); - let data = if let Ok(stored_versions) = - stored_artifact_versions(state, group_path, artifact_id).await +fn merge_group_metadata(path: &str, documents: &[Bytes]) -> Option { + let levels: Vec = documents + .iter() + .map(|document| classify_metadata_level(document)) + .collect::>()?; + if levels + .iter() + .all(|level| *level == MavenMetadataLevel::Version) { - merge_artifact_metadata( - &group_path.replace('/', "."), - artifact_id, - Some(upstream), - &stored_versions, - last_updated.as_deref(), - ) - .map_or_else(|| Bytes::copy_from_slice(upstream), Bytes::from) - } else { - Bytes::copy_from_slice(upstream) - }; - - if let Err(error) = state.storage.put(&key, &data).await { - tracing::warn!(key = %key, error = %error, "maven: failed to cache metadata"); - } else { - compute_and_store_checksums(&state.storage, &key, &data).await; + return merge_group_version_metadata(path, documents); + } + if levels.contains(&MavenMetadataLevel::Version) { + return None; } - data -} - -async fn update_artifact_metadata(state: &AppState, group_path: &str, artifact_id: &str) { - let versions = match stored_artifact_versions(state, group_path, artifact_id).await { - Ok(versions) => versions, - Err(e) => { - tracing::warn!(error = ?e, group_path, artifact_id, "maven: failed to list storage for metadata generation"); - return; - } + let artifact_documents: Vec = documents + .iter() + .zip(&levels) + .filter(|(_, level)| { + matches!( + level, + MavenMetadataLevel::Artifact | MavenMetadataLevel::ArtifactAndGroup + ) + }) + .map(|(document, _)| document.clone()) + .collect(); + let artifact = if artifact_documents.is_empty() { + None + } else { + let document = merge_group_artifact_metadata(path, &artifact_documents)?; + Some(String::from_utf8(document.to_vec()).ok()?) }; + let plugins = merge_group_plugins(documents); - if versions.is_empty() { - return; - } + combine_metadata_sections(artifact, plugins.as_deref()).map(Bytes::from) +} - let prefix = format!("maven/{}/{}/", group_path, artifact_id); - let metadata_key = format!("{}maven-metadata.xml", prefix); - let current = state.storage.get(&metadata_key).await.ok(); - let group_id_dotted = group_path.replace('/', "."); - let last_updated = chrono::Utc::now().format("%Y%m%d%H%M%S").to_string(); - let xml = merge_artifact_metadata( - &group_id_dotted, +fn merge_group_artifact_metadata(path: &str, documents: &[Bytes]) -> Option { + let MavenPathKind::ArtifactMeta { + group_path, artifact_id, - current.as_deref(), - &versions, - Some(&last_updated), - ) - .unwrap_or_else(|| { - let mut sorted: Vec = versions.into_iter().collect(); - sort_maven_versions(&mut sorted); - generate_metadata_xml(&group_id_dotted, artifact_id, &sorted) - }); - - if state - .storage - .put(&metadata_key, xml.as_bytes()) - .await - .is_err() + .. + } = classify_path(path) + else { + return None; + }; + let parsed: Vec = documents + .iter() + .map(|document| parse_artifact_metadata(document)) + .collect::>()?; + if parsed + .iter() + .any(|metadata| !artifact_metadata_matches_path(metadata, &group_path, &artifact_id)) { - tracing::error!(key = %metadata_key, "Failed to write maven-metadata.xml"); - return; + return None; } - - compute_and_store_checksums(&state.storage, &metadata_key, xml.as_bytes()).await; + let mut versions: Vec = parsed + .iter() + .flat_map(|metadata| metadata.versions.iter().cloned()) + .collect::>() + .into_iter() + .collect(); + sort_maven_versions(&mut versions); + let latest = parsed + .iter() + .filter_map(|metadata| metadata.latest.as_ref()) + .max_by(|left, right| compare_maven_versions(left, right)) + .cloned(); + let release = parsed + .iter() + .filter_map(|metadata| metadata.release.as_ref()) + .filter(|version| !is_snapshot(version)) + .max_by(|left, right| compare_maven_versions(left, right)) + .cloned() + .or_else(|| { + versions + .iter() + .rev() + .find(|version| !is_snapshot(version)) + .cloned() + }); + let last_updated = parsed + .iter() + .filter_map(|metadata| metadata.last_updated.as_deref()) + .max(); + Some(Bytes::from(generate_metadata_xml_with_versioning( + &group_path.replace('/', "."), + &artifact_id, + &versions, + latest.as_deref(), + release.as_deref(), + last_updated, + ))) } -fn sort_maven_versions(versions: &mut [String]) { - versions.sort_by(|a, b| compare_maven_versions(a, b)); +#[derive(Clone, Default)] +struct MavenSnapshot { + timestamp: Option, + build_number: Option, + local_copy: Option, } -/// Compare Maven versions: split on `.`/`-`, compare numeric segments numerically. -/// SNAPSHOT sorts before release for the same base version. -fn compare_maven_versions(a: &str, b: &str) -> std::cmp::Ordering { - let a_base = a.strip_suffix("-SNAPSHOT").unwrap_or(a); - let b_base = b.strip_suffix("-SNAPSHOT").unwrap_or(b); - - let a_parts: Vec<&str> = a_base.split(['.', '-']).collect(); - let b_parts: Vec<&str> = b_base.split(['.', '-']).collect(); +#[derive(Clone, Default)] +struct MavenSnapshotVersion { + classifier: Option, + extension: String, + value: String, + updated: Option, +} - for (ap, bp) in a_parts.iter().zip(b_parts.iter()) { - let ord = match (ap.parse::(), bp.parse::()) { - (Ok(an), Ok(bn)) => an.cmp(&bn), - _ => ap.cmp(bp), - }; - if ord != std::cmp::Ordering::Equal { - return ord; - } - } +#[derive(Clone, Default)] +struct MavenVersionMetadata { + group_id: Option, + artifact_id: Option, + version: Option, + snapshot: MavenSnapshot, + last_updated: Option, + snapshot_versions: Vec, +} - // If all compared parts equal, shorter version is less (1.0 < 1.0.1) - let base_ord = a_parts.len().cmp(&b_parts.len()); - if base_ord != std::cmp::Ordering::Equal { - return base_ord; +fn parse_version_metadata(data: &[u8]) -> Option { + if classify_metadata_level(data) != Some(MavenMetadataLevel::Version) { + return None; } - // Same base: SNAPSHOT before release (1.0-SNAPSHOT < 1.0) - let a_snap = a.ends_with("-SNAPSHOT"); - let b_snap = b.ends_with("-SNAPSHOT"); - b_snap.cmp(&a_snap) -} + let mut reader = Reader::from_reader(data); + reader.config_mut().trim_text(true); + let mut metadata = MavenVersionMetadata::default(); + let mut in_snapshot = false; + let mut current_snapshot_version: Option = None; -/// Escape XML special characters in interpolated values. -fn xml_escape(s: &str) -> String { - s.replace('&', "&") - .replace('<', "<") - .replace('>', ">") - .replace('"', """) -} + loop { + match reader.read_event() { + Ok(Event::Start(element)) if element.local_name().as_ref() == b"snapshot" => { + in_snapshot = true; + } + Ok(Event::Start(element)) if element.local_name().as_ref() == b"snapshotVersion" => { + current_snapshot_version = Some(MavenSnapshotVersion::default()); + } + Ok(Event::Start(element)) => { + let name = element.local_name(); + let field = name.as_ref(); + if matches!( + field, + b"groupId" + | b"artifactId" + | b"version" + | b"timestamp" + | b"buildNumber" + | b"localCopy" + | b"lastUpdated" + | b"classifier" + | b"extension" + | b"value" + | b"updated" + ) { + let text = reader.read_text(element.name()).ok()?.decode().ok()?; + let value = quick_xml::escape::unescape(&text).ok()?.into_owned(); + if let Some(snapshot_version) = current_snapshot_version.as_mut() { + match field { + b"classifier" => { + if !value.is_empty() { + snapshot_version.classifier = Some(value); + } + } + b"extension" => snapshot_version.extension = value, + b"value" => snapshot_version.value = value, + b"updated" if !value.is_empty() => { + snapshot_version.updated = Some(value); + } + _ => {} + } + } else if in_snapshot { + match field { + b"timestamp" => { + if !value.is_empty() { + metadata.snapshot.timestamp = Some(value); + } + } + b"buildNumber" => { + metadata.snapshot.build_number = value.parse().ok(); + } + b"localCopy" => { + metadata.snapshot.local_copy = value.parse().ok(); + } + _ => {} + } + } else { + match field { + b"groupId" => metadata.group_id = Some(value), + b"artifactId" => metadata.artifact_id = Some(value), + b"version" => metadata.version = Some(value), + b"lastUpdated" if !value.is_empty() => { + metadata.last_updated = Some(value); + } + _ => {} + } + } + } + } + Ok(Event::End(element)) if element.local_name().as_ref() == b"snapshot" => { + in_snapshot = false; + } + Ok(Event::End(element)) if element.local_name().as_ref() == b"snapshotVersion" => { + let snapshot_version = current_snapshot_version.take()?; + if snapshot_version.extension.is_empty() || snapshot_version.value.is_empty() { + return None; + } + metadata.snapshot_versions.push(snapshot_version); + } + Ok(Event::Eof) => break, + Err(_) => return None, + _ => {} + } + } -fn generate_metadata_xml(group_id: &str, artifact_id: &str, versions: &[String]) -> String { - let latest = versions.last().map(|s| s.as_str()).unwrap_or(""); - let release = versions + Some(metadata) +} + +fn version_metadata_path(path: &str) -> Option<(String, String, String)> { + let document_path = metadata_document_key(path).unwrap_or(path); + let segments: Vec<&str> = document_path + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + if segments.len() < 4 || segments.last().copied() != Some("maven-metadata.xml") { + return None; + } + let version = segments[segments.len() - 2]; + if !is_snapshot(version) { + return None; + } + Some(( + segments[..segments.len() - 3].join("/"), + segments[segments.len() - 3].to_string(), + version.to_string(), + )) +} + +fn merge_group_version_metadata(path: &str, documents: &[Bytes]) -> Option { + let (group_path, artifact_id, version) = version_metadata_path(path)?; + let group_id = group_path.replace('/', "."); + let parsed: Vec = documents .iter() - .rev() - .find(|v| !v.ends_with("-SNAPSHOT")) - .map(|s| s.as_str()) - .unwrap_or(""); + .map(|document| parse_version_metadata(document)) + .collect::>()?; + if parsed.iter().any(|metadata| { + metadata.group_id.as_deref() != Some(group_id.as_str()) + || metadata.artifact_id.as_deref() != Some(artifact_id.as_str()) + || metadata.version.as_deref() != Some(version.as_str()) + }) { + return None; + } - generate_metadata_xml_with_versioning(group_id, artifact_id, versions, latest, release, None) + let first = parsed.first()?; + let selected_snapshot = parsed + .iter() + .skip(1) + .fold(first, |selected, candidate| { + let ordering = ( + candidate.snapshot.timestamp.as_deref().unwrap_or(""), + candidate.snapshot.build_number.unwrap_or(0), + candidate.last_updated.as_deref().unwrap_or(""), + ) + .cmp(&( + selected.snapshot.timestamp.as_deref().unwrap_or(""), + selected.snapshot.build_number.unwrap_or(0), + selected.last_updated.as_deref().unwrap_or(""), + )); + if ordering.is_gt() { + candidate + } else { + selected + } + }) + .snapshot + .clone(); + let last_updated = parsed + .iter() + .filter_map(|metadata| metadata.last_updated.as_ref()) + .max() + .cloned(); + let mut snapshot_versions: BTreeMap<(String, String), MavenSnapshotVersion> = BTreeMap::new(); + for metadata in &parsed { + for candidate in &metadata.snapshot_versions { + let key = ( + candidate.extension.clone(), + candidate.classifier.clone().unwrap_or_default(), + ); + let replace = snapshot_versions.get(&key).is_none_or(|current| { + candidate.updated.as_deref().unwrap_or("") + > current.updated.as_deref().unwrap_or("") + }); + if replace { + snapshot_versions.insert(key, candidate.clone()); + } + } + } + + Some(Bytes::from(generate_version_metadata_xml( + &group_id, + &artifact_id, + &version, + &selected_snapshot, + last_updated.as_deref(), + snapshot_versions.values(), + ))) } -fn generate_metadata_xml_with_versioning( +fn generate_version_metadata_xml<'a>( group_id: &str, artifact_id: &str, - versions: &[String], - latest: &str, - release: &str, + version: &str, + snapshot: &MavenSnapshot, last_updated: Option<&str>, + snapshot_versions: impl Iterator, ) -> String { - let generated_last_updated; - let last_updated = match last_updated { - Some(last_updated) => last_updated, - None => { - generated_last_updated = chrono::Utc::now().format("%Y%m%d%H%M%S").to_string(); - &generated_last_updated + let mut versioning = String::new(); + if snapshot.timestamp.is_some() + || snapshot.build_number.is_some() + || snapshot.local_copy.is_some() + { + versioning.push_str(" \n"); + if let Some(timestamp) = &snapshot.timestamp { + versioning.push_str(&format!( + " {}\n", + xml_escape(timestamp) + )); } - }; - - let version_elements: String = versions - .iter() - .map(|v| format!(" {}", xml_escape(v))) - .collect::>() - .join("\n"); + if let Some(build_number) = snapshot.build_number { + versioning.push_str(&format!( + " {build_number}\n" + )); + } + if let Some(local_copy) = snapshot.local_copy { + versioning.push_str(&format!(" {local_copy}\n")); + } + versioning.push_str(" \n"); + } + if let Some(last_updated) = last_updated { + versioning.push_str(&format!( + " {}\n", + xml_escape(last_updated) + )); + } + let snapshot_versions: Vec<&MavenSnapshotVersion> = snapshot_versions.collect(); + if !snapshot_versions.is_empty() { + versioning.push_str(" \n"); + for snapshot_version in snapshot_versions { + versioning.push_str(" \n"); + if let Some(classifier) = &snapshot_version.classifier { + versioning.push_str(&format!( + " {}\n", + xml_escape(classifier) + )); + } + versioning.push_str(&format!( + " {}\n {}\n", + xml_escape(&snapshot_version.extension), + xml_escape(&snapshot_version.value) + )); + if let Some(updated) = &snapshot_version.updated { + versioning.push_str(&format!( + " {}\n", + xml_escape(updated) + )); + } + versioning.push_str(" \n"); + } + versioning.push_str(" \n"); + } format!( - r#" - - {} - {} - - {} - {} - -{} - - {} - - -"#, + "\n\n {}\n {}\n {}\n \n{} \n\n", xml_escape(group_id), xml_escape(artifact_id), - xml_escape(latest), - xml_escape(release), - version_elements, - last_updated + xml_escape(version), + versioning, ) } -// ============================================================================ -// Content type -// ============================================================================ - -fn with_content_type( - path: &str, - data: Bytes, -) -> (StatusCode, [(header::HeaderName, &'static str); 2], Bytes) { - let content_type = if ends_with_ci(path, ".pom") { - "application/xml" - } else if ends_with_ci(path, ".jar") { - "application/java-archive" - } else if ends_with_ci(path, ".xml") { - "application/xml" - } else if ends_with_ci(path, ".sha1") - || ends_with_ci(path, ".md5") - || ends_with_ci(path, ".sha256") - || ends_with_ci(path, ".sha512") - { - "text/plain" - } else { - "application/octet-stream" - }; - - // maven-metadata.xml is mutable; release artifacts are immutable - let cache_control = if ends_with_ci(path, "maven-metadata.xml") - || ends_with_ci(path, "maven-metadata.xml.sha1") - || ends_with_ci(path, "maven-metadata.xml.md5") - { - "public, max-age=60, must-revalidate" - } else { - "public, max-age=31536000, immutable" - }; - - ( - StatusCode::OK, - [ - (header::CONTENT_TYPE, content_type), - (header::CACHE_CONTROL, cache_control), - ], - data, - ) +#[derive(Clone)] +struct MavenPlugin { + name: String, + prefix: String, + artifact_id: String, } -// ============================================================================ -// Unit Tests -// ============================================================================ - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_url_is_maven_central() { - // canonical Central hosts → true (date source available) - assert!(url_is_maven_central("https://repo1.maven.org/maven2")); - assert!(url_is_maven_central("https://repo.maven.apache.org/maven2")); - assert!(url_is_maven_central("https://search.maven.org")); - assert!(url_is_maven_central("https://central.sonatype.com")); - // private mirrors → false: their coordinates must NEVER reach search.maven.org (#68/#733) - assert!(!url_is_maven_central( - "https://nexus.internal.corp/repository/maven" - )); - assert!(!url_is_maven_central("https://artifactory.acme.io/maven")); - assert!(!url_is_maven_central("https://maven.pkg.github.com/acme")); - assert!(!url_is_maven_central("")); +fn parse_group_plugins(document: &[u8]) -> Option> { + if !matches!( + classify_metadata_level(document), + Some(MavenMetadataLevel::Group | MavenMetadataLevel::ArtifactAndGroup) + ) { + return None; } + let mut reader = Reader::from_reader(document); + reader.config_mut().trim_text(true); + let mut plugins = Vec::new(); + let mut current: Option = None; - #[test] - fn test_content_type_pom() { - let (status, headers, _) = - with_content_type("com/example/1.0/example-1.0.pom", Bytes::from("data")); - assert_eq!(status, StatusCode::OK); - assert_eq!(headers[0].1, "application/xml"); + loop { + match reader.read_event() { + Ok(Event::Start(element)) if element.local_name().as_ref() == b"plugin" => { + current = Some(MavenPlugin { + name: String::new(), + prefix: String::new(), + artifact_id: String::new(), + }); + } + Ok(Event::Start(element)) if current.is_some() => { + let field = element.local_name(); + if matches!(field.as_ref(), b"name" | b"prefix" | b"artifactId") { + let value = reader.read_text(element.name()).ok()?.decode().ok()?; + let value = quick_xml::escape::unescape(&value).ok()?.into_owned(); + let plugin = current.as_mut()?; + match field.as_ref() { + b"name" => plugin.name = value, + b"prefix" => plugin.prefix = value, + b"artifactId" => plugin.artifact_id = value, + _ => {} + } + } + } + Ok(Event::End(element)) if element.local_name().as_ref() == b"plugin" => { + let plugin = current.take()?; + if plugin.prefix.is_empty() || plugin.artifact_id.is_empty() { + return None; + } + plugins.push(plugin); + } + Ok(Event::Eof) => break, + Err(_) => return None, + _ => {} + } } + (!plugins.is_empty()).then_some(plugins) +} - #[test] - fn test_content_type_jar() { - let (_, headers, _) = - with_content_type("com/example/1.0/example-1.0.jar", Bytes::from("data")); - assert_eq!(headers[0].1, "application/java-archive"); +fn extend_plugins_first_wins( + plugins: &mut Vec, + seen_prefixes: &mut BTreeSet, + source: impl IntoIterator, +) { + for plugin in source { + if seen_prefixes.insert(plugin.prefix.clone()) { + plugins.push(plugin); + } } +} - #[test] - fn test_content_type_xml() { - let (_, headers, _) = - with_content_type("com/example/maven-metadata.xml", Bytes::from("data")); - assert_eq!(headers[0].1, "application/xml"); +fn merge_group_plugins(documents: &[Bytes]) -> Option> { + let mut plugins = Vec::new(); + let mut seen_prefixes = BTreeSet::new(); + let mut found_plugin_section = false; + for document in documents { + if matches!( + classify_metadata_level(document), + Some(MavenMetadataLevel::Group | MavenMetadataLevel::ArtifactAndGroup) + ) { + found_plugin_section = true; + extend_plugins_first_wins( + &mut plugins, + &mut seen_prefixes, + parse_group_plugins(document)?, + ); + } } + found_plugin_section.then_some(plugins) +} - #[test] - fn test_content_type_sha1() { - let (_, headers, _) = - with_content_type("com/example/1.0/example-1.0.jar.sha1", Bytes::from("data")); - assert_eq!(headers[0].1, "text/plain"); - } +fn plugins_section(plugins: &[MavenPlugin]) -> String { + let entries = plugins + .iter() + .map(|plugin| { + format!( + " \n {}\n {}\n {}\n ", + xml_escape(&plugin.name), + xml_escape(&plugin.prefix), + xml_escape(&plugin.artifact_id), + ) + }) + .collect::>() + .join("\n"); + format!(" \n{entries}\n \n") +} - #[test] - fn test_content_type_md5() { - let (_, headers, _) = - with_content_type("com/example/1.0/example-1.0.jar.md5", Bytes::from("data")); - assert_eq!(headers[0].1, "text/plain"); +fn combine_metadata_sections( + artifact_metadata: Option, + plugins: Option<&[MavenPlugin]>, +) -> Option { + match (artifact_metadata, plugins) { + (Some(mut artifact), Some(plugins)) => { + let closing = artifact.rfind("")?; + artifact.insert_str(closing, &plugins_section(plugins)); + Some(artifact) + } + (Some(artifact), None) => Some(artifact), + (None, Some(plugins)) => Some(format!( + "\n\n{}\n", + plugins_section(plugins) + )), + (None, None) => None, } +} - #[test] - fn test_content_type_sha256() { - let (_, headers, _) = with_content_type( - "com/example/1.0/example-1.0.jar.sha256", - Bytes::from("data"), - ); - assert_eq!(headers[0].1, "text/plain"); - } +async fn merge_and_cache_proxy_metadata( + state: &AppState, + storage_prefix: &str, + group_path: &str, + artifact_id: &str, + document_path: &str, + upstream: &[u8], + version_policy: MavenVersionPolicy, +) -> Bytes { + let key = format!("{storage_prefix}{document_path}"); - #[test] - fn test_content_type_unknown() { - let (_, headers, _) = with_content_type("some/random/file.bin", Bytes::from("data")); - assert_eq!(headers[0].1, "application/octet-stream"); - } + // Serialize the read -> merge -> write -> checksums cycle with the upload-side + // regeneration (`update_artifact_metadata`) and any concurrent proxy merge: all of them + // write the same `maven-metadata.xml` key and its four checksum sidecars. Without a shared + // lock those five independent `put`s interleave, leaving a stored `.sha1`/`.md5`/… that + // corresponds to different bytes than the stored `.xml` (checksum-mismatch window). The + // upload path locks the artifact's `maven-metadata.xml` key; locking the document key here + // — identical for artifact-level metadata — keeps `publish_lock serializes all writes to the + // same artifact path` intact on the proxy path too (#886). Held across the writes below. + let lock = state.publish_lock(&key); + let _guard = lock.lock().await; - #[test] - fn test_content_type_preserves_body() { - let body = Bytes::from("test-jar-content"); - let (_, _, data) = with_content_type("test.jar", body.clone()); - assert_eq!(data, body); - } + let cached = state.storage.get(&key).await.ok(); + let last_updated = [Some(upstream), cached.as_deref()] + .into_iter() + .flatten() + .filter_map(parse_artifact_metadata) + .filter_map(|metadata| metadata.last_updated) + .max(); + let stored_versions = + match stored_artifact_versions(state, storage_prefix, group_path, artifact_id).await { + Ok(versions) => versions, + Err(error) => { + tracing::warn!( + %error, + group_path, + artifact_id, + "maven: failed to list cached versions while merging proxy metadata" + ); + BTreeSet::new() + } + }; + let artifact = merge_artifact_metadata( + &group_path.replace('/', "."), + artifact_id, + Some(upstream), + &stored_versions, + last_updated.as_deref(), + version_policy, + ); + let plugin_documents: Vec = [Some(Bytes::copy_from_slice(upstream)), cached.clone()] + .into_iter() + .flatten() + .collect(); + let plugins = merge_group_plugins(&plugin_documents); + let data = combine_metadata_sections(artifact, plugins.as_deref()) + .map_or_else(|| Bytes::copy_from_slice(upstream), Bytes::from); - #[test] - fn test_is_mutable_maven_path() { - // maven-metadata.xml and its checksums are mutable (rewritten as versions deploy). - assert!(is_mutable_maven_path( - "com/example/mylib/maven-metadata.xml" - )); - assert!(is_mutable_maven_path( - "com/example/mylib/maven-metadata.xml.sha1" - )); - // SNAPSHOT version files are republished in place → mutable. - assert!(is_mutable_maven_path( - "com/example/mylib/1.0.0-SNAPSHOT/mylib-1.0.0-SNAPSHOT.jar" - )); - // Released artifacts are immutable. - assert!(!is_mutable_maven_path( - "com/example/mylib/1.0.0/mylib-1.0.0.jar" - )); - assert!(!is_mutable_maven_path( - "com/example/mylib/1.0.0/mylib-1.0.0.pom" - )); + if let Err(error) = state.storage.put(&key, &data).await { + tracing::warn!(key = %key, error = %error, "maven: failed to cache metadata"); + } else if let Err(error) = compute_and_store_checksums(&state.storage, &key, &data).await { + tracing::warn!(key = %key, error = %error, "maven: failed to cache metadata checksums"); } - // ── Path classification ───────────────────────────────────────────── + data +} - #[test] - fn test_classify_version_file() { - match classify_path("com/example/mylib/1.0.0/mylib-1.0.0.jar") { - MavenPathKind::VersionFile(c) => { - assert_eq!(c.group_path, "com/example"); - assert_eq!(c.artifact_id, "mylib"); - assert_eq!(c.version, "1.0.0"); - assert_eq!(c.filename, "mylib-1.0.0.jar"); - } - _ => panic!("expected VersionFile"), +async fn delete_existing_maven_object( + storage: &crate::storage::Storage, + key: &str, +) -> crate::storage::Result> { + match storage.get(key).await { + Ok(data) => { + let size = data.len() as u64; + storage.delete(key).await?; + Ok(Some(size)) } + Err(StorageError::NotFound) => Ok(None), + Err(error) => Err(error), } +} - #[test] - fn test_classify_version_checksum() { - match classify_path("com/example/mylib/1.0.0/mylib-1.0.0.jar.sha1") { - MavenPathKind::VersionFile(c) => { - assert!(is_checksum_file(&c.filename)); - assert_eq!(c.version, "1.0.0"); - } - _ => panic!("expected VersionFile"), +async fn delete_artifact_metadata_sidecars( + storage: &crate::storage::Storage, + metadata_key: &str, +) -> crate::storage::Result<(usize, u64)> { + let mut removed_keys = 0; + let mut removed_bytes = 0; + for suffix in ["md5", "sha1", "sha256", "sha512"] { + if let Some(size) = + delete_existing_maven_object(storage, &format!("{metadata_key}.{suffix}")).await? + { + removed_keys += 1; + removed_bytes += size; } } + Ok((removed_keys, removed_bytes)) +} - #[test] - fn test_classify_artifact_metadata() { - match classify_path("com/example/mylib/maven-metadata.xml") { - MavenPathKind::ArtifactMeta { - group_path, - artifact_id, - filename, - } => { - assert_eq!(group_path, "com/example"); - assert_eq!(artifact_id, "mylib"); - assert_eq!(filename, "maven-metadata.xml"); - } - _ => panic!("expected ArtifactMeta"), - } +/// Regenerate one hosted GA's A-level metadata before retention deletes +/// `removed_version`. The caller must hold the GA metadata publish lock and +/// must not call this for proxy repositories. +/// +/// Sidecars are removed before the base is replaced/deleted, so readers never +/// observe a stale checksum for the new document. Mixed G-level `` are +/// preserved. The returned counters only include A-level objects permanently +/// deleted when no metadata sections remain; regenerated documents report zero. +pub(crate) async fn update_hosted_metadata_after_retention( + storage: &crate::storage::Storage, + storage_prefix: &str, + group_path: &str, + artifact_id: &str, + removed_version: &str, +) -> crate::storage::Result<(usize, u64)> { + let metadata_key = format!("{storage_prefix}{group_path}/{artifact_id}/maven-metadata.xml"); + let current = match storage.get(&metadata_key).await { + Ok(current) => Some(current), + Err(StorageError::NotFound) => None, + Err(error) => return Err(error), + }; + let current_artifact = current.as_deref().and_then(parse_artifact_metadata); + let current_plugins = current.as_deref().and_then(parse_group_plugins); + if current.is_some() && current_artifact.is_none() && current_plugins.is_none() { + return Err(StorageError::IntegrityViolation); + } + if current_artifact + .as_ref() + .is_some_and(|metadata| !artifact_metadata_matches_path(metadata, group_path, artifact_id)) + { + return Err(StorageError::IntegrityViolation); } - #[test] - fn test_classify_metadata_checksum() { - match classify_path("com/example/mylib/maven-metadata.xml.sha256") { - MavenPathKind::ArtifactMeta { - artifact_id, - filename, - .. - } => { - assert_eq!(artifact_id, "mylib"); - assert_eq!(filename, "maven-metadata.xml.sha256"); - } - _ => panic!("expected ArtifactMeta"), + let mut remaining = + stored_artifact_versions_in_storage(storage, storage_prefix, group_path, artifact_id) + .await?; + remaining.remove(removed_version); + + let removed_sidecars = delete_artifact_metadata_sidecars(storage, &metadata_key).await?; + if remaining.is_empty() { + if let Some(plugins) = current_plugins.as_deref() { + let document = combine_metadata_sections(None, Some(plugins)) + .ok_or(StorageError::IntegrityViolation)?; + storage.put(&metadata_key, document.as_bytes()).await?; + compute_and_store_checksums(storage, &metadata_key, document.as_bytes()).await?; + return Ok((0, 0)); } - } - #[test] - fn test_classify_deep_group() { - match classify_path("org/apache/maven/plugins/maven-compiler-plugin/3.11.0/maven-compiler-plugin-3.11.0.jar") { - MavenPathKind::VersionFile(c) => { - assert_eq!(c.group_path, "org/apache/maven/plugins"); - assert_eq!(c.artifact_id, "maven-compiler-plugin"); - assert_eq!(c.version, "3.11.0"); - } - _ => panic!("expected VersionFile"), + let mut removed_keys = removed_sidecars.0; + let mut removed_bytes = removed_sidecars.1; + if let Some(size) = delete_existing_maven_object(storage, &metadata_key).await? { + removed_keys += 1; + removed_bytes += size; } + return Ok((removed_keys, removed_bytes)); } - #[test] - fn test_classify_snapshot() { - match classify_path("com/example/mylib/1.0-SNAPSHOT/mylib-1.0-SNAPSHOT.jar") { - MavenPathKind::VersionFile(c) => { - assert!(is_snapshot(&c.version)); + let mut versions = Vec::new(); + let mut known = BTreeSet::new(); + if let Some(metadata) = current_artifact.as_ref() { + for version in &metadata.versions { + if remaining.contains(version) { + push_unique(&mut versions, &mut known, version); } - _ => panic!("expected VersionFile"), } } - - #[test] - fn test_classify_opaque_short_path() { - assert!(matches!(classify_path("a"), MavenPathKind::Opaque)); + let mut additional: Vec = remaining + .iter() + .filter(|version| !known.contains(*version)) + .cloned() + .collect(); + sort_maven_versions(&mut additional); + for version in additional { + push_unique(&mut versions, &mut known, &version); } - // ── Checksum detection ────────────────────────────────────────────── + let latest = current_artifact + .as_ref() + .and_then(|metadata| metadata.latest.as_ref()) + .filter(|version| known.contains(*version)) + .cloned() + .or_else(|| { + versions + .iter() + .max_by(|left, right| compare_maven_versions(left, right)) + .cloned() + }); + let release = current_artifact + .as_ref() + .and_then(|metadata| metadata.release.as_ref()) + .filter(|version| !is_snapshot(version) && known.contains(*version)) + .cloned() + .or_else(|| { + versions + .iter() + .filter(|version| !is_snapshot(version)) + .max_by(|left, right| compare_maven_versions(left, right)) + .cloned() + }); + let last_updated = chrono::Utc::now().format("%Y%m%d%H%M%S").to_string(); + let artifact = generate_metadata_xml_with_versioning( + &group_path.replace('/', "."), + artifact_id, + &versions, + latest.as_deref(), + release.as_deref(), + Some(&last_updated), + ); + let document = combine_metadata_sections(Some(artifact), current_plugins.as_deref()) + .ok_or(StorageError::IntegrityViolation)?; + storage.put(&metadata_key, document.as_bytes()).await?; + compute_and_store_checksums(storage, &metadata_key, document.as_bytes()).await?; + Ok((0, 0)) +} - #[test] - fn test_is_checksum_file() { - assert!(is_checksum_file("foo.md5")); - assert!(is_checksum_file("foo.sha1")); - assert!(is_checksum_file("foo.sha256")); - assert!(is_checksum_file("foo.sha512")); - assert!(!is_checksum_file("foo.jar")); - assert!(!is_checksum_file("foo.pom")); +async fn update_artifact_metadata( + state: &AppState, + storage_prefix: &str, + group_path: &str, + artifact_id: &str, + deployed_version: Option<&str>, +) -> crate::storage::Result<()> { + let versions = stored_artifact_versions(state, storage_prefix, group_path, artifact_id).await?; + + if versions.is_empty() { + return Ok(()); } - // ── Version sorting ───────────────────────────────────────────────── + let prefix = format!("{storage_prefix}{group_path}/{artifact_id}/"); + let metadata_key = format!("{}maven-metadata.xml", prefix); + let current = match state.storage.get(&metadata_key).await { + Ok(current) => Some(current), + Err(StorageError::NotFound) => None, + Err(error) => return Err(error), + }; + let current_artifact_document = current + .as_deref() + .filter(|document| parse_artifact_metadata(document).is_some()); + let current_plugins = current.as_deref().and_then(parse_group_plugins); + let group_id_dotted = group_path.replace('/', "."); + // A completed exact retry passes no deployed version. Keep the existing + // timestamp in that case so the retry remains byte-stable while still + // repairing missing versions/checksums. A real deployment, or an exact + // retry whose metadata is incomplete, passes the version and advances the + // timestamp. + let last_updated = + deployed_version.map(|_| chrono::Utc::now().format("%Y%m%d%H%M%S").to_string()); + let artifact_xml = merge_hosted_artifact_metadata( + &group_id_dotted, + artifact_id, + current_artifact_document, + None, + &versions, + deployed_version, + last_updated.as_deref(), + ) + .unwrap_or_else(|| { + let mut sorted: Vec = versions.into_iter().collect(); + sort_maven_versions(&mut sorted); + generate_metadata_xml(&group_id_dotted, artifact_id, &sorted) + }); + let xml = combine_metadata_sections(Some(artifact_xml), current_plugins.as_deref()) + .expect("artifact metadata section is present"); - #[test] - fn test_sort_versions_lexicographic() { - let mut v = vec!["1.0.0".into(), "0.9.0".into(), "1.1.0".into()]; - sort_maven_versions(&mut v); - assert_eq!(v, vec!["0.9.0", "1.0.0", "1.1.0"]); - } + state.storage.put(&metadata_key, xml.as_bytes()).await?; + compute_and_store_checksums(&state.storage, &metadata_key, xml.as_bytes()).await?; + Ok(()) +} - #[test] - fn test_sort_snapshot_before_release() { - let mut v = vec!["1.0.0-SNAPSHOT".into(), "1.0.0".into(), "0.9.0".into()]; - sort_maven_versions(&mut v); - assert_eq!(v, vec!["0.9.0", "1.0.0-SNAPSHOT", "1.0.0"]); - } +fn sort_maven_versions(versions: &mut [String]) { + versions.sort_by(|a, b| compare_maven_versions(a, b)); +} - #[test] - fn test_sort_numeric_segments() { - let mut v = vec!["10.0.0".into(), "9.0.0".into(), "2.1.0".into()]; - sort_maven_versions(&mut v); - assert_eq!(v, vec!["2.1.0", "9.0.0", "10.0.0"]); +#[derive(Debug, Clone, Eq, PartialEq)] +enum MavenVersionItem { + Numeric(String), + Qualifier(String), + List(Vec), +} + +fn canonical_maven_qualifier(value: &str, followed_by_digit: bool) -> String { + let value = if followed_by_digit && value.len() == 1 { + match value { + "a" => "alpha", + "b" => "beta", + "m" => "milestone", + other => other, + } + } else { + value + }; + + match value { + "cr" | "rc" => "rc", + "ga" | "final" | "release" => "", + other => other, } + .to_string() +} - // ── Metadata XML generation ───────────────────────────────────────── +fn compare_numeric_strings(left: &str, right: &str) -> std::cmp::Ordering { + left.len().cmp(&right.len()).then_with(|| left.cmp(right)) +} - #[test] - fn test_generate_metadata_xml() { - let xml = generate_metadata_xml("com.example", "mylib", &["0.9.0".into(), "1.0.0".into()]); - assert!(xml.contains("com.example")); - assert!(xml.contains("mylib")); - assert!(xml.contains("1.0.0")); - assert!(xml.contains("1.0.0")); - assert!(xml.contains("0.9.0")); - assert!(xml.contains("1.0.0")); - assert!(xml.contains("")); +fn maven_qualifier_key(value: &str) -> (u8, &str) { + match value { + "alpha" => (0, ""), + "beta" => (1, ""), + "milestone" => (2, ""), + "rc" => (3, ""), + "snapshot" => (4, ""), + "" => (5, ""), + "sp" => (6, ""), + other => (7, other), } +} - #[test] - fn test_generate_metadata_snapshot_only() { - let xml = generate_metadata_xml("com.example", "mylib", &["1.0.0-SNAPSHOT".into()]); - assert!(xml.contains("1.0.0-SNAPSHOT")); - assert!(xml.contains("")); +impl MavenVersionItem { + fn is_null(&self) -> bool { + match self { + Self::Numeric(value) => value == "0", + Self::Qualifier(value) => value.is_empty(), + Self::List(items) => items.is_empty(), + } } - #[test] - fn test_merge_artifact_metadata_keeps_public_and_hosted_versions() { - let upstream = br#" - - com.example - library - - 2.0.0 - 2.0.0 - - 1.0.0 - 2.0.0 - - 20260728010000 - - -"#; - let stored = BTreeSet::from(["0.5.0-internal".to_string(), "9.0.0-internal".to_string()]); + fn compare_to(&self, other: Option<&Self>) -> std::cmp::Ordering { + use MavenVersionItem::{List, Numeric, Qualifier}; - let merged = - merge_artifact_metadata("com.example", "library", Some(upstream), &stored, None) - .unwrap(); + match (self, other) { + (Numeric(left), Some(Numeric(right))) => compare_numeric_strings(left, right), + (Numeric(_), Some(Qualifier(_) | List(_))) => std::cmp::Ordering::Greater, + (Numeric(left), None) => compare_numeric_strings(left, "0"), - assert!(merged.contains("1.0.0")); - assert!(merged.contains("2.0.0")); - assert!(merged.contains("0.5.0-internal")); - assert!(merged.contains("9.0.0-internal")); - assert!(merged.contains("9.0.0-internal")); - assert!(merged.contains("9.0.0-internal")); - assert!(merged.contains("20260728010000")); + (Qualifier(left), Some(Qualifier(right))) => { + maven_qualifier_key(left).cmp(&maven_qualifier_key(right)) + } + (Qualifier(_), Some(Numeric(_) | List(_))) => std::cmp::Ordering::Less, + (Qualifier(left), None) => maven_qualifier_key(left).cmp(&maven_qualifier_key("")), + + (List(left), Some(List(right))) => { + for index in 0..left.len().max(right.len()) { + let ordering = match (left.get(index), right.get(index)) { + (Some(left), right) => left.compare_to(right), + (None, Some(right)) => right.compare_to(None).reverse(), + (None, None) => std::cmp::Ordering::Equal, + }; + if ordering != std::cmp::Ordering::Equal { + return ordering; + } + } + std::cmp::Ordering::Equal + } + (List(_), Some(Numeric(_))) => std::cmp::Ordering::Less, + (List(_), Some(Qualifier(_))) => std::cmp::Ordering::Greater, + (List(items), None) => items + .iter() + .map(|item| item.compare_to(None)) + .find(|ordering| *ordering != std::cmp::Ordering::Equal) + .unwrap_or(std::cmp::Ordering::Equal), + } + } +} - let lower_only = BTreeSet::from(["0.5.0-internal".to_string()]); - let merged = - merge_artifact_metadata("com.example", "library", Some(upstream), &lower_only, None) - .unwrap(); - assert!(merged.contains("2.0.0")); - assert!(merged.contains("2.0.0")); +fn normalize_maven_version_list(items: &mut Vec) { + let mut index = items.len(); + while index > 0 { + index -= 1; + if items[index].is_null() { + items.remove(index); + } else if !matches!(items[index], MavenVersionItem::List(_)) { + break; + } } +} - #[test] - fn test_classify_artifact_level_metadata() { - let xml = br#" - - com.example - library - - 1.0.0 - - - "#; - assert_eq!( - classify_metadata_level(xml), - Some(MavenMetadataLevel::Artifact) - ); +fn maven_version_items(version: &str) -> MavenVersionItem { + fn numeric_item(value: &str) -> MavenVersionItem { + let normalized = value.trim_start_matches('0'); + MavenVersionItem::Numeric(if normalized.is_empty() { + "0".to_string() + } else { + normalized.to_string() + }) } - #[test] - fn test_classify_version_level_metadata() { + fn parsed_item(value: &str, is_digit: bool) -> MavenVersionItem { + if is_digit { + numeric_item(value) + } else { + MavenVersionItem::Qualifier(canonical_maven_qualifier(value, false)) + } + } + + let version = version.to_ascii_lowercase(); + let mut lists = vec![Vec::new()]; + let mut is_digit = false; + let mut start_index = 0; + + for (index, character) in version.char_indices() { + if character == '.' { + let item = if index == start_index { + numeric_item("0") + } else { + parsed_item(&version[start_index..index], is_digit) + }; + lists.last_mut().expect("root list exists").push(item); + start_index = index + character.len_utf8(); + } else if character == '-' { + let item = if index == start_index { + numeric_item("0") + } else { + parsed_item(&version[start_index..index], is_digit) + }; + lists.last_mut().expect("root list exists").push(item); + start_index = index + character.len_utf8(); + lists.push(Vec::new()); + } else if character.is_ascii_digit() { + if !is_digit && index > start_index { + if !lists.last().expect("root list exists").is_empty() { + lists.push(Vec::new()); + } + lists + .last_mut() + .expect("root list exists") + .push(MavenVersionItem::Qualifier(canonical_maven_qualifier( + &version[start_index..index], + true, + ))); + start_index = index; + lists.push(Vec::new()); + } + is_digit = true; + } else { + if is_digit && index > start_index { + lists + .last_mut() + .expect("root list exists") + .push(numeric_item(&version[start_index..index])); + start_index = index; + lists.push(Vec::new()); + } + is_digit = false; + } + } + + if version.len() > start_index { + if !is_digit && !lists.last().expect("root list exists").is_empty() { + lists.push(Vec::new()); + } + lists + .last_mut() + .expect("root list exists") + .push(parsed_item(&version[start_index..], is_digit)); + } + + while lists.len() > 1 { + let mut child = lists.pop().expect("child list exists"); + normalize_maven_version_list(&mut child); + lists + .last_mut() + .expect("parent list exists") + .push(MavenVersionItem::List(child)); + } + let mut root = lists.pop().expect("root list exists"); + normalize_maven_version_list(&mut root); + MavenVersionItem::List(root) +} + +/// Maven `ComparableVersion` ordering used by repository metadata, including +/// qualifier aliases, numeric normalization, and hyphen sub-list precedence. +fn compare_maven_versions(a: &str, b: &str) -> std::cmp::Ordering { + let left = maven_version_items(a); + let right = maven_version_items(b); + left.compare_to(Some(&right)) +} + +/// Escape XML special characters in interpolated values. +fn xml_escape(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +fn generate_metadata_xml(group_id: &str, artifact_id: &str, versions: &[String]) -> String { + let latest = versions.last().map(String::as_str); + let release = versions + .iter() + .rev() + .find(|v| !v.ends_with("-SNAPSHOT")) + .map(String::as_str); + + generate_metadata_xml_with_versioning(group_id, artifact_id, versions, latest, release, None) +} + +fn generate_metadata_xml_with_versioning( + group_id: &str, + artifact_id: &str, + versions: &[String], + latest: Option<&str>, + release: Option<&str>, + last_updated: Option<&str>, +) -> String { + let generated_last_updated; + let last_updated = match last_updated { + Some(last_updated) => last_updated, + None => { + generated_last_updated = chrono::Utc::now().format("%Y%m%d%H%M%S").to_string(); + &generated_last_updated + } + }; + + let version_elements: String = versions + .iter() + .map(|v| format!(" {}", xml_escape(v))) + .collect::>() + .join("\n"); + let latest_element = latest.map_or_else(String::new, |latest| { + format!(" {}\n", xml_escape(latest)) + }); + let release_element = release.map_or_else(String::new, |release| { + format!(" {}\n", xml_escape(release)) + }); + + format!( + r#" + + {} + {} + +{}{} + +{} + + {} + + +"#, + xml_escape(group_id), + xml_escape(artifact_id), + latest_element, + release_element, + version_elements, + last_updated + ) +} + +// ============================================================================ +// Content type +// ============================================================================ + +fn with_content_type( + path: &str, + data: Bytes, +) -> (StatusCode, [(header::HeaderName, &'static str); 2], Bytes) { + let content_type = if ends_with_ci(path, ".pom") { + "application/xml" + } else if ends_with_ci(path, ".jar") { + "application/java-archive" + } else if ends_with_ci(path, ".xml") { + "application/xml" + } else if ends_with_ci(path, ".sha1") + || ends_with_ci(path, ".md5") + || ends_with_ci(path, ".sha256") + || ends_with_ci(path, ".sha512") + { + "text/plain" + } else { + "application/octet-stream" + }; + + // Metadata and SNAPSHOT assets are mutable; release artifacts are immutable. + let cache_control = if is_mutable_maven_path(path) { + "public, max-age=60, must-revalidate" + } else { + "public, max-age=31536000, immutable" + }; + + ( + StatusCode::OK, + [ + (header::CONTENT_TYPE, content_type), + (header::CACHE_CONTROL, cache_control), + ], + data, + ) +} + +// ============================================================================ +// Unit Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_url_is_maven_central() { + // canonical Central hosts → true (date source available) + assert!(url_is_maven_central("https://repo1.maven.org/maven2")); + assert!(url_is_maven_central("https://repo.maven.apache.org/maven2")); + assert!(url_is_maven_central("https://search.maven.org")); + assert!(url_is_maven_central("https://central.sonatype.com")); + // private mirrors → false: their coordinates must NEVER reach search.maven.org (#68/#733) + assert!(!url_is_maven_central( + "https://nexus.internal.corp/repository/maven" + )); + assert!(!url_is_maven_central("https://artifactory.acme.io/maven")); + assert!(!url_is_maven_central("https://maven.pkg.github.com/acme")); + assert!(!url_is_maven_central("")); + } + + #[test] + fn test_content_type_pom() { + let (status, headers, _) = + with_content_type("com/example/1.0/example-1.0.pom", Bytes::from("data")); + assert_eq!(status, StatusCode::OK); + assert_eq!(headers[0].1, "application/xml"); + } + + #[test] + fn test_content_type_jar() { + let (_, headers, _) = + with_content_type("com/example/1.0/example-1.0.jar", Bytes::from("data")); + assert_eq!(headers[0].1, "application/java-archive"); + } + + #[test] + fn test_content_type_xml() { + let (_, headers, _) = + with_content_type("com/example/maven-metadata.xml", Bytes::from("data")); + assert_eq!(headers[0].1, "application/xml"); + } + + #[test] + fn test_content_type_sha1() { + let (_, headers, _) = + with_content_type("com/example/1.0/example-1.0.jar.sha1", Bytes::from("data")); + assert_eq!(headers[0].1, "text/plain"); + } + + #[test] + fn test_content_type_md5() { + let (_, headers, _) = + with_content_type("com/example/1.0/example-1.0.jar.md5", Bytes::from("data")); + assert_eq!(headers[0].1, "text/plain"); + } + + #[test] + fn test_content_type_sha256() { + let (_, headers, _) = with_content_type( + "com/example/1.0/example-1.0.jar.sha256", + Bytes::from("data"), + ); + assert_eq!(headers[0].1, "text/plain"); + } + + #[test] + fn test_content_type_unknown() { + let (_, headers, _) = with_content_type("some/random/file.bin", Bytes::from("data")); + assert_eq!(headers[0].1, "application/octet-stream"); + } + + #[test] + fn test_content_type_preserves_body() { + let body = Bytes::from("test-jar-content"); + let (_, _, data) = with_content_type("test.jar", body.clone()); + assert_eq!(data, body); + } + + #[test] + fn test_is_mutable_maven_path() { + // maven-metadata.xml and its checksums are mutable (rewritten as versions deploy). + assert!(is_mutable_maven_path( + "com/example/mylib/maven-metadata.xml" + )); + assert!(is_mutable_maven_path( + "com/example/mylib/maven-metadata.xml.sha1" + )); + // SNAPSHOT version files are republished in place → mutable. + assert!(is_mutable_maven_path( + "com/example/mylib/1.0.0-SNAPSHOT/mylib-1.0.0-SNAPSHOT.jar" + )); + // Released artifacts are immutable. + assert!(!is_mutable_maven_path( + "com/example/mylib/1.0.0/mylib-1.0.0.jar" + )); + assert!(!is_mutable_maven_path( + "com/example/mylib/1.0.0/mylib-1.0.0.pom" + )); + } + + #[test] + fn test_negative_cache_is_bounded_and_prunes_expired_entries() { + let mut cache = HashMap::new(); + cache.insert( + "expired".to_string(), + Instant::now() - Duration::from_secs(10), + ); + insert_negative_cache_entry(&mut cache, "first".to_string(), 1, 2); + insert_negative_cache_entry(&mut cache, "second".to_string(), 1, 2); + insert_negative_cache_entry(&mut cache, "third".to_string(), 1, 2); + assert_eq!(cache.len(), 2); + assert!(!cache.contains_key("expired")); + assert!(cache.contains_key("third")); + } + + #[test] + fn test_all_metadata_checksum_cache_headers_revalidate() { + for suffix in ["md5", "sha1", "sha256", "sha512"] { + let (_, headers, _) = with_content_type( + &format!("com/example/lib/maven-metadata.xml.{suffix}"), + Bytes::new(), + ); + assert_eq!(headers[1].1, "public, max-age=60, must-revalidate"); + } + } + + #[test] + fn test_snapshot_cache_headers_revalidate_and_release_is_immutable() { + for path in [ + "com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar", + "com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar.sha512", + ] { + let (_, headers, _) = with_content_type(path, Bytes::new()); + assert_eq!(headers[1].1, "public, max-age=60, must-revalidate"); + } + let (_, headers, _) = with_content_type("com/example/lib/1.0/lib-1.0.jar", Bytes::new()); + assert_eq!(headers[1].1, "public, max-age=31536000, immutable"); + } + + // ── Path classification ───────────────────────────────────────────── + + #[test] + fn test_classify_version_file() { + match classify_path("com/example/mylib/1.0.0/mylib-1.0.0.jar") { + MavenPathKind::VersionFile(c) => { + assert_eq!(c.group_path, "com/example"); + assert_eq!(c.artifact_id, "mylib"); + assert_eq!(c.version, "1.0.0"); + } + _ => panic!("expected VersionFile"), + } + } + + #[test] + fn test_classify_version_checksum() { + match classify_path("com/example/mylib/1.0.0/mylib-1.0.0.jar.sha1") { + MavenPathKind::VersionFile(c) => { + assert_eq!(c.version, "1.0.0"); + assert_eq!( + checksum_suffix("com/example/mylib/1.0.0/mylib-1.0.0.jar.sha1"), + Some("sha1") + ); + } + _ => panic!("expected VersionFile"), + } + } + + #[test] + fn test_classify_artifact_metadata() { + match classify_path("com/example/mylib/maven-metadata.xml") { + MavenPathKind::ArtifactMeta { + group_path, + artifact_id, + filename, + } => { + assert_eq!(group_path, "com/example"); + assert_eq!(artifact_id, "mylib"); + assert_eq!(filename, "maven-metadata.xml"); + } + _ => panic!("expected ArtifactMeta"), + } + } + + #[test] + fn test_classify_metadata_checksum() { + match classify_path("com/example/mylib/maven-metadata.xml.sha256") { + MavenPathKind::ArtifactMeta { + artifact_id, + filename, + .. + } => { + assert_eq!(artifact_id, "mylib"); + assert_eq!(filename, "maven-metadata.xml.sha256"); + } + _ => panic!("expected ArtifactMeta"), + } + } + + #[test] + fn test_classify_deep_group() { + match classify_path("org/apache/maven/plugins/maven-compiler-plugin/3.11.0/maven-compiler-plugin-3.11.0.jar") { + MavenPathKind::VersionFile(c) => { + assert_eq!(c.group_path, "org/apache/maven/plugins"); + assert_eq!(c.artifact_id, "maven-compiler-plugin"); + assert_eq!(c.version, "3.11.0"); + } + _ => panic!("expected VersionFile"), + } + } + + #[test] + fn test_classify_snapshot() { + match classify_path("com/example/mylib/1.0-SNAPSHOT/mylib-1.0-SNAPSHOT.jar") { + MavenPathKind::VersionFile(c) => { + assert!(is_snapshot(&c.version)); + } + _ => panic!("expected VersionFile"), + } + } + + #[test] + fn test_classify_opaque_short_path() { + assert!(matches!(classify_path("a"), MavenPathKind::Opaque)); + } + + // ── Checksum detection ────────────────────────────────────────────── + + #[test] + fn test_checksum_suffix() { + assert_eq!(checksum_suffix("foo.md5"), Some("md5")); + assert_eq!(checksum_suffix("foo.sha1"), Some("sha1")); + assert_eq!(checksum_suffix("foo.sha256"), Some("sha256")); + assert_eq!(checksum_suffix("foo.sha512"), Some("sha512")); + assert_eq!(checksum_suffix("foo.jar"), None); + assert_eq!(checksum_suffix("foo.pom"), None); + } + + // ── Version sorting ───────────────────────────────────────────────── + + #[test] + fn test_sort_versions_lexicographic() { + let mut v = vec!["1.0.0".into(), "0.9.0".into(), "1.1.0".into()]; + sort_maven_versions(&mut v); + assert_eq!(v, vec!["0.9.0", "1.0.0", "1.1.0"]); + } + + #[test] + fn test_sort_snapshot_before_release() { + let mut v = vec!["1.0.0-SNAPSHOT".into(), "1.0.0".into(), "0.9.0".into()]; + sort_maven_versions(&mut v); + assert_eq!(v, vec!["0.9.0", "1.0.0-SNAPSHOT", "1.0.0"]); + } + + #[test] + fn test_sort_numeric_segments() { + let mut v = vec!["10.0.0".into(), "9.0.0".into(), "2.1.0".into()]; + sort_maven_versions(&mut v); + assert_eq!(v, vec!["2.1.0", "9.0.0", "10.0.0"]); + } + + #[test] + fn test_maven_version_qualifiers_and_equivalent_forms() { + assert_eq!( + compare_maven_versions("1", "1.0.0"), + std::cmp::Ordering::Equal + ); + assert_eq!( + compare_maven_versions("1.0-1", "1.0.1"), + std::cmp::Ordering::Less + ); + assert_eq!( + compare_maven_versions("1-1", "1.1"), + std::cmp::Ordering::Less + ); + assert_eq!( + compare_maven_versions("1.0-final", "1.0-ga"), + std::cmp::Ordering::Equal + ); + assert_eq!( + compare_maven_versions("1.0.0", "1.0.0-0"), + std::cmp::Ordering::Equal + ); + assert_eq!( + compare_maven_versions("1.0-alpha", "1.0.alpha"), + std::cmp::Ordering::Equal + ); + assert_eq!( + compare_maven_versions("1alpha", "1-alpha"), + std::cmp::Ordering::Equal + ); + assert_eq!( + compare_maven_versions("1-alpha", "1.alpha"), + std::cmp::Ordering::Equal + ); + assert_eq!( + compare_maven_versions("1.0alpha1", "1.0-alpha1"), + std::cmp::Ordering::Equal + ); + let mut versions = vec![ + "1.0-sp".into(), + "1.0".into(), + "1.0-SNAPSHOT".into(), + "1.0-rc1".into(), + "1.0-beta1".into(), + "1.0-alpha1".into(), + ]; + sort_maven_versions(&mut versions); + assert_eq!( + versions, + vec![ + "1.0-alpha1", + "1.0-beta1", + "1.0-rc1", + "1.0-SNAPSHOT", + "1.0", + "1.0-sp", + ] + ); + } + + #[test] + fn test_maven_comparable_version_reference_order() { + let versions = [ + "1-alpha2snapshot", + "1-alpha2", + "1-alpha-123", + "1-beta-2", + "1-beta123", + "1-m2", + "1-m11", + "1-rc", + "1-cr2", + "1-rc123", + "1-SNAPSHOT", + "1", + "1-sp", + "1-sp2", + "1-sp123", + "1-abc", + "1-def", + "1-pom-1", + "1-1-snapshot", + "1-1", + "1-2", + "1-123", + ]; + for (index, lower) in versions.iter().enumerate() { + for higher in &versions[index + 1..] { + assert_eq!( + compare_maven_versions(lower, higher), + std::cmp::Ordering::Less, + "expected {lower} < {higher}" + ); + } + } + + let numeric_versions = [ + "2.0", "2.0.a", "2-1", "2.0.2", "2.0.123", "2.1.0", "2.1-a", "2.1b", "2.1-c", "2.1-1", + "2.1.0.1", "2.2", "2.123", "11.a2", "11.a11", "11.b2", "11.b11", "11.m2", "11.m11", + "11", "11.a", "11b", "11c", "11m", + ]; + for (index, lower) in numeric_versions.iter().enumerate() { + for higher in &numeric_versions[index + 1..] { + assert_eq!( + compare_maven_versions(lower, higher), + std::cmp::Ordering::Less, + "expected {lower} < {higher}" + ); + } + } + + for (left, right) in [ + ("1", "1.0"), + ("1", "1-0"), + ("1a", "1.0.0-a"), + ("1x", "1.0.0-x"), + ("1ga", "1"), + ("1release", "1"), + ("1cr", "1rc"), + ("1a1", "1-alpha-1"), + ("1b2", "1-beta-2"), + ("1m3", "1-milestone-3"), + ] { + assert_eq!( + compare_maven_versions(left, right), + std::cmp::Ordering::Equal, + "expected {left} == {right}" + ); + } + } + + // ── Metadata XML generation ───────────────────────────────────────── + + #[test] + fn test_generate_metadata_xml() { + let xml = generate_metadata_xml("com.example", "mylib", &["0.9.0".into(), "1.0.0".into()]); + assert!(xml.contains("com.example")); + assert!(xml.contains("mylib")); + assert!(xml.contains("1.0.0")); + assert!(xml.contains("1.0.0")); + assert!(xml.contains("0.9.0")); + assert!(xml.contains("1.0.0")); + assert!(xml.contains("")); + } + + #[test] + fn test_generate_metadata_snapshot_only() { + let xml = generate_metadata_xml("com.example", "mylib", &["1.0.0-SNAPSHOT".into()]); + assert!(xml.contains("1.0.0-SNAPSHOT")); + assert!(!xml.contains("")); + } + + #[test] + fn test_merge_artifact_metadata_keeps_public_and_hosted_versions() { + let upstream = br#" + + com.example + library + + 2.0.0 + 2.0.0 + + 1.0.0 + 2.0.0 + + 20260728010000 + + +"#; + let stored = BTreeSet::from(["0.5.0-internal".to_string(), "9.0.0-internal".to_string()]); + + let merged = merge_artifact_metadata( + "com.example", + "library", + Some(upstream), + &stored, + None, + MavenVersionPolicy::Mixed, + ) + .unwrap(); + + assert!(merged.contains("1.0.0")); + assert!(merged.contains("2.0.0")); + assert!(merged.contains("0.5.0-internal")); + assert!(merged.contains("9.0.0-internal")); + assert!(merged.contains("2.0.0")); + assert!(merged.contains("2.0.0")); + assert!(merged.contains("20260728010000")); + + let lower_only = BTreeSet::from(["0.5.0-internal".to_string()]); + let merged = merge_artifact_metadata( + "com.example", + "library", + Some(upstream), + &lower_only, + None, + MavenVersionPolicy::Mixed, + ) + .unwrap(); + assert!(merged.contains("2.0.0")); + assert!(merged.contains("2.0.0")); + } + + #[test] + fn test_hosted_metadata_latest_and_release_follow_deployment_order() { + let versions = BTreeSet::from(["1.0".to_string(), "2.0".to_string()]); + let first = merge_hosted_artifact_metadata( + "com.example", + "library", + None, + None, + &versions, + Some("2.0"), + Some("20260730010000"), + ) + .unwrap(); + let second = merge_hosted_artifact_metadata( + "com.example", + "library", + Some(first.as_bytes()), + None, + &versions, + Some("1.0"), + Some("20260730020000"), + ) + .unwrap(); + assert!(second.contains("1.0")); + assert!(second.contains("1.0")); + assert!(second.find("2.0") < second.find("1.0")); + } + + #[test] + fn test_group_version_metadata_merges_newest_per_extension_and_classifier() { + let older = Bytes::from_static( + br#"com.examplelibrary1.0-SNAPSHOT20260730.010000120260730010000jar1.0-20260730.010000-120260730010000sourcesjar1.0-20260730.010000-120260730010000"#, + ); + let newer = Bytes::from_static( + br#"com.examplelibrary1.0-SNAPSHOT20260730.020000220260730020000jar1.0-20260730.020000-220260730020000javadocjar1.0-20260730.020000-220260730020000"#, + ); + let merged = merge_group_version_metadata( + "com/example/library/1.0-SNAPSHOT/maven-metadata.xml", + &[older, newer], + ) + .unwrap(); + let merged = String::from_utf8(merged.to_vec()).unwrap(); + assert!(merged.contains("20260730.020000")); + assert!(merged.contains("2")); + assert!(merged.contains("1.0-20260730.020000-2")); + assert!(merged.contains("sources")); + assert!(merged.contains("javadoc")); + assert_eq!(merged.matches("1.0-20260730.010000-1").count(), 1); + assert_eq!(merged.matches("1.0-20260730.020000-2").count(), 2); + } + + #[test] + fn test_classify_artifact_level_metadata() { + let xml = br#" + + com.example + library + + 1.0.0 + + + "#; + assert_eq!( + classify_metadata_level(xml), + Some(MavenMetadataLevel::Artifact) + ); + } + + #[test] + fn test_classify_version_level_metadata() { let xml = br#" com.example @@ -1582,46 +3549,1492 @@ mod tests { "#; assert_eq!( - classify_metadata_level(xml), - Some(MavenMetadataLevel::Version) + classify_metadata_level(xml), + Some(MavenMetadataLevel::Version) + ); + } + + #[test] + fn test_classify_group_level_metadata() { + let xml = br#" + + + + example + example-maven-plugin + + + + "#; + assert_eq!( + classify_metadata_level(xml), + Some(MavenMetadataLevel::Group) + ); + } + + #[test] + fn test_classify_combined_artifact_and_group_metadata() { + let xml = br#" + + org.example + plugins + + 1.0 + 1.0 + + + + example + example-maven-plugin + + + + "#; + assert_eq!( + classify_metadata_level(xml), + Some(MavenMetadataLevel::ArtifactAndGroup) + ); + assert_eq!(parse_artifact_metadata(xml).unwrap().versions, ["1.0"]); + assert_eq!(parse_group_plugins(xml).unwrap().len(), 1); + } + + #[test] + fn test_reject_malformed_metadata() { + assert_eq!(classify_metadata_level(b""), None); + assert_eq!(classify_metadata_level(b""), None); + } +} + +// ============================================================================ +// Integration Tests +// ============================================================================ + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod integration_tests { + use super::{ + checksum_hex, compute_and_store_checksums, parse_artifact_metadata, repository_storage_key, + }; + use crate::config::{MavenRepository, MavenVersionPolicy, MavenWritePolicy}; + use crate::storage::{ + FileMeta, Result as StorageResult, Storage, StorageBackend, StorageError, + }; + use crate::test_helpers::{ + body_bytes, create_test_context, create_test_context_with_config, send, + }; + use axum::body::{Body, Bytes}; + use axum::http::{Method, StatusCode}; + use parking_lot::Mutex; + use std::collections::HashMap; + use std::path::Path as FsPath; + use std::pin::Pin; + use std::sync::Arc; + use tokio::io::AsyncRead; + + #[derive(Clone, Copy, PartialEq, Eq)] + enum InjectedOperation { + Put, + Get, + List, + } + + struct InjectedFailure { + operation: InjectedOperation, + key: String, + skip_matches: usize, + } + + #[derive(Default)] + struct FaultInjectingBackend { + objects: Mutex>, + failures: Mutex>, + } + + impl FaultInjectingBackend { + fn fail_once(&self, operation: InjectedOperation, key: impl Into) { + self.fail_after(operation, key, 0); + } + + fn fail_after( + &self, + operation: InjectedOperation, + key: impl Into, + skip_matches: usize, + ) { + self.failures.lock().push(InjectedFailure { + operation, + key: key.into(), + skip_matches, + }); + } + + fn take_failure(&self, operation: InjectedOperation, key: &str) -> bool { + let mut failures = self.failures.lock(); + let Some(index) = failures + .iter() + .position(|failure| failure.operation == operation && failure.key == key) + else { + return false; + }; + if failures[index].skip_matches > 0 { + failures[index].skip_matches -= 1; + return false; + } + failures.remove(index); + true + } + + fn injected_error() -> StorageError { + StorageError::Io(std::io::Error::other("injected Maven storage failure")) + } + } + + #[async_trait::async_trait] + impl StorageBackend for FaultInjectingBackend { + async fn put(&self, key: &str, data: &[u8]) -> StorageResult<()> { + if self.take_failure(InjectedOperation::Put, key) { + return Err(Self::injected_error()); + } + self.objects + .lock() + .insert(key.to_string(), Bytes::copy_from_slice(data)); + Ok(()) + } + + async fn put_if_absent(&self, key: &str, data: &[u8]) -> StorageResult<()> { + let mut objects = self.objects.lock(); + if objects.contains_key(key) { + return Err(StorageError::AlreadyExists); + } + objects.insert(key.to_string(), Bytes::copy_from_slice(data)); + Ok(()) + } + + async fn get(&self, key: &str) -> StorageResult { + if self.take_failure(InjectedOperation::Get, key) { + return Err(Self::injected_error()); + } + self.objects + .lock() + .get(key) + .cloned() + .ok_or(StorageError::NotFound) + } + + async fn delete(&self, key: &str) -> StorageResult<()> { + self.objects + .lock() + .remove(key) + .map(|_| ()) + .ok_or(StorageError::NotFound) + } + + async fn list(&self, prefix: &str) -> StorageResult> { + if self.take_failure(InjectedOperation::List, prefix) { + return Err(Self::injected_error()); + } + let mut keys: Vec = self + .objects + .lock() + .keys() + .filter(|key| key.starts_with(prefix)) + .cloned() + .collect(); + keys.sort(); + Ok(keys) + } + + async fn stat(&self, key: &str) -> Option { + self.objects.lock().get(key).map(|data| FileMeta { + size: data.len() as u64, + modified: 1, + }) + } + + async fn health_check(&self) -> bool { + true + } + + async fn total_size(&self) -> u64 { + self.objects + .lock() + .values() + .map(|data| data.len() as u64) + .sum() + } + + fn backend_name(&self) -> &'static str { + "fault-injecting-maven-test" + } + + async fn put_from_path(&self, _key: &str, _src: &FsPath) -> StorageResult<()> { + Err(StorageError::Network( + "put_from_path is not used by Maven fault tests".to_string(), + )) + } + + async fn get_reader( + &self, + _key: &str, + ) -> StorageResult<(u64, Pin>)> { + Err(StorageError::NotFound) + } + } + + async fn upload_with_state( + state: &crate::AppState, + path: &str, + data: &'static [u8], + ) -> axum::response::Response { + super::upload_legacy( + axum::extract::State(state.clone()), + axum::extract::Path(path.to_string()), + axum::Extension(crate::auth::NamespaceAuthority::Unrestricted), + Bytes::from_static(data), + ) + .await + } + + fn named_repository_context() -> crate::test_helpers::TestContext { + create_test_context_with_config(|config| { + config.maven.repositories = vec![ + MavenRepository::Hosted { + name: "maven-releases".to_string(), + version_policy: MavenVersionPolicy::Release, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Hosted { + name: "maven-snapshots".to_string(), + version_policy: MavenVersionPolicy::Snapshot, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Hosted { + name: "maven-open".to_string(), + version_policy: MavenVersionPolicy::Release, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Group { + name: "maven-public".to_string(), + members: vec![ + "maven-open".to_string(), + "maven-releases".to_string(), + "maven-snapshots".to_string(), + ], + }, + ]; + config.maven.default_repository = Some("maven-public".to_string()); + }) + } + + fn proxy_group_context( + upstream_url: String, + negative_ttl: i64, + ) -> crate::test_helpers::TestContext { + create_test_context_with_config(move |config| { + config.maven.repositories = vec![ + MavenRepository::Proxy { + name: "proxy".to_string(), + url: upstream_url, + auth: None, + version_policy: MavenVersionPolicy::Release, + metadata_ttl: Some(0), + negative_ttl, + }, + MavenRepository::Hosted { + name: "hosted".to_string(), + version_policy: MavenVersionPolicy::Release, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Group { + name: "public".to_string(), + members: vec!["proxy".to_string(), "hosted".to_string()], + }, + ]; + }) + } + + #[tokio::test] + async fn test_named_allow_write_policy_replaces_release_and_derived_checksums() { + let ctx = named_repository_context(); + let path = "com/example/redeploy/1.0/redeploy-1.0.jar"; + let uri = format!("/repository/maven-releases/{path}"); + + for body in ["first", "replacement"] { + assert_eq!( + send(&ctx.app, Method::PUT, &uri, body).await.status(), + StatusCode::CREATED + ); + } + + let artifact = send(&ctx.app, Method::GET, &uri, "").await; + assert_eq!(artifact.status(), StatusCode::OK); + assert_eq!(body_bytes(artifact).await, "replacement"); + + let checksum = send(&ctx.app, Method::GET, &format!("{uri}.sha256"), "").await; + assert_eq!(checksum.status(), StatusCode::OK); + assert_eq!( + String::from_utf8_lossy(&body_bytes(checksum).await), + checksum_hex("sha256", b"replacement").unwrap() + ); + } + + #[tokio::test] + async fn test_proxy_non_404_client_errors_propagate_and_never_negative_cache() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let artifact_path = "com/example/client-error/1.0/client-error-1.0.jar"; + let upstream_path = format!("/{artifact_path}"); + let metadata_path = "com/example/client-error/maven-metadata.xml"; + let upstream_metadata_path = format!("/{metadata_path}"); + for status in [ + StatusCode::UNAUTHORIZED, + StatusCode::FORBIDDEN, + StatusCode::TOO_MANY_REQUESTS, + ] { + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path(upstream_path.as_str())) + .respond_with(ResponseTemplate::new(status.as_u16())) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path(upstream_metadata_path.as_str())) + .respond_with(ResponseTemplate::new(status.as_u16())) + .mount(&upstream) + .await; + let ctx = proxy_group_context(upstream.uri(), 60); + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/repository/hosted/{artifact_path}"), + "hosted-fallback-must-not-be-served", + ) + .await + .status(), + StatusCode::CREATED + ); + let metadata_key = repository_storage_key("proxy", metadata_path); + ctx.state + .storage + .put( + &metadata_key, + br#"com.exampleclient-error0.9"#, + ) + .await + .unwrap(); + + let direct = send( + &ctx.app, + Method::GET, + &format!("/repository/proxy/{artifact_path}"), + "", + ) + .await; + assert_eq!(direct.status(), status); + + let group = send( + &ctx.app, + Method::GET, + &format!("/repository/public/{artifact_path}"), + "", + ) + .await; + assert_eq!( + group.status(), + status, + "a group must not treat upstream {status} as a member miss" + ); + + let cache_key = repository_storage_key("proxy", artifact_path); + assert!( + !ctx.state + .maven_negative_cache + .lock() + .contains_key(&cache_key), + "upstream {status} must not poison the Maven negative cache" + ); + + let stale = send( + &ctx.app, + Method::GET, + &format!("/repository/proxy/{metadata_path}"), + "", + ) + .await; + assert_eq!( + stale.status(), + status, + "upstream {status} must propagate instead of serving stale metadata" + ); + assert!(stale.headers().get("x-nora-stale").is_none()); + + upstream.reset().await; + Mock::given(method("GET")) + .and(path(upstream_path.as_str())) + .respond_with(ResponseTemplate::new(200).set_body_bytes("upstream")) + .mount(&upstream) + .await; + let recovered = send( + &ctx.app, + Method::GET, + &format!("/repository/proxy/{artifact_path}"), + "", + ) + .await; + assert_eq!(recovered.status(), StatusCode::OK); + assert_eq!(body_bytes(recovered).await, "upstream"); + } + } + + #[tokio::test] + async fn test_proxy_exact_404_negative_caches_and_group_falls_back() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let artifact_path = "com/example/missing/1.0/missing-1.0.jar"; + let upstream_path = format!("/{artifact_path}"); + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path(upstream_path.as_str())) + .respond_with(ResponseTemplate::new(404)) + .mount(&upstream) + .await; + let ctx = proxy_group_context(upstream.uri(), 60); + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/repository/hosted/{artifact_path}"), + "hosted-fallback", + ) + .await + .status(), + StatusCode::CREATED + ); + + let direct = send( + &ctx.app, + Method::GET, + &format!("/repository/proxy/{artifact_path}"), + "", + ) + .await; + assert_eq!(direct.status(), StatusCode::NOT_FOUND); + + let cache_key = repository_storage_key("proxy", artifact_path); + assert!(ctx + .state + .maven_negative_cache + .lock() + .contains_key(&cache_key)); + + upstream.reset().await; + Mock::given(method("GET")) + .and(path(upstream_path.as_str())) + .respond_with(ResponseTemplate::new(200).set_body_bytes("late-upstream")) + .mount(&upstream) + .await; + + let group = send( + &ctx.app, + Method::GET, + &format!("/repository/public/{artifact_path}"), + "", + ) + .await; + assert_eq!(group.status(), StatusCode::OK); + assert_eq!(body_bytes(group).await, "hosted-fallback"); + + let still_cached = send( + &ctx.app, + Method::GET, + &format!("/repository/proxy/{artifact_path}"), + "", + ) + .await; + assert_eq!(still_cached.status(), StatusCode::NOT_FOUND); + assert!( + upstream.received_requests().await.unwrap().is_empty(), + "the exact 404 must suppress upstream requests until its negative TTL expires" + ); + } + + #[tokio::test] + async fn test_proxy_policy_blocked_404_never_negative_caches() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let cases = [ + ( + "com/example/policy-blocked/1.0/policy-blocked-1.0.jar", + "must-not-fall-through", + "recovered-artifact", + ), + ( + "com/example/policy-blocked/maven-metadata.xml", + "com.examplepolicy-blocked1.01.01.020260731000000", + "com.examplepolicy-blocked2.02.02.020260731010000", + ), + ]; + let upstream = MockServer::start().await; + for (artifact_path, _, _) in cases { + Mock::given(method("GET")) + .and(path(format!("/{artifact_path}"))) + .respond_with(ResponseTemplate::new(404).insert_header("x-amzn-waf-reason", "geo")) + .mount(&upstream) + .await; + } + let ctx = proxy_group_context(upstream.uri(), 60); + + for (artifact_path, hosted_body, _) in cases { + let blocked = send( + &ctx.app, + Method::GET, + &format!("/repository/proxy/{artifact_path}"), + "", + ) + .await; + assert_eq!(blocked.status(), StatusCode::BAD_GATEWAY); + + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/repository/hosted/{artifact_path}"), + hosted_body, + ) + .await + .status(), + StatusCode::CREATED + ); + let blocked_group = send( + &ctx.app, + Method::GET, + &format!("/repository/public/{artifact_path}"), + "", + ) + .await; + assert_eq!(blocked_group.status(), StatusCode::BAD_GATEWAY); + + let cache_key = repository_storage_key("proxy", artifact_path); + assert!( + !ctx.state + .maven_negative_cache + .lock() + .contains_key(&cache_key), + "a policy/WAF 404 is an upstream failure, not a negative-cacheable miss" + ); + } + + upstream.reset().await; + for (artifact_path, _, recovered_body) in cases { + Mock::given(method("GET")) + .and(path(format!("/{artifact_path}"))) + .respond_with( + ResponseTemplate::new(200).set_body_bytes(recovered_body.as_bytes().to_vec()), + ) + .mount(&upstream) + .await; + } + for (artifact_path, _, recovered_body) in cases { + let recovered = send( + &ctx.app, + Method::GET, + &format!("/repository/proxy/{artifact_path}"), + "", + ) + .await; + assert_eq!(recovered.status(), StatusCode::OK); + let recovered = body_bytes(recovered).await; + if artifact_path.ends_with("maven-metadata.xml") { + let recovered = String::from_utf8_lossy(&recovered); + assert!(recovered.contains("2.0")); + assert!(recovered.contains("20260731010000")); + } else { + assert_eq!(recovered, recovered_body); + } + } + } + + #[tokio::test] + async fn test_named_repositories_isolate_same_path_and_group_preserves_order() { + let ctx = named_repository_context(); + let path = "com/example/shared/1.0/shared-1.0.jar"; + + for (repository, body) in [ + ("maven-releases", "main-bytes"), + ("maven-open", "open-bytes"), + ] { + let response = send( + &ctx.app, + Method::PUT, + &format!("/repository/{repository}/{path}"), + body, + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + } + + let direct = send( + &ctx.app, + Method::GET, + &format!("/repository/maven-releases/{path}"), + "", + ) + .await; + assert_eq!(body_bytes(direct).await, "main-bytes"); + + let redeploy = send( + &ctx.app, + Method::PUT, + &format!("/repository/maven-releases/{path}"), + "main-bytes-v2", + ) + .await; + assert_eq!(redeploy.status(), StatusCode::CREATED); + let direct = send( + &ctx.app, + Method::GET, + &format!("/repository/maven-releases/{path}"), + "", + ) + .await; + assert_eq!(body_bytes(direct).await, "main-bytes-v2"); + + let grouped = send( + &ctx.app, + Method::GET, + &format!("/repository/maven-public/{path}"), + "", + ) + .await; + assert_eq!(body_bytes(grouped).await, "open-bytes"); + + let legacy = send(&ctx.app, Method::GET, &format!("/maven2/{path}"), "").await; + assert_eq!(body_bytes(legacy).await, "open-bytes"); + } + + #[tokio::test] + async fn test_named_repository_version_policies() { + let ctx = named_repository_context(); + + let release_to_snapshots = send( + &ctx.app, + Method::PUT, + "/repository/maven-snapshots/com/example/lib/1.0/lib-1.0.jar", + "release", + ) + .await; + assert_eq!(release_to_snapshots.status(), StatusCode::BAD_REQUEST); + + let snapshot_to_releases = send( + &ctx.app, + Method::PUT, + "/repository/maven-releases/com/example/lib/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar", + "snapshot", + ) + .await; + assert_eq!(snapshot_to_releases.status(), StatusCode::BAD_REQUEST); + } + + #[tokio::test] + async fn test_group_merges_artifact_metadata_and_checksum() { + let ctx = named_repository_context(); + for (repository, version) in [ + ("maven-releases", "1.0"), + ("maven-snapshots", "2.0-SNAPSHOT"), + ] { + let response = send( + &ctx.app, + Method::PUT, + &format!("/repository/{repository}/com/example/lib/{version}/lib-{version}.jar"), + Body::from(version.to_string()), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + } + + let metadata = send( + &ctx.app, + Method::GET, + "/repository/maven-public/com/example/lib/maven-metadata.xml", + "", + ) + .await; + let metadata = body_bytes(metadata).await; + let metadata_text = String::from_utf8(metadata.to_vec()).unwrap(); + assert!(metadata_text.contains("1.0")); + assert!(metadata_text.contains("2.0-SNAPSHOT")); + + let checksum = send( + &ctx.app, + Method::GET, + "/repository/maven-public/com/example/lib/maven-metadata.xml.sha1", + "", + ) + .await; + let checksum = body_bytes(checksum).await; + assert_eq!( + String::from_utf8(checksum.to_vec()).unwrap(), + hex::encode(sha1::Sha1::digest(&metadata)) + ); + } + + #[tokio::test] + async fn test_group_artifact_metadata_uses_highest_member_release() { + let ctx = named_repository_context(); + for (repository, version) in [ + ("maven-releases", "2.0"), + ("maven-releases", "1.0"), + ("maven-open", "1.5"), + ] { + let response = send( + &ctx.app, + Method::PUT, + &format!( + "/repository/{repository}/com/example/group-order/{version}/group-order-{version}.jar" + ), + Body::from(version.to_string()), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + } + + let metadata = body_bytes( + send( + &ctx.app, + Method::GET, + "/repository/maven-public/com/example/group-order/maven-metadata.xml", + "", + ) + .await, + ) + .await; + let metadata = String::from_utf8_lossy(&metadata); + assert!(metadata.contains("1.5")); + assert!(metadata.contains("1.5")); + for version in ["1.0", "1.5", "2.0"] { + assert!(metadata.contains(&format!("{version}"))); + } + } + + #[tokio::test] + async fn test_group_merges_version_metadata_and_checksum() { + let ctx = create_test_context_with_config(|config| { + config.maven.repositories = vec![ + MavenRepository::Hosted { + name: "snapshots-a".to_string(), + version_policy: MavenVersionPolicy::Snapshot, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Hosted { + name: "snapshots-b".to_string(), + version_policy: MavenVersionPolicy::Snapshot, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Group { + name: "snapshots".to_string(), + members: vec!["snapshots-a".to_string(), "snapshots-b".to_string()], + }, + ]; + }); + let older = r#"com.examplelibrary1.0-SNAPSHOT20260730.010000120260730010000jar1.0-20260730.010000-120260730010000sourcesjar1.0-20260730.010000-120260730010000"#; + let newer = r#"com.examplelibrary1.0-SNAPSHOT20260730.020000220260730020000jar1.0-20260730.020000-220260730020000"#; + let path = "com/example/library/1.0-SNAPSHOT/maven-metadata.xml"; + for (repository, body) in [("snapshots-a", older), ("snapshots-b", newer)] { + let response = send( + &ctx.app, + Method::PUT, + &format!("/repository/{repository}/{path}"), + Body::from(body), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + } + + let merged = body_bytes( + send( + &ctx.app, + Method::GET, + &format!("/repository/snapshots/{path}"), + "", + ) + .await, + ) + .await; + let merged_text = String::from_utf8_lossy(&merged); + assert!(merged_text.contains("20260730.020000")); + assert_eq!(merged_text.matches("1.0-20260730.010000-1").count(), 1); + assert_eq!(merged_text.matches("1.0-20260730.020000-2").count(), 1); + + let checksum = body_bytes( + send( + &ctx.app, + Method::GET, + &format!("/repository/snapshots/{path}.sha512"), + "", + ) + .await, + ) + .await; + assert_eq!( + String::from_utf8_lossy(&checksum), + checksum_hex("sha512", &merged).unwrap() + ); + } + + #[tokio::test] + async fn test_group_merges_plugin_metadata() { + let ctx = named_repository_context(); + for (repository, name, prefix, artifact_id) in [ + ("maven-open", "Open Plugin", "open", "open-maven-plugin"), + ("maven-releases", "Main Plugin", "main", "main-maven-plugin"), + ] { + let metadata = format!( + "\n{name}{prefix}{artifact_id}" + ); + let response = send( + &ctx.app, + Method::PUT, + &format!("/repository/{repository}/org/example/plugins/maven-metadata.xml"), + metadata, + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + } + + let response = send( + &ctx.app, + Method::GET, + "/repository/maven-public/org/example/plugins/maven-metadata.xml", + "", + ) + .await; + let metadata = String::from_utf8(body_bytes(response).await.to_vec()).unwrap(); + assert!(metadata.contains("open")); + assert!(metadata.contains("main")); + } + + #[tokio::test] + async fn test_combined_artifact_and_plugin_metadata_survives_hosted_and_group_merge() { + let ctx = create_test_context_with_config(|config| { + config.maven.repositories = vec![ + MavenRepository::Hosted { + name: "first".to_string(), + version_policy: MavenVersionPolicy::Mixed, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Hosted { + name: "second".to_string(), + version_policy: MavenVersionPolicy::Mixed, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Group { + name: "public".to_string(), + members: vec!["first".to_string(), "second".to_string()], + }, + ]; + }); + let path = "org/example/plugins/maven-metadata.xml"; + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/first/org/example/plugins/1.0/plugins-1.0.jar", + "artifact", + ) + .await + .status(), + StatusCode::CREATED + ); + + let combined = r#" + org.example + plugins + + 1.0 + 1.0 + 1.0 + 20260730010101 + + + Firstcollisionfirst-plugin + First onlyfirstfirst-only-plugin + +"#; + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/repository/first/{path}"), + combined, + ) + .await + .status(), + StatusCode::CREATED + ); + + let second_plugins = r#" + Second collisioncollisionsecond-plugin + Second onlysecondsecond-only-plugin +"#; + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/repository/second/{path}"), + second_plugins, + ) + .await + .status(), + StatusCode::CREATED + ); + + let hosted = body_bytes( + send( + &ctx.app, + Method::GET, + &format!("/repository/first/{path}"), + "", + ) + .await, + ) + .await; + let hosted = String::from_utf8_lossy(&hosted); + assert!(hosted.contains("1.0")); + assert!(hosted.contains("1.0")); + assert!(hosted.contains("collision")); + + let grouped = body_bytes( + send( + &ctx.app, + Method::GET, + &format!("/repository/public/{path}"), + "", + ) + .await, + ) + .await; + let grouped_text = String::from_utf8_lossy(&grouped); + assert!(grouped_text.contains("1.0")); + assert!(grouped_text.contains("1.0")); + assert_eq!( + grouped_text.matches("collision").count(), + 1 + ); + assert!(grouped_text.contains("first-plugin")); + assert!(!grouped_text.contains("second-plugin")); + assert!(grouped_text.contains("second-only-plugin")); + + let checksum = body_bytes( + send( + &ctx.app, + Method::GET, + &format!("/repository/public/{path}.sha256"), + "", + ) + .await, + ) + .await; + assert_eq!( + String::from_utf8_lossy(&checksum), + checksum_hex("sha256", &grouped).unwrap() ); } - #[test] - fn test_classify_group_level_metadata() { - let xml = br#" - - - - example - example-maven-plugin - - - - "#; + #[tokio::test] + async fn test_retention_metadata_helper_preserves_plugins_and_deletes_empty_document() { + let ctx = create_test_context(); + for version in ["1.0", "2.0"] { + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/maven2/org/example/retain/{version}/retain-{version}.jar"), + version, + ) + .await + .status(), + StatusCode::CREATED + ); + } + let plugins = r#"Retainedretainedretained-plugin"#; assert_eq!( - classify_metadata_level(xml), - Some(MavenMetadataLevel::Group) + send( + &ctx.app, + Method::PUT, + "/maven2/org/example/retain/maven-metadata.xml", + plugins, + ) + .await + .status(), + StatusCode::CREATED + ); + + assert_eq!( + super::update_hosted_metadata_after_retention( + &ctx.state.storage, + "maven/", + "org/example", + "retain", + "1.0", + ) + .await + .unwrap(), + (0, 0) + ); + let metadata_key = "maven/org/example/retain/maven-metadata.xml"; + let metadata = ctx.state.storage.get(metadata_key).await.unwrap(); + let metadata_text = String::from_utf8_lossy(&metadata); + assert!(!metadata_text.contains("1.0")); + assert!(metadata_text.contains("2.0")); + assert!(metadata_text.contains("retained")); + assert_eq!( + String::from_utf8_lossy( + &ctx.state + .storage + .get(&format!("{metadata_key}.sha256")) + .await + .unwrap() + ), + checksum_hex("sha256", &metadata).unwrap() + ); + + for key in ctx + .state + .storage + .list("maven/org/example/retain/1.0/") + .await + .unwrap() + { + ctx.state.storage.delete(&key).await.unwrap(); + } + assert_eq!( + super::update_hosted_metadata_after_retention( + &ctx.state.storage, + "maven/", + "org/example", + "retain", + "2.0", + ) + .await + .unwrap(), + (0, 0) + ); + let plugin_only = ctx.state.storage.get(metadata_key).await.unwrap(); + let plugin_only_text = String::from_utf8_lossy(&plugin_only); + assert!(!plugin_only_text.contains("")); + assert!(plugin_only_text.contains("retained")); + assert_eq!( + String::from_utf8_lossy( + &ctx.state + .storage + .get(&format!("{metadata_key}.sha1")) + .await + .unwrap() + ), + checksum_hex("sha1", &plugin_only).unwrap() + ); + + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/maven2/org/example/empty/1.0/empty-1.0.jar", + "empty", + ) + .await + .status(), + StatusCode::CREATED ); + let removed = super::update_hosted_metadata_after_retention( + &ctx.state.storage, + "maven/", + "org/example", + "empty", + "1.0", + ) + .await + .unwrap(); + assert_eq!(removed.0, 5); + assert!(removed.1 > 0); + let empty_metadata_key = "maven/org/example/empty/maven-metadata.xml"; + for suffix in ["", ".md5", ".sha1", ".sha256", ".sha512"] { + assert!(ctx + .state + .storage + .get(&format!("{empty_metadata_key}{suffix}")) + .await + .is_err()); + } } - #[test] - fn test_reject_malformed_metadata() { - assert_eq!(classify_metadata_level(b""), None); - assert_eq!(classify_metadata_level(b""), None); + #[tokio::test] + async fn test_group_keeps_proxy_first_match_and_merges_hosted_metadata() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + let metadata = r#" + + com.example + collision + + 1.0 + 1.0 + 1.0 + 20260728010000 + + +"#; + Mock::given(method("GET")) + .and(path("/com/example/collision/1.0/collision-1.0.jar")) + .respond_with(ResponseTemplate::new(200).set_body_bytes("public")) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/com/example/collision/maven-metadata.xml")) + .respond_with(ResponseTemplate::new(200).set_body_raw(metadata, "application/xml")) + .mount(&upstream) + .await; + + let upstream_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + config.maven.repositories = vec![ + MavenRepository::Proxy { + name: "central".to_string(), + url: upstream_url, + auth: None, + version_policy: MavenVersionPolicy::Release, + metadata_ttl: Some(0), + negative_ttl: 0, + }, + MavenRepository::Hosted { + name: "releases".to_string(), + version_policy: MavenVersionPolicy::Release, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Group { + name: "public".to_string(), + members: vec!["central".to_string(), "releases".to_string()], + }, + ]; + }); + + for (version, body) in [("1.0", "hosted-same-path"), ("9.0", "hosted-new")] { + let response = send( + &ctx.app, + Method::PUT, + &format!( + "/repository/releases/com/example/collision/{version}/collision-{version}.jar" + ), + body, + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + } + + let artifact = send( + &ctx.app, + Method::GET, + "/repository/public/com/example/collision/1.0/collision-1.0.jar", + "", + ) + .await; + assert_eq!(body_bytes(artifact).await, "public"); + + let metadata = send( + &ctx.app, + Method::GET, + "/repository/public/com/example/collision/maven-metadata.xml", + "", + ) + .await; + let metadata = String::from_utf8(body_bytes(metadata).await.to_vec()).unwrap(); + assert!(metadata.contains("1.0")); + assert!(metadata.contains("9.0")); } -} -// ============================================================================ -// Integration Tests -// ============================================================================ + #[tokio::test] + async fn test_group_continues_after_unavailable_proxy_member() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod integration_tests { - use crate::test_helpers::{ - body_bytes, create_test_context, create_test_context_with_config, send, - }; + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/com/example/failover/1.0/failover-1.0.jar")) + .respond_with(ResponseTemplate::new(500)) + .mount(&upstream) + .await; + let upstream_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + config.maven.repositories = vec![ + MavenRepository::Proxy { + name: "unavailable".to_string(), + url: upstream_url, + auth: None, + version_policy: MavenVersionPolicy::Release, + metadata_ttl: Some(0), + negative_ttl: 0, + }, + MavenRepository::Hosted { + name: "hosted".to_string(), + version_policy: MavenVersionPolicy::Release, + write_policy: MavenWritePolicy::Allow, + }, + MavenRepository::Group { + name: "public".to_string(), + members: vec!["unavailable".to_string(), "hosted".to_string()], + }, + ]; + }); + let path = "com/example/failover/1.0/failover-1.0.jar"; + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/repository/hosted/{path}"), + "hosted", + ) + .await + .status(), + StatusCode::CREATED + ); + let response = send( + &ctx.app, + Method::GET, + &format!("/repository/public/{path}"), + "", + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(body_bytes(response).await, "hosted"); + } + + #[tokio::test] + async fn test_group_stops_on_hosted_storage_error_before_proxy() { + use crate::storage::{ + FileMeta, Result as StorageResult, Storage, StorageBackend, StorageError, + }; + use axum::body::Bytes; + use std::path::Path; + use std::pin::Pin; + use std::sync::Arc; + use tokio::io::AsyncRead; + use wiremock::MockServer; + + struct FailingGetBackend; + + #[async_trait::async_trait] + impl StorageBackend for FailingGetBackend { + async fn put(&self, _key: &str, _data: &[u8]) -> StorageResult<()> { + Ok(()) + } + + async fn get(&self, _key: &str) -> StorageResult { + Err(StorageError::Io(std::io::Error::other( + "injected hosted read failure", + ))) + } + + async fn delete(&self, _key: &str) -> StorageResult<()> { + Ok(()) + } + + async fn list(&self, _prefix: &str) -> StorageResult> { + Ok(Vec::new()) + } + + async fn stat(&self, _key: &str) -> Option { + None + } + + async fn health_check(&self) -> bool { + true + } + + async fn total_size(&self) -> u64 { + 0 + } + + fn backend_name(&self) -> &'static str { + "failing-maven-get-test" + } + + async fn put_from_path(&self, _key: &str, _src: &Path) -> StorageResult<()> { + Ok(()) + } + + async fn get_reader( + &self, + _key: &str, + ) -> StorageResult<(u64, Pin>)> { + Err(StorageError::Io(std::io::Error::other( + "injected hosted read failure", + ))) + } + } + + let upstream = MockServer::start().await; + let ctx = create_test_context(); + let mut config = (*ctx.state.config).clone(); + config.maven.repositories = vec![ + MavenRepository::Hosted { + name: "hosted".to_string(), + version_policy: MavenVersionPolicy::Release, + write_policy: MavenWritePolicy::AllowOnce, + }, + MavenRepository::Proxy { + name: "proxy".to_string(), + url: upstream.uri(), + auth: None, + version_policy: MavenVersionPolicy::Release, + metadata_ttl: None, + negative_ttl: 60, + }, + MavenRepository::Group { + name: "public".to_string(), + members: vec!["hosted".to_string(), "proxy".to_string()], + }, + ]; + let mut state = ctx.state.clone(); + state.config = Arc::new(config); + state.storage = Storage::from_backend(Arc::new(FailingGetBackend)); + + let members = ["hosted".to_string(), "proxy".to_string()]; + let response = super::download_group( + state, + axum::http::HeaderMap::new(), + &members, + "com/example/fail/1.0/fail-1.0.jar".to_string(), + ) + .await; + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!( + upstream.received_requests().await.unwrap().is_empty(), + "a hosted storage error must stop the group before proxy fallback" + ); + } + + #[tokio::test] + async fn test_proxy_and_group_artifact_metadata_respect_version_policy() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + let metadata_path = "/com/example/policy/maven-metadata.xml"; + let upstream_metadata = r#" + com.example + policy + + 2.0-SNAPSHOT + 1.0 + + 1.0 + 2.0-SNAPSHOT + + 20260730020202 + +"#; + Mock::given(method("GET")) + .and(path(metadata_path)) + .respond_with( + ResponseTemplate::new(200).set_body_raw(upstream_metadata, "application/xml"), + ) + .mount(&upstream) + .await; + let upstream_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + config.maven.repositories = vec![ + MavenRepository::Proxy { + name: "releases".to_string(), + url: upstream_url.clone(), + auth: None, + version_policy: MavenVersionPolicy::Release, + metadata_ttl: Some(0), + negative_ttl: 0, + }, + MavenRepository::Proxy { + name: "snapshots".to_string(), + url: upstream_url, + auth: None, + version_policy: MavenVersionPolicy::Snapshot, + metadata_ttl: Some(0), + negative_ttl: 0, + }, + MavenRepository::Group { + name: "public".to_string(), + members: vec!["releases".to_string(), "snapshots".to_string()], + }, + ]; + }); + + let release = body_bytes( + send( + &ctx.app, + Method::GET, + "/repository/releases/com/example/policy/maven-metadata.xml", + "", + ) + .await, + ) + .await; + let release_text = String::from_utf8_lossy(&release); + assert!(release_text.contains("1.0")); + assert!(!release_text.contains("2.0-SNAPSHOT")); + assert!(release_text.contains("1.0")); + assert!(release_text.contains("1.0")); + + let snapshot = body_bytes( + send( + &ctx.app, + Method::GET, + "/repository/snapshots/com/example/policy/maven-metadata.xml", + "", + ) + .await, + ) + .await; + let snapshot_text = String::from_utf8_lossy(&snapshot); + assert!(!snapshot_text.contains("1.0")); + assert!(snapshot_text.contains("2.0-SNAPSHOT")); + assert!(snapshot_text.contains("2.0-SNAPSHOT")); + assert!(!snapshot_text.contains("")); + + let grouped = body_bytes( + send( + &ctx.app, + Method::GET, + "/repository/public/com/example/policy/maven-metadata.xml", + "", + ) + .await, + ) + .await; + let grouped_text = String::from_utf8_lossy(&grouped); + assert!(grouped_text.contains("1.0")); + assert!(grouped_text.contains("2.0-SNAPSHOT")); + assert!(grouped_text.contains("2.0-SNAPSHOT")); + assert!(grouped_text.contains("1.0")); + + let grouped_checksum = body_bytes( + send( + &ctx.app, + Method::GET, + "/repository/public/com/example/policy/maven-metadata.xml.sha1", + "", + ) + .await, + ) + .await; + assert_eq!( + String::from_utf8_lossy(&grouped_checksum), + checksum_hex("sha1", &grouped).unwrap() + ); + } #[tokio::test] async fn test_maven_namespace_scope_enforced() { @@ -1640,7 +5053,7 @@ mod integration_tests { ); // Out of scope (different group) -> 403. - let resp = super::upload( + let resp = super::upload_legacy( State(ctx.state.clone()), Path("com/other/lib/1.0/lib-1.0.jar".to_string()), Extension(scoped.clone()), @@ -1650,7 +5063,7 @@ mod integration_tests { assert_eq!(resp.status(), StatusCode::FORBIDDEN); // Opaque (unrecognized) path under a real scope -> fail-closed 403. - let resp = super::upload( + let resp = super::upload_legacy( State(ctx.state.clone()), Path("foo".to_string()), Extension(scoped.clone()), @@ -1660,7 +5073,7 @@ mod integration_tests { assert_eq!(resp.status(), StatusCode::FORBIDDEN); // In scope -> enforcement passes (not 403). - let resp = super::upload( + let resp = super::upload_legacy( State(ctx.state.clone()), Path("com/myorg/lib/1.0/lib-1.0.jar".to_string()), Extension(scoped), @@ -1669,8 +5082,7 @@ mod integration_tests { .await; assert_ne!(resp.status(), StatusCode::FORBIDDEN); } - use axum::body::Body; - use axum::http::{header, Method, StatusCode}; + use axum::http::header; use sha2::Digest; #[tokio::test] @@ -1771,42 +5183,25 @@ mod integration_tests { ) .await; - // SHA-256 - let resp = send( - &ctx.app, - Method::GET, - "/maven2/com/example/ck/1.0/ck-1.0.jar.sha256", - "", - ) - .await; - assert_eq!(resp.status(), StatusCode::OK); - let hash = body_bytes(resp).await; - let expected = hex::encode(sha2::Sha256::digest(data)); - assert_eq!(String::from_utf8_lossy(&hash), expected); - - // SHA-1 - let resp = send( - &ctx.app, - Method::GET, - "/maven2/com/example/ck/1.0/ck-1.0.jar.sha1", - "", - ) - .await; - assert_eq!(resp.status(), StatusCode::OK); - - // MD5 - let resp = send( - &ctx.app, - Method::GET, - "/maven2/com/example/ck/1.0/ck-1.0.jar.md5", - "", - ) - .await; - assert_eq!(resp.status(), StatusCode::OK); + for suffix in ["md5", "sha1", "sha256", "sha512"] { + let resp = send( + &ctx.app, + Method::GET, + &format!("/maven2/com/example/ck/1.0/ck-1.0.jar.{suffix}"), + "", + ) + .await; + assert_eq!(resp.status(), StatusCode::OK); + let hash = body_bytes(resp).await; + assert_eq!( + String::from_utf8_lossy(&hash), + checksum_hex(suffix, data).unwrap() + ); + } } #[tokio::test] - async fn test_maven_checksum_verify_ok() { + async fn test_maven_checksum_upload_accepts_all_algorithms() { let ctx = create_test_context(); let data = b"checksum-test-jar"; @@ -1818,19 +5213,20 @@ mod integration_tests { ) .await; - let sha1 = hex::encode(sha1::Sha1::digest(data)); - let resp = send( - &ctx.app, - Method::PUT, - "/maven2/com/example/cv/1.0/cv-1.0.jar.sha1", - Body::from(sha1), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); + for suffix in ["md5", "sha1", "sha256", "sha512"] { + let resp = send( + &ctx.app, + Method::PUT, + &format!("/maven2/com/example/cv/1.0/cv-1.0.jar.{suffix}"), + Body::from(checksum_hex(suffix, data).unwrap()), + ) + .await; + assert_eq!(resp.status(), StatusCode::CREATED); + } } #[tokio::test] - async fn test_maven_checksum_verify_mismatch() { + async fn test_maven_checksum_upload_rejects_mismatch() { let ctx = create_test_context(); let data = b"checksum-mismatch-test"; @@ -1852,6 +5248,90 @@ mod integration_tests { assert_eq!(resp.status(), StatusCode::BAD_REQUEST); } + #[tokio::test] + async fn test_maven_checksum_requires_base_object() { + let ctx = create_test_context(); + for method in [Method::GET, Method::PUT] { + let resp = send( + &ctx.app, + method, + "/maven2/com/example/missing/1.0/missing-1.0.jar.sha256", + Body::from("00"), + ) + .await; + assert_eq!(resp.status(), StatusCode::NOT_FOUND); + } + } + + #[tokio::test] + async fn test_maven_checksum_get_repairs_stale_sidecar() { + let ctx = create_test_context(); + let path = "com/example/repair/1.0/repair-1.0.jar"; + let data = b"authoritative"; + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/maven2/{path}"), + Body::from(&data[..]), + ) + .await + .status(), + StatusCode::CREATED + ); + let sidecar_key = format!("maven/{path}.sha256"); + ctx.state.storage.put(&sidecar_key, b"stale").await.unwrap(); + + let response = send(&ctx.app, Method::GET, &format!("/maven2/{path}.sha256"), "").await; + assert_eq!(response.status(), StatusCode::OK); + let expected = checksum_hex("sha256", data).unwrap(); + assert_eq!( + String::from_utf8_lossy(&body_bytes(response).await), + expected + ); + assert_eq!( + String::from_utf8_lossy(&ctx.state.storage.get(&sidecar_key).await.unwrap()), + expected + ); + } + + #[tokio::test] + async fn test_maven_checksum_reread_error_fails_instead_of_hashing_prelock_bytes() { + let ctx = create_test_context(); + let backend = Arc::new(FaultInjectingBackend::default()); + let mut state = ctx.state.clone(); + state.storage = Storage::from_backend(backend.clone()); + let path = "com/example/locked/1.0/locked-1.0.jar"; + let key = format!("maven/{path}"); + let data = b"authoritative-under-lock"; + state.storage.put(&key, data).await.unwrap(); + + // The recursive base GET consumes the first read. Fail the authoritative + // re-read after the mutation lock is acquired. + backend.fail_after(InjectedOperation::Get, &key, 1); + let response = super::download_direct( + state.clone(), + axum::http::HeaderMap::new(), + super::DirectRepository::legacy(&state), + format!("{path}.sha256"), + ) + .await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let retry = super::download_direct( + state.clone(), + axum::http::HeaderMap::new(), + super::DirectRepository::legacy(&state), + format!("{path}.sha256"), + ) + .await; + assert_eq!(retry.status(), StatusCode::OK); + assert_eq!( + String::from_utf8_lossy(&body_bytes(retry).await), + checksum_hex("sha256", data).unwrap() + ); + } + // ── Immutability ──────────────────────────────────────────────────── #[tokio::test] @@ -1887,6 +5367,301 @@ mod integration_tests { assert_eq!(&body[..], b"v1"); } + #[tokio::test] + async fn test_maven_exact_release_retry_repairs_derived_state() { + let ctx = create_test_context(); + let path = "com/example/retry/1.0.0/retry-1.0.0.jar"; + let key = format!("maven/{path}"); + let data = b"immutable-release"; + + // Simulate interruption immediately after the atomic create: the base + // object exists, but none of its derived state was written. + ctx.state.storage.put_if_absent(&key, data).await.unwrap(); + for suffix in ["md5", "sha1", "sha256", "sha512"] { + assert!(ctx + .state + .storage + .get(&format!("{key}.{suffix}")) + .await + .is_err()); + } + let metadata_key = "maven/com/example/retry/maven-metadata.xml"; + assert!(ctx.state.storage.get(metadata_key).await.is_err()); + + let retry = send( + &ctx.app, + Method::PUT, + &format!("/maven2/{path}"), + Body::from(&data[..]), + ) + .await; + assert_eq!(retry.status(), StatusCode::CREATED); + + for suffix in ["md5", "sha1", "sha256", "sha512"] { + assert_eq!( + String::from_utf8_lossy( + &ctx.state + .storage + .get(&format!("{key}.{suffix}")) + .await + .unwrap() + ), + checksum_hex(suffix, data).unwrap() + ); + } + let metadata = ctx.state.storage.get(metadata_key).await.unwrap(); + let metadata = String::from_utf8_lossy(&metadata); + assert!(metadata.contains("1.0.0")); + assert!(metadata.contains("1.0.0")); + for suffix in ["md5", "sha1", "sha256", "sha512"] { + assert_eq!( + String::from_utf8_lossy( + &ctx.state + .storage + .get(&format!("{metadata_key}.{suffix}")) + .await + .unwrap() + ), + checksum_hex(suffix, metadata.as_bytes()).unwrap() + ); + } + } + + #[tokio::test] + async fn test_maven_completed_exact_release_retry_keeps_metadata_byte_stable() { + let ctx = create_test_context(); + let older_path = "com/example/stable-retry/1.0.0/stable-retry-1.0.0.jar"; + let current_path = "com/example/stable-retry/2.0.0/stable-retry-2.0.0.jar"; + let older_key = format!("maven/{older_path}"); + let metadata_key = "maven/com/example/stable-retry/maven-metadata.xml"; + let older_data = b"immutable-release-1"; + let current_data = b"immutable-release-2"; + + for (path, data) in [ + (older_path, older_data.as_slice()), + (current_path, current_data.as_slice()), + ] { + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/maven2/{path}"), + Body::from(data.to_vec()), + ) + .await + .status(), + StatusCode::CREATED + ); + } + + let original = + String::from_utf8(ctx.state.storage.get(metadata_key).await.unwrap().to_vec()).unwrap(); + let generated_timestamp = parse_artifact_metadata(original.as_bytes()) + .and_then(|metadata| metadata.last_updated) + .unwrap(); + let original = original.replace( + &format!("{generated_timestamp}"), + "20000101000000", + ); + assert!(parse_artifact_metadata(original.as_bytes()).is_some()); + ctx.state + .storage + .put(metadata_key, original.as_bytes()) + .await + .unwrap(); + compute_and_store_checksums(&ctx.state.storage, metadata_key, original.as_bytes()) + .await + .unwrap(); + ctx.state + .storage + .put(&format!("{metadata_key}.sha512"), b"corrupt") + .await + .unwrap(); + + assert_eq!( + send( + &ctx.app, + Method::PUT, + &format!("/maven2/{older_path}"), + Body::from(&older_data[..]), + ) + .await + .status(), + StatusCode::CREATED + ); + + assert_eq!( + ctx.state.storage.get(&older_key).await.unwrap(), + older_data.as_slice() + ); + assert_eq!( + ctx.state.storage.get(metadata_key).await.unwrap(), + original.as_bytes(), + "a completed exact retry must not create a new metadata generation" + ); + let metadata = parse_artifact_metadata(original.as_bytes()).unwrap(); + assert_eq!(metadata.latest.as_deref(), Some("2.0.0")); + assert_eq!(metadata.release.as_deref(), Some("2.0.0")); + assert_eq!(metadata.last_updated.as_deref(), Some("20000101000000")); + for suffix in ["md5", "sha1", "sha256", "sha512"] { + assert_eq!( + String::from_utf8_lossy( + &ctx.state + .storage + .get(&format!("{metadata_key}.{suffix}")) + .await + .unwrap() + ), + checksum_hex(suffix, original.as_bytes()).unwrap() + ); + } + } + + #[tokio::test] + async fn test_maven_derived_failures_return_500_and_exact_retry_repairs() { + let ctx = create_test_context(); + let backend = Arc::new(FaultInjectingBackend::default()); + let mut state = ctx.state.clone(); + state.storage = Storage::from_backend(backend.clone()); + + let checksum_path = "com/example/checksum-failure/1.0.0/checksum-failure-1.0.0.jar"; + let checksum_key = format!("maven/{checksum_path}"); + let checksum_data = b"checksum-body"; + backend.fail_once(InjectedOperation::Put, format!("{checksum_key}.md5")); + assert_eq!( + upload_with_state(&state, checksum_path, checksum_data) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + state.storage.get(&checksum_key).await.unwrap(), + checksum_data.as_slice() + ); + assert_eq!( + upload_with_state(&state, checksum_path, checksum_data) + .await + .status(), + StatusCode::CREATED + ); + for suffix in ["md5", "sha1", "sha256", "sha512"] { + assert_eq!( + String::from_utf8_lossy( + &state + .storage + .get(&format!("{checksum_key}.{suffix}")) + .await + .unwrap() + ), + checksum_hex(suffix, checksum_data).unwrap() + ); + } + assert!(state + .storage + .get("maven/com/example/checksum-failure/maven-metadata.xml") + .await + .is_ok()); + + let list_path = "com/example/list-failure/1.0.0/list-failure-1.0.0.jar"; + let list_key = format!("maven/{list_path}"); + let list_data = b"list-body"; + backend.fail_once(InjectedOperation::List, "maven/com/example/list-failure/"); + assert_eq!( + upload_with_state(&state, list_path, list_data) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + state.storage.get(&list_key).await.unwrap(), + list_data.as_slice() + ); + assert_eq!( + upload_with_state(&state, list_path, list_data) + .await + .status(), + StatusCode::CREATED + ); + assert!(state + .storage + .get("maven/com/example/list-failure/maven-metadata.xml") + .await + .is_ok()); + + let metadata_write_path = + "com/example/metadata-write-failure/1.0.0/metadata-write-failure-1.0.0.jar"; + let metadata_write_key = "maven/com/example/metadata-write-failure/maven-metadata.xml"; + let metadata_write_data = b"metadata-write-body"; + backend.fail_once(InjectedOperation::Put, metadata_write_key); + assert_eq!( + upload_with_state(&state, metadata_write_path, metadata_write_data) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + state + .storage + .get(&format!("maven/{metadata_write_path}")) + .await + .unwrap(), + metadata_write_data.as_slice() + ); + assert!(state.storage.get(metadata_write_key).await.is_err()); + assert_eq!( + upload_with_state(&state, metadata_write_path, metadata_write_data) + .await + .status(), + StatusCode::CREATED + ); + + let metadata_path = "com/example/metadata-failure/1.0.0/metadata-failure-1.0.0.jar"; + let metadata_key = "maven/com/example/metadata-failure/maven-metadata.xml"; + let metadata_data = b"metadata-body"; + backend.fail_once(InjectedOperation::Get, metadata_key); + assert_eq!( + upload_with_state(&state, metadata_path, metadata_data) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + state + .storage + .get(&format!("maven/{metadata_path}")) + .await + .unwrap(), + metadata_data.as_slice() + ); + assert!(state.storage.get(metadata_key).await.is_err()); + assert_eq!( + upload_with_state(&state, metadata_path, metadata_data) + .await + .status(), + StatusCode::CREATED + ); + + let metadata_before = state.storage.get(metadata_key).await.unwrap(); + backend.fail_once(InjectedOperation::Get, metadata_key); + assert_eq!( + upload_with_state(&state, metadata_path, metadata_data) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + state.storage.get(metadata_key).await.unwrap(), + metadata_before, + "an exact-retry metadata read failure must not regenerate or overwrite metadata" + ); + assert_eq!( + upload_with_state(&state, metadata_path, metadata_data) + .await + .status(), + StatusCode::CREATED + ); + } + #[tokio::test] async fn test_maven_snapshot_overwrite() { let ctx = create_test_context(); @@ -1961,6 +5736,117 @@ mod integration_tests { assert!(xml.contains("2.0.0")); } + #[tokio::test] + async fn test_maven_metadata_release_tracks_lower_version_deployed_last() { + let ctx = create_test_context(); + for version in ["2.0.0", "1.0.0"] { + let response = send( + &ctx.app, + Method::PUT, + &format!("/maven2/com/example/order/{version}/order-{version}.jar"), + Body::from(version.to_string()), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + } + + let metadata = body_bytes( + send( + &ctx.app, + Method::GET, + "/maven2/com/example/order/maven-metadata.xml", + "", + ) + .await, + ) + .await; + let metadata = String::from_utf8_lossy(&metadata); + assert!(metadata.contains("1.0.0")); + assert!(metadata.contains("1.0.0")); + assert!(metadata.contains("2.0.0")); + assert!(metadata.contains("1.0.0")); + } + + #[tokio::test] + async fn test_ordinary_metadata_latest_and_client_checksum_deploy_sequence() { + let ctx = create_test_context(); + let client_metadata = r#" + + com.example + ordinary + + 1.0.0 + 1.0.0 + 1.0.0 + 20260729010101 + + +"#; + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/maven2/com/example/ordinary/1.0.0/ordinary-1.0.0.pom", + Body::from(""), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/maven2/com/example/ordinary/maven-metadata.xml", + Body::from(client_metadata), + ) + .await + .status(), + StatusCode::CREATED + ); + + let stored = body_bytes( + send( + &ctx.app, + Method::GET, + "/maven2/com/example/ordinary/maven-metadata.xml", + "", + ) + .await, + ) + .await; + assert!(String::from_utf8_lossy(&stored).contains("1.0.0")); + assert_ne!(stored.as_ref(), client_metadata.as_bytes()); + + let client_sha1 = checksum_hex("sha1", client_metadata.as_bytes()).unwrap(); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/maven2/com/example/ordinary/maven-metadata.xml.sha1", + Body::from(client_sha1.clone()), + ) + .await + .status(), + StatusCode::CREATED + ); + let served_sha1 = body_bytes( + send( + &ctx.app, + Method::GET, + "/maven2/com/example/ordinary/maven-metadata.xml.sha1", + "", + ) + .await, + ) + .await; + assert_eq!( + String::from_utf8_lossy(&served_sha1), + checksum_hex("sha1", &stored).unwrap() + ); + assert_ne!(String::from_utf8_lossy(&served_sha1), client_sha1); + } + #[tokio::test] async fn test_maven_metadata_checksums() { let ctx = create_test_context(); @@ -2096,7 +5982,56 @@ mod integration_tests { } #[tokio::test] - async fn test_client_artifact_metadata_does_not_overwrite_generated_metadata() { + async fn test_proxy_stale_metadata_only_on_error_not_upstream_404() { + use crate::config::MavenProxyEntry; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + let metadata = r#"com.examplestale1.01.0"#; + let metadata_path = "/com/example/stale/maven-metadata.xml"; + Mock::given(method("GET")) + .and(path(metadata_path)) + .respond_with(ResponseTemplate::new(200).set_body_raw(metadata, "application/xml")) + .mount(&upstream) + .await; + let upstream_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + config.maven.proxies = vec![MavenProxyEntry::Simple(upstream_url)]; + config.maven.metadata_ttl = 0; + }); + let request_path = "/maven2/com/example/stale/maven-metadata.xml"; + assert_eq!( + send(&ctx.app, Method::GET, request_path, "").await.status(), + StatusCode::OK + ); + + upstream.reset().await; + Mock::given(method("GET")) + .and(path(metadata_path)) + .respond_with(ResponseTemplate::new(404)) + .mount(&upstream) + .await; + let not_found = send(&ctx.app, Method::GET, request_path, "").await; + assert_eq!(not_found.status(), StatusCode::NOT_FOUND); + assert!(not_found.headers().get("x-nora-stale").is_none()); + + upstream.reset().await; + Mock::given(method("GET")) + .and(path(metadata_path)) + .respond_with(ResponseTemplate::new(500)) + .mount(&upstream) + .await; + let stale = send(&ctx.app, Method::GET, request_path, "").await; + assert_eq!(stale.status(), StatusCode::OK); + assert_eq!(stale.headers().get("x-nora-stale").unwrap(), "true"); + assert!( + String::from_utf8_lossy(&body_bytes(stale).await).contains("1.0") + ); + } + + #[tokio::test] + async fn test_client_artifact_metadata_controls_release_without_losing_versions() { let ctx = create_test_context(); send( @@ -2148,12 +6083,12 @@ mod integration_tests { let metadata = String::from_utf8_lossy(&metadata); assert!(metadata.contains("1.0.0")); assert!(metadata.contains("2.0.0")); - assert!(metadata.contains("2.0.0")); - assert!(metadata.contains("2.0.0")); + assert!(metadata.contains("1.0.0")); + assert!(metadata.contains("1.0.0")); } #[tokio::test] - async fn test_client_artifact_metadata_checksum_does_not_overwrite_generated_checksum() { + async fn test_stale_client_artifact_metadata_checksum_is_acknowledged_but_not_stored() { let ctx = create_test_context(); send( diff --git a/nora-registry/src/registry/mod.rs b/nora-registry/src/registry/mod.rs index 41c788da..7a1be66f 100644 --- a/nora-registry/src/registry/mod.rs +++ b/nora-registry/src/registry/mod.rs @@ -10,6 +10,7 @@ pub mod docker_auth; pub(crate) mod gems; mod go; mod maven; +mod named; mod npm; pub(crate) mod nuget; pub(crate) mod pub_dart; @@ -31,12 +32,14 @@ pub use docker_auth::DockerAuth; pub use gems::routes as gems_routes; pub use go::routes as go_routes; pub use maven::routes as maven_routes; +pub use named::routes as named_repository_routes; pub use npm::routes as npm_routes; // Storage-key builders reused by `nora import` so imported keys are // byte-identical to the keys these handlers serve — GC/retention/UI browse walk // keys as strings (review R7, contract `import-key-format-equals-handler-key-format`). pub(crate) use maven::storage_key as maven_storage_key; +pub(crate) use maven::update_hosted_metadata_after_retention; pub use nuget::alias_routes as nuget_alias_routes; pub use nuget::routes as nuget_routes; pub use pub_dart::routes as pub_dart_routes; @@ -196,7 +199,8 @@ where // operator cannot tell it apart from a genuine 404. Surface it // distinctly when the response carries a block signature (#881). // A plain 4xx (no signature) stays silent, exactly as before. - if let Some(reason) = policy_block_reason(response.headers()) { + let policy_block = policy_block_reason(response.headers()); + if let Some(reason) = policy_block { UPSTREAM_POLICY_BLOCKED_TOTAL .with_label_values(&[registry_str, reason]) .inc(); @@ -205,7 +209,7 @@ where url, status, reason, - "upstream returned a policy/geo block, relayed as 404 (not a genuine not-found) — check egress/region" + "upstream returned a policy/geo block (not a genuine not-found) — check egress/region" ); } // A 4xx means the upstream is alive and answered — not an @@ -214,7 +218,13 @@ where // is a no-op in Closed, so a 4xx never clears a real failure // tally (#606). cb.record_alive(registry_str, probe); - return Err(ProxyError::NotFound); + return if response.status() == reqwest::StatusCode::NOT_FOUND + && policy_block.is_none() + { + Err(ProxyError::NotFound) + } else { + Err(ProxyError::Upstream(status)) + }; } if attempt == 0 { UPSTREAM_REQUEST_DURATION @@ -272,6 +282,315 @@ pub(crate) async fn proxy_fetch( .await } +fn is_followable_redirect(status: reqwest::StatusCode) -> bool { + matches!( + status, + reqwest::StatusCode::MOVED_PERMANENTLY + | reqwest::StatusCode::FOUND + | reqwest::StatusCode::SEE_OTHER + | reqwest::StatusCode::TEMPORARY_REDIRECT + | reqwest::StatusCode::PERMANENT_REDIRECT + ) +} + +fn validate_initial_url(url: &str, allowed: &F) -> Result +where + F: Fn(&reqwest::Url) -> bool, +{ + let parsed = reqwest::Url::parse(url) + .map_err(|error| ProxyError::Network(format!("invalid upstream URL: {error}")))?; + if !allowed(&parsed) { + return Err(ProxyError::Network( + "initial upstream URL rejected by repository URL policy".to_string(), + )); + } + Ok(parsed) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RedirectTargetError { + MissingOrInvalidLocation, + LimitExceeded, + Disallowed, +} + +impl RedirectTargetError { + fn reason(self) -> &'static str { + match self { + Self::MissingOrInvalidLocation => "missing or invalid Location", + Self::LimitExceeded => "redirect limit exceeded", + Self::Disallowed => "target rejected by repository URL policy", + } + } +} + +fn validated_redirect_target( + current_url: &reqwest::Url, + response: &reqwest::Response, + followed: usize, + max_redirects: usize, + allowed: &F, +) -> Result +where + F: Fn(&reqwest::Url) -> bool, +{ + if followed >= max_redirects { + return Err(RedirectTargetError::LimitExceeded); + } + let location = response + .headers() + .get(reqwest::header::LOCATION) + .and_then(|value| value.to_str().ok()) + .ok_or(RedirectTargetError::MissingOrInvalidLocation)?; + let target = current_url + .join(location) + .map_err(|_| RedirectTargetError::MissingOrInvalidLocation)?; + if !allowed(&target) { + return Err(RedirectTargetError::Disallowed); + } + Ok(target) +} + +/// Fetch binary content while following only the initial URL and redirect +/// targets accepted by `redirect_allowed`. +/// +/// `client` MUST have reqwest's automatic redirect policy disabled. The +/// initial URL and every resolved `Location` are checked before the request +/// (and its Authorization header) is built. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn proxy_fetch_with_validated_redirects( + client: &reqwest::Client, + url: &str, + timeout: Duration, + auth: Option<&str>, + cb: &CircuitBreakerRegistry, + registry: RegistryType, + max_redirects: usize, + redirect_allowed: F, +) -> Result, ProxyError> +where + F: Fn(&reqwest::Url) -> bool, +{ + proxy_fetch_with_validated_redirects_impl( + client, + url, + timeout, + auth, + cb, + registry, + max_redirects, + None, + redirect_allowed, + ) + .await +} + +/// The redirect-validated GET flow with a strict response-body budget. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn proxy_fetch_with_validated_redirects_bounded( + client: &reqwest::Client, + url: &str, + timeout: Duration, + auth: Option<&str>, + cb: &CircuitBreakerRegistry, + registry: RegistryType, + max_redirects: usize, + body_cap: usize, + redirect_allowed: F, +) -> Result, ProxyError> +where + F: Fn(&reqwest::Url) -> bool, +{ + proxy_fetch_with_validated_redirects_impl( + client, + url, + timeout, + auth, + cb, + registry, + max_redirects, + Some(body_cap), + redirect_allowed, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn proxy_fetch_with_validated_redirects_impl( + client: &reqwest::Client, + url: &str, + timeout: Duration, + auth: Option<&str>, + cb: &CircuitBreakerRegistry, + registry: RegistryType, + max_redirects: usize, + body_cap: Option, + redirect_allowed: F, +) -> Result, ProxyError> +where + F: Fn(&reqwest::Url) -> bool, +{ + let initial_url = validate_initial_url(url, &redirect_allowed)?; + let registry_str = registry.as_str(); + let probe = cb.check(registry_str)?; + + for attempt in 0..2 { + let mut current_url = initial_url.clone(); + let mut followed = 0; + + loop { + let mut request = client.get(current_url.clone()).timeout(timeout); + if let Some(credentials) = auth { + request = request.header("Authorization", basic_auth_header(credentials)); + } + + let upstream_start = Instant::now(); + match request.send().await { + Ok(response) => { + let elapsed = upstream_start.elapsed().as_secs_f64(); + let status = response.status(); + if status.is_success() { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "2xx"]) + .observe(elapsed); + let result = match body_cap { + Some(body_cap) => { + read_response_body_bounded( + response, + body_cap, + "validated proxy response", + ) + .await + } + None => response + .bytes() + .await + .map(|bytes| bytes.to_vec()) + .map_err(|error| ProxyError::Network(error.to_string())), + }; + if result.is_ok() { + cb.record_success(registry_str, probe); + } else { + cb.record_failure(registry_str, probe); + } + return result; + } + + if is_followable_redirect(status) { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "3xx"]) + .observe(elapsed); + let next_url = match validated_redirect_target( + ¤t_url, + &response, + followed, + max_redirects, + &redirect_allowed, + ) { + Ok(target) => target, + Err(error) => { + cb.record_alive(registry_str, probe); + tracing::warn!( + registry = registry_str, + status = status.as_u16(), + reason = error.reason(), + "upstream redirect rejected" + ); + return Err(ProxyError::Upstream(status.as_u16())); + } + }; + followed += 1; + current_url = next_url; + continue; + } + if status.is_redirection() { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "3xx"]) + .observe(elapsed); + cb.record_alive(registry_str, probe); + tracing::warn!( + registry = registry_str, + status = status.as_u16(), + "unsupported upstream redirect status" + ); + return Err(ProxyError::Upstream(status.as_u16())); + } + + let status_code = status.as_u16(); + if status.is_client_error() { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "4xx"]) + .observe(elapsed); + let policy_block = policy_block_reason(response.headers()); + if let Some(reason) = policy_block { + UPSTREAM_POLICY_BLOCKED_TOTAL + .with_label_values(&[registry_str, reason]) + .inc(); + tracing::warn!( + registry = registry_str, + url = %current_url, + status = status_code, + reason, + "upstream returned a policy/geo block (not a genuine not-found) — check egress/region" + ); + } + cb.record_alive(registry_str, probe); + return if status == reqwest::StatusCode::NOT_FOUND && policy_block.is_none() + { + Err(ProxyError::NotFound) + } else { + Err(ProxyError::Upstream(status_code)) + }; + } + + if attempt == 0 { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "5xx"]) + .observe(elapsed); + tracing::debug!( + url = %current_url, + status = status_code, + "upstream 5xx, retrying in 1s" + ); + tokio::time::sleep(Duration::from_secs(1)).await; + break; + } + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "5xx"]) + .observe(elapsed); + cb.record_failure(registry_str, probe); + return Err(ProxyError::Upstream(status_code)); + } + Err(error) => { + let elapsed = upstream_start.elapsed().as_secs_f64(); + let status_label = if error.is_timeout() { + "timeout" + } else { + "error" + }; + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, status_label]) + .observe(elapsed); + if attempt == 0 { + tracing::debug!( + url = %current_url, + error = %error, + "upstream error, retrying in 1s" + ); + tokio::time::sleep(Duration::from_secs(1)).await; + break; + } + cb.record_failure(registry_str, probe); + return Err(ProxyError::Network(error.to_string())); + } + } + } + } + + cb.record_failure(registry_str, probe); + Err(ProxyError::Network("max retries exceeded".into())) +} + /// Fetch text content from upstream proxy with timeout and 1 retry. pub(crate) async fn proxy_fetch_text( client: &reqwest::Client, @@ -458,10 +777,43 @@ pub(crate) async fn repo_proxy_download( } } +async fn read_response_body_bounded( + response: reqwest::Response, + cap: usize, + description: &str, +) -> Result, ProxyError> { + let content_length = response.content_length(); + if content_length.is_some_and(|length| length > cap as u64) { + return Err(ProxyError::Network(format!( + "upstream {description} exceeds {cap} byte limit" + ))); + } + + let mut body = Vec::with_capacity(content_length.unwrap_or_default().min(cap as u64) as usize); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|error| ProxyError::Network(error.to_string()))?; + if chunk.len() > cap.saturating_sub(body.len()) { + return Err(ProxyError::Network(format!( + "upstream {description} exceeds {cap} byte limit" + ))); + } + body.extend_from_slice(&chunk); + } + Ok(body) +} + +const NPM_AUDIT_RESPONSE_BODY_CAP: usize = 8 * 1024 * 1024; + +async fn read_npm_audit_response_body(response: reqwest::Response) -> Result, ProxyError> { + read_response_body_bounded(response, NPM_AUDIT_RESPONSE_BODY_CAP, "npm audit response").await +} + /// Forward a POST (request body + an allowlist of headers) to an upstream and -/// return its `(status, body, content-type)` verbatim. Mirrors `proxy_fetch_core`'s -/// circuit-breaker discipline (`check` → send → `record_success`/`record_alive`/ -/// `record_failure`, one retry on 5xx/network). +/// return its `(status, body, content-type)` verbatim. Mirrors +/// `proxy_fetch_core`'s circuit-breaker discipline (`check` → send → +/// `record_success`/`record_alive`/`record_failure`). Request-send and 5xx +/// failures get one retry; response-body failures fail immediately. /// /// Used for `npm audit` (#597): a query POST that must return the upstream's answer /// as-is — including a 4xx (a real audit response, upstream is alive) — with only @@ -469,9 +821,17 @@ pub(crate) async fn repo_proxy_download( /// /// `auth` is the configured proxy credential (Basic); the caller's own /// `Authorization` is never forwarded — pass only the intended headers in -/// `fwd_headers` (allowlist). The body is not inspected or decompressed here. +/// `fwd_headers` (allowlist). A 2xx or 4xx response is accepted only after its +/// body has been read completely within the strict 8 MiB npm-audit response +/// budget. A truncated or oversized body is an upstream failure, never an +/// empty successful response, and is recorded against the circuit breaker. +/// +/// `client` must have automatic redirects disabled. The initial URL and every +/// redirect target are policy-checked before a request or Authorization header +/// is built. 307/308 preserve the method, body and headers. 301/302/303 switch +/// permanently to a bodyless GET and retain only `Accept` plus configured auth. #[allow(clippy::too_many_arguments)] -pub(crate) async fn proxy_forward_post( +pub(crate) async fn proxy_forward_post( client: &reqwest::Client, url: &str, timeout: Duration, @@ -480,82 +840,170 @@ pub(crate) async fn proxy_forward_post( body: &[u8], cb: &CircuitBreakerRegistry, registry: RegistryType, -) -> Result<(u16, Vec, Option), ProxyError> { + max_redirects: usize, + redirect_allowed: F, +) -> Result<(u16, Vec, Option), ProxyError> +where + F: Fn(&reqwest::Url) -> bool, +{ + let initial_url = validate_initial_url(url, &redirect_allowed)?; let registry_str = registry.as_str(); let probe = cb.check(registry_str)?; for attempt in 0..2 { - let mut request = client.post(url).timeout(timeout).body(body.to_vec()); - if let Some(credentials) = auth { - request = request.header("Authorization", basic_auth_header(credentials)); - } - for (k, v) in fwd_headers { - request = request.header(*k, *v); - } + let mut current_url = initial_url.clone(); + let mut followed = 0; + let mut send_body = true; + + loop { + let mut request = if send_body { + client + .post(current_url.clone()) + .timeout(timeout) + .body(body.to_vec()) + } else { + client.get(current_url.clone()).timeout(timeout) + }; + if let Some(credentials) = auth { + request = request.header(header::AUTHORIZATION, basic_auth_header(credentials)); + } + for (name, value) in fwd_headers { + if send_body || name.eq_ignore_ascii_case("accept") { + request = request.header(*name, *value); + } + } - let upstream_start = Instant::now(); - match request.send().await { - Ok(response) => { - let elapsed = upstream_start.elapsed().as_secs_f64(); - let code = response.status().as_u16(); - let content_type = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - .map(str::to_owned); - if response.status().is_success() { - UPSTREAM_REQUEST_DURATION - .with_label_values(&[registry_str, "2xx"]) - .observe(elapsed); - match response.bytes().await { - Ok(b) => { - cb.record_success(registry_str, probe); - return Ok((code, b.to_vec(), content_type)); + let upstream_start = Instant::now(); + match request.send().await { + Ok(response) => { + let elapsed = upstream_start.elapsed().as_secs_f64(); + let status = response.status(); + let code = status.as_u16(); + if is_followable_redirect(status) { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "3xx"]) + .observe(elapsed); + let next_url = match validated_redirect_target( + ¤t_url, + &response, + followed, + max_redirects, + &redirect_allowed, + ) { + Ok(target) => target, + Err(error) => { + cb.record_alive(registry_str, probe); + tracing::warn!( + registry = registry_str, + status = code, + reason = error.reason(), + "upstream POST redirect rejected" + ); + return Err(ProxyError::Upstream(code)); + } + }; + if send_body + && matches!( + status, + reqwest::StatusCode::MOVED_PERMANENTLY + | reqwest::StatusCode::FOUND + | reqwest::StatusCode::SEE_OTHER + ) + { + send_body = false; } - Err(e) => { - cb.record_failure(registry_str, probe); - return Err(ProxyError::Network(e.to_string())); + followed += 1; + current_url = next_url; + continue; + } + if status.is_redirection() { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "3xx"]) + .observe(elapsed); + cb.record_alive(registry_str, probe); + tracing::warn!( + registry = registry_str, + status = code, + "unsupported upstream POST redirect status" + ); + return Err(ProxyError::Upstream(code)); + } + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + if status.is_success() { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "2xx"]) + .observe(elapsed); + match read_npm_audit_response_body(response).await { + Ok(response_body) => { + cb.record_success(registry_str, probe); + return Ok((code, response_body, content_type)); + } + Err(error) => { + cb.record_failure(registry_str, probe); + return Err(error); + } } } - } - if (400..500).contains(&code) { + if status.is_client_error() { + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, "4xx"]) + .observe(elapsed); + // A fully read 4xx audit response is a real answer and + // proves that the upstream is alive. A truncated or + // oversized body is instead an availability failure. + return match read_npm_audit_response_body(response).await { + Ok(response_body) => { + cb.record_alive(registry_str, probe); + Ok((code, response_body, content_type)) + } + Err(error) => { + cb.record_failure(registry_str, probe); + Err(error) + } + }; + } UPSTREAM_REQUEST_DURATION - .with_label_values(&[registry_str, "4xx"]) + .with_label_values(&[registry_str, "5xx"]) .observe(elapsed); - // Upstream is alive and answered — a 4xx audit response is a real - // answer, forward it verbatim (not an availability failure). #606. - cb.record_alive(registry_str, probe); - let b = response - .bytes() - .await - .map(|b| b.to_vec()) - .unwrap_or_default(); - return Ok((code, b, content_type)); - } - UPSTREAM_REQUEST_DURATION - .with_label_values(&[registry_str, "5xx"]) - .observe(elapsed); - if attempt == 0 { - tracing::debug!(url, status = code, "upstream 5xx on POST, retrying in 1s"); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; + if attempt == 0 { + tracing::debug!( + url = %current_url, + status = code, + "upstream 5xx on POST, retrying in 1s" + ); + tokio::time::sleep(Duration::from_secs(1)).await; + break; + } + cb.record_failure(registry_str, probe); + return Err(ProxyError::Upstream(code)); } - cb.record_failure(registry_str, probe); - return Err(ProxyError::Upstream(code)); - } - Err(e) => { - let elapsed = upstream_start.elapsed().as_secs_f64(); - let status_label = if e.is_timeout() { "timeout" } else { "error" }; - UPSTREAM_REQUEST_DURATION - .with_label_values(&[registry_str, status_label]) - .observe(elapsed); - if attempt == 0 { - tracing::debug!(url, error = %e, "upstream error on POST, retrying in 1s"); - tokio::time::sleep(Duration::from_secs(1)).await; - continue; + Err(error) => { + let elapsed = upstream_start.elapsed().as_secs_f64(); + let status_label = if error.is_timeout() { + "timeout" + } else { + "error" + }; + UPSTREAM_REQUEST_DURATION + .with_label_values(&[registry_str, status_label]) + .observe(elapsed); + if attempt == 0 { + tracing::debug!( + url = %current_url, + error = %error, + "upstream error on POST, retrying in 1s" + ); + tokio::time::sleep(Duration::from_secs(1)).await; + break; + } + cb.record_failure(registry_str, probe); + return Err(ProxyError::Network(error.to_string())); } - cb.record_failure(registry_str, probe); - return Err(ProxyError::Network(e.to_string())); } } } @@ -807,7 +1255,14 @@ pub(crate) async fn proxy_fetch_conditional( // 4xx — upstream alive; recover the breaker without clearing a // real failure tally, consistent with proxy_fetch_core (#606). cb.record_alive(registry_str, probe); - return Err(ProxyError::NotFound); + // Only an actual 404 is safe to feed a negative cache. Auth, + // throttling and policy failures must remain distinguishable + // from package absence. + return if status == reqwest::StatusCode::NOT_FOUND { + Err(ProxyError::NotFound) + } else { + Err(ProxyError::Upstream(code)) + }; } cb.record_failure(registry_str, probe); Err(ProxyError::Upstream(code)) @@ -819,6 +1274,138 @@ pub(crate) async fn proxy_fetch_conditional( } } +/// Conditional upstream fetch with explicit validation of the initial URL and +/// every redirect target. +/// +/// The client must have automatic redirects disabled. Conditional validators +/// are retained on every GET hop; the final response alone determines whether +/// the result is `NotModified` or captures a new body and validators. Like +/// [`proxy_fetch_conditional`], this helper never retries. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn proxy_fetch_conditional_with_validated_redirects( + client: &reqwest::Client, + url: &str, + timeout: Duration, + auth: Option<&str>, + validators: &Validators, + cb: &CircuitBreakerRegistry, + registry: RegistryType, + max_redirects: usize, + redirect_allowed: F, +) -> Result +where + F: Fn(&reqwest::Url) -> bool, +{ + let initial_url = validate_initial_url(url, &redirect_allowed)?; + let registry_str = registry.as_str(); + let probe = cb.check(registry_str)?; + let mut current_url = initial_url; + let mut followed = 0; + + loop { + let mut request = client.get(current_url.clone()).timeout(timeout); + if let Some(credentials) = auth { + request = request.header(header::AUTHORIZATION, basic_auth_header(credentials)); + } + if let Some(ref etag) = validators.etag { + request = request.header(header::IF_NONE_MATCH, etag); + } + if let Some(ref last_modified) = validators.last_modified { + request = request.header(header::IF_MODIFIED_SINCE, last_modified); + } + + match request.send().await { + Ok(response) => { + let status = response.status(); + if status == reqwest::StatusCode::NOT_MODIFIED { + cb.record_success(registry_str, probe); + return Ok(Revalidation::NotModified); + } + if status.is_success() { + let new_validators = Validators { + etag: header_string(&response, header::ETAG), + last_modified: header_string(&response, header::LAST_MODIFIED), + }; + let body = match response.bytes().await { + Ok(body) => body, + Err(error) => { + cb.record_failure(registry_str, probe); + return Err(ProxyError::Network(error.to_string())); + } + }; + cb.record_success(registry_str, probe); + return Ok(Revalidation::Modified { + body: body.to_vec(), + validators: new_validators, + }); + } + if is_followable_redirect(status) { + let next_url = match validated_redirect_target( + ¤t_url, + &response, + followed, + max_redirects, + &redirect_allowed, + ) { + Ok(target) => target, + Err(error) => { + cb.record_alive(registry_str, probe); + tracing::warn!( + registry = registry_str, + status = status.as_u16(), + reason = error.reason(), + "conditional upstream redirect rejected" + ); + return Err(ProxyError::Upstream(status.as_u16())); + } + }; + followed += 1; + current_url = next_url; + continue; + } + + let code = status.as_u16(); + if status.is_redirection() { + cb.record_alive(registry_str, probe); + tracing::warn!( + registry = registry_str, + status = code, + "unsupported conditional upstream redirect status" + ); + return Err(ProxyError::Upstream(code)); + } + if status.is_client_error() { + let policy_block = policy_block_reason(response.headers()); + if let Some(reason) = policy_block { + UPSTREAM_POLICY_BLOCKED_TOTAL + .with_label_values(&[registry_str, reason]) + .inc(); + tracing::warn!( + registry = registry_str, + url = %current_url, + status = code, + reason, + "conditional upstream returned a policy/geo block (not a genuine not-found) — check egress/region" + ); + } + cb.record_alive(registry_str, probe); + return if status == reqwest::StatusCode::NOT_FOUND && policy_block.is_none() { + Err(ProxyError::NotFound) + } else { + Err(ProxyError::Upstream(code)) + }; + } + cb.record_failure(registry_str, probe); + return Err(ProxyError::Upstream(code)); + } + Err(error) => { + cb.record_failure(registry_str, probe); + return Err(ProxyError::Network(error.to_string())); + } + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -841,31 +1428,76 @@ mod tests { assert!(matches!(result, Err(ProxyError::Network(_)))); } - // --- Policy/geo upstream block observability (#881) --- - - #[test] - fn policy_block_reason_detects_waf_geo() { - use reqwest::header::HeaderMap; - let mut geo = HeaderMap::new(); - geo.insert("x-amzn-waf-reason", "geo".parse().unwrap()); - assert_eq!(policy_block_reason(&geo), Some("geo")); - - let mut other = HeaderMap::new(); - other.insert("x-amzn-waf-reason", "rate-based".parse().unwrap()); - assert_eq!(policy_block_reason(&other), Some("waf")); - - // A genuine 4xx (no WAF signature) is not a policy block. - assert_eq!(policy_block_reason(&HeaderMap::new()), None); - } - - /// A geo-blocked upstream 4xx (WAF `x-amzn-waf-reason: geo`) bumps the policy-block - /// metric and is still relayed as `NotFound`; a plain 4xx does neither (#881). #[tokio::test] - async fn upstream_waf_geo_block_surfaced_but_plain_4xx_silent() { + async fn proxy_fetch_only_classifies_exact_404_as_not_found() { use wiremock::matchers::any; use wiremock::{Mock, MockServer, ResponseTemplate}; - // Geo-blocked upstream: 404 + x-amzn-waf-reason: geo. + let upstream = MockServer::start().await; + for status in [401_u16, 403, 429, 451] { + upstream.reset().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(status)) + .mount(&upstream) + .await; + + let result = proxy_fetch( + &reqwest::Client::new(), + &upstream.uri(), + Duration::from_secs(5), + None, + &noop_cb(), + RegistryType::Maven, + ) + .await; + assert!( + matches!(result, Err(ProxyError::Upstream(actual)) if actual == status), + "HTTP {status} must remain distinguishable from a missing artifact" + ); + } + + upstream.reset().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(404)) + .mount(&upstream) + .await; + let result = proxy_fetch( + &reqwest::Client::new(), + &upstream.uri(), + Duration::from_secs(5), + None, + &noop_cb(), + RegistryType::Maven, + ) + .await; + assert!(matches!(result, Err(ProxyError::NotFound))); + } + + // --- Policy/geo upstream block observability (#881) --- + + #[test] + fn policy_block_reason_detects_waf_geo() { + use reqwest::header::HeaderMap; + let mut geo = HeaderMap::new(); + geo.insert("x-amzn-waf-reason", "geo".parse().unwrap()); + assert_eq!(policy_block_reason(&geo), Some("geo")); + + let mut other = HeaderMap::new(); + other.insert("x-amzn-waf-reason", "rate-based".parse().unwrap()); + assert_eq!(policy_block_reason(&other), Some("waf")); + + // A genuine 4xx (no WAF signature) is not a policy block. + assert_eq!(policy_block_reason(&HeaderMap::new()), None); + } + + /// A geo-blocked upstream 4xx (WAF `x-amzn-waf-reason: geo`) bumps the policy-block + /// metric and remains distinguishable from `NotFound`; a plain 404 does neither (#881). + #[tokio::test] + async fn upstream_waf_geo_block_surfaced_but_plain_4xx_silent() { + use wiremock::matchers::any; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + // Geo-blocked upstream: 404 + x-amzn-waf-reason: geo. let blocked = MockServer::start().await; Mock::given(any()) .respond_with(ResponseTemplate::new(404).insert_header("x-amzn-waf-reason", "geo")) @@ -885,7 +1517,7 @@ mod tests { reg, ) .await; - assert!(matches!(r, Err(ProxyError::NotFound))); + assert!(matches!(r, Err(ProxyError::Upstream(404)))); assert_eq!( UPSTREAM_POLICY_BLOCKED_TOTAL .with_label_values(&[reg.as_str(), "geo"]) @@ -930,6 +1562,40 @@ mod tests { CircuitBreakerRegistry::new(crate::config::CircuitBreakerConfig::default()) } + fn one_failure_cb() -> CircuitBreakerRegistry { + CircuitBreakerRegistry::new(crate::config::CircuitBreakerConfig { + enabled: true, + failure_threshold: 1, + reset_timeout: 30, + overrides: std::collections::HashMap::new(), + }) + } + + fn assert_npm_breaker_open(cb: &CircuitBreakerRegistry) { + let health = cb + .health_snapshot(RegistryType::Npm.as_str()) + .expect("npm breaker must have recorded the response-body failure"); + assert_eq!(health.status, "open"); + assert_eq!(health.failure_count, 1); + assert!(matches!( + cb.check(RegistryType::Npm.as_str()), + Err(ProxyError::CircuitOpen(_)) + )); + } + + fn no_redirect_client() -> reqwest::Client { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap() + } + + fn same_origin(expected: &reqwest::Url, candidate: &reqwest::Url) -> bool { + expected.scheme() == candidate.scheme() + && expected.host_str() == candidate.host_str() + && expected.port_or_known_default() == candidate.port_or_known_default() + } + /// With no stored validators, the conditional fetch sends no `If-None-Match`, /// always gets a 200, and captures the upstream validators (this is also the /// full-fetch path that seeds the sidecar for next time). @@ -1005,6 +1671,772 @@ mod tests { assert!(matches!(out, Revalidation::NotModified)); } + #[tokio::test] + async fn conditional_non_404_client_error_is_not_negative_cacheable() { + use wiremock::matchers::any; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(401)) + .mount(&upstream) + .await; + + let result = proxy_fetch_conditional( + &reqwest::Client::new(), + &upstream.uri(), + Duration::from_secs(5), + None, + &Validators::default(), + &noop_cb(), + RegistryType::Npm, + ) + .await; + assert!(matches!(result, Err(ProxyError::Upstream(401)))); + } + + #[tokio::test] + async fn validated_fetch_rejects_initial_url_before_any_request() { + use wiremock::MockServer; + + let upstream = MockServer::start().await; + let result = proxy_fetch_with_validated_redirects( + &no_redirect_client(), + &format!("{}/artifact", upstream.uri()), + Duration::from_secs(5), + Some("user:password"), + &noop_cb(), + RegistryType::Npm, + 3, + |_| false, + ) + .await; + + assert!(matches!(result, Err(ProxyError::Network(_)))); + assert!( + upstream + .received_requests() + .await + .unwrap_or_default() + .is_empty(), + "a rejected initial URL must not receive a request or credentials" + ); + } + + #[tokio::test] + async fn validated_fetch_resolves_relative_location_and_rejects_invalid_location() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/repo/start")) + .respond_with(ResponseTemplate::new(302).insert_header("location", "artifact")) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/repo/artifact")) + .respond_with(ResponseTemplate::new(200).set_body_string("artifact-body")) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/invalid")) + .respond_with(ResponseTemplate::new(302).insert_header("location", "http://[")) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/missing")) + .respond_with(ResponseTemplate::new(302)) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let fetched = proxy_fetch_with_validated_redirects( + &no_redirect_client(), + &format!("{}/repo/start", upstream.uri()), + Duration::from_secs(5), + None, + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await + .unwrap(); + assert_eq!(fetched, b"artifact-body"); + + let invalid = proxy_fetch_with_validated_redirects( + &no_redirect_client(), + &format!("{}/invalid", upstream.uri()), + Duration::from_secs(5), + None, + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + assert!(matches!(invalid, Err(ProxyError::Upstream(302)))); + + let missing = proxy_fetch_with_validated_redirects( + &no_redirect_client(), + &format!("{}/missing", upstream.uri()), + Duration::from_secs(5), + None, + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + assert!(matches!(missing, Err(ProxyError::Upstream(302)))); + } + + #[tokio::test] + async fn validated_fetch_only_classifies_plain_404_as_not_found() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + for (request_path, status) in [("/unauthorized", 401), ("/throttled", 429)] { + Mock::given(method("GET")) + .and(path(request_path)) + .respond_with(ResponseTemplate::new(status)) + .expect(1) + .mount(&upstream) + .await; + } + Mock::given(method("GET")) + .and(path("/blocked")) + .respond_with( + ResponseTemplate::new(404).insert_header("x-amzn-waf-reason", "rate-based"), + ) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/missing")) + .respond_with(ResponseTemplate::new(404)) + .expect(1) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + for (request_path, expected) in [ + ("/unauthorized", ProxyError::Upstream(401)), + ("/throttled", ProxyError::Upstream(429)), + ("/blocked", ProxyError::Upstream(404)), + ("/missing", ProxyError::NotFound), + ] { + let result = proxy_fetch_with_validated_redirects( + &no_redirect_client(), + &format!("{}{request_path}", upstream.uri()), + Duration::from_secs(5), + None, + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + assert!( + matches!( + (&result, &expected), + (Err(ProxyError::Upstream(actual)), ProxyError::Upstream(wanted)) + if actual == wanted + ) || matches!( + (&result, &expected), + (Err(ProxyError::NotFound), ProxyError::NotFound) + ), + "unexpected classification for {request_path}" + ); + } + upstream.verify().await; + } + + #[tokio::test] + async fn validated_bounded_fetch_rejects_oversized_response_body() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/search")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"oversized")) + .expect(1) + .mount(&upstream) + .await; + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let result = proxy_fetch_with_validated_redirects_bounded( + &no_redirect_client(), + &format!("{}/search", upstream.uri()), + Duration::from_secs(5), + None, + &noop_cb(), + RegistryType::Npm, + 3, + 4, + |candidate| same_origin(&origin, candidate), + ) + .await; + assert!(matches!(result, Err(ProxyError::Network(_)))); + upstream.verify().await; + } + + #[tokio::test] + async fn conditional_redirect_preserves_validators_and_final_304() { + use wiremock::matchers::{header, header_exists, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + for request_path in ["/metadata", "/revalidated"] { + let response = if request_path == "/metadata" { + ResponseTemplate::new(302).insert_header("location", "revalidated") + } else { + ResponseTemplate::new(304) + }; + Mock::given(method("GET")) + .and(path(request_path)) + .and(header("if-none-match", "\"v1\"")) + .and(header_exists("if-modified-since")) + .and(header("authorization", basic_auth_header("user:password"))) + .respond_with(response) + .mount(&upstream) + .await; + } + + let validators = Validators { + etag: Some("\"v1\"".to_string()), + last_modified: Some("Wed, 21 Oct 2026 07:28:00 GMT".to_string()), + }; + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let result = proxy_fetch_conditional_with_validated_redirects( + &no_redirect_client(), + &format!("{}/metadata", upstream.uri()), + Duration::from_secs(5), + Some("user:password"), + &validators, + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await + .unwrap(); + + assert!(matches!(result, Revalidation::NotModified)); + let requests = upstream.received_requests().await.unwrap_or_default(); + assert_eq!( + requests.len(), + 2, + "conditional redirects must not trigger a retry" + ); + assert!(requests.iter().all(|request| { + request + .headers + .get("if-modified-since") + .and_then(|value| value.to_str().ok()) + == Some("Wed, 21 Oct 2026 07:28:00 GMT") + })); + } + + #[tokio::test] + async fn conditional_validated_fetch_only_classifies_plain_404_as_not_found() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + for (request_path, status) in [("/unauthorized", 401), ("/throttled", 429)] { + Mock::given(method("GET")) + .and(path(request_path)) + .respond_with(ResponseTemplate::new(status)) + .expect(1) + .mount(&upstream) + .await; + } + Mock::given(method("GET")) + .and(path("/blocked")) + .respond_with( + ResponseTemplate::new(404).insert_header("x-amzn-waf-reason", "rate-based"), + ) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/missing")) + .respond_with(ResponseTemplate::new(404)) + .expect(1) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + for (request_path, expected) in [ + ("/unauthorized", ProxyError::Upstream(401)), + ("/throttled", ProxyError::Upstream(429)), + ("/blocked", ProxyError::Upstream(404)), + ("/missing", ProxyError::NotFound), + ] { + let result = proxy_fetch_conditional_with_validated_redirects( + &no_redirect_client(), + &format!("{}{request_path}", upstream.uri()), + Duration::from_secs(5), + None, + &Validators::default(), + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + assert!( + matches!( + (&result, &expected), + (Err(ProxyError::Upstream(actual)), ProxyError::Upstream(wanted)) + if actual == wanted + ) || matches!( + (&result, &expected), + (Err(ProxyError::NotFound), ProxyError::NotFound) + ), + "unexpected classification for {request_path}" + ); + } + upstream.verify().await; + } + + #[tokio::test] + async fn conditional_validated_fetch_does_not_retry_5xx() { + use wiremock::matchers::any; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + Mock::given(any()) + .respond_with(ResponseTemplate::new(503)) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let result = proxy_fetch_conditional_with_validated_redirects( + &no_redirect_client(), + &format!("{}/metadata", upstream.uri()), + Duration::from_secs(5), + None, + &Validators::default(), + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + + assert!(matches!(result, Err(ProxyError::Upstream(503)))); + assert_eq!( + upstream.received_requests().await.unwrap_or_default().len(), + 1, + "conditional fetches must not retry upstream failures" + ); + } + + #[tokio::test] + async fn post_forwards_fully_read_4xx_body_and_content_type_exactly() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + let expected = vec![0, 1, 2, b'{', b'}', 0xff]; + Mock::given(method("POST")) + .and(path("/audit")) + .respond_with( + ResponseTemplate::new(422) + .insert_header("content-type", "application/problem+json") + .set_body_bytes(expected.clone()), + ) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let result = proxy_forward_post( + &no_redirect_client(), + &format!("{}/audit", upstream.uri()), + Duration::from_secs(5), + None, + &[("content-type", "application/json")], + b"{}", + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await + .unwrap(); + + assert_eq!( + result, + (422, expected, Some("application/problem+json".into())) + ); + } + + #[tokio::test] + async fn post_rejects_oversized_2xx_body_and_opens_breaker() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/audit")) + .respond_with( + ResponseTemplate::new(200) + .insert_header("transfer-encoding", "chunked") + .set_body_bytes(vec![b'x'; NPM_AUDIT_RESPONSE_BODY_CAP + 1]), + ) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let cb = one_failure_cb(); + let result = proxy_forward_post( + &no_redirect_client(), + &format!("{}/audit", upstream.uri()), + Duration::from_secs(5), + None, + &[("content-type", "application/json")], + b"{}", + &cb, + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + + assert!( + matches!(result, Err(ProxyError::Network(message)) if message.contains("exceeds")), + "an oversized 2xx must fail specifically at the response-body cap" + ); + assert_npm_breaker_open(&cb); + } + + #[tokio::test] + async fn post_rejects_oversized_4xx_body_and_opens_breaker() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/audit")) + .respond_with( + ResponseTemplate::new(400) + .insert_header("transfer-encoding", "chunked") + .set_body_bytes(vec![b'x'; NPM_AUDIT_RESPONSE_BODY_CAP + 1]), + ) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let cb = one_failure_cb(); + let result = proxy_forward_post( + &no_redirect_client(), + &format!("{}/audit", upstream.uri()), + Duration::from_secs(5), + None, + &[("content-type", "application/json")], + b"{}", + &cb, + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + + assert!( + matches!(result, Err(ProxyError::Network(message)) if message.contains("exceeds")), + "an oversized 4xx must fail specifically at the response-body cap" + ); + assert_npm_breaker_open(&cb); + } + + #[tokio::test] + async fn post_rejects_truncated_4xx_body_and_opens_breaker() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0_u8; 4096]; + let _ = socket.read(&mut request).await.unwrap(); + socket + .write_all( + b"HTTP/1.1 400 Bad Request\r\n\ + Content-Type: application/json\r\n\ + Content-Length: 64\r\n\ + Connection: close\r\n\ + \r\n\ + {\"partial\":true}", + ) + .await + .unwrap(); + socket.shutdown().await.unwrap(); + }); + + let url = format!("http://{address}/audit"); + let origin = reqwest::Url::parse(&url).unwrap(); + let cb = one_failure_cb(); + let result = proxy_forward_post( + &no_redirect_client(), + &url, + Duration::from_secs(5), + None, + &[("content-type", "application/json")], + b"{}", + &cb, + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + server.await.unwrap(); + + assert!(matches!(result, Err(ProxyError::Network(_)))); + assert_npm_breaker_open(&cb); + } + + #[tokio::test] + async fn post_307_and_308_preserve_body_and_payload_headers() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + for status in [307, 308] { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/audit")) + .respond_with( + ResponseTemplate::new(status).insert_header("location", "/audit/final"), + ) + .mount(&upstream) + .await; + Mock::given(method("POST")) + .and(path("/audit/final")) + .respond_with(ResponseTemplate::new(200).set_body_string("audit-result")) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let result = proxy_forward_post( + &no_redirect_client(), + &format!("{}/audit", upstream.uri()), + Duration::from_secs(5), + Some("user:password"), + &[ + ("content-type", "application/json"), + ("content-encoding", "gzip"), + ("accept", "application/json"), + ], + b"audit-body", + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await + .unwrap(); + assert_eq!(result.0, 200); + assert_eq!(result.1, b"audit-result"); + + let requests = upstream.received_requests().await.unwrap_or_default(); + let final_request = requests + .iter() + .find(|request| request.url.path() == "/audit/final") + .expect("redirect target received request"); + assert_eq!(final_request.method.as_str(), "POST"); + assert_eq!(final_request.body, b"audit-body"); + assert_eq!( + final_request.headers["content-type"], "application/json", + "{status} must preserve Content-Type" + ); + assert_eq!( + final_request.headers["content-encoding"], "gzip", + "{status} must preserve Content-Encoding" + ); + assert_eq!(final_request.headers["accept"], "application/json"); + assert_eq!( + final_request.headers["authorization"], + basic_auth_header("user:password") + ); + } + } + + #[tokio::test] + async fn post_301_302_303_switch_permanently_to_bodyless_get() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + for status in [301, 302, 303] { + let upstream = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/audit")) + .respond_with( + ResponseTemplate::new(status).insert_header("location", "/audit/get-hop"), + ) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/audit/get-hop")) + .respond_with(ResponseTemplate::new(307).insert_header("location", "/audit/final")) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/audit/final")) + .respond_with(ResponseTemplate::new(200).set_body_string("audit-result")) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let result = proxy_forward_post( + &no_redirect_client(), + &format!("{}/audit", upstream.uri()), + Duration::from_secs(5), + Some("user:password"), + &[ + ("content-type", "application/json"), + ("content-encoding", "gzip"), + ("accept", "application/json"), + ], + b"audit-body", + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await + .unwrap(); + assert_eq!(result.1, b"audit-result"); + + let requests = upstream.received_requests().await.unwrap_or_default(); + for request_path in ["/audit/get-hop", "/audit/final"] { + let request = requests + .iter() + .find(|request| request.url.path() == request_path) + .expect("redirect target received request"); + assert_eq!( + request.method.as_str(), + "GET", + "{status} must switch POST to GET permanently" + ); + assert!( + request.body.is_empty(), + "{status} redirected GET must have no body" + ); + assert!( + !request.headers.contains_key("content-type"), + "{status} redirected GET must drop Content-Type" + ); + assert!( + !request.headers.contains_key("content-encoding"), + "{status} redirected GET must drop Content-Encoding" + ); + assert_eq!(request.headers["accept"], "application/json"); + assert_eq!( + request.headers["authorization"], + basic_auth_header("user:password") + ); + } + } + } + + #[tokio::test] + async fn post_redirect_rejects_disallowed_target_before_sending_auth_or_body() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + let disallowed = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/audit")) + .respond_with( + ResponseTemplate::new(307) + .insert_header("location", format!("{}/capture", disallowed.uri())), + ) + .mount(&upstream) + .await; + Mock::given(method("POST")) + .and(path("/capture")) + .respond_with(ResponseTemplate::new(200)) + .mount(&disallowed) + .await; + + let allowed_origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let result = proxy_forward_post( + &no_redirect_client(), + &format!("{}/audit", upstream.uri()), + Duration::from_secs(5), + Some("user:password"), + &[("content-type", "application/json")], + b"sensitive-audit-body", + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&allowed_origin, candidate), + ) + .await; + + assert!(matches!(result, Err(ProxyError::Upstream(307)))); + assert!( + disallowed + .received_requests() + .await + .unwrap_or_default() + .is_empty(), + "a disallowed target must receive neither configured auth nor the POST body" + ); + } + + #[tokio::test] + async fn post_redirect_rejects_fourth_hop_without_sending_fifth_request() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + for hop in 0..=3 { + Mock::given(method("POST")) + .and(path(format!("/hop/{hop}"))) + .respond_with( + ResponseTemplate::new(307) + .insert_header("location", format!("/hop/{}", hop + 1)), + ) + .mount(&upstream) + .await; + } + Mock::given(method("POST")) + .and(path("/hop/4")) + .respond_with(ResponseTemplate::new(200)) + .mount(&upstream) + .await; + + let origin = reqwest::Url::parse(&upstream.uri()).unwrap(); + let result = proxy_forward_post( + &no_redirect_client(), + &format!("{}/hop/0", upstream.uri()), + Duration::from_secs(5), + None, + &[], + b"audit-body", + &noop_cb(), + RegistryType::Npm, + 3, + |candidate| same_origin(&origin, candidate), + ) + .await; + + assert!(matches!(result, Err(ProxyError::Upstream(307)))); + let requests = upstream.received_requests().await.unwrap_or_default(); + assert_eq!(requests.len(), 4, "only the initial request plus 3 hops"); + assert!( + requests + .iter() + .all(|request| request.url.path() != "/hop/4"), + "the fourth redirect target must be rejected before request construction" + ); + } + /// Validators round-trip through storage (the sidecar lives on disk, so they /// survive a restart) — acceptance criterion for #596. #[tokio::test] diff --git a/nora-registry/src/registry/named.rs b/nora-registry/src/registry/named.rs new file mode 100644 index 00000000..7c127a08 --- /dev/null +++ b/nora-registry/src/registry/named.rs @@ -0,0 +1,271 @@ +// Copyright (c) 2026 The NORA Authors +// SPDX-License-Identifier: MIT + +//! Shared Nexus-compatible `/repository/{repository}/...` dispatcher. +//! +//! Maven and npm intentionally use the same public repository namespace. +//! Mounting one wildcard route per protocol makes Axum's routers conflict and +//! leaves the format ambiguous. Configuration validation guarantees global +//! name uniqueness; this dispatcher resolves the concrete name and invokes the +//! matching protocol handler without guessing from naming conventions. + +use super::method_not_allowed; +use crate::auth::{AuthenticatedUser, NamespaceAuthority}; +use crate::AppState; +use axum::{ + body::{Body, Bytes}, + extract::{OriginalUri, Path, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::get, + Extension, Router, +}; + +pub fn routes() -> Router { + Router::new().route( + "/repository/{repository}/{*path}", + get(download) + .put(upload) + .post(post) + .fallback(|| async { method_not_allowed("GET, PUT, POST") }), + ) +} + +async fn download( + State(state): State, + Path((repository, path)): Path<(String, String)>, + OriginalUri(uri): OriginalUri, + headers: HeaderMap, + Extension(user): Extension, +) -> Response { + if state.config.maven.enabled && state.config.maven.repository(&repository).is_some() { + return super::maven::download_named(State(state), headers, Path((repository, path))).await; + } + if state.config.npm.enabled && state.config.npm.repository(&repository).is_some() { + return super::npm::named_get_request(state, repository, path, uri, headers, user).await; + } + StatusCode::NOT_FOUND.into_response() +} + +async fn upload( + State(state): State, + Path((repository, path)): Path<(String, String)>, + Extension(authority): Extension, + body: Bytes, +) -> Response { + if state.config.maven.enabled && state.config.maven.repository(&repository).is_some() { + return super::maven::upload_named( + State(state), + Path((repository, path)), + Extension(authority), + body, + ) + .await; + } + if state.config.npm.enabled && state.config.npm.repository(&repository).is_some() { + return super::npm::named_put_request(state, repository, path, authority, body).await; + } + StatusCode::NOT_FOUND.into_response() +} + +async fn post( + State(state): State, + Path((repository, path)): Path<(String, String)>, + headers: HeaderMap, + body: Body, +) -> Response { + if state.config.maven.enabled && state.config.maven.repository(&repository).is_some() { + return method_not_allowed("GET, PUT"); + } + if state.config.npm.enabled && state.config.npm.repository(&repository).is_some() { + return super::npm::named_post_request(state, repository, path, headers, body).await; + } + StatusCode::NOT_FOUND.into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::{ + MavenRepository, MavenVersionPolicy, MavenWritePolicy, NpmRepository, NpmWritePolicy, + }; + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + + fn combined_named_config(config: &mut crate::config::Config) { + config.maven.repositories = vec![MavenRepository::Hosted { + name: "maven-releases".to_string(), + version_policy: MavenVersionPolicy::Release, + write_policy: MavenWritePolicy::AllowOnce, + }]; + config.maven.default_repository = Some("maven-releases".to_string()); + config.npm.repositories = vec![NpmRepository::Hosted { + name: "npm-private".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }]; + config.npm.default_repository = Some("npm-private".to_string()); + } + + #[tokio::test] + async fn shared_dispatcher_reaches_maven_and_npm_without_route_conflict() { + // Constructing the combined test router is itself a regression for the + // Axum wildcard conflict that separate protocol routers caused. + let context = create_test_context_with_config(combined_named_config); + + let maven_path = "/repository/maven-releases/com/example/app/1.0/app-1.0.jar"; + let uploaded = send( + &context.app, + Method::PUT, + maven_path, + b"maven-bytes".as_slice(), + ) + .await; + assert_eq!(uploaded.status(), StatusCode::CREATED); + let downloaded = send(&context.app, Method::GET, maven_path, "").await; + assert_eq!(downloaded.status(), StatusCode::OK); + assert_eq!(body_bytes(downloaded).await.as_ref(), b"maven-bytes"); + + let manifest = br#"{"name":"pkg","version":"1.0.0","dist":{}}"#; + context + .state + .storage + .put( + "npm/repositories/npm-private/pkg/versions/1.0.0.json", + manifest, + ) + .await + .unwrap(); + let completion = crate::npm_layout::hosted_manifest_digest(manifest); + context + .state + .storage + .put( + "npm/repositories/npm-private/pkg/publish-complete/1.0.0", + completion.as_bytes(), + ) + .await + .unwrap(); + let npm_path = "/repository/npm-private/pkg"; + let deprecated = send( + &context.app, + Method::PUT, + npm_path, + br#"{"name":"pkg","versions":{"1.0.0":{"deprecated":"old"}}}"#.as_slice(), + ) + .await; + assert_eq!(deprecated.status(), StatusCode::CREATED); + assert_eq!( + context + .state + .storage + .get("npm/repositories/npm-private/pkg/deprecations/1.0.0") + .await + .unwrap() + .as_ref(), + b"old" + ); + let npm = send(&context.app, Method::GET, npm_path, "").await; + assert_eq!(npm.status(), StatusCode::OK); + let npm_json: serde_json::Value = serde_json::from_slice(&body_bytes(npm).await).unwrap(); + assert_eq!(npm_json["versions"]["1.0.0"]["name"], "pkg"); + + context + .state + .storage + .put("npm/repositories/npm-private/pkg/dist-tags/next", b"1.0.0") + .await + .unwrap(); + let deleted = send( + &context.app, + Method::DELETE, + "/repository/npm-private/-/package/pkg/dist-tags/next", + "", + ) + .await; + assert_eq!(deleted.status(), StatusCode::NO_CONTENT); + assert!(context + .state + .storage + .get("npm/repositories/npm-private/pkg/dist-tags/next") + .await + .is_err()); + + assert_eq!( + send(&context.app, Method::DELETE, maven_path, "") + .await + .status(), + StatusCode::METHOD_NOT_ALLOWED + ); + + let unknown = send( + &context.app, + Method::GET, + "/repository/no-such-repository/pkg", + "", + ) + .await; + assert_eq!(unknown.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn shared_dispatcher_blocks_configured_npm_when_protocol_is_disabled() { + let context = create_test_context_with_config(|config| { + combined_named_config(config); + config.npm.enabled = false; + }); + + let maven_path = "/repository/maven-releases/com/example/app/1.0/app-1.0.jar"; + assert_eq!( + send( + &context.app, + Method::PUT, + maven_path, + b"maven-bytes".as_slice(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send(&context.app, Method::GET, "/repository/npm-private/pkg", "",) + .await + .status(), + StatusCode::NOT_FOUND + ); + } + + #[tokio::test] + async fn shared_dispatcher_blocks_configured_maven_when_protocol_is_disabled() { + let context = create_test_context_with_config(|config| { + combined_named_config(config); + config.maven.enabled = false; + }); + context + .state + .storage + .put( + "npm/repositories/npm-private/pkg/versions/1.0.0.json", + br#"{"name":"pkg","version":"1.0.0","dist":{}}"#, + ) + .await + .unwrap(); + + assert_eq!( + send(&context.app, Method::GET, "/repository/npm-private/pkg", "",) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + send( + &context.app, + Method::GET, + "/repository/maven-releases/com/example/app/1.0/app-1.0.jar", + "", + ) + .await + .status(), + StatusCode::NOT_FOUND + ); + } +} diff --git a/nora-registry/src/registry/npm.rs b/nora-registry/src/registry/npm.rs index 595bf1fd..053eb854 100644 --- a/nora-registry/src/registry/npm.rs +++ b/nora-registry/src/registry/npm.rs @@ -1,2626 +1,6958 @@ // Copyright (c) 2026 The NORA Authors // SPDX-License-Identifier: MIT +//! npm registry with explicit hosted, proxy and group repositories. +//! +//! Hosted and proxy state deliberately live in different namespaces. A group +//! owns no package metadata: its packument is synthesized for every request +//! using member order as the conflict-resolution rule. + use crate::activity_log::{ActionType, ActivityEntry}; use crate::audit::AuditEntry; use crate::auth::{enforce_namespace_scope, AuthenticatedUser, NamespaceAuthority}; -use crate::metrics::METADATA_CORRUPT_TOTAL; +use crate::config::{NpmRepository, NpmWritePolicy}; use crate::registry::{ - circuit_open_response, method_not_allowed, nora_base_url, proxy_fetch, proxy_fetch_conditional, + circuit_open_response, method_not_allowed, proxy_fetch_conditional_with_validated_redirects, + proxy_fetch_with_validated_redirects, proxy_fetch_with_validated_redirects_bounded, proxy_forward_post, read_validators, write_validators, ProxyError, Revalidation, Validators, }; use crate::registry_type::RegistryType; use crate::secrets::expose_opt; +use crate::storage::StorageError; use crate::AppState; use axum::{ - body::Bytes, - extract::{Path, State}, - http::{header, HeaderMap, HeaderValue, StatusCode}, + body::{Body, Bytes}, + extract::{OriginalUri, Path, State}, + http::{header, HeaderMap, HeaderValue, StatusCode, Uri}, response::{IntoResponse, Response}, routing::get, Extension, Router, }; use base64::Engine; +use futures::{stream, StreamExt}; use sha2::Digest; -use std::sync::Arc; -use std::time::Duration; +use std::collections::HashSet; +use std::io::{Read, Write}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +const NPM_AUDIT_BODY_CAP: usize = 8 * 1024 * 1024; +const NPM_SEARCH_BODY_CAP: usize = 8 * 1024 * 1024; +const NPM_SEARCH_SCAN_RESULT_CAP: usize = 10_000; +const NPM_SEARCH_SCAN_PAGE_CAP: usize = 40; +const NPM_SEARCH_SCAN_TIMEOUT: Duration = Duration::from_secs(30); +const PACKAGE_JSON_CAP: u64 = 2 * 1024 * 1024; +const TAR_SCAN_CAP: u64 = 64 * 1024 * 1024; +const MAX_NPM_PROXY_REDIRECTS: usize = 3; +const HOSTED_PACKUMENT_READ_CONCURRENCY: usize = 8; +const LEGACY_HOSTED: &str = "npm-private"; +const LEGACY_PROXY: &str = "npm-registry"; pub fn routes() -> Router { - Router::new().route( - "/npm/{*path}", - get(handle_request) - .put(handle_publish) - .post(handle_npm_post) - .fallback(|| async { method_not_allowed("GET, PUT, POST") }), - ) + Router::new() + .route( + "/repository/{repository}/-/package/{package}/dist-tags", + get(named_dist_tags_get).fallback(|| async { method_not_allowed("GET") }), + ) + .route( + "/repository/{repository}/-/package/{package}/dist-tags/{tag}", + axum::routing::put(named_dist_tag_put) + .delete(named_dist_tag_delete) + .fallback(|| async { method_not_allowed("PUT, DELETE") }), + ) + // Compatibility alias. It resolves to npm.default_repository when one + // is configured. Otherwise it uses isolated synthetic hosted/proxy + // namespaces; it never reads the pre-named `npm//metadata.json` + // layout. + .route( + "/npm/-/package/{package}/dist-tags", + get(alias_dist_tags_get).fallback(|| async { method_not_allowed("GET") }), + ) + .route( + "/npm/-/package/{package}/dist-tags/{tag}", + axum::routing::put(alias_dist_tag_put) + .delete(alias_dist_tag_delete) + .fallback(|| async { method_not_allowed("PUT, DELETE") }), + ) + .route( + "/npm/{*path}", + get(alias_get) + .put(alias_put) + .post(alias_post) + .fallback(|| async { method_not_allowed("GET, PUT, POST") }), + ) } -/// Max body accepted on the `npm audit` POST. Real audit payloads are well under -/// 1 MB; the global `body_limit_mb` (default 2 GB, for tarball PUTs) must not let a -/// client shove a huge body through NORA into the upstream (#597 SEC). -const NPM_AUDIT_BODY_CAP: usize = 8 * 1024 * 1024; +#[derive(Clone)] +enum RepositoryTarget { + Named(NpmRepository), + Legacy, +} -/// `npm audit` proxy (#597). npm sends its audit request as a POST — npm7 to -/// `/-/npm/v1/security/advisories/bulk` (body `{"":["",…]}`), npm6 to -/// `/-/npm/v1/security/audits/quick` (body = a possibly-gzipped lockfile). -/// -/// NORA keeps no advisory DB, so for a remote/proxy repo it forwards the request to -/// the configured upstream and returns the response verbatim. Any other POST path -/// is a 405. Dependency-confusion guard (#68/#733): internal-namespace package -/// names must not leak upstream — see `strip_internal_bulk` / the quick refuse. -async fn handle_npm_post( - State(state): State, - Path(path): Path, - headers: HeaderMap, - body: axum::body::Body, -) -> Response { - let is_bulk = path == "-/npm/v1/security/advisories/bulk"; - let is_quick = path == "-/npm/v1/security/audits/quick"; - if !is_bulk && !is_quick { - // POST is only meaningful on the audit endpoints. - return method_not_allowed("GET, PUT"); - } +#[derive(Clone)] +struct ProxyRepository { + name: String, + url: String, + auth: Option, + metadata_ttl: i64, + negative_ttl: i64, +} - // Bound the body AT READ TIME. The global DefaultBodyLimit is 2 GB (for tarball - // PUTs); audit bodies are <1 MB. `to_bytes` aborts collection past the cap, so a - // chunked / no-Content-Length client cannot force NORA to buffer gigabytes (#597). - let body = match axum::body::to_bytes(body, NPM_AUDIT_BODY_CAP).await { - Ok(b) => b, - Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), - }; +#[derive(Debug)] +enum ReadError { + NotFound, + Unavailable, + CircuitOpen(String), + Corrupt, + SearchScanLimit, +} - // Only a remote/proxy repo can answer audits (advisories come from upstream). - // Hosted-only (no proxy) → npm-compatible empty result so `npm audit` doesn't - // hard-fail — npm treats a 200 `{}` as "no advisories". - let Some(proxy_url) = state.config.npm.proxy.clone() else { - return npm_empty_audit(); - }; +struct PackumentRead { + value: serde_json::Value, + stale: bool, +} - // #68/#733 dependency-confusion: never send internal package names upstream. - // When a namespace filter is configured we must SEE plaintext names to strip - // them, so ANY body we cannot verify — the quick lockfile, an encoded bulk body, - // or a bulk body that is not the expected JSON object — is refused (fail CLOSED, - // symmetric across both paths). With no filter, nothing is internal → forward - // verbatim. - let engine = &state.curation().curation_engine; - let filter_active = crate::curation::namespace_filter_active(engine); - let content_encoded = headers - .get(header::CONTENT_ENCODING) - .and_then(|v| v.to_str().ok()) - .is_some_and(|s| !s.eq_ignore_ascii_case("identity")); - let forward_body: Vec = if !filter_active { - body.to_vec() - } else if is_quick || content_encoded { - return npm_empty_audit(); - } else { - match strip_internal_bulk(&body, engine) { - Some(stripped) => stripped, - None => return npm_empty_audit(), // unparsable under a filter → refuse +impl PackumentRead { + fn fresh(value: serde_json::Value) -> Self { + Self { + value, + stale: false, } - }; + } +} - // Allowlist the forwarded headers; carry the configured proxy credential only — - // NEVER the client's Authorization (it's the caller's NORA token). - let mut fwd: Vec<(&str, &str)> = Vec::new(); - if let Some(v) = headers - .get(header::CONTENT_TYPE) - .and_then(|v| v.to_str().ok()) - { - fwd.push(("content-type", v)); +fn storage_read_error(error: StorageError) -> ReadError { + match error { + StorageError::NotFound => ReadError::NotFound, + StorageError::IntegrityViolation => ReadError::Corrupt, + _ => ReadError::Unavailable, } - if let Some(v) = headers - .get(header::CONTENT_ENCODING) - .and_then(|v| v.to_str().ok()) - { - fwd.push(("content-encoding", v)); +} + +async fn optional_storage_get(state: &AppState, key: &str) -> Result, ReadError> { + match state.storage.get(key).await { + Ok(bytes) => Ok(Some(bytes)), + Err(StorageError::NotFound) => Ok(None), + Err(error) => Err(storage_read_error(error)), } - if let Some(v) = headers.get(header::ACCEPT).and_then(|v| v.to_str().ok()) { - fwd.push(("accept", v)); +} + +fn named_target(state: &AppState, repository: &str) -> Option { + state + .config + .npm + .repository(repository) + .cloned() + .map(RepositoryTarget::Named) +} + +fn alias_target(state: &AppState) -> Option { + match state.config.npm.default_repository.as_deref() { + Some(name) => named_target(state, name), + None if !state.config.npm.repositories.is_empty() => None, + None => Some(RepositoryTarget::Legacy), } +} - let url = format!("{}/{}", proxy_url.trim_end_matches('/'), path); - match proxy_forward_post( - &state.http_client, - &url, - Duration::from_secs(state.config.npm.proxy_timeout), - expose_opt(&state.config.npm.proxy_auth), - &fwd, - &forward_body, - &state.circuit_breaker, - RegistryType::Npm, - ) - .await - { - Ok((code, resp_body, resp_ct)) => { - state - .audit - .log(AuditEntry::new("proxy_fetch", "api", "", "npm", "audit")); - let status = StatusCode::from_u16(code).unwrap_or(StatusCode::BAD_GATEWAY); - let ct = resp_ct - .as_deref() - .and_then(|v| HeaderValue::from_str(v).ok()) - .unwrap_or_else(|| HeaderValue::from_static("application/json")); - (status, [(header::CONTENT_TYPE, ct)], resp_body).into_response() - } - Err(ProxyError::CircuitOpen(reg)) => circuit_open_response(®), - // Upstream failure must NOT be masked as `200 {}` — that renders in npm as - // "0 vulnerabilities" (a false-clean security signal). Surface 502 so npm - // reports an audit-endpoint error instead (non-fatal to the install), and - // log it so the failure is observable. - Err(e) => { - tracing::warn!(error = ?e, "npm audit upstream forward failed"); - StatusCode::BAD_GATEWAY.into_response() - } +fn public_base(state: &AppState, route_repository: Option<&str>) -> String { + let server = state.config.server.public_base_url(); + match route_repository { + Some(name) => format!("{}/repository/{name}", server.trim_end_matches('/')), + None => format!("{}/npm", server.trim_end_matches('/')), } } -/// npm-compatible "no advisories" response — a `200 {}` that `npm audit` accepts -/// without erroring. -fn npm_empty_audit() -> Response { - ( - StatusCode::OK, - [( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), - )], - b"{}".to_vec(), - ) - .into_response() +fn repository_prefix(repository: &str) -> String { + format!("npm/repositories/{repository}") } -/// Strip internal-namespace package keys from an npm7 bulk-advisories body -/// (`{"":[…]}`). `Some(bytes)` = the (possibly unchanged) body safe to forward; -/// `None` = the body is NOT the expected JSON object, so the caller must fail closed -/// under an active namespace filter (we could not verify no internal name is present). -fn strip_internal_bulk(body: &[u8], engine: &crate::curation::CurationEngine) -> Option> { - let mut map = - serde_json::from_slice::>(body).ok()?; - let before = map.len(); - map.retain(|k, _| { - !crate::curation::is_internal_namespace(engine, crate::curation::RegistryType::Npm, k) - }); - if map.len() == before { - Some(body.to_vec()) - } else { - // Re-serializing a just-parsed JSON map cannot fail in practice, but if it - // ever did, refuse (`None`) rather than fall back to the ORIGINAL body — - // that still contains the internal names. Keep the invariant fail-closed. - serde_json::to_vec(&map).ok() - } -} - -/// Rewrite tarball URLs in npm metadata to point to NORA. -/// -/// Replaces upstream registry URLs (e.g. `https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz`) -/// with NORA URLs (e.g. `http://nora:5000/npm/lodash/-/lodash-4.17.21.tgz`). -/// -/// Two-layer approach (#439): -/// 1. Targeted: parse JSON, rewrite `versions.*.dist.tarball` -/// 2. Safety net: byte-level replace of upstream URL prefix in serialized output -fn rewrite_tarball_urls(data: &[u8], nora_base: &str, upstream_url: &str) -> Result, ()> { - let mut json: serde_json::Value = serde_json::from_slice(data).map_err(|e| { - tracing::warn!(error = %e, "npm: JSON parse failed in rewrite_tarball_urls"); - })?; +fn package_prefix(repository: &str, package: &str) -> String { + format!("{}/{package}", repository_prefix(repository)) +} - let upstream_trimmed = upstream_url.trim_end_matches('/'); - let nora_npm_base = format!("{}/npm", nora_base.trim_end_matches('/')); +fn hosted_version_key(repository: &str, package: &str, version: &str) -> String { + format!( + "{}/versions/{version}.json", + package_prefix(repository, package) + ) +} - if let Some(versions) = json.get_mut("versions").and_then(|v| v.as_object_mut()) { - for (_ver, version_data) in versions.iter_mut() { - if let Some(tarball_url) = version_data - .get("dist") - .and_then(|d| d.get("tarball")) - .and_then(|t| t.as_str()) - .map(|s| s.to_string()) - { - let rewritten = tarball_url.replace(upstream_trimmed, &nora_npm_base); - if let Some(dist) = version_data.get_mut("dist") { - dist["tarball"] = serde_json::Value::String(rewritten); - } - } - } - } +fn hosted_publish_complete_key(repository: &str, package: &str, version: &str) -> String { + format!( + "{}/publish-complete/{version}", + package_prefix(repository, package) + ) +} - let output = serde_json::to_vec(&json).map_err(|e| { - tracing::warn!(error = %e, "npm: JSON serialize failed in rewrite_tarball_urls"); - })?; +fn hosted_publish_pending_prefix(repository: &str, package: &str) -> String { + format!("{}/publish-pending/", package_prefix(repository, package)) +} - // Safety net: byte-level replace of any remaining upstream URL prefix (#439). - // Catches edge cases where targeted rewrite missed (e.g. new npm metadata fields). - Ok(replace_upstream_bytes( - &output, - upstream_trimmed, - &nora_npm_base, - )) -} - -/// Byte-level replace of upstream URL prefix in response body (#439). -/// -/// Used as safety net after targeted JSON rewrite, and as fallback when JSON -/// parsing fails. Replaces full URL prefix (e.g. `https://registry.npmjs.org`) -/// rather than bare hostname to avoid corrupting unrelated fields. -fn replace_upstream_bytes(data: &[u8], upstream_url: &str, nora_npm_base: &str) -> Vec { - if upstream_url.is_empty() { - return data.to_vec(); - } - let needle = upstream_url.as_bytes(); - if memchr::memmem::find(data, needle).is_none() { - return data.to_vec(); - } - // Replace all occurrences of the upstream URL prefix - let replacement = nora_npm_base.as_bytes(); - let mut result = Vec::with_capacity(data.len()); - let mut start = 0; - let finder = memchr::memmem::Finder::new(needle); - while let Some(pos) = finder.find(&data[start..]) { - result.extend_from_slice(&data[start..start + pos]); - result.extend_from_slice(replacement); - start += pos + needle.len(); - } - result.extend_from_slice(&data[start..]); - result -} - -/// npm whoami handler: returns `{"username": "..."}` -async fn handle_whoami(user: &AuthenticatedUser) -> Response { - // Serialize via serde_json, NOT format!: a username can contain `"` (e.g. the - // OIDC `sub` claim, which is not charset-validated) and would otherwise break - // the JSON or inject extra fields into the response. - axum::Json(serde_json::json!({ "username": user.0 })).into_response() -} - -// LOCK-SAFE: cache-through proxy — get miss → fetch upstream → put; no RMW race -async fn handle_request( - State(state): State, - headers: axum::http::HeaderMap, - Path(path): Path, - Extension(user): Extension, -) -> Response { - // Handle npm whoami endpoint - if path == "-/whoami" { - return handle_whoami(&user).await; - } +fn hosted_publish_pending_key(repository: &str, package: &str, version: &str) -> String { + format!( + "{}{version}", + hosted_publish_pending_prefix(repository, package) + ) +} - let is_tarball = path.contains("/-/"); +fn hosted_publish_pending_index_key(repository: &str, package: &str) -> String { + format!( + "{}/publish-pending-index-v1", + package_prefix(repository, package) + ) +} - let key = if is_tarball { - let parts: Vec<&str> = path.splitn(2, "/-/").collect(); - if parts.len() == 2 { - format!("npm/{}/tarballs/{}", parts[0], parts[1]) - } else { - format!("npm/{}", path) - } - } else { - format!("npm/{}/metadata.json", path) - }; +fn hosted_tag_key(repository: &str, package: &str, tag: &str) -> String { + format!("{}/dist-tags/{tag}", package_prefix(repository, package)) +} - let package_name = if is_tarball { - path.split("/-/").next().unwrap_or(&path).to_string() - } else { - path.clone() - }; +fn hosted_deprecation_key(repository: &str, package: &str, version: &str) -> String { + format!( + "{}/deprecations/{version}", + package_prefix(repository, package) + ) +} - // Parse tarball version (used for both pre-download and integrity checks) - let tarball_version = if is_tarball { - let filename = path.split("/-/").nth(1).unwrap_or(""); - crate::curation::parse_npm_tarball_version(&package_name, filename) - } else { - None - }; - - // Extract publish date from cached metadata (npm `time` field). Hoisted to - // function scope so the digest-quarantine serve gate below can seed first_seen - // from a trusted upstream release date (#750). Non-tarball requests have no - // version, so this is `None` without any extra storage read. - let publish_date = if let Some(ref ver) = tarball_version { - let meta_key = format!("npm/{}/metadata.json", package_name); - extract_npm_publish_date( - &state.storage, - &meta_key, - ver, - state.config.server.trust_upstream_dates, - ) - .await - } else { - None - }; - - // Curation check — tarball downloads only (metadata passes through) - if is_tarball { - // #733: an internal-namespace package is operator-owned — skip curation; the cache-hit - // serve below handles a locally-published tarball, and the namespace guard at the - // cache-miss boundary (below) blocks the upstream branch for internal names. - if !crate::curation::is_internal_namespace( - &state.curation().curation_engine, - crate::curation::RegistryType::Npm, - &package_name, - ) { - if let Some(response) = crate::curation::check_download( - &state.curation().curation_engine, - state.bypass_token().as_deref(), - &headers, - crate::curation::RegistryType::Npm, - &package_name, - tarball_version.as_deref(), - publish_date, - ) { - return response; - } - } - } +fn hosted_package_key(repository: &str, package: &str) -> String { + format!("{}/pkg.json", package_prefix(repository, package)) +} - // --- Cache hit path --- - // get_verified discharges the integrity witness at the serve site (compile-time - // guarantee — see crate::verified). Both the tarball serve (pinned, Verified arm) - // and the metadata serve (unpinned, Unpinned arm) flow the discharged bytes. The - // .sha256 sidecar check below stays: it is the integrity check on S3 (storage - // pins are local-only). - if let Ok(outcome) = state.storage.get_verified(&key).await { - use nora_registry::verified::{verified_body, GateOutcome}; - let data = match outcome { - GateOutcome::Verified(blob) => verified_body(blob), - GateOutcome::Unpinned(blob) => blob.into_inner(), +async fn pending_publish_versions( + state: &AppState, + repository: &str, + package: &str, +) -> Result, StorageError> { + let prefix = hosted_publish_pending_prefix(repository, package); + let mut incomplete = HashSet::new(); + for pending_key in state.storage.list(&prefix).await? { + let Some(version) = pending_key + .strip_prefix(&prefix) + .filter(|relative| !relative.contains('/')) + .filter(|version| !version.is_empty()) + else { + return Err(StorageError::IntegrityViolation); }; - // Metadata TTL: if stale, try to refetch from upstream - if !is_tarball { - let ttl = state.config.npm.metadata_ttl; - if let Some(meta) = state.storage.stat(&key).await { - if !crate::cache_ttl::is_within_ttl(meta.modified, ttl) { - // #68 namespace isolation: a stale internal-namespace package must - // NOT be revalidated upstream (dependency confusion) — refetch_metadata - // proxies the name. Serve the cached copy already in hand instead; an - // internal package is owned/hosted locally, never upstream-refreshed. - // (The cache-miss proxy path is guarded separately below.) - if crate::curation::is_internal_namespace( - &state.curation().curation_engine, - crate::curation::RegistryType::Npm, - &package_name, - ) { - return with_content_type(false, data).into_response(); - } - // Single-flight: when a popular packument expires and a CI - // fleet stampedes the same key, one request revalidates - // upstream and the rest serve its in-memory result (#595). - let fresh = if state.config.server.proxy_coalesce { - let budget = - crate::proxy_coalesce::follower_budget(state.config.npm.proxy_timeout); - state - .proxy_coalesce - .coalesced(&key, "npm", budget, || async { - refetch_metadata(&state, &path, &key).await.map(Bytes::from) - }) - .await - } else { - refetch_metadata(&state, &path, &key).await.map(Bytes::from) - }; - if let Some(fresh) = fresh { - return with_content_type(false, fresh).into_response(); - } - // Upstream failed — serve stale if configured, otherwise 502 - if state.config.npm.serve_stale { - tracing::warn!( - registry = "npm", - path = %path, - "npm upstream unavailable, serving stale metadata" - ); - return ( - StatusCode::OK, - [ - ( - header::CONTENT_TYPE, - axum::http::HeaderValue::from_static("application/json"), - ), - ( - header::CACHE_CONTROL, - axum::http::HeaderValue::from_static( - "public, max-age=0, must-revalidate", - ), - ), - ( - axum::http::header::HeaderName::from_static("x-nora-stale"), - axum::http::HeaderValue::from_static("true"), - ), - ], - data.to_vec(), - ) - .into_response(); - } - return StatusCode::BAD_GATEWAY.into_response(); - } + let expected = state.storage.get(&pending_key).await?; + let expected = std::str::from_utf8(&expected) + .ok() + .filter(|digest| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + }) + .ok_or(StorageError::IntegrityViolation)?; + let manifest = match state + .storage + .get(&hosted_version_key(repository, package, version)) + .await + { + Ok(manifest) => manifest, + Err(StorageError::NotFound) => { + incomplete.insert(version.to_string()); + continue; } - return with_content_type(false, data).into_response(); + Err(error) => return Err(error), + }; + let manifest_digest = crate::npm_layout::hosted_manifest_digest(&manifest); + if manifest_digest != expected { + incomplete.insert(version.to_string()); + continue; } - - // Tarball: integrity check if hash exists - let hash_key = format!("{}.sha256", key); - if let Ok(stored_hash) = state.storage.get(&hash_key).await { - let computed = hex::encode(sha2::Sha256::digest(&data)); - let expected = String::from_utf8_lossy(&stored_hash); - if computed != expected.as_ref() { - tracing::error!( - key = %key, - expected = %expected, - computed = %computed, - "SECURITY: npm tarball integrity check FAILED — possible tampering" - ); - return (StatusCode::INTERNAL_SERVER_ERROR, "Integrity check failed") - .into_response(); + match state + .storage + .get(&hosted_publish_complete_key(repository, package, version)) + .await + { + Ok(completion) if completion.as_ref() == expected.as_bytes() => { + match state.storage.delete(&pending_key).await { + Ok(()) | Err(StorageError::NotFound) => {} + Err(error) => return Err(error), + } } + Ok(_) | Err(StorageError::NotFound) => { + incomplete.insert(version.to_string()); + } + Err(error) => return Err(error), } + } + Ok(incomplete) +} - // Curation integrity verification (issue #189) - if let Some(response) = crate::curation::verify_integrity( - &state.curation().curation_engine, - crate::curation::RegistryType::Npm, - &package_name, - tarball_version.as_deref(), - &data, - ) { - return response; - } - - state.metrics.record_download("npm"); - state.metrics.record_cache_hit("npm"); - state.activity.push(ActivityEntry::new( - ActionType::CacheHit, - package_name, - crate::registry_type::RegistryType::Npm, - "CACHE", - )); - state - .audit - .log(AuditEntry::new("cache_hit", "api", "", "npm", "")); - let (q_mode, q_secs) = crate::digest_quarantine::resolve_global( - state.config.curation.npm.quarantine.as_ref().or(state - .config - .curation - .quarantine - .as_ref()), - state.config.curation.npm.quarantine_ttl.as_deref().or(state - .config - .curation - .quarantine_ttl - .as_deref()), - ); - if let Some(resp) = crate::digest_quarantine::proxy_gate_dated( - &state.digest_store, - "npm", - &data, - &q_mode, - q_secs, - "cache", - publish_date, - ) { - return resp; - } - return with_content_type(true, data).into_response(); +async fn incomplete_publish_versions( + state: &AppState, + repository: &str, + package: &str, +) -> Result, StorageError> { + let index_key = hosted_publish_pending_index_key(repository, package); + match state.storage.get(&index_key).await { + Ok(index) if index.as_ref() == b"1" => {} + Ok(_) => return Err(StorageError::IntegrityViolation), + Err(StorageError::NotFound) => state.storage.put(&index_key, b"1").await?, + Err(error) => return Err(error), } + pending_publish_versions(state, repository, package).await +} - // --- Namespace isolation: prevent proxying internal namespaces --- - // Metadata requests skip the curation check_download (which only runs for - // tarballs), so we must protect the proxy path separately. This runs after - // cache lookup so locally-published packages are still served from cache. - if let Some(response) = crate::curation::check_namespace_isolation( - &state.curation().curation_engine, - crate::curation::RegistryType::Npm, - &package_name, - ) { - return response; +async fn restore_publish_pending( + state: &AppState, + pending_key: &str, + previous: Option<&[u8]>, +) -> Result<(), StorageError> { + match previous { + Some(previous) => state.storage.put(pending_key, previous).await, + None => match state.storage.delete(pending_key).await { + Ok(()) | Err(StorageError::NotFound) => Ok(()), + Err(error) => Err(error), + }, } +} - // --- Proxy fetch path --- - if let Some(proxy_url) = &state.config.npm.proxy { - let url = format!("{}/{}", proxy_url.trim_end_matches('/'), path); +fn incomplete_publish_response() -> Response { + ( + StatusCode::CONFLICT, + "Package has an incomplete publish; retry that exact publish before mutating package metadata", + ) + .into_response() +} - match proxy_fetch( - &state.http_client, - &url, - Duration::from_secs(state.config.npm.proxy_timeout), - expose_opt(&state.config.npm.proxy_auth), - &state.circuit_breaker, - RegistryType::Npm, - ) - .await - { - Ok(data) => { - let data_to_cache; - let data_to_serve; - - if is_tarball { - // Compute and store sha256 - let hash = hex::encode(sha2::Sha256::digest(&data)); - let hash_key = format!("{}.sha256", key); - let storage = state.storage.clone(); - tokio::spawn(async move { - if let Err(e) = storage.put(&hash_key, hash.as_bytes()).await { - tracing::warn!(key = %hash_key, error = ?e, "npm proxy: failed to cache hash sidecar"); - } - }); - - state.metrics.record_download("npm"); - state.metrics.record_cache_miss("npm"); - state.activity.push(ActivityEntry::new( - ActionType::ProxyFetch, - package_name, - crate::registry_type::RegistryType::Npm, - "PROXY", - )); - state - .audit - .log(AuditEntry::new("proxy_fetch", "api", "", "npm", "")); +fn proxy_packument_key(repository: &str, package: &str) -> String { + format!( + "{}/proxy/packuments/{package}.json", + repository_prefix(repository) + ) +} - data_to_cache = data.clone(); - data_to_serve = data; - } else { - // Metadata: rewrite tarball URLs to point to NORA - let nora_base = nora_base_url(&state); - let rewritten = rewrite_tarball_urls(&data, &nora_base, proxy_url) - .unwrap_or_else(|()| { - tracing::warn!( - path = %path, - "npm metadata JSON parse failed, using byte-level URL rewrite" - ); - let upstream_trimmed = proxy_url.trim_end_matches('/'); - let nora_npm_base = format!("{}/npm", nora_base.trim_end_matches('/')); - replace_upstream_bytes(&data, upstream_trimmed, &nora_npm_base) - }); - - data_to_cache = rewritten.clone(); - data_to_serve = rewritten; - } +fn proxy_negative_key(repository: &str, package: &str) -> String { + format!("{}/proxy/negative/{package}", repository_prefix(repository)) +} - // Cache in background, invalidate index AFTER write completes - let storage = state.storage.clone(); - let key_clone = key.clone(); - let invalidate_npm = is_tarball; - let repo_index = Arc::clone(&state.repo_index); - tokio::spawn(async move { - if let Err(e) = storage.put(&key_clone, &data_to_cache).await { - tracing::warn!(key = %key_clone, error = ?e, "npm proxy: failed to cache artifact"); - } else if invalidate_npm { - repo_index.invalidate("npm"); - } - }); - - if is_tarball { - let (q_mode, q_secs) = crate::digest_quarantine::resolve_global( - state.config.curation.npm.quarantine.as_ref().or(state - .config - .curation - .quarantine - .as_ref()), - state.config.curation.npm.quarantine_ttl.as_deref().or(state - .config - .curation - .quarantine_ttl - .as_deref()), - ); - if let Some(resp) = crate::digest_quarantine::proxy_gate_dated( - &state.digest_store, - "npm", - &data_to_serve, - &q_mode, - q_secs, - &url, - publish_date, - ) { - return resp; - } - } - return with_content_type(is_tarball, data_to_serve.into()).into_response(); - } - Err(ProxyError::CircuitOpen(reg)) => return circuit_open_response(®), - Err(e) => { - tracing::debug!(error = ?e, path = %path, "npm proxy fetch failed"); - } - } - tracing::warn!(registry = "npm", path = %path, "Proxy failed, returning 404"); - } +fn proxy_tarball_key(repository: &str, package: &str, filename: &str) -> String { + format!( + "{}/proxy/tarballs/{package}/{filename}", + repository_prefix(repository) + ) +} - StatusCode::NOT_FOUND.into_response() +fn legacy_proxy(state: &AppState) -> Option { + Some(ProxyRepository { + name: LEGACY_PROXY.to_string(), + url: state.config.npm.proxy.clone()?, + auth: state.config.npm.proxy_auth.clone(), + metadata_ttl: state.config.npm.metadata_ttl, + negative_ttl: 300, + }) } -/// Refetch metadata from upstream, rewrite URLs, update cache. -/// Returns None if upstream is unavailable (caller serves stale cache). -async fn refetch_metadata(state: &AppState, path: &str, key: &str) -> Option> { - let proxy_url = state.config.npm.proxy.as_ref()?; - let url = format!("{}/{}", proxy_url.trim_end_matches('/'), path); +fn configured_proxy(state: &AppState, repository: &NpmRepository) -> Option { + match repository { + NpmRepository::Proxy { + name, + url, + auth, + metadata_ttl, + negative_ttl, + } => Some(ProxyRepository { + name: name.clone(), + url: url.clone(), + auth: auth.clone(), + metadata_ttl: metadata_ttl.unwrap_or(state.config.npm.metadata_ttl), + negative_ttl: *negative_ttl, + }), + _ => None, + } +} - // Revalidate with a conditional request when enabled and we have stored - // validators. Empty validators ⇒ no conditional headers ⇒ always a 200, - // which is also how the first fetch captures validators for next time (#596). - let validators = if state.config.npm.revalidate { - read_validators(&state.storage, key) - .await - .unwrap_or_default() - } else { - Validators::default() - }; - let had_validators = validators.is_some(); +/// Resolve an npm upstream reference only within the configured proxy origin +/// and base path. Every initial request and redirect target passes through this +/// boundary before a request (and its configured Authorization header) is +/// built. +fn validated_proxy_url(repository: &ProxyRepository, candidate: &str) -> Option { + let base = reqwest::Url::parse(&repository.url).ok()?; + let mut join_base = base.clone(); + if !join_base.path().ends_with('/') { + join_base.set_path(&format!("{}/", join_base.path())); + } + let target = join_base.join(candidate).ok()?; + if !matches!(target.scheme(), "http" | "https") { + return None; + } + if !target.username().is_empty() || target.password().is_some() { + return None; + } + let same_origin = base.scheme() == target.scheme() + && base.host_str() == target.host_str() + && base.port_or_known_default() == target.port_or_known_default(); + if !same_origin { + return None; + } + let base_path = base.path().trim_end_matches('/'); + if !base_path.is_empty() + && base_path != "/" + && target.path() != base_path + && !target + .path() + .strip_prefix(base_path) + .is_some_and(|suffix| suffix.starts_with('/')) + { + return None; + } + Some(target) +} - match proxy_fetch_conditional( - &state.http_client, - &url, - Duration::from_secs(state.config.npm.proxy_timeout), - expose_opt(&state.config.npm.proxy_auth), - &validators, - &state.circuit_breaker, - RegistryType::Npm, +fn is_internal(state: &AppState, package: &str) -> bool { + crate::curation::is_internal_namespace( + &state.curation().curation_engine, + crate::curation::RegistryType::Npm, + package, ) - .await - { - // Upstream unchanged — serve the cached (already-rewritten) body and - // refresh its freshness so we don't revalidate again until the next TTL - // window. No body was downloaded. - Ok(Revalidation::NotModified) => { - let cached = state.storage.get(key).await.ok()?; // body gone → fail-open - crate::metrics::PROXY_UPSTREAM_304_TOTAL - .with_label_values(&["npm"]) - .inc(); - crate::metrics::PROXY_REVALIDATION_BYTES_SAVED_TOTAL - .with_label_values(&["npm"]) - .inc_by(cached.len() as u64); - // Re-put bumps the file mtime (the freshness source) without an - // upstream download. - let storage = state.storage.clone(); - let key_clone = key.to_string(); - let body = cached.clone(); - tokio::spawn(async move { - let _ = storage.put(&key_clone, &body).await; - }); - Some(cached.to_vec()) - } - // New body — rewrite, cache it, then persist the fresh validators. - Ok(Revalidation::Modified { body, validators }) => { - let nora_base = nora_base_url(state); - let rewritten = - rewrite_tarball_urls(&body, &nora_base, proxy_url).unwrap_or_else(|()| { - tracing::warn!( - path = %path, - "npm metadata refetch: JSON parse failed, using byte-level URL rewrite" - ); - let upstream_trimmed = proxy_url.trim_end_matches('/'); - let nora_npm_base = format!("{}/npm", nora_base.trim_end_matches('/')); - replace_upstream_bytes(&body, upstream_trimmed, &nora_npm_base) - }); - - let storage = state.storage.clone(); - let key_clone = key.to_string(); - let cache_data = rewritten.clone(); - tokio::spawn(async move { - // Body first; the validator sidecar must never advertise - // freshness for a body that isn't there (#596). - if let Err(e) = storage.put(&key_clone, &cache_data).await { - tracing::warn!(key = %key_clone, error = ?e, "npm proxy: failed to cache metadata"); - return; - } - write_validators(&storage, &key_clone, &validators).await; - }); +} - Some(rewritten) - } - // Upstream unavailable / error — fall back to the caller's serve_stale / - // 502 path exactly as before. - Err(_) => { - if had_validators { - crate::metrics::PROXY_REVALIDATION_ERRORS_TOTAL - .with_label_values(&["npm"]) - .inc(); - } - None - } +fn is_valid_npm_package_name(name: &str) -> bool { + // A residual percent sign means the route was encoded more than once. + // Reject it instead of decoding repeatedly: repeated decoding can turn an + // apparently public name into an internal scoped package only after the + // namespace-isolation decision. + if name.is_empty() || name.len() > 214 || name.contains(['%', '\\', '\0']) { + return false; + } + if let Some(scoped) = name.strip_prefix('@') { + let Some((scope, package)) = scoped.split_once('/') else { + return false; + }; + !scope.is_empty() + && !package.is_empty() + && !matches!(scope, "." | "..") + && !matches!(package, "." | "..") + && !package.contains('/') + } else { + !matches!(name, "." | "..") && !name.contains('/') } } -// ============================================================================ -// npm publish -// ============================================================================ +fn is_valid_dist_tag(tag: &str) -> bool { + !tag.is_empty() + && tag.len() <= 214 + && !matches!(tag, "." | "..") + && !tag.contains(['/', '\\', '\0']) + && semver::VersionReq::parse(tag).is_err() + && semver::Version::parse(tag.trim_start_matches('v')).is_err() +} + +fn is_valid_npm_version(version: &str) -> bool { + semver::Version::parse(version.trim_start_matches('v')).is_ok() +} -/// Validate attachment filename: only safe characters, no path traversal. fn is_valid_attachment_name(name: &str) -> bool { !name.is_empty() && !name.contains("..") - && !name.contains('/') - && !name.contains('\\') - && !name.contains('\0') + && !name.contains(['/', '\\', '\0']) && name .chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '@')) } -async fn handle_publish( - State(state): State, - Path(path): Path, - Extension(authority): Extension, - body: Bytes, -) -> Response { - let package_name = path; - - // Enforce OIDC namespace_scope on the package coordinate (#583). - if enforce_namespace_scope(&authority, &package_name).is_err() { - return StatusCode::FORBIDDEN.into_response(); +fn normalize_attachment_name<'a>(package: &str, filename: &'a str) -> &'a str { + if let Some(scope_end) = package.find('/') { + filename + .strip_prefix(&package[..=scope_end]) + .unwrap_or(filename) + } else { + filename } +} - let payload: serde_json::Value = match serde_json::from_slice(&body) { - Ok(v) => v, - Err(e) => return (StatusCode::BAD_REQUEST, format!("Invalid JSON: {}", e)).into_response(), - }; +fn decode_package_name(raw: &str) -> Option { + let decoded = percent_encoding::percent_decode_str(raw) + .decode_utf8() + .ok()? + .into_owned(); + is_valid_npm_package_name(&decoded).then_some(decoded) +} - // Security: verify payload name matches URL path (required field) - match payload.get("name").and_then(|n| n.as_str()) { - Some(payload_name) if payload_name == package_name => {} - Some(payload_name) => { - tracing::warn!( - url_name = %package_name, - payload_name = %payload_name, - "SECURITY: npm publish name mismatch — possible spoofing attempt" - ); - return ( - StatusCode::BAD_REQUEST, - "Package name in URL does not match payload", - ) - .into_response(); - } - None => { - return ( - StatusCode::BAD_REQUEST, - "Missing required 'name' field in publish payload", - ) - .into_response(); - } +fn parse_package_path(path: &str) -> Option<(String, Option)> { + if let Some((package, filename)) = path.split_once("/-/") { + let package = decode_package_name(package)?; + let filename = percent_encoding::percent_decode_str(filename) + .decode_utf8() + .ok()? + .into_owned(); + is_valid_attachment_name(&filename).then_some((package, Some(filename))) + } else { + decode_package_name(path).map(|package| (package, None)) } +} - let attachments = match payload.get("_attachments").and_then(|a| a.as_object()) { - Some(a) => a, - None => return (StatusCode::BAD_REQUEST, "Missing _attachments").into_response(), - }; +fn upstream_package_path(package: &str) -> String { + package.replace('/', "%2F") +} - let new_versions = match payload.get("versions").and_then(|v| v.as_object()) { - Some(v) => v, - None => return (StatusCode::BAD_REQUEST, "Missing versions").into_response(), - }; +fn canonical_tarball_filename(package: &str, version: &str) -> String { + format!( + "{}-{version}.tgz", + package.split('/').next_back().unwrap_or(package) + ) +} - // Lock per package to serialize the packument regeneration within one process. - let metadata_key = format!("npm/{}/metadata.json", package_name); - let lock = state.publish_lock(&metadata_key); - let _guard = lock.lock().await; +fn set_tarball_url( + version_data: &mut serde_json::Value, + public_base: &str, + package: &str, + version: &str, +) { + let Some(object) = version_data.as_object_mut() else { + return; + }; + let dist = object + .entry("dist") + .or_insert_with(|| serde_json::json!({})); + if let Some(dist) = dist.as_object_mut() { + dist.insert( + "tarball".to_string(), + serde_json::Value::String(format!( + "{public_base}/{package}/-/{}", + canonical_tarball_filename(package, version) + )), + ); + } +} - // Read the existing packument ONCE — only to (a) refuse on corruption and (b) enforce version - // immutability against versions still embedded in an un-migrated packument. The new versions - // are NOT merged into it: each is written to its own immutable key and the packument is - // regenerated by listing those keys, so concurrent publishers never read-modify-write the same - // shared file (the multi-replica lost-update of #39). - let existing_meta: Option = match state.storage.get(&metadata_key).await { - Ok(existing) => match serde_json::from_slice::(&existing) { - Ok(val) => Some(val), - Err(e) => { - // Corrupt metadata — refuse publish to protect existing versions (#533). - tracing::error!( - registry = "npm", - key = %metadata_key, - error = %e, - bytes = existing.len(), - "Corrupt metadata detected during publish — refusing to overwrite" - ); - METADATA_CORRUPT_TOTAL.with_label_values(&["npm"]).inc(); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - "Existing package metadata is corrupt; publish blocked to prevent data loss", - ) - .into_response(); - } - }, - Err(_) => None, // No existing metadata — first publish +fn json_response(headers: &HeaderMap, value: &serde_json::Value) -> Response { + let Ok(bytes) = serde_json::to_vec(value) else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); }; + let etag = format!("\"{}\"", hex::encode(sha2::Sha256::digest(&bytes))); + if headers + .get(header::IF_NONE_MATCH) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.split(',').any(|candidate| candidate.trim() == etag)) + { + return (StatusCode::NOT_MODIFIED, [(header::ETAG, etag)]).into_response(); + } + ( + StatusCode::OK, + [ + (header::CONTENT_TYPE, "application/json".to_string()), + (header::CACHE_CONTROL, "no-cache".to_string()), + (header::ETAG, etag), + ], + bytes, + ) + .into_response() +} - // Version immutability: a version already published — as its own immutable key, or still - // embedded in an un-migrated packument — may not be overwritten. - for ver in new_versions.keys() { - let version_key = format!("npm/{}/versions/{}.json", package_name, ver); - let in_keys = state.storage.stat(&version_key).await.is_some(); - let in_embedded = existing_meta - .as_ref() - .and_then(|m| m.get("versions")) - .and_then(|v| v.as_object()) - .map(|o| o.contains_key(ver)) - .unwrap_or(false); - if in_keys || in_embedded { - return ( - StatusCode::CONFLICT, - format!("Version {} already exists", ver), - ) - .into_response(); - } - } - - // Lazily migrate an old embedded-packument package to per-version keys BEFORE writing the new - // version, so the regenerate below lists both the migrated and the new versions (no loss). - if let Some(existing) = &existing_meta { - migrate_embedded_packument(&state, &package_name, existing).await; - } - - // Store tarballs - for (filename, attachment_data) in attachments { - // Scoped packages (e.g. @scope/name) may have attachment filenames - // like "@scope/name-1.0.0.tgz". Strip the scope prefix since it is - // already captured in the package name — the filename part must be a - // flat name without path separators to prevent path traversal. - let normalized_name = if let Some(scope_end) = package_name.find('/') { - let scope_prefix = &package_name[..=scope_end]; - filename.strip_prefix(scope_prefix).unwrap_or(filename) - } else { - filename - }; +fn json_response_with_stale( + headers: &HeaderMap, + value: &serde_json::Value, + stale: bool, +) -> Response { + let mut response = json_response(headers, value); + if stale { + response + .headers_mut() + .insert("x-nora-stale", HeaderValue::from_static("true")); + } + response +} - if !is_valid_attachment_name(normalized_name) { - tracing::warn!( - filename = %filename, - package = %package_name, - "SECURITY: npm publish rejected — invalid attachment filename" - ); - return (StatusCode::BAD_REQUEST, "Invalid attachment filename").into_response(); - } +fn tarball_response(data: Bytes) -> Response { + ( + StatusCode::OK, + [ + (header::CONTENT_TYPE, "application/octet-stream"), + (header::CACHE_CONTROL, "public, max-age=31536000, immutable"), + ], + data, + ) + .into_response() +} - let base64_data = match attachment_data.get("data").and_then(|d| d.as_str()) { - Some(d) => d, - None => continue, - }; +fn read_string(bytes: Bytes) -> Option { + String::from_utf8(bytes.to_vec()).ok() +} - let tarball_bytes = match base64::engine::general_purpose::STANDARD.decode(base64_data) { - Ok(b) => b, - Err(_) => { - return (StatusCode::BAD_REQUEST, "Invalid base64 in attachment").into_response() - } - }; +fn npm_publish_date_from_packument(packument: &serde_json::Value, version: &str) -> Option { + packument + .get("time") + .and_then(|value| value.get(version)) + .and_then(|value| value.as_str()) + .and_then(crate::curation::parse_iso8601_to_unix) +} - let tarball_key = format!("npm/{}/tarballs/{}", package_name, normalized_name); - if let Err(e) = state.storage.put(&tarball_key, &tarball_bytes).await { - tracing::error!(key = %tarball_key, error = ?e, "npm publish: failed to store tarball"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); +async fn cached_proxy_publish_date( + state: &AppState, + repository: &str, + package: &str, + version: &str, + filename: &str, +) -> Option { + if state.config.server.trust_upstream_dates { + let packument = state + .storage + .get(&proxy_packument_key(repository, package)) + .await + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + if let Some(date) = packument + .as_ref() + .and_then(|value| npm_publish_date_from_packument(value, version)) + { + return Some(date); } + } + crate::curation::extract_mtime_as_publish_date( + &state.storage, + &proxy_tarball_key(repository, package, filename), + ) + .await +} - // Store sha256 - let hash = hex::encode(sha2::Sha256::digest(&tarball_bytes)); - let hash_key = format!("{}.sha256", tarball_key); - if let Err(e) = state.storage.put(&hash_key, hash.as_bytes()).await { - tracing::warn!(key = %hash_key, error = ?e, "npm publish: failed to store hash sidecar"); - } - } - - // Write each new version as its OWN immutable key, with the tarball URL rewritten to point at - // this registry. Concurrent publishes of DIFFERENT versions write distinct keys, so none is - // lost — the heart of the #39 fix (vs the old read-merge-write of the shared packument). - let nora_base = nora_base_url(&state); - let short_name = package_name.split('/').next_back().unwrap_or(&package_name); - for (ver, ver_data) in new_versions { - let mut vd = ver_data.clone(); - if let Some(dist) = vd.get_mut("dist") { - let tarball_url = format!( - "{}/npm/{}/-/{}-{}.tgz", - nora_base.trim_end_matches('/'), - package_name, - short_name, - ver - ); - dist["tarball"] = serde_json::Value::String(tarball_url); +async fn hosted_blob_key_for_version( + state: &AppState, + repository: &str, + package: &str, + version: &str, +) -> Result { + let manifest = state + .storage + .get(&hosted_version_key(repository, package, version)) + .await + .map_err(storage_read_error)?; + crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) + .ok_or(ReadError::Corrupt) +} + +async fn target_publish_date( + state: &AppState, + target: &RepositoryTarget, + package: &str, + version: &str, + filename: &str, +) -> Result, ReadError> { + match target { + RepositoryTarget::Named(NpmRepository::Hosted { name, .. }) => { + let blob_key = hosted_blob_key_for_version(state, name, package, version).await?; + Ok(crate::curation::extract_mtime_as_publish_date(&state.storage, &blob_key).await) } - let version_key = format!("npm/{}/versions/{}.json", package_name, ver); - match serde_json::to_vec(&vd) { - Ok(bytes) => { - if let Err(e) = state.storage.put(&version_key, &bytes).await { - tracing::error!(key = %version_key, error = ?e, "npm publish: failed to store version"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - } - Err(e) => { - tracing::error!(error = ?e, "npm publish: failed to serialize version"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } + RepositoryTarget::Named(NpmRepository::Proxy { name, .. }) => { + Ok(cached_proxy_publish_date(state, name, package, version, filename).await) } - } - - // dist-tags: one small mutable pointer per tag (tag -> version). Last-writer-wins on a single - // tag is an acceptable pointer-flip — it never loses a version (each version is its own key). - if let Some(new_dist_tags) = payload.get("dist-tags").and_then(|d| d.as_object()) { - for (tag, ver) in new_dist_tags { - if let Some(ver_str) = ver.as_str() { - let tag_key = format!("npm/{}/dist-tags/{}", package_name, tag); - if let Err(e) = state.storage.put(&tag_key, ver_str.as_bytes()).await { - tracing::warn!(key = %tag_key, error = ?e, "npm publish: failed to store dist-tag"); + RepositoryTarget::Named(NpmRepository::Group { members, .. }) => { + for member in members { + let Some(repository) = state.config.npm.repository(member) else { + continue; + }; + match repository { + NpmRepository::Hosted { name, .. } => { + if hosted_has_version(state, name, package, version).await? { + let blob_key = + hosted_blob_key_for_version(state, name, package, version).await?; + return Ok(crate::curation::extract_mtime_as_publish_date( + &state.storage, + &blob_key, + ) + .await); + } + } + NpmRepository::Proxy { name, .. } if !is_internal(state, package) => { + let packument = + optional_storage_get(state, &proxy_packument_key(name, package)) + .await? + .map(|bytes| { + serde_json::from_slice::(&bytes) + .map_err(|_| ReadError::Corrupt) + }) + .transpose()?; + if packument.as_ref().is_some_and(|value| { + value + .get("versions") + .and_then(|versions| versions.get(version)) + .is_some() + }) { + return Ok(cached_proxy_publish_date( + state, name, package, version, filename, + ) + .await); + } + } + _ => {} } } + Ok(None) } - } - - // Package-level descriptive fields (not per-version) — overwrite (last-writer-wins is fine). - let mut pkg_fields = serde_json::Map::new(); - for field in &["name", "_id", "description", "readme", "license"] { - if let Some(val) = payload.get(*field) { - pkg_fields.insert(field.to_string(), val.clone()); - } - } - if let Ok(bytes) = serde_json::to_vec(&serde_json::Value::Object(pkg_fields)) { - let pkg_key = format!("npm/{}/pkg.json", package_name); - if let Err(e) = state.storage.put(&pkg_key, &bytes).await { - tracing::warn!(key = %pkg_key, error = ?e, "npm publish: failed to store package fields"); + RepositoryTarget::Legacy => { + if hosted_has_version(state, LEGACY_HOSTED, package, version).await? { + let blob_key = + hosted_blob_key_for_version(state, LEGACY_HOSTED, package, version).await?; + Ok(crate::curation::extract_mtime_as_publish_date(&state.storage, &blob_key).await) + } else { + Ok( + cached_proxy_publish_date(state, LEGACY_PROXY, package, version, filename) + .await, + ) + } } } +} - // Regenerate the packument (metadata.json) by listing the immutable per-version keys. - if regenerate_packument(&state, &package_name).await.is_err() { - tracing::error!(package = %package_name, "npm publish: failed to regenerate packument"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); +fn curated_tarball_response( + state: &AppState, + package: &str, + version: &str, + data: Bytes, + source: &str, + publish_date: Option, +) -> Response { + if let Some(response) = crate::curation::verify_integrity( + &state.curation().curation_engine, + crate::curation::RegistryType::Npm, + package, + Some(version), + &data, + ) { + return response; } - - // Fail-closed (#878): a 201 MUST mean the published version is actually visible in - // the packument. scan-regenerate derives `versions` by LISTING the per-version keys - // and reading each back; a storage key-encoding round-trip bug (the object_store - // `%40`->`%2540` double-encoding for `@scope/...` keys) silently produced an empty - // `versions` map, yet publish still returned 201 and `npm install` then failed - // ENOVERSIONS. Read the regenerated packument back and refuse (500) unless every - // just-published version is present, so a regen round-trip defect can never again - // hide behind a 201. - let published_visible = match state.storage.get(&metadata_key).await { - Ok(bytes) => serde_json::from_slice::(&bytes) - .ok() - .and_then(|m| { - m.get("versions") - .and_then(|v| v.as_object()) - .map(|vs| new_versions.keys().all(|v| vs.contains_key(v))) - }) - .unwrap_or(false), - Err(_) => false, - }; - if !published_visible { - tracing::error!( - package = %package_name, - "npm publish: regenerated packument does not list the just-published version(s) — refusing 201 (scan-regenerate round-trip broke, see #878)" + let (mode, ttl) = + crate::digest_quarantine::resolve_global( + state.config.curation.npm.quarantine.as_ref().or(state + .config + .curation + .quarantine + .as_ref()), + state.config.curation.npm.quarantine_ttl.as_deref().or(state + .config + .curation + .quarantine_ttl + .as_deref()), ); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + if let Some(response) = crate::digest_quarantine::proxy_gate_dated( + &state.digest_store, + "npm", + &data, + &mode, + ttl, + source, + publish_date, + ) { + return response; } - - state.metrics.record_upload("npm"); - state - .audit - .log(AuditEntry::new("push", "api", &package_name, "npm", "")); - state.activity.push(ActivityEntry::new( - ActionType::Push, - package_name, - crate::registry_type::RegistryType::Npm, - "LOCAL", - )); - state.repo_index.invalidate("npm"); - - StatusCode::CREATED.into_response() + tarball_response(data) } -// ============================================================================ -// Helpers -// ============================================================================ - -/// Regenerate the npm packument (`metadata.json`) by LISTING the immutable per-version keys, the -/// dist-tag pointers and the package-level fields — never by read-modify-write of the shared file. -/// Concurrent publishers each write their own `versions/{v}.json`; whoever regenerates last lists -/// them all, so no version is permanently lost (the maven scan-regenerate guarantee, #39). Old -/// embedded-packument packages are migrated to per-version keys by `migrate_embedded_packument` -/// in the publish handler BEFORE the new version is written, so this stays a pure list-derive. -async fn regenerate_packument(state: &AppState, package_name: &str) -> Result<(), ()> { - let versions_prefix = format!("npm/{}/versions/", package_name); +async fn hosted_packument( + state: &AppState, + repository: &str, + package: &str, + response_base: &str, +) -> Result { + let prefix = package_prefix(repository, package); + let version_prefix = format!("{prefix}/versions/"); let version_keys = state .storage - .list(&versions_prefix) + .list(&version_prefix) .await - .unwrap_or_default(); + .map_err(|_| ReadError::Unavailable)?; + + let version_reads = stream::iter(version_keys.into_iter().filter_map(|key| { + let version = key + .strip_prefix(&version_prefix)? + .strip_suffix(".json")? + .to_string(); + (!version.is_empty() && !version.contains('/')).then_some((version, key)) + })) + .map(|(version, key)| async move { + let bytes = state.storage.get(&key).await.map_err(|error| { + match storage_read_error(error) { + // LIST advertised this committed manifest. Disappearing between + // LIST and GET is an incomplete hosted read, not permission to + // let a lower-priority group member shadow the version. + ReadError::NotFound => ReadError::Unavailable, + other => other, + } + })?; + let mut value = serde_json::from_slice::(&bytes).map_err(|_| { + crate::metrics::METADATA_CORRUPT_TOTAL + .with_label_values(&["npm"]) + .inc(); + ReadError::Corrupt + })?; + if let Some(deprecated) = optional_storage_get( + state, + &hosted_deprecation_key(repository, package, &version), + ) + .await? + { + let Some(message) = read_string(deprecated) else { + return Err(ReadError::Corrupt); + }; + let Some(object) = value.as_object_mut() else { + return Err(ReadError::Corrupt); + }; + object.insert("deprecated".to_string(), serde_json::Value::String(message)); + } + set_tarball_url(&mut value, response_base, package, &version); + Ok::<_, ReadError>((version, value)) + }) + .buffered(HOSTED_PACKUMENT_READ_CONCURRENCY) + .collect::>() + .await; - // versions map, keyed by the filename (version) without the .json suffix. let mut versions = serde_json::Map::new(); - for key in &version_keys { - let Ok(data) = state.storage.get(key).await else { + for result in version_reads { + let (version, value) = result?; + versions.insert(version, value); + } + + let mut tags = serde_json::Map::new(); + let tag_prefix = format!("{prefix}/dist-tags/"); + for key in state + .storage + .list(&tag_prefix) + .await + .map_err(|_| ReadError::Unavailable)? + { + let Some(tag) = key.rsplit('/').next() else { continue; }; - let Ok(vd) = serde_json::from_slice::(&data) else { - continue; + let value = + state + .storage + .get(&key) + .await + .map_err(|error| match storage_read_error(error) { + ReadError::NotFound => ReadError::Unavailable, + other => other, + })?; + let Some(version) = read_string(value) else { + return Err(ReadError::Corrupt); }; - if let Some(ver) = key.rsplit('/').next().and_then(|f| f.strip_suffix(".json")) { - versions.insert(ver.to_string(), vd); - } + tags.insert(tag.to_string(), serde_json::Value::String(version)); } - // dist-tags from the pointer keys (+ derive `latest` if publish left it unset). - let dt_prefix = format!("npm/{}/dist-tags/", package_name); - let mut dist_tags = serde_json::Map::new(); - for key in state.storage.list(&dt_prefix).await.unwrap_or_default() { - if let Ok(data) = state.storage.get(&key).await { - if let (Some(tag), Ok(ver)) = (key.rsplit('/').next(), String::from_utf8(data.to_vec())) - { - dist_tags.insert(tag.to_string(), serde_json::Value::String(ver)); - } - } - } - if !dist_tags.contains_key("latest") { - if let Some(latest) = max_semver(versions.keys()) { - dist_tags.insert("latest".to_string(), serde_json::Value::String(latest)); - } + let package_fields = + match optional_storage_get(state, &hosted_package_key(repository, package)).await? { + Some(bytes) => Some(serde_json::from_slice::(&bytes).map_err( + |_| { + crate::metrics::METADATA_CORRUPT_TOTAL + .with_label_values(&["npm"]) + .inc(); + ReadError::Corrupt + }, + )?), + None => None, + }; + + if versions.is_empty() && tags.is_empty() && package_fields.is_none() { + return Err(ReadError::NotFound); } - // Package-level fields + the assembled maps -> the packument. - let mut packument = match state - .storage - .get(&format!("npm/{}/pkg.json", package_name)) - .await - { - Ok(d) => serde_json::from_slice(&d).unwrap_or_else(|_| serde_json::json!({})), - Err(_) => serde_json::json!({}), + let mut packument = package_fields.unwrap_or_else(|| serde_json::json!({})); + let Some(object) = packument.as_object_mut() else { + return Err(ReadError::Corrupt); }; - let obj = packument.as_object_mut().ok_or(())?; - obj.insert( + object.insert( "name".to_string(), - serde_json::Value::String(package_name.to_string()), + serde_json::Value::String(package.to_string()), ); - obj.insert( - "dist-tags".to_string(), - serde_json::Value::Object(dist_tags), - ); - obj.insert("versions".to_string(), serde_json::Value::Object(versions)); + object.insert("versions".to_string(), serde_json::Value::Object(versions)); + object.insert("dist-tags".to_string(), serde_json::Value::Object(tags)); + Ok(packument) +} - let bytes = serde_json::to_vec(&packument).map_err(|_| ())?; - state - .storage - .put(&format!("npm/{}/metadata.json", package_name), &bytes) - .await - .map_err(|_| ()) +fn negative_fresh(modified: u64, ttl: i64) -> bool { + if ttl <= 0 { + return false; + } + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + now.saturating_sub(modified) < ttl as u64 } -/// Seed per-version keys from an old embedded packument (versions inside `metadata.json`, no -/// per-version keys) so the scan-regenerate path does not drop them. Idempotent; a no-op once the -/// package has per-version keys. Runs BEFORE the new version is written. -async fn migrate_embedded_packument( +async fn proxy_packument_raw( state: &AppState, - package_name: &str, - existing: &serde_json::Value, -) { - let versions_prefix = format!("npm/{}/versions/", package_name); - if !state + repository: &ProxyRepository, + package: &str, +) -> Result { + if is_internal(state, package) { + return Err(ReadError::NotFound); + } + + let key = proxy_packument_key(&repository.name, package); + let negative_key = proxy_negative_key(&repository.name, package); + if state .storage - .list(&versions_prefix) + .stat(&negative_key) .await - .unwrap_or_default() - .is_empty() + .is_some_and(|meta| negative_fresh(meta.modified, repository.negative_ttl)) { - return; // already migrated - } - if let Some(embedded) = existing.get("versions").and_then(|v| v.as_object()) { - for (ver, data) in embedded { - if let Ok(bytes) = serde_json::to_vec(data) { - let vkey = format!("npm/{}/versions/{}.json", package_name, ver); - let _ = state.storage.put(&vkey, &bytes).await; - } - } - } - if let Some(tags) = existing.get("dist-tags").and_then(|d| d.as_object()) { - for (tag, ver) in tags { - if let Some(vs) = ver.as_str() { - let tkey = format!("npm/{}/dist-tags/{}", package_name, tag); - let _ = state.storage.put(&tkey, vs.as_bytes()).await; - } - } + return Err(ReadError::NotFound); } -} - -/// Highest version by a naive numeric semver comparison (a release outranks a prerelease at the -/// same core). Used only as the `latest` dist-tag fallback when publish did not set one. -fn max_semver<'a>(versions: impl Iterator) -> Option { - versions - .max_by(|a, b| semver_key(a).cmp(&semver_key(b))) - .cloned() -} -fn semver_key(v: &str) -> (u64, u64, u64, bool) { - let core = v.split(['-', '+']).next().unwrap_or(v); - let mut it = core.trim_start_matches('v').split('.'); - let n = |x: Option<&str>| x.and_then(|s| s.parse::().ok()).unwrap_or(0); - (n(it.next()), n(it.next()), n(it.next()), !v.contains('-')) -} + let cached = state.storage.get(&key).await.ok(); + let fresh = cached.is_some() + && state + .storage + .stat(&key) + .await + .is_some_and(|meta| negative_fresh(meta.modified, repository.metadata_ttl)); + if fresh { + return serde_json::from_slice(cached.as_ref().expect("cached when fresh")) + .map(PackumentRead::fresh) + .map_err(|_| ReadError::Corrupt); + } -/// Extract publish date for a specific version from cached npm metadata. -/// -/// npm metadata JSON has a `time` object mapping versions to ISO 8601 dates: -/// ```json -/// { "time": { "1.0.0": "2024-01-15T10:30:00.000Z" } } -/// ``` -async fn extract_npm_publish_date( - storage: &crate::storage::Storage, - metadata_key: &str, - version: &str, - trust_upstream: bool, -) -> Option { - // #513: when upstream dates are not trusted, derive age from NORA's own - // cache mtime instead of the (spoofable) upstream metadata date. Never fall - // back to the upstream date here — that would reopen the spoof vector. - if !trust_upstream { - return crate::curation::extract_mtime_as_publish_date(storage, metadata_key).await; + // A singleton Nora still receives concurrent cold misses. Serialize the + // refresh for this package so a slower, older upstream response cannot + // overwrite a newer one. + let refresh_lock = state.publish_lock(&format!("npm-proxy:{}:{package}", repository.name)); + let _refresh_guard = refresh_lock.lock().await; + let cached = state.storage.get(&key).await.ok(); + let fresh = cached.is_some() + && state + .storage + .stat(&key) + .await + .is_some_and(|meta| negative_fresh(meta.modified, repository.metadata_ttl)); + if fresh { + return serde_json::from_slice(cached.as_ref().expect("cached when fresh")) + .map(PackumentRead::fresh) + .map_err(|_| ReadError::Corrupt); } - let data = storage.get(metadata_key).await.ok()?; - let json: serde_json::Value = serde_json::from_slice(&data).ok()?; - let date_str = json.get("time")?.get(version)?.as_str()?; - crate::curation::parse_iso8601_to_unix(date_str) -} -fn with_content_type( - is_tarball: bool, - data: Bytes, -) -> (StatusCode, [(header::HeaderName, &'static str); 2], Bytes) { - let (content_type, cache_control) = if is_tarball { - ( - "application/octet-stream", - "public, max-age=31536000, immutable", - ) + let url = format!( + "{}/{}", + repository.url.trim_end_matches('/'), + upstream_package_path(package) + ); + let validators = if state.config.npm.revalidate { + read_validators(&state.storage, &key) + .await + .unwrap_or_default() } else { - ("application/json", "public, max-age=60, must-revalidate") + Validators::default() }; - - ( - StatusCode::OK, - [ - (header::CONTENT_TYPE, content_type), - (header::CACHE_CONTROL, cache_control), - ], - data, + let had_validators = validators.is_some(); + let fetched = proxy_fetch_conditional_with_validated_redirects( + &state.no_redirect_http_client, + &url, + Duration::from_secs(state.config.npm.proxy_timeout), + expose_opt(&repository.auth), + &validators, + &state.circuit_breaker, + RegistryType::Npm, + MAX_NPM_PROXY_REDIRECTS, + |next_url| validated_proxy_url(repository, next_url.as_str()).is_some(), ) -} - -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod tests { - use super::*; - - #[test] - fn test_rewrite_tarball_urls_regular_package() { - let metadata = serde_json::json!({ - "name": "lodash", - "versions": { - "4.17.21": { - "dist": { - "tarball": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "shasum": "abc123" - } - } - } - }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:5000", "https://registry.npmjs.org").unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - - assert_eq!( - json["versions"]["4.17.21"]["dist"]["tarball"], - "http://nora:5000/npm/lodash/-/lodash-4.17.21.tgz" - ); - assert_eq!(json["versions"]["4.17.21"]["dist"]["shasum"], "abc123"); - } + .await; - #[test] - fn test_rewrite_tarball_urls_scoped_package() { - let metadata = serde_json::json!({ - "name": "@babel/core", - "versions": { - "7.26.0": { - "dist": { - "tarball": "https://registry.npmjs.org/@babel/core/-/core-7.26.0.tgz", - "integrity": "sha512-test" - } + match fetched { + Ok(Revalidation::NotModified) => { + let Some(data) = cached else { + if had_validators { + crate::metrics::PROXY_REVALIDATION_ERRORS_TOTAL + .with_label_values(&["npm"]) + .inc(); } + return Err(ReadError::Unavailable); + }; + crate::metrics::PROXY_UPSTREAM_304_TOTAL + .with_label_values(&["npm"]) + .inc(); + crate::metrics::PROXY_REVALIDATION_BYTES_SAVED_TOTAL + .with_label_values(&["npm"]) + .inc_by(data.len() as u64); + // Touching the body after 304 is intentional: its mtime is the + // freshness marker, while the validator sidecar remains unchanged. + state + .storage + .put(&key, &data) + .await + .map_err(|_| ReadError::Unavailable)?; + serde_json::from_slice(&data) + .map(PackumentRead::fresh) + .map_err(|_| ReadError::Corrupt) + } + Ok(Revalidation::Modified { body, validators }) => { + let value = serde_json::from_slice::(&body) + .map_err(|_| ReadError::Corrupt)?; + state + .storage + .put(&key, &body) + .await + .map_err(|_| ReadError::Unavailable)?; + write_validators(&state.storage, &key, &validators).await; + state.repo_index.invalidate("npm"); + if state.storage.stat(&negative_key).await.is_some() { + let _ = state.storage.delete(&negative_key).await; } - }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:5000", "https://registry.npmjs.org").unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - - assert_eq!( - json["versions"]["7.26.0"]["dist"]["tarball"], - "http://nora:5000/npm/@babel/core/-/core-7.26.0.tgz" - ); - } - - #[test] - fn test_rewrite_tarball_urls_multiple_versions() { - let metadata = serde_json::json!({ - "name": "express", - "versions": { - "4.18.2": { "dist": { "tarball": "https://registry.npmjs.org/express/-/express-4.18.2.tgz" } }, - "4.19.0": { "dist": { "tarball": "https://registry.npmjs.org/express/-/express-4.19.0.tgz" } } + Ok(PackumentRead::fresh(value)) + } + Err(ProxyError::NotFound) => { + if had_validators { + crate::metrics::PROXY_REVALIDATION_ERRORS_TOTAL + .with_label_values(&["npm"]) + .inc(); } - }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = rewrite_tarball_urls( - &data, - "https://demo.getnora.io", - "https://registry.npmjs.org", - ) - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - - assert_eq!( - json["versions"]["4.18.2"]["dist"]["tarball"], - "https://demo.getnora.io/npm/express/-/express-4.18.2.tgz" - ); - assert_eq!( - json["versions"]["4.19.0"]["dist"]["tarball"], - "https://demo.getnora.io/npm/express/-/express-4.19.0.tgz" - ); + if repository.negative_ttl > 0 { + let _ = state.storage.put(&negative_key, b"not-found").await; + } + Err(ReadError::NotFound) + } + Err(ProxyError::CircuitOpen(name)) => { + if had_validators { + crate::metrics::PROXY_REVALIDATION_ERRORS_TOTAL + .with_label_values(&["npm"]) + .inc(); + } + if state.config.npm.serve_stale { + if let Some(data) = cached { + return serde_json::from_slice(&data) + .map(|value| PackumentRead { value, stale: true }) + .map_err(|_| ReadError::Corrupt); + } + } + Err(ReadError::CircuitOpen(name)) + } + Err(_) => { + if had_validators { + crate::metrics::PROXY_REVALIDATION_ERRORS_TOTAL + .with_label_values(&["npm"]) + .inc(); + } + if state.config.npm.serve_stale { + if let Some(data) = cached { + return serde_json::from_slice(&data) + .map(|value| PackumentRead { value, stale: true }) + .map_err(|_| ReadError::Corrupt); + } + } + Err(ReadError::Unavailable) + } } +} - #[test] - fn test_rewrite_tarball_urls_no_versions() { - let metadata = serde_json::json!({ "name": "empty-pkg" }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:5000", "https://registry.npmjs.org").unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - assert_eq!(json["name"], "empty-pkg"); +fn rewrite_packument_urls( + mut packument: serde_json::Value, + response_base: &str, + package: &str, + upstream_base: &str, +) -> serde_json::Value { + fn rewrite_upstream_values(value: &mut serde_json::Value, upstream: &str, public: &str) { + match value { + serde_json::Value::String(text) => { + let upstream = upstream.trim_end_matches('/'); + if text == upstream { + *text = public.trim_end_matches('/').to_string(); + } else if let Some(suffix) = text.strip_prefix(upstream) { + if suffix.starts_with('/') || suffix.starts_with('?') || suffix.starts_with('#') + { + *text = format!("{}{}", public.trim_end_matches('/'), suffix); + } + } + } + serde_json::Value::Array(values) => { + for value in values { + rewrite_upstream_values(value, upstream, public); + } + } + serde_json::Value::Object(object) => { + for value in object.values_mut() { + rewrite_upstream_values(value, upstream, public); + } + } + _ => {} + } } - #[test] - fn test_rewrite_invalid_json() { - assert!(rewrite_tarball_urls( - b"not json", - "http://nora:5000", - "https://registry.npmjs.org" - ) - .is_err()); + // Proxy-specific custom fields can contain absolute Nexus/upstream URLs. + // Rewrite complete string values with a path-boundary prefix; never byte + // replace arbitrary substrings inside descriptions or scripts. + rewrite_upstream_values(&mut packument, upstream_base, response_base); + if let Some(versions) = packument + .get_mut("versions") + .and_then(serde_json::Value::as_object_mut) + { + for (version, data) in versions { + set_tarball_url(data, response_base, package, version); + } } + packument +} - #[test] - fn test_valid_attachment_names() { - assert!(is_valid_attachment_name("lodash-4.17.21.tgz")); - assert!(is_valid_attachment_name("core-7.26.0.tgz")); - assert!(is_valid_attachment_name("my_package-1.0.0.tgz")); - assert!(is_valid_attachment_name("@scope-pkg-1.0.0.tgz")); - } +fn merge_packuments( + package: &str, + response_base: &str, + packuments: Vec, +) -> Result { + let mut result = serde_json::Map::new(); + let mut versions = serde_json::Map::new(); + let mut tags = serde_json::Map::new(); + let mut any = false; - #[test] - fn test_path_traversal_attachment_names() { - assert!(!is_valid_attachment_name("../../etc/passwd")); - assert!(!is_valid_attachment_name( - "../docker/nginx/manifests/latest.json" - )); - assert!(!is_valid_attachment_name("foo/bar.tgz")); - assert!(!is_valid_attachment_name("foo\\bar.tgz")); + for packument in packuments { + let Some(mut object) = packument.as_object().cloned() else { + return Err(ReadError::Corrupt); + }; + any = true; + if let Some(member_versions) = object + .remove("versions") + .and_then(|value| value.as_object().cloned()) + { + for (version, mut data) in member_versions { + if versions.contains_key(&version) { + continue; + } + set_tarball_url(&mut data, response_base, package, &version); + versions.insert(version, data); + } + } + if let Some(member_tags) = object + .remove("dist-tags") + .and_then(|value| value.as_object().cloned()) + { + for (tag, target) in member_tags { + tags.entry(tag).or_insert(target); + } + } + for (field, value) in object { + result.entry(field).or_insert(value); + } } - #[test] - fn test_empty_and_null_attachment_names() { - assert!(!is_valid_attachment_name("")); - assert!(!is_valid_attachment_name("foo\0bar.tgz")); + if !any { + return Err(ReadError::NotFound); } + result.insert( + "name".to_string(), + serde_json::Value::String(package.to_string()), + ); + result.insert("versions".to_string(), serde_json::Value::Object(versions)); + result.insert("dist-tags".to_string(), serde_json::Value::Object(tags)); + Ok(serde_json::Value::Object(result)) +} - #[test] - fn test_with_content_type_tarball() { - let data = Bytes::from("tarball-data"); - let (status, headers, body) = with_content_type(true, data.clone()); - assert_eq!(status, StatusCode::OK); - assert_eq!(headers[0].1, "application/octet-stream"); - assert_eq!(body, data); +async fn group_packument( + state: &AppState, + members: &[String], + package: &str, + response_base: &str, +) -> Result { + let mut packuments = Vec::new(); + let mut stale = false; + for member in members { + let Some(repository) = state.config.npm.repository(member).cloned() else { + continue; + }; + let result = match repository { + NpmRepository::Hosted { name, .. } => { + hosted_packument(state, &name, package, response_base) + .await + .map(PackumentRead::fresh) + } + NpmRepository::Proxy { .. } if is_internal(state, package) => Err(ReadError::NotFound), + NpmRepository::Proxy { .. } => { + let proxy = configured_proxy(state, &repository).expect("proxy config"); + proxy_packument_raw(state, &proxy, package) + .await + .map(|read| PackumentRead { + value: rewrite_packument_urls( + read.value, + response_base, + package, + &proxy.url, + ), + stale: read.stale, + }) + } + NpmRepository::Group { .. } => continue, + }; + match result { + Ok(packument) => { + stale |= packument.stale; + packuments.push(packument.value); + } + Err(ReadError::NotFound) => {} + Err(error) if packuments.is_empty() => return Err(error), + // A later member cannot override data already selected from an + // earlier member. Return the safe hosted/warm prefix rather than + // turning a lower-priority proxy outage into hosted downtime. + Err(_) => break, + } } - - #[test] - fn test_with_content_type_json() { - let data = Bytes::from("json-data"); - let (status, headers, body) = with_content_type(false, data.clone()); - assert_eq!(status, StatusCode::OK); - assert_eq!(headers[0].1, "application/json"); - assert_eq!(body, data); + if packuments.is_empty() { + return Err(ReadError::NotFound); } + merge_packuments(package, response_base, packuments).map(|value| PackumentRead { value, stale }) +} - #[test] - fn test_rewrite_tarball_urls_trailing_slash() { - let metadata = serde_json::json!({ - "name": "test", - "versions": { - "1.0.0": { - "dist": { - "tarball": "https://registry.npmjs.org/test/-/test-1.0.0.tgz" +async fn target_packument( + state: &AppState, + target: &RepositoryTarget, + package: &str, + response_base: &str, +) -> Result { + match target { + RepositoryTarget::Named(NpmRepository::Hosted { name, .. }) => { + hosted_packument(state, name, package, response_base) + .await + .map(PackumentRead::fresh) + } + RepositoryTarget::Named(repository @ NpmRepository::Proxy { .. }) => { + let proxy = configured_proxy(state, repository).expect("proxy config"); + proxy_packument_raw(state, &proxy, package) + .await + .map(|read| PackumentRead { + value: rewrite_packument_urls(read.value, response_base, package, &proxy.url), + stale: read.stale, + }) + } + RepositoryTarget::Named(NpmRepository::Group { members, .. }) => { + group_packument(state, members, package, response_base).await + } + RepositoryTarget::Legacy => { + let mut packuments = Vec::new(); + let mut stale = false; + match hosted_packument(state, LEGACY_HOSTED, package, response_base).await { + Ok(hosted) => packuments.push(hosted), + Err(ReadError::NotFound) => {} + Err(error) => return Err(error), + } + if !is_internal(state, package) { + if let Some(proxy) = legacy_proxy(state) { + match proxy_packument_raw(state, &proxy, package).await { + Ok(read) => { + stale |= read.stale; + packuments.push(rewrite_packument_urls( + read.value, + response_base, + package, + &proxy.url, + )) + } + Err(ReadError::NotFound) => {} + Err(error) if packuments.is_empty() => return Err(error), + Err(_) => {} } } } - }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:5000/", "https://registry.npmjs.org/") - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - let tarball = json["versions"]["1.0.0"]["dist"]["tarball"] - .as_str() - .unwrap(); - assert!(tarball.starts_with("http://nora:5000/npm/")); + if packuments.is_empty() { + return Err(ReadError::NotFound); + } + merge_packuments(package, response_base, packuments) + .map(|value| PackumentRead { value, stale }) + } } +} - #[test] - fn test_rewrite_tarball_urls_preserves_other_fields() { - let metadata = serde_json::json!({ - "name": "test", - "description": "A test package", - "versions": { - "1.0.0": { - "dist": { - "tarball": "https://registry.npmjs.org/test/-/test-1.0.0.tgz", - "shasum": "abc123" - }, - "dependencies": {"lodash": "^4.0.0"} - } - } - }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:5000", "https://registry.npmjs.org").unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - assert_eq!(json["description"], "A test package"); - assert_eq!(json["versions"]["1.0.0"]["dist"]["shasum"], "abc123"); +fn read_error_response(error: ReadError) -> Response { + match error { + ReadError::NotFound => StatusCode::NOT_FOUND.into_response(), + ReadError::CircuitOpen(name) => circuit_open_response(&name), + ReadError::Unavailable => StatusCode::BAD_GATEWAY.into_response(), + ReadError::Corrupt => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + ReadError::SearchScanLimit => ( + StatusCode::BAD_GATEWAY, + "upstream npm search exceeded the bounded scan budget", + ) + .into_response(), } +} - // ── Safety net tests (#439) ── +async fn hosted_has_version( + state: &AppState, + repository: &str, + package: &str, + version: &str, +) -> Result { + optional_storage_get(state, &hosted_version_key(repository, package, version)) + .await + .map(|manifest| manifest.is_some()) +} - #[test] - fn test_replace_upstream_bytes_basic() { - let data = b"https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz"; - let result = - replace_upstream_bytes(data, "https://registry.npmjs.org", "http://nora:5000/npm"); - assert_eq!( - String::from_utf8(result).unwrap(), - "http://nora:5000/npm/lodash/-/lodash-4.17.21.tgz" +fn dist_digest_matches(data: &[u8], version_data: &serde_json::Value) -> bool { + let Some(dist) = version_data.get("dist") else { + return false; + }; + let mut verified = false; + if let Some(shasum) = dist.get("shasum").and_then(|value| value.as_str()) { + // SHA-1 is required by the npm packument protocol for legacy clients. + if !shasum.eq_ignore_ascii_case(&hex::encode(sha1::Sha1::digest(data))) { + return false; + } + verified = true; + } + if let Some(integrity) = dist.get("integrity").and_then(|value| value.as_str()) { + let expected = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(data)) ); + let has_sha512 = integrity + .split_ascii_whitespace() + .any(|candidate| candidate.starts_with("sha512-")); + if has_sha512 + && !integrity + .split_ascii_whitespace() + .any(|candidate| candidate == expected) + { + return false; + } + verified |= has_sha512; } + verified +} - #[test] - fn test_replace_upstream_bytes_no_match() { - let data = b"no upstream urls here"; - let result = replace_upstream_bytes(data, "https://registry.npmjs.org", "http://nora/npm"); - assert_eq!(result, data); +async fn serve_hosted_tarball( + state: &AppState, + repository: &str, + package: &str, + filename: &str, + publish_date: Option, +) -> Response { + let Some(version) = crate::curation::parse_npm_tarball_version(package, filename) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let manifest = match state + .storage + .get(&hosted_version_key(repository, package, &version)) + .await + { + Ok(manifest) => manifest, + Err(error) => return read_error_response(storage_read_error(error)), + }; + let Ok(version_data) = serde_json::from_slice::(&manifest) else { + crate::metrics::METADATA_CORRUPT_TOTAL + .with_label_values(&["npm"]) + .inc(); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let Some(key) = + crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) + else { + crate::metrics::METADATA_CORRUPT_TOTAL + .with_label_values(&["npm"]) + .inc(); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let data = match state.storage.get(&key).await { + Ok(data) => data, + Err(StorageError::IntegrityViolation) => { + return StatusCode::INTERNAL_SERVER_ERROR.into_response() + } + Err(_) => return StatusCode::BAD_GATEWAY.into_response(), + }; + if !dist_digest_matches(&data, &version_data) { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } + curated_tarball_response(state, package, &version, data, "hosted", publish_date) +} - #[test] - fn test_replace_upstream_bytes_empty_upstream() { - let data = b"https://registry.npmjs.org/test"; - let result = replace_upstream_bytes(data, "", "http://nora/npm"); - assert_eq!(result, data); +async fn serve_proxy_tarball( + state: &AppState, + repository: &ProxyRepository, + package: &str, + filename: &str, + publish_date: Option, +) -> Response { + if is_internal(state, package) { + return StatusCode::NOT_FOUND.into_response(); } - - #[test] - fn test_replace_upstream_bytes_multiple_occurrences() { - let data = b"url1: https://registry.npmjs.org/a url2: https://registry.npmjs.org/b"; - let result = - replace_upstream_bytes(data, "https://registry.npmjs.org", "http://nora:5000/npm"); - let s = String::from_utf8(result).unwrap(); - assert!(!s.contains("registry.npmjs.org")); - assert!(s.contains("http://nora:5000/npm/a")); - assert!(s.contains("http://nora:5000/npm/b")); + let Some(version) = crate::curation::parse_npm_tarball_version(package, filename) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let packument = match proxy_packument_raw(state, repository, package).await { + Ok(value) => value, + Err(error) => return read_error_response(error), + }; + let Some(version_data) = packument + .value + .get("versions") + .and_then(|value| value.get(&version)) + else { + return StatusCode::NOT_FOUND.into_response(); + }; + let Some(candidate_url) = version_data + .get("dist") + .and_then(|value| value.get("tarball")) + .and_then(|value| value.as_str()) + else { + return StatusCode::BAD_GATEWAY.into_response(); + }; + let Some(url) = validated_proxy_url(repository, candidate_url) else { + tracing::warn!( + repository = %repository.name, + package, + version, + "npm proxy rejected tarball URL outside configured upstream origin/base path" + ); + return StatusCode::BAD_GATEWAY.into_response(); + }; + let key = proxy_tarball_key(&repository.name, package, filename); + let cached = state.storage.get(&key).await.ok(); + if let Some(data) = &cached { + if dist_digest_matches(data, version_data) { + state.metrics.record_cache_hit("npm"); + state.activity.push(ActivityEntry::new( + ActionType::CacheHit, + package.to_string(), + RegistryType::Npm, + "CACHE", + )); + state.audit.log(AuditEntry::new( + "cache_hit", + "api", + package, + "npm", + &repository.name, + )); + return curated_tarball_response( + state, + package, + &version, + data.clone(), + "cache", + publish_date, + ); + } } - #[test] - fn test_rewrite_tarball_urls_safety_net_catches_unknown_fields() { - // Simulate metadata with upstream URL in an unexpected field - let metadata = serde_json::json!({ - "name": "test", - "versions": { - "1.0.0": { - "dist": { - "tarball": "https://registry.npmjs.org/test/-/test-1.0.0.tgz" - }, - "_resolved": "https://registry.npmjs.org/test/-/test-1.0.0.tgz" + // Packument refresh has its own package lock, but the immutable tarball is + // a separate cold miss. Preserve the existing single-flight behavior so a + // CI fan-out performs one upstream transfer and one cache create. + let circuit_name = std::sync::Arc::new(parking_lot::Mutex::new(None::)); + let observed_circuit = std::sync::Arc::clone(&circuit_name); + let fetch = || async { + match proxy_fetch_with_validated_redirects( + &state.no_redirect_http_client, + url.as_str(), + Duration::from_secs(state.config.npm.proxy_timeout), + expose_opt(&repository.auth), + &state.circuit_breaker, + RegistryType::Npm, + MAX_NPM_PROXY_REDIRECTS, + |next_url| validated_proxy_url(repository, next_url.as_str()).is_some(), + ) + .await + { + Ok(data) if dist_digest_matches(&data, version_data) => { + let bytes = Bytes::from(data); + if state.storage.put(&key, &bytes).await.is_err() { + return None; } + state.repo_index.invalidate("npm"); + state.metrics.record_cache_miss("npm"); + state.activity.push(ActivityEntry::new( + ActionType::ProxyFetch, + package.to_string(), + RegistryType::Npm, + "PROXY", + )); + state.audit.log(AuditEntry::new( + "proxy_fetch", + "api", + package, + "npm", + &repository.name, + )); + Some(bytes) } - }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:5000", "https://registry.npmjs.org").unwrap(); - let body = String::from_utf8(result).unwrap(); - // Safety net should catch the _resolved field too - assert!( - !body.contains("registry.npmjs.org"), - "upstream URL leaked through _resolved field: {}", - &body[..body.len().min(500)] + Ok(_) => { + tracing::warn!( + repository = %repository.name, + package, + version, + "npm proxy tarball digest mismatch" + ); + None + } + Err(ProxyError::CircuitOpen(name)) => { + *observed_circuit.lock() = Some(name); + None + } + Err(error) => { + tracing::debug!( + repository = %repository.name, + package, + version, + error = ?error, + "npm proxy tarball fetch failed" + ); + None + } + } + }; + let fetched = if state.config.server.proxy_coalesce { + state + .proxy_coalesce + .coalesced( + &key, + "npm", + crate::proxy_coalesce::follower_budget(state.config.npm.proxy_timeout), + fetch, + ) + .await + } else { + fetch().await + }; + if let Some(data) = fetched { + return curated_tarball_response( + state, + package, + &version, + data, + url.as_str(), + publish_date, ); } + let circuit_name = circuit_name.lock().clone(); + if let Some(name) = circuit_name { + circuit_open_response(&name) + } else { + StatusCode::BAD_GATEWAY.into_response() + } +} - #[test] - fn test_rewrite_tarball_urls_preserves_non_upstream_urls() { - // homepage and repository.url should NOT be mangled - let metadata = serde_json::json!({ - "name": "test", - "homepage": "https://github.com/test/test", - "repository": { "url": "git+https://github.com/test/test.git" }, - "versions": { - "1.0.0": { - "dist": { - "tarball": "https://registry.npmjs.org/test/-/test-1.0.0.tgz" +async fn group_tarball( + state: &AppState, + members: &[String], + package: &str, + filename: &str, + publish_date: Option, +) -> Response { + let Some(version) = crate::curation::parse_npm_tarball_version(package, filename) else { + return StatusCode::NOT_FOUND.into_response(); + }; + for member in members { + let Some(repository) = state.config.npm.repository(member).cloned() else { + continue; + }; + match repository { + NpmRepository::Hosted { name, .. } => { + match hosted_has_version(state, &name, package, &version).await { + Ok(true) => { + // Once a member claims the version, its tarball is the + // only legal origin. Do not fall through after an + // incomplete or corrupt member. + return serve_hosted_tarball(state, &name, package, filename, publish_date) + .await; } + Ok(false) => {} + Err(error) => return read_error_response(error), } } - }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:5000", "https://registry.npmjs.org").unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - assert_eq!(json["homepage"], "https://github.com/test/test"); - assert_eq!( - json["repository"]["url"], - "git+https://github.com/test/test.git" - ); + NpmRepository::Proxy { .. } if !is_internal(state, package) => { + let proxy = configured_proxy(state, &repository).expect("proxy config"); + match proxy_packument_raw(state, &proxy, package).await { + Ok(packument) + if packument + .value + .get("versions") + .and_then(|value| value.get(&version)) + .is_some() => + { + return serve_proxy_tarball(state, &proxy, package, filename, publish_date) + .await; + } + Ok(_) | Err(ReadError::NotFound) => {} + Err(error) => return read_error_response(error), + } + } + _ => {} + } } + StatusCode::NOT_FOUND.into_response() +} - #[test] - fn test_is_valid_attachment_name_valid() { - assert!(is_valid_attachment_name("package-1.0.0.tgz")); - assert!(is_valid_attachment_name("@scope-pkg-2.0.tgz")); - assert!(is_valid_attachment_name("my_pkg.tgz")); +async fn target_tarball( + state: &AppState, + target: &RepositoryTarget, + package: &str, + filename: &str, + publish_date: Option, +) -> Response { + match target { + RepositoryTarget::Named(NpmRepository::Hosted { name, .. }) => { + serve_hosted_tarball(state, name, package, filename, publish_date).await + } + RepositoryTarget::Named(repository @ NpmRepository::Proxy { .. }) => { + let proxy = configured_proxy(state, repository).expect("proxy config"); + serve_proxy_tarball(state, &proxy, package, filename, publish_date).await + } + RepositoryTarget::Named(NpmRepository::Group { members, .. }) => { + group_tarball(state, members, package, filename, publish_date).await + } + RepositoryTarget::Legacy => { + let Some(version) = crate::curation::parse_npm_tarball_version(package, filename) + else { + return StatusCode::NOT_FOUND.into_response(); + }; + match hosted_has_version(state, LEGACY_HOSTED, package, &version).await { + Ok(true) => { + return serve_hosted_tarball( + state, + LEGACY_HOSTED, + package, + filename, + publish_date, + ) + .await + } + Ok(false) => {} + Err(error) => return read_error_response(error), + } + if let Some(proxy) = legacy_proxy(state) { + serve_proxy_tarball(state, &proxy, package, filename, publish_date).await + } else { + StatusCode::NOT_FOUND.into_response() + } + } } +} - #[test] - fn test_is_valid_attachment_name_traversal() { - assert!(!is_valid_attachment_name("../etc/passwd")); - assert!(!is_valid_attachment_name("foo/../bar")); - } +#[derive(Debug, Clone)] +struct SearchRequest { + text: String, + from: usize, + size: usize, +} - #[test] - fn test_is_valid_attachment_name_slash() { - assert!(!is_valid_attachment_name("path/file.tgz")); - assert!(!is_valid_attachment_name("path\\file.tgz")); +fn parse_search_request(query: Option<&str>) -> SearchRequest { + let mut request = SearchRequest { + text: String::new(), + from: 0, + size: 20, + }; + let Ok(url) = reqwest::Url::parse(&format!("http://localhost/?{}", query.unwrap_or_default())) + else { + return request; + }; + for (key, value) in url.query_pairs() { + match key.as_ref() { + "text" => request.text = value.into_owned(), + "from" => request.from = value.parse().unwrap_or(0), + "size" => request.size = value.parse().unwrap_or(20), + _ => {} + } } + request.size = request.size.clamp(1, 250); + request +} - #[test] - fn test_is_valid_attachment_name_null_byte() { - assert!(!is_valid_attachment_name("file\0.tgz")); +fn search_query_with_window(query: Option<&str>, from: usize, size: usize) -> String { + let mut url = reqwest::Url::parse("http://localhost/").expect("static URL"); + { + let mut pairs = url.query_pairs_mut(); + if let Some(query) = query { + if let Ok(source) = reqwest::Url::parse(&format!("http://localhost/?{query}")) { + for (key, value) in source.query_pairs() { + if key != "from" && key != "size" { + pairs.append_pair(&key, &value); + } + } + } + } + pairs.append_pair("from", &from.to_string()); + pairs.append_pair("size", &size.clamp(1, 250).to_string()); } + url.query().unwrap_or_default().to_string() +} - #[test] - fn test_is_valid_attachment_name_empty() { - assert!(!is_valid_attachment_name("")); +fn search_matches(packument: &serde_json::Value, latest: &serde_json::Value, text: &str) -> bool { + let terms: Vec = text + .split_whitespace() + .map(|term| term.to_ascii_lowercase()) + .collect(); + if terms.is_empty() { + return true; + } + let mut fields = Vec::new(); + for field in ["name", "description"] { + if let Some(value) = packument + .get(field) + .or_else(|| latest.get(field)) + .and_then(|value| value.as_str()) + { + fields.push(value.to_ascii_lowercase()); + } + } + for source in [packument, latest] { + if let Some(keywords) = source.get("keywords") { + match keywords { + serde_json::Value::Array(values) => fields.extend( + values + .iter() + .filter_map(|value| value.as_str()) + .map(str::to_ascii_lowercase), + ), + serde_json::Value::String(value) => fields.push(value.to_ascii_lowercase()), + _ => {} + } + } } + terms + .iter() + .all(|term| fields.iter().any(|field| field.contains(term))) +} - #[test] - fn test_is_valid_attachment_name_special_chars() { - assert!(!is_valid_attachment_name("file name.tgz")); // space - assert!(!is_valid_attachment_name("file;cmd.tgz")); // semicolon +fn search_targets_internal(state: &AppState, text: &str) -> bool { + let text = text.trim(); + if text.is_empty() { + return false; + } + if text_contains_internal_package(text, &state.curation().curation_engine) { + return true; } + text.split_whitespace().any(|term| { + let term = term.trim_matches(|character| { + matches!( + character, + '"' | '\'' | '(' | ')' | '[' | ']' | '{' | '}' | ',' + ) + }); + if is_internal(state, term) { + return true; + } + let Some(scope) = term.strip_prefix("scope:") else { + return false; + }; + let scope = if scope.starts_with('@') { + scope.to_string() + } else { + format!("@{scope}") + }; + is_internal(state, &format!("{scope}/__nora_search__")) + }) } -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod integration_tests { - use crate::test_helpers::{ - body_bytes, create_test_context, create_test_context_with_auth, send, send_with_headers, +fn search_query_targets_internal(state: &AppState, query: Option<&str>) -> bool { + let Some(query) = query.filter(|query| !query.is_empty()) else { + return false; }; + let Ok(url) = reqwest::Url::parse(&format!("http://localhost/?{query}")) else { + // A query we cannot classify must never be forwarded: doing so would + // turn parser disagreement into a namespace-isolation bypass. + return true; + }; + url.query_pairs().any(|(key, value)| { + (key == "text" && search_targets_internal(state, &value)) + || text_contains_internal_package(&key, &state.curation().curation_engine) + || text_contains_internal_package(&value, &state.curation().curation_engine) + }) +} + +async fn hosted_search_object( + state: &AppState, + repository: &str, + package: &str, + response_base: &str, + request: &SearchRequest, +) -> Result, ReadError> { + let version_prefix = format!("{}/versions/", package_prefix(repository, package)); + let version_keys = state + .storage + .list(&version_prefix) + .await + .map_err(|_| ReadError::Unavailable)?; + if version_keys.len() > NPM_SEARCH_SCAN_RESULT_CAP { + return Err(ReadError::SearchScanLimit); + } + let versions: HashSet = version_keys + .iter() + .filter_map(|key| { + key.strip_prefix(&version_prefix)? + .strip_suffix(".json") + .filter(|version| !version.is_empty() && !version.contains('/')) + .map(str::to_string) + }) + .collect(); + if versions.is_empty() { + return Ok(None); + } + let tagged_latest = + match optional_storage_get(state, &hosted_tag_key(repository, package, "latest")).await? { + Some(bytes) => Some(read_string(bytes).ok_or(ReadError::Corrupt)?), + None => None, + }; + let latest_version = tagged_latest + .filter(|version| versions.contains(version)) + .or_else(|| { + versions + .iter() + .filter_map(|version| { + semver::Version::parse(version.trim_start_matches('v')) + .ok() + .map(|parsed| (parsed, version.clone())) + }) + .max_by(|left, right| left.0.cmp(&right.0)) + .map(|(_, version)| version) + }); + let Some(version) = latest_version else { + return Ok(None); + }; + let manifest = state + .storage + .get(&hosted_version_key(repository, package, &version)) + .await + .map_err(storage_read_error)?; + let latest = + serde_json::from_slice::(&manifest).map_err(|_| ReadError::Corrupt)?; + let packument = + match optional_storage_get(state, &hosted_package_key(repository, package)).await? { + Some(bytes) => serde_json::from_slice::(&bytes) + .map_err(|_| ReadError::Corrupt)?, + None => serde_json::json!({"name": package}), + }; + if !search_matches(&packument, &latest, &request.text) { + return Ok(None); + } + let mut package_data = serde_json::Map::new(); + package_data.insert( + "name".to_string(), + serde_json::Value::String(package.to_string()), + ); + package_data.insert("version".to_string(), serde_json::Value::String(version)); + for field in ["description", "keywords", "publisher", "maintainers"] { + if let Some(value) = packument.get(field).or_else(|| latest.get(field)) { + package_data.insert(field.to_string(), value.clone()); + } + } + package_data + .entry("maintainers".to_string()) + .or_insert_with(|| serde_json::Value::Array(Vec::new())); + package_data.insert( + "links".to_string(), + serde_json::json!({"npm": format!("{response_base}/{package}")}), + ); + Ok(Some(serde_json::json!({ + "package": package_data, + "score": { + "final": 1.0, + "detail": {"quality": 1.0, "popularity": 0.0, "maintenance": 1.0} + }, + "searchScore": 1.0 + }))) +} + +async fn hosted_search_objects( + state: &AppState, + repository: &str, + response_base: &str, + request: &SearchRequest, + deadline: Instant, +) -> Result, ReadError> { + // Protocol search needs a strict index read: stale-on-error is useful for + // the UI, but would silently turn a hosted member failure into an empty + // successful protocol result. + let index = tokio::time::timeout_at( + tokio::time::Instant::from_std(deadline), + state.repo_index.get_strict("npm", &state.storage), + ) + .await + .map_err(|_| ReadError::SearchScanLimit)? + .map_err(|_| ReadError::Unavailable)?; + let prefix = format!("repositories/{repository}/"); + let packages: Vec = index + .iter() + .filter_map(|entry| entry.name.strip_prefix(&prefix).map(str::to_string)) + .collect(); + if packages.len() > NPM_SEARCH_SCAN_RESULT_CAP { + return Err(ReadError::SearchScanLimit); + } + let mut objects = Vec::new(); + for package in packages { + let object = tokio::time::timeout_at( + tokio::time::Instant::from_std(deadline), + hosted_search_object(state, repository, &package, response_base, request), + ) + .await + .map_err(|_| ReadError::SearchScanLimit)??; + if let Some(object) = object { + objects.push(object); + } + } + objects.sort_by(|left, right| { + left["package"]["name"] + .as_str() + .cmp(&right["package"]["name"].as_str()) + }); + Ok(objects) +} + +#[derive(Debug)] +struct ProxySearchPage { + objects: Vec, + raw_count: usize, + total: usize, + total_is_approximate: bool, +} + +fn public_proxy_search_page( + state: &AppState, + response: serde_json::Value, +) -> Result { + let Some(objects) = response.get("objects").and_then(|value| value.as_array()) else { + return Err(ReadError::Corrupt); + }; + let Some(total) = response + .get("total") + .and_then(|value| value.as_u64()) + .and_then(|value| usize::try_from(value).ok()) + else { + return Err(ReadError::Corrupt); + }; + let public = objects + .iter() + .filter(|object| { + object + .get("package") + .and_then(|package| package.get("name")) + .and_then(|value| value.as_str()) + .is_some_and(|package| !is_internal(state, package)) + }) + .cloned() + .collect(); + Ok(ProxySearchPage { + objects: public, + raw_count: objects.len(), + total, + total_is_approximate: false, + }) +} + +async fn proxy_search_page( + state: &AppState, + repository: &ProxyRepository, + query: Option<&str>, +) -> Result { + let candidate = match query { + Some(query) if !query.is_empty() => format!("-/v1/search?{query}"), + _ => "-/v1/search".to_string(), + }; + let Some(url) = validated_proxy_url(repository, &candidate) else { + return Err(ReadError::Unavailable); + }; + let body = proxy_fetch_with_validated_redirects_bounded( + &state.no_redirect_http_client, + url.as_str(), + Duration::from_secs(state.config.npm.proxy_timeout), + expose_opt(&repository.auth), + &state.circuit_breaker, + RegistryType::Npm, + MAX_NPM_PROXY_REDIRECTS, + NPM_SEARCH_BODY_CAP, + |next_url| validated_proxy_url(repository, next_url.as_str()).is_some(), + ) + .await + .map_err(|error| match error { + ProxyError::NotFound => ReadError::NotFound, + ProxyError::CircuitOpen(name) => ReadError::CircuitOpen(name), + _ => ReadError::Unavailable, + })?; + let response = serde_json::from_slice(&body).map_err(|_| ReadError::Corrupt)?; + public_proxy_search_page(state, response) +} + +async fn proxy_search_page_before( + state: &AppState, + repository: &ProxyRepository, + query: Option<&str>, + deadline: Instant, +) -> Result { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return Err(ReadError::SearchScanLimit); + }; + tokio::time::timeout(remaining, proxy_search_page(state, repository, query)) + .await + .map_err(|_| ReadError::SearchScanLimit)? +} + +async fn proxy_search_window( + state: &AppState, + repository: &ProxyRepository, + query: Option<&str>, + request: &SearchRequest, + scan_deadline: Option, +) -> Result { + let filter_active = crate::curation::namespace_filter_active(&state.curation().curation_engine); + if !filter_active { + let query = search_query_with_window(query, request.from, request.size); + return proxy_search_page(state, repository, Some(&query)).await; + } + let deadline = scan_deadline.ok_or(ReadError::SearchScanLimit)?; + let mut cursor = 0usize; + let mut total = None; + let needed = request.from.saturating_add(request.size); + let mut objects = Vec::with_capacity(needed.min(250)); + let mut scanned = 0usize; + let mut pages = 0usize; + while objects.len() < needed { + if pages >= NPM_SEARCH_SCAN_PAGE_CAP || scanned >= NPM_SEARCH_SCAN_RESULT_CAP { + return Err(ReadError::SearchScanLimit); + } + let page_size = (NPM_SEARCH_SCAN_RESULT_CAP - scanned).min(250); + let query = search_query_with_window(query, cursor, page_size); + let page = proxy_search_page_before(state, repository, Some(&query), deadline).await?; + pages = pages.saturating_add(1); + let upstream_total = *total.get_or_insert(page.total); + let raw_count = page.raw_count; + scanned = scanned + .checked_add(raw_count) + .filter(|scanned| *scanned <= NPM_SEARCH_SCAN_RESULT_CAP) + .ok_or(ReadError::SearchScanLimit)?; + let remaining = needed - objects.len(); + objects.extend(page.objects.into_iter().take(remaining)); + if raw_count == 0 || cursor.saturating_add(raw_count) >= upstream_total { + break; + } + cursor = cursor.saturating_add(raw_count); + } + let objects: Vec = objects + .into_iter() + .skip(request.from) + .take(request.size) + .collect(); + Ok(ProxySearchPage { + raw_count: objects.len(), + objects, + total: total.unwrap_or(0), + total_is_approximate: filter_active, + }) +} + +async fn proxy_search_group_prefix( + state: &AppState, + repository: &ProxyRepository, + query: Option<&str>, + already_seen: &HashSet, + already_collected: usize, + needed: usize, + deadline: Instant, +) -> Result { + let mut cursor = 0usize; + let mut total = None; + let mut objects = Vec::new(); + let mut seen = already_seen.clone(); + let mut scanned = 0usize; + let mut pages = 0usize; + loop { + if pages >= NPM_SEARCH_SCAN_PAGE_CAP || scanned >= NPM_SEARCH_SCAN_RESULT_CAP { + return Err(ReadError::SearchScanLimit); + } + let query = search_query_with_window( + query, + cursor, + (NPM_SEARCH_SCAN_RESULT_CAP - scanned).min(250), + ); + let page = proxy_search_page_before(state, repository, Some(&query), deadline).await?; + pages = pages.saturating_add(1); + let upstream_total = *total.get_or_insert(page.total); + let raw_count = page.raw_count; + scanned = scanned + .checked_add(raw_count) + .filter(|scanned| *scanned <= NPM_SEARCH_SCAN_RESULT_CAP) + .ok_or(ReadError::SearchScanLimit)?; + for object in page.objects { + let Some(package) = object + .get("package") + .and_then(|package| package.get("name")) + .and_then(|value| value.as_str()) + else { + continue; + }; + if seen.insert(package.to_string()) { + objects.push(object); + } + } + if already_collected.saturating_add(objects.len()) >= needed + || cursor.saturating_add(raw_count) >= upstream_total + { + break; + } + if raw_count == 0 { + return Err(ReadError::Corrupt); + } + cursor = cursor.saturating_add(raw_count); + } + Ok(ProxySearchPage { + raw_count: objects.len(), + objects, + total: total.unwrap_or(0), + total_is_approximate: true, + }) +} + +fn search_response( + headers: &HeaderMap, + objects: Vec, + request: &SearchRequest, + total: usize, + already_paged: bool, + total_is_approximate: bool, +) -> Response { + let page = if already_paged { + objects.into_iter().take(request.size).collect::>() + } else { + objects + .into_iter() + .skip(request.from) + .take(request.size) + .collect::>() + }; + json_response( + headers, + &serde_json::json!({ + "objects": page, + "total": total, + "time": "0ms", + "totalIsApproximate": total_is_approximate + }), + ) +} + +async fn handle_search( + state: &AppState, + target: &RepositoryTarget, + response_base: &str, + headers: &HeaderMap, + query: Option<&str>, +) -> Response { + let request = parse_search_request(query); + let internal_only = search_query_targets_internal(state, query); + let needed = request.from.saturating_add(request.size); + let filter_active = crate::curation::namespace_filter_active(&state.curation().curation_engine); + let needs_bounded_proxy_scan = match target { + RepositoryTarget::Named(NpmRepository::Proxy { .. }) => filter_active && !internal_only, + RepositoryTarget::Named(NpmRepository::Group { members, .. }) => { + !internal_only + && members.iter().any(|member| { + matches!( + state.config.npm.repository(member), + Some(NpmRepository::Proxy { .. }) + ) + }) + } + RepositoryTarget::Legacy => !internal_only && legacy_proxy(state).is_some(), + RepositoryTarget::Named(NpmRepository::Hosted { .. }) => false, + }; + if needs_bounded_proxy_scan && needed > NPM_SEARCH_SCAN_RESULT_CAP { + return ( + StatusCode::BAD_REQUEST, + "npm search from + size exceeds the 10000-result group/filter scan window", + ) + .into_response(); + } + let scan_deadline = Instant::now() + NPM_SEARCH_SCAN_TIMEOUT; + let result = match target { + RepositoryTarget::Named(NpmRepository::Hosted { name, .. }) => { + hosted_search_objects(state, name, response_base, &request, scan_deadline) + .await + .map(|objects| { + let total = objects.len(); + (objects, total, false, false) + }) + } + RepositoryTarget::Named(repository @ NpmRepository::Proxy { .. }) => { + if internal_only { + Ok((Vec::new(), 0, true, false)) + } else { + let proxy = configured_proxy(state, repository).expect("proxy config"); + proxy_search_window( + state, + &proxy, + query, + &request, + filter_active.then_some(scan_deadline), + ) + .await + .map(|page| (page.objects, page.total, true, page.total_is_approximate)) + } + } + RepositoryTarget::Named(NpmRepository::Group { members, .. }) => { + let mut merged = Vec::new(); + let mut seen = HashSet::new(); + let mut first_error = None; + let mut any_success = false; + let mut total_upper_bound = 0usize; + let mut has_proxy = false; + let mut member_failed = false; + for member in members { + let Some(repository) = state.config.npm.repository(member).cloned() else { + continue; + }; + let member_page = match repository { + NpmRepository::Hosted { name, .. } => { + hosted_search_objects(state, &name, response_base, &request, scan_deadline) + .await + .map(|objects| { + let total = objects.len(); + (objects, total) + }) + } + repository @ NpmRepository::Proxy { .. } if !internal_only => { + has_proxy = true; + let proxy = configured_proxy(state, &repository).expect("proxy config"); + proxy_search_group_prefix( + state, + &proxy, + query, + &seen, + merged.len(), + needed, + scan_deadline, + ) + .await + .map(|page| (page.objects, page.total)) + } + NpmRepository::Proxy { .. } => continue, + NpmRepository::Group { .. } => continue, + }; + match member_page { + Ok((objects, member_total)) => { + any_success = true; + total_upper_bound = total_upper_bound.saturating_add(member_total); + for object in objects { + let Some(package) = object + .get("package") + .and_then(|package| package.get("name")) + .and_then(|value| value.as_str()) + else { + continue; + }; + if seen.insert(package.to_string()) { + merged.push(object); + } + } + } + Err(error @ (ReadError::SearchScanLimit | ReadError::Corrupt)) => { + return read_error_response(error) + } + Err(error) if merged.is_empty() && first_error.is_none() => { + member_failed = true; + first_error = Some(error); + } + Err(_) => member_failed = true, + } + } + if merged.is_empty() && (internal_only || any_success) { + Ok(( + Vec::new(), + total_upper_bound, + false, + has_proxy || member_failed, + )) + } else if merged.is_empty() { + Err(first_error.unwrap_or(ReadError::NotFound)) + } else { + let total = if has_proxy { + total_upper_bound.max(merged.len()) + } else { + merged.len() + }; + Ok((merged, total, false, has_proxy || member_failed)) + } + } + RepositoryTarget::Legacy => { + let hosted = + hosted_search_objects(state, LEGACY_HOSTED, response_base, &request, scan_deadline) + .await; + let mut any_success = hosted.is_ok(); + let mut member_failed = hosted.is_err(); + let mut merged = hosted.unwrap_or_default(); + let mut total_upper_bound = merged.len(); + let mut has_proxy = false; + let mut seen: HashSet = merged + .iter() + .filter_map(|object| object["package"]["name"].as_str().map(str::to_string)) + .collect(); + if !internal_only { + if let Some(proxy) = legacy_proxy(state) { + has_proxy = true; + match proxy_search_group_prefix( + state, + &proxy, + query, + &seen, + merged.len(), + needed, + scan_deadline, + ) + .await + { + Ok(page) => { + any_success = true; + total_upper_bound = total_upper_bound.saturating_add(page.total); + for object in page.objects { + if let Some(package) = object["package"]["name"].as_str() { + if seen.insert(package.to_string()) { + merged.push(object); + } + } + } + } + Err(error @ (ReadError::SearchScanLimit | ReadError::Corrupt)) => { + return read_error_response(error) + } + Err(_) => member_failed = true, + } + } + } + if merged.is_empty() && (internal_only || any_success) { + Ok(( + Vec::new(), + total_upper_bound, + false, + has_proxy || member_failed, + )) + } else if merged.is_empty() { + Err(ReadError::NotFound) + } else { + let total = if has_proxy { + total_upper_bound.max(merged.len()) + } else { + merged.len() + }; + Ok((merged, total, false, has_proxy || member_failed)) + } + } + }; + match result { + Ok((objects, total, already_paged, total_is_approximate)) => search_response( + headers, + objects, + &request, + total, + already_paged, + total_is_approximate, + ), + Err(error) => read_error_response(error), + } +} + +async fn handle_get( + state: AppState, + target: RepositoryTarget, + response_base: String, + headers: HeaderMap, + path: String, + query: Option, + user: AuthenticatedUser, +) -> Response { + if path == "-/ping" { + return axum::Json(serde_json::json!({})).into_response(); + } + if path == "-/whoami" { + return axum::Json(serde_json::json!({ "username": user.0 })).into_response(); + } + if path == "-/v1/search" { + return handle_search(&state, &target, &response_base, &headers, query.as_deref()).await; + } + let Some((package, filename)) = parse_package_path(&path) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let response = if let Some(filename) = filename { + let Some(version) = crate::curation::parse_npm_tarball_version(&package, &filename) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let publish_date = + match target_publish_date(&state, &target, &package, &version, &filename).await { + Ok(date) => date, + Err(error) => return read_error_response(error), + }; + // Internal namespaces are operator-owned and may be served from hosted + // storage; their no-proxy boundary is enforced by target resolution. + // Preserve that behavior while applying normal policy to public + // hosted/proxy/group tarball reads. + if !is_internal(&state, &package) { + if let Some(response) = crate::curation::check_download( + &state.curation().curation_engine, + state.bypass_token().as_deref(), + &headers, + crate::curation::RegistryType::Npm, + &package, + Some(&version), + publish_date, + ) { + return response; + } + } + target_tarball(&state, &target, &package, &filename, publish_date).await + } else { + match target_packument(&state, &target, &package, &response_base).await { + Ok(packument) => json_response_with_stale(&headers, &packument.value, packument.stale), + Err(error) => read_error_response(error), + } + }; + if response.status().is_success() { + state.metrics.record_download("npm"); + } + response +} + +pub(crate) async fn named_get_request( + state: AppState, + repository: String, + path: String, + uri: Uri, + headers: HeaderMap, + user: AuthenticatedUser, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_get( + state.clone(), + target, + public_base(&state, Some(&repository)), + headers, + path, + uri.query().map(str::to_string), + user, + ) + .await +} + +async fn alias_get( + State(state): State, + headers: HeaderMap, + Path(path): Path, + OriginalUri(uri): OriginalUri, + Extension(user): Extension, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_get( + state.clone(), + target, + public_base(&state, None), + headers, + path, + uri.query().map(str::to_string), + user, + ) + .await +} + +struct WritableHosted { + name: String, + write_policy: NpmWritePolicy, +} + +fn writable_hosted( + state: &AppState, + target: &RepositoryTarget, + group_publish: bool, +) -> Result { + let hosted = match target { + RepositoryTarget::Legacy => WritableHosted { + name: LEGACY_HOSTED.to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }, + RepositoryTarget::Named(NpmRepository::Hosted { name, write_policy }) => WritableHosted { + name: name.clone(), + write_policy: *write_policy, + }, + RepositoryTarget::Named(NpmRepository::Proxy { .. }) => Err(NpmHttpError::new( + StatusCode::CONFLICT, + "Repository is a pull-through proxy (read-only)", + ))?, + RepositoryTarget::Named(NpmRepository::Group { + writable_member, .. + }) if group_publish => { + let Some(name) = writable_member else { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Group repository has no writable_member", + )); + }; + if let Some(NpmRepository::Hosted { name, write_policy }) = + state.config.npm.repository(name) + { + WritableHosted { + name: name.clone(), + write_policy: *write_policy, + } + } else { + return Err(NpmHttpError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "Invalid writable_member configuration", + )); + } + } + RepositoryTarget::Named(NpmRepository::Group { .. }) => { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Dist-tag mutations must target a hosted repository", + )) + } + }; + if hosted.write_policy == NpmWritePolicy::Deny { + Err(NpmHttpError::new( + StatusCode::METHOD_NOT_ALLOWED, + "Repository write policy denies mutations", + )) + } else { + Ok(hosted) + } +} + +#[derive(Debug)] +struct NpmHttpError { + status: StatusCode, + message: &'static str, +} + +impl NpmHttpError { + const fn new(status: StatusCode, message: &'static str) -> Self { + Self { status, message } + } +} + +impl IntoResponse for NpmHttpError { + fn into_response(self) -> Response { + (self.status, self.message).into_response() + } +} + +#[derive(Debug, PartialEq, Eq)] +enum ImmutableWrite { + Created, + ExistingSame, + Conflict, +} + +async fn put_immutable( + state: &AppState, + key: &str, + data: &[u8], +) -> Result { + match state.storage.put_if_absent(key, data).await { + Ok(()) => Ok(ImmutableWrite::Created), + Err(StorageError::AlreadyExists) => match state.storage.get(key).await { + Ok(existing) if existing.as_ref() == data => Ok(ImmutableWrite::ExistingSame), + Ok(_) => Ok(ImmutableWrite::Conflict), + Err(error) => Err(error), + }, + Err(error) => Err(error), + } +} + +struct ValidatedPublish { + version: String, + manifest: Vec, + tarball: Vec, + blob_digest: String, + tags: Vec<(String, String)>, + deprecation: Option, + package_fields: Vec, +} + +fn inspect_tarball_package_json(tarball: &[u8]) -> Result<(String, String), &'static str> { + // npm only needs package/package.json here. Bound decompression as well as + // the file itself so a small gzip bomb cannot make validation consume an + // unbounded amount of CPU or memory while searching the archive. + let decoder = flate2::read::GzDecoder::new(tarball).take(TAR_SCAN_CAP); + let mut archive = tar::Archive::new(decoder); + let entries = archive.entries().map_err(|_| "Invalid npm tarball")?; + for entry in entries { + let mut entry = entry.map_err(|_| "Invalid npm tarball")?; + let path = entry.path().map_err(|_| "Invalid npm tarball")?; + if path.as_ref() != std::path::Path::new("package/package.json") { + continue; + } + if entry.size() > PACKAGE_JSON_CAP { + return Err("package.json is too large"); + } + let mut body = Vec::with_capacity(entry.size() as usize); + entry + .read_to_end(&mut body) + .map_err(|_| "Invalid package.json")?; + let json: serde_json::Value = + serde_json::from_slice(&body).map_err(|_| "Invalid package.json")?; + let name = json + .get("name") + .and_then(|value| value.as_str()) + .ok_or("package.json is missing name")?; + let version = json + .get("version") + .and_then(|value| value.as_str()) + .ok_or("package.json is missing version")?; + return Ok((name.to_string(), version.to_string())); + } + Err("npm tarball is missing package/package.json") +} + +fn validate_publish( + package: &str, + payload: &serde_json::Value, +) -> Result { + if !is_valid_npm_package_name(package) + || payload.get("name").and_then(|value| value.as_str()) != Some(package) + { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Package name in URL does not match payload", + )); + } + let Some(versions) = payload.get("versions").and_then(|value| value.as_object()) else { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Missing versions", + )); + }; + let Some(attachments) = payload + .get("_attachments") + .and_then(|value| value.as_object()) + else { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Missing _attachments", + )); + }; + if versions.len() != 1 || attachments.len() != 1 { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "npm publish must contain exactly one version and one attachment", + )); + } + let (version, version_data) = versions.iter().next().expect("one version"); + if !is_valid_npm_version(version) + || version_data.get("name").and_then(|value| value.as_str()) != Some(package) + || version_data.get("version").and_then(|value| value.as_str()) != Some(version) + { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Version metadata does not match the publish coordinate", + )); + } + let (attachment_name, attachment) = attachments.iter().next().expect("one attachment"); + let normalized = normalize_attachment_name(package, attachment_name); + let canonical = canonical_tarball_filename(package, version); + if normalized != canonical || !is_valid_attachment_name(normalized) { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Attachment filename is not canonical", + )); + } + let Some(encoded) = attachment.get("data").and_then(|value| value.as_str()) else { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Attachment is missing data", + )); + }; + let tarball = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|_| NpmHttpError::new(StatusCode::BAD_REQUEST, "Invalid attachment base64"))?; + let Some(length) = attachment.get("length").and_then(|value| value.as_u64()) else { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Attachment is missing length", + )); + }; + if length != tarball.len() as u64 { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Attachment length mismatch", + )); + } + let (tar_name, tar_version) = inspect_tarball_package_json(&tarball) + .map_err(|message| NpmHttpError::new(StatusCode::BAD_REQUEST, message))?; + if tar_name != package || tar_version != *version { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "package.json coordinate does not match publish coordinate", + )); + } + + let shasum = hex::encode(sha1::Sha1::digest(&tarball)); + let blob_digest = hex::encode(sha2::Sha512::digest(&tarball)); + let integrity = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(&tarball)) + ); + let supplied_dist = version_data.get("dist"); + if supplied_dist + .and_then(|dist| dist.get("shasum")) + .and_then(|value| value.as_str()) + .is_some_and(|value| !value.eq_ignore_ascii_case(&shasum)) + { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Tarball shasum mismatch", + )); + } + if let Some(supplied) = supplied_dist + .and_then(|dist| dist.get("integrity")) + .and_then(|value| value.as_str()) + { + let sha512_supplied = supplied + .split_ascii_whitespace() + .find(|candidate| candidate.starts_with("sha512-")); + if sha512_supplied.is_some_and(|candidate| candidate != integrity) { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Tarball integrity mismatch", + )); + } + } + + let mut manifest = version_data.clone(); + let Some(manifest_object) = manifest.as_object_mut() else { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Invalid version metadata", + )); + }; + // Deprecation is mutable npm metadata. Keep it out of the immutable + // version manifest so a later explicit undeprecation cannot reveal the + // original publish-time value again. + let deprecation = match manifest_object.remove("deprecated") { + Some(serde_json::Value::String(message)) => Some(message), + Some(_) => { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Invalid deprecated message", + )) + } + None => None, + }; + let dist = manifest_object + .entry("dist") + .or_insert_with(|| serde_json::json!({})); + let Some(dist) = dist.as_object_mut() else { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Invalid dist metadata", + )); + }; + dist.remove("tarball"); + dist.insert("shasum".to_string(), serde_json::Value::String(shasum)); + dist.insert( + "integrity".to_string(), + serde_json::Value::String(integrity), + ); + let manifest = serde_json::to_vec(&manifest).map_err(|_| { + NpmHttpError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to serialize version metadata", + ) + })?; + + let mut tags = Vec::new(); + if let Some(dist_tags) = payload.get("dist-tags").and_then(|value| value.as_object()) { + for (tag, target) in dist_tags { + let Some(target) = target.as_str() else { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Invalid dist-tag target", + )); + }; + if !is_valid_dist_tag(tag) || target != version { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Invalid dist-tag", + )); + } + tags.push((tag.clone(), target.to_string())); + } + } + + let mut package_fields = serde_json::Map::new(); + for field in ["name", "_id", "description", "readme", "license"] { + if let Some(value) = payload.get(field) { + package_fields.insert(field.to_string(), value.clone()); + } + } + let package_fields = + serde_json::to_vec(&serde_json::Value::Object(package_fields)).map_err(|_| { + NpmHttpError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to serialize package metadata", + ) + })?; + + Ok(ValidatedPublish { + version: version.clone(), + manifest, + tarball, + blob_digest, + tags, + deprecation, + package_fields, + }) +} + +async fn publish( + state: &AppState, + repository: &str, + write_policy: NpmWritePolicy, + package: &str, + payload: &serde_json::Value, +) -> Response { + let validated = match validate_publish(package, payload) { + Ok(validated) => validated, + Err(error) => return error.into_response(), + }; + let lock_key = format!("npm:{repository}:{package}"); + let lock = state.publish_lock(&lock_key); + let _guard = lock.lock().await; + match incomplete_publish_versions(state, repository, package).await { + Ok(incomplete) + if incomplete + .iter() + .any(|version| version != &validated.version) => + { + return incomplete_publish_response() + } + Ok(_) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + + let pending_key = hosted_publish_pending_key(repository, package, &validated.version); + let completion_digest = crate::npm_layout::hosted_manifest_digest(&validated.manifest); + let previous_pending = match state.storage.get(&pending_key).await { + Ok(previous) if previous.as_ref() == completion_digest.as_bytes() => Some(previous), + Ok(_) => return incomplete_publish_response(), + Err(StorageError::NotFound) => None, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if state + .storage + .put(&pending_key, completion_digest.as_bytes()) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + + let blob_key = + crate::npm_layout::hosted_blob_key_for_digest(repository, package, &validated.blob_digest); + match put_immutable(state, &blob_key, &validated.tarball).await { + Ok(ImmutableWrite::Created | ImmutableWrite::ExistingSame) => {} + Ok(ImmutableWrite::Conflict) => { + if restore_publish_pending(state, &pending_key, previous_pending.as_deref()) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + tracing::error!(key = %blob_key, "npm content digest collision"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + Err(error) => { + if restore_publish_pending(state, &pending_key, previous_pending.as_deref()) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + tracing::error!(key = %blob_key, error = ?error, "npm tarball blob create failed"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + } + + // The version manifest is the sole visibility/commit point. The referenced + // tarball blob is content-addressed and is always durable before this write. + let manifest_key = hosted_version_key(repository, package, &validated.version); + let manifest_outcome = + match commit_hosted_manifest(state, &manifest_key, &validated.manifest, write_policy).await + { + Ok(outcome) => outcome, + Err(ManifestCommitError::Conflict) => { + if restore_publish_pending(state, &pending_key, previous_pending.as_deref()) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + return ( + StatusCode::CONFLICT, + "Version already exists with other metadata or tarball bytes", + ) + .into_response(); + } + Err(ManifestCommitError::Storage(error)) => { + if restore_publish_pending(state, &pending_key, previous_pending.as_deref()) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + tracing::error!(key = %manifest_key, error = ?error, "npm version commit failed"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + + let completion_key = hosted_publish_complete_key(repository, package, &validated.version); + let needs_completion = if manifest_outcome == ManifestCommit::ExistingSame + && write_policy != NpmWritePolicy::Allow + { + match state.storage.get(&completion_key).await { + Ok(existing) if existing.as_ref() == completion_digest.as_bytes() => false, + Ok(_) => { + tracing::error!( + key = %completion_key, + "npm publish completion marker does not match the committed manifest" + ); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + Err(StorageError::NotFound) => { + // An allow-once retry must repair an interrupted first + // publish without overwriting a newer explicit + // tag/deprecation value. + if fill_missing_retry_state(state, repository, package, &validated) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + true + } + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } else { + // `allow` is a real redeploy even when the immutable version + // manifest happens to be byte-identical: npm publish's selected + // dist-tag and package fields are mutable payload state. Remove + // the completion marker before touching that state so a failed + // post-commit phase cannot be acknowledged as complete on retry. + match state.storage.delete(&completion_key).await { + Ok(()) | Err(StorageError::NotFound) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + if replace_publish_state(state, repository, package, &validated) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + true + }; + if needs_completion + && state + .storage + .put(&completion_key, completion_digest.as_bytes()) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + match state.storage.delete(&pending_key).await { + Ok(()) | Err(StorageError::NotFound) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + + state.metrics.record_upload("npm"); + state + .audit + .log(AuditEntry::new("push", "api", package, "npm", repository)); + state.activity.push(ActivityEntry::new( + ActionType::Push, + package.to_string(), + RegistryType::Npm, + "LOCAL", + )); + state.repo_index.invalidate("npm"); + StatusCode::CREATED.into_response() +} + +async fn replace_publish_state( + state: &AppState, + repository: &str, + package: &str, + validated: &ValidatedPublish, +) -> Result<(), ()> { + state + .storage + .put( + &hosted_package_key(repository, package), + &validated.package_fields, + ) + .await + .map_err(|_| ())?; + for (tag, version) in &validated.tags { + state + .storage + .put( + &hosted_tag_key(repository, package, tag), + version.as_bytes(), + ) + .await + .map_err(|_| ())?; + } + if let Some(message) = &validated.deprecation { + let key = hosted_deprecation_key(repository, package, &validated.version); + if message.is_empty() { + match state.storage.delete(&key).await { + Ok(()) | Err(StorageError::NotFound) => {} + Err(_) => return Err(()), + } + } else { + state + .storage + .put(&key, message.as_bytes()) + .await + .map_err(|_| ())?; + } + } + Ok(()) +} + +#[derive(Debug, PartialEq, Eq)] +enum ManifestCommit { + Created, + ExistingSame, + Replaced, +} + +#[derive(Debug)] +enum ManifestCommitError { + Conflict, + Storage(StorageError), +} + +async fn commit_hosted_manifest( + state: &AppState, + key: &str, + manifest: &[u8], + write_policy: NpmWritePolicy, +) -> Result { + match write_policy { + NpmWritePolicy::Deny => Err(ManifestCommitError::Conflict), + NpmWritePolicy::AllowOnce => match put_immutable(state, key, manifest) + .await + .map_err(ManifestCommitError::Storage)? + { + ImmutableWrite::Created => Ok(ManifestCommit::Created), + ImmutableWrite::ExistingSame => Ok(ManifestCommit::ExistingSame), + ImmutableWrite::Conflict => Err(ManifestCommitError::Conflict), + }, + NpmWritePolicy::Allow => match state.storage.get(key).await { + Ok(existing) if existing.as_ref() == manifest => Ok(ManifestCommit::ExistingSame), + Ok(_) => { + state + .storage + .put(key, manifest) + .await + .map_err(ManifestCommitError::Storage)?; + Ok(ManifestCommit::Replaced) + } + Err(StorageError::NotFound) => { + state + .storage + .put(key, manifest) + .await + .map_err(ManifestCommitError::Storage)?; + Ok(ManifestCommit::Created) + } + Err(error) => Err(ManifestCommitError::Storage(error)), + }, + } +} + +async fn fill_missing_retry_state( + state: &AppState, + repository: &str, + package: &str, + validated: &ValidatedPublish, +) -> Result<(), ()> { + let package_key = hosted_package_key(repository, package); + match state.storage.get(&package_key).await { + Ok(existing) => { + let mut current = + serde_json::from_slice::(&existing).map_err(|_| ())?; + let candidate = serde_json::from_slice::(&validated.package_fields) + .map_err(|_| ())?; + let (Some(current), Some(candidate)) = (current.as_object_mut(), candidate.as_object()) + else { + return Err(()); + }; + let mut changed = false; + for (field, value) in candidate { + if !current.contains_key(field) { + current.insert(field.clone(), value.clone()); + changed = true; + } + } + if changed { + let merged = serde_json::to_vec(&serde_json::Value::Object(current.clone())) + .map_err(|_| ())?; + state + .storage + .put(&package_key, &merged) + .await + .map_err(|_| ())?; + } + } + Err(StorageError::NotFound) => state + .storage + .put(&package_key, &validated.package_fields) + .await + .map_err(|_| ())?, + Err(_) => return Err(()), + } + + for (tag, version) in &validated.tags { + let key = hosted_tag_key(repository, package, tag); + match state.storage.get(&key).await { + Ok(_) => {} + Err(StorageError::NotFound) => state + .storage + .put(&key, version.as_bytes()) + .await + .map_err(|_| ())?, + Err(_) => return Err(()), + } + } + if let Some(message) = validated + .deprecation + .as_deref() + .filter(|message| !message.is_empty()) + { + let key = hosted_deprecation_key(repository, package, &validated.version); + match state.storage.get(&key).await { + Ok(_) => {} + Err(StorageError::NotFound) => state + .storage + .put(&key, message.as_bytes()) + .await + .map_err(|_| ())?, + Err(_) => return Err(()), + } + } + Ok(()) +} + +async fn deprecate( + state: &AppState, + repository: &str, + package: &str, + payload: &serde_json::Value, +) -> Response { + if payload.get("name").and_then(|value| value.as_str()) != Some(package) { + return (StatusCode::BAD_REQUEST, "Package name mismatch").into_response(); + } + let Some(versions) = payload.get("versions").and_then(|value| value.as_object()) else { + return (StatusCode::BAD_REQUEST, "Missing versions").into_response(); + }; + let lock = state.publish_lock(&format!("npm:{repository}:{package}")); + let _guard = lock.lock().await; + match incomplete_publish_versions(state, repository, package).await { + Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), + Ok(_) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + let mut changed = 0usize; + for (version, data) in versions { + match hosted_has_version(state, repository, package, version).await { + Ok(true) => {} + Ok(false) => continue, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + let Some(message) = data.get("deprecated").and_then(|value| value.as_str()) else { + continue; + }; + let key = hosted_deprecation_key(repository, package, version); + let result = if message.is_empty() { + match state.storage.delete(&key).await { + Ok(()) | Err(StorageError::NotFound) => Ok(()), + Err(error) => Err(error), + } + } else { + state.storage.put(&key, message.as_bytes()).await + }; + if result.is_err() { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + changed += 1; + } + if changed == 0 { + return StatusCode::NOT_FOUND.into_response(); + } + state.repo_index.invalidate("npm"); + StatusCode::CREATED.into_response() +} + +async fn handle_put( + state: AppState, + target: RepositoryTarget, + path: String, + authority: NamespaceAuthority, + body: Bytes, +) -> Response { + let Some(package) = decode_package_name(&path) else { + return StatusCode::NOT_FOUND.into_response(); + }; + if enforce_namespace_scope(&authority, &package).is_err() { + return StatusCode::FORBIDDEN.into_response(); + } + let hosted = match writable_hosted(&state, &target, true) { + Ok(hosted) => hosted, + Err(error) => return error.into_response(), + }; + let payload = match serde_json::from_slice::(&body) { + Ok(value) => value, + Err(_) => return (StatusCode::BAD_REQUEST, "Invalid JSON").into_response(), + }; + if payload.get("_attachments").is_some() { + publish( + &state, + &hosted.name, + hosted.write_policy, + &package, + &payload, + ) + .await + } else { + deprecate(&state, &hosted.name, &package, &payload).await + } +} + +pub(crate) async fn named_put_request( + state: AppState, + repository: String, + path: String, + authority: NamespaceAuthority, + body: Bytes, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_put(state, target, path, authority, body).await +} + +async fn alias_put( + State(state): State, + Path(path): Path, + Extension(authority): Extension, + body: Bytes, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_put(state, target, path, authority, body).await +} + +async fn handle_dist_tags_get( + state: AppState, + target: RepositoryTarget, + response_base: String, + headers: HeaderMap, + package: String, +) -> Response { + let Some(package) = decode_package_name(&package) else { + return StatusCode::NOT_FOUND.into_response(); + }; + match target_packument(&state, &target, &package, &response_base).await { + Ok(packument) => { + let tags = packument + .value + .get("dist-tags") + .cloned() + .unwrap_or_else(|| serde_json::json!({})); + json_response_with_stale(&headers, &tags, packument.stale) + } + Err(error) => read_error_response(error), + } +} + +async fn named_dist_tags_get( + State(state): State, + headers: HeaderMap, + Path((repository, package)): Path<(String, String)>, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tags_get( + state.clone(), + target, + public_base(&state, Some(&repository)), + headers, + package, + ) + .await +} + +async fn alias_dist_tags_get( + State(state): State, + headers: HeaderMap, + Path(package): Path, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tags_get( + state.clone(), + target, + public_base(&state, None), + headers, + package, + ) + .await +} + +async fn handle_dist_tag_put( + state: AppState, + target: RepositoryTarget, + package: String, + tag: String, + authority: NamespaceAuthority, + body: Bytes, +) -> Response { + let Some(package) = decode_package_name(&package) else { + return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + }; + if !is_valid_dist_tag(&tag) { + return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + } + if enforce_namespace_scope(&authority, &package).is_err() { + return StatusCode::FORBIDDEN.into_response(); + } + let repository = match writable_hosted(&state, &target, false) { + Ok(hosted) => hosted.name, + Err(error) => return error.into_response(), + }; + let version = match serde_json::from_slice::(&body) { + Ok(version) if is_valid_npm_version(&version) => version, + _ => return (StatusCode::BAD_REQUEST, "Invalid version").into_response(), + }; + match hosted_has_version(&state, &repository, &package, &version).await { + Ok(true) => {} + Ok(false) => return StatusCode::NOT_FOUND.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + let lock = state.publish_lock(&format!("npm:{repository}:{package}")); + let _guard = lock.lock().await; + match incomplete_publish_versions(&state, &repository, &package).await { + Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), + Ok(_) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + match hosted_has_version(&state, &repository, &package, &version).await { + Ok(true) => {} + Ok(false) => return StatusCode::NOT_FOUND.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + if state + .storage + .put( + &hosted_tag_key(&repository, &package, &tag), + version.as_bytes(), + ) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + state.repo_index.invalidate("npm"); + StatusCode::CREATED.into_response() +} + +async fn named_dist_tag_put( + State(state): State, + Path((repository, package, tag)): Path<(String, String, String)>, + Extension(authority): Extension, + body: Bytes, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tag_put(state, target, package, tag, authority, body).await +} + +async fn alias_dist_tag_put( + State(state): State, + Path((package, tag)): Path<(String, String)>, + Extension(authority): Extension, + body: Bytes, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tag_put(state, target, package, tag, authority, body).await +} + +async fn handle_dist_tag_delete( + state: AppState, + target: RepositoryTarget, + package: String, + tag: String, + authority: NamespaceAuthority, +) -> Response { + let Some(package) = decode_package_name(&package) else { + return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + }; + if !is_valid_dist_tag(&tag) { + return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + } + if tag == "latest" { + return ( + StatusCode::BAD_REQUEST, + "The latest dist-tag cannot be deleted", + ) + .into_response(); + } + if enforce_namespace_scope(&authority, &package).is_err() { + return StatusCode::FORBIDDEN.into_response(); + } + let repository = match writable_hosted(&state, &target, false) { + Ok(hosted) => hosted.name, + Err(error) => return error.into_response(), + }; + let lock = state.publish_lock(&format!("npm:{repository}:{package}")); + let _guard = lock.lock().await; + match incomplete_publish_versions(&state, &repository, &package).await { + Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), + Ok(_) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + let key = hosted_tag_key(&repository, &package, &tag); + match state.storage.delete(&key).await { + Ok(()) | Err(StorageError::NotFound) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + state.repo_index.invalidate("npm"); + StatusCode::NO_CONTENT.into_response() +} + +async fn named_dist_tag_delete( + State(state): State, + Path((repository, package, tag)): Path<(String, String, String)>, + Extension(authority): Extension, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tag_delete(state, target, package, tag, authority).await +} + +async fn alias_dist_tag_delete( + State(state): State, + Path((package, tag)): Path<(String, String)>, + Extension(authority): Extension, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tag_delete(state, target, package, tag, authority).await +} + +fn proxy_for_audit(state: &AppState, target: &RepositoryTarget) -> Option { + match target { + RepositoryTarget::Legacy => legacy_proxy(state), + RepositoryTarget::Named(repository @ NpmRepository::Proxy { .. }) => { + configured_proxy(state, repository) + } + RepositoryTarget::Named(NpmRepository::Hosted { .. }) => None, + RepositoryTarget::Named(NpmRepository::Group { members, .. }) => members + .iter() + .filter_map(|name| state.config.npm.repository(name)) + .find_map(|repository| configured_proxy(state, repository)), + } +} + +fn npm_audit_error(status: StatusCode, message: &'static str) -> Response { + ( + status, + [( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + )], + serde_json::to_vec(&serde_json::json!({"error": message})) + .expect("static audit error JSON"), + ) + .into_response() +} + +#[derive(Debug, PartialEq, Eq)] +enum AuditBodyError { + Invalid, + TooLarge, +} + +fn decode_audit_body(headers: &HeaderMap, body: &[u8]) -> Result<(Vec, bool), AuditBodyError> { + let encoding = headers + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .unwrap_or("identity") + .trim(); + if encoding.eq_ignore_ascii_case("identity") || encoding.is_empty() { + if body.is_empty() { + return Err(AuditBodyError::Invalid); + } + return Ok((body.to_vec(), false)); + } + if !encoding.eq_ignore_ascii_case("gzip") { + return Err(AuditBodyError::Invalid); + } + let mut decoded = Vec::new(); + flate2::read::GzDecoder::new(body) + .take((NPM_AUDIT_BODY_CAP + 1) as u64) + .read_to_end(&mut decoded) + .map_err(|_| AuditBodyError::Invalid)?; + if decoded.len() > NPM_AUDIT_BODY_CAP { + return Err(AuditBodyError::TooLarge); + } + if decoded.is_empty() { + return Err(AuditBodyError::Invalid); + } + Ok((decoded, true)) +} + +fn gzip_audit_body(body: &[u8]) -> Result, AuditBodyError> { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(body) + .map_err(|_| AuditBodyError::Invalid)?; + encoder.finish().map_err(|_| AuditBodyError::Invalid) +} + +fn retain_public_dependencies( + value: &mut serde_json::Value, + engine: &crate::curation::CurationEngine, +) { + let Some(object) = value.as_object_mut() else { + return; + }; + for field in ["dependencies", "requires"] { + if let Some(dependencies) = object + .get_mut(field) + .and_then(serde_json::Value::as_object_mut) + { + dependencies.retain(|package, _| { + !crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + package, + ) + }); + for dependency in dependencies.values_mut() { + retain_public_dependencies(dependency, engine); + } + } + } + if let Some(packages) = object + .get_mut("packages") + .and_then(serde_json::Value::as_object_mut) + { + packages.retain(|path, _| { + let package = path + .rsplit_once("node_modules/") + .map(|(_, package)| package) + .unwrap_or(path); + package.is_empty() + || !crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + package, + ) + }); + for package in packages.values_mut() { + retain_public_dependencies(package, engine); + } + } + if object + .get("name") + .and_then(|value| value.as_str()) + .is_some_and(|package| { + crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + package, + ) + }) + { + object.remove("name"); + } + for (field, child) in object { + if field != "dependencies" && field != "requires" && field != "packages" { + match child { + serde_json::Value::Array(values) => { + for value in values { + retain_public_dependencies(value, engine); + } + } + serde_json::Value::Object(_) => retain_public_dependencies(child, engine), + _ => {} + } + } + } +} + +fn text_contains_internal_package(text: &str, engine: &crate::curation::CurationEngine) -> bool { + let decoded = percent_encoding::percent_decode_str(text) + .decode_utf8_lossy() + .into_owned(); + if crate::curation::is_internal_namespace(engine, crate::curation::RegistryType::Npm, &decoded) + { + return true; + } + let is_internal_candidate = |candidate: &str| { + !candidate.is_empty() + && crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + candidate, + ) + }; + for token in decoded.split(|character: char| { + !(character.is_ascii_alphanumeric() + || matches!(character, '@' | '/' | '.' | '_' | '-' | '~')) + }) { + let token = token.trim_matches('/'); + if is_internal_candidate(token) { + return true; + } + let segments: Vec<&str> = token + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + for (index, segment) in segments.iter().enumerate() { + if is_internal_candidate(segment) { + return true; + } + if segment.starts_with('@') + && index + 1 < segments.len() + && is_internal_candidate(&format!("{segment}/{}", segments[index + 1])) + { + return true; + } + } + } + let bytes = decoded.as_bytes(); + for start in 0..bytes.len() { + if bytes[start] != b'@' { + continue; + } + let mut end = start + 1; + while end < bytes.len() + && (bytes[end].is_ascii_alphanumeric() + || matches!(bytes[end], b'@' | b'/' | b'.' | b'_' | b'-')) + { + end += 1; + } + if let Some(candidate) = decoded.get(start..end) { + if crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + candidate, + ) { + return true; + } + } + } + false +} + +fn audit_json_contains_internal( + value: &serde_json::Value, + engine: &crate::curation::CurationEngine, +) -> bool { + match value { + serde_json::Value::String(text) => text_contains_internal_package(text, engine), + serde_json::Value::Array(values) => values + .iter() + .any(|value| audit_json_contains_internal(value, engine)), + serde_json::Value::Object(object) => object.iter().any(|(key, value)| { + text_contains_internal_package(key, engine) + || audit_json_contains_internal(value, engine) + }), + _ => false, + } +} + +fn filter_audit_json( + path: &str, + body: &[u8], + engine: &crate::curation::CurationEngine, + filter_active: bool, +) -> Option> { + if path == "-/npm/v1/security/advisories/bulk" { + let mut map = + serde_json::from_slice::>(body).ok()?; + if filter_active { + map.retain(|package, _| { + !crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + package, + ) + }); + } + if map.is_empty() { + return None; + } + let value = serde_json::Value::Object(map); + if filter_active && audit_json_contains_internal(&value, engine) { + return None; + } + return serde_json::to_vec(&value).ok(); + } + + let mut value = serde_json::from_slice::(body).ok()?; + if !value.is_object() { + return None; + } + if filter_active { + retain_public_dependencies(&mut value, engine); + if audit_json_contains_internal(&value, engine) { + return None; + } + } + serde_json::to_vec(&value).ok() +} + +async fn handle_post( + state: AppState, + target: RepositoryTarget, + path: String, + headers: HeaderMap, + body: Body, +) -> Response { + let is_bulk = path == "-/npm/v1/security/advisories/bulk"; + let is_quick = path == "-/npm/v1/security/audits/quick"; + let is_full = path == "-/npm/v1/security/audits"; + if !is_bulk && !is_quick && !is_full { + return method_not_allowed("GET, PUT"); + } + let body = match axum::body::to_bytes(body, NPM_AUDIT_BODY_CAP).await { + Ok(body) => body, + Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), + }; + let Some(proxy) = proxy_for_audit(&state, &target) else { + return npm_audit_error( + StatusCode::BAD_REQUEST, + "Audit requires a configured proxy repository", + ); + }; + let engine = &state.curation().curation_engine; + let filter_active = crate::curation::namespace_filter_active(engine); + let (decoded, was_gzip) = match decode_audit_body(&headers, &body) { + Ok(decoded) => decoded, + Err(AuditBodyError::TooLarge) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), + Err(AuditBodyError::Invalid) => { + return npm_audit_error(StatusCode::BAD_REQUEST, "Invalid audit request body") + } + }; + let Some(filtered) = filter_audit_json(&path, &decoded, engine, filter_active) else { + return npm_audit_error(StatusCode::BAD_REQUEST, "Empty or invalid audit request"); + }; + let forward = if was_gzip { + match gzip_audit_body(&filtered) { + Ok(body) => body, + Err(_) => { + return npm_audit_error(StatusCode::BAD_REQUEST, "Invalid audit request body") + } + } + } else { + filtered + }; + let mut forwarded_headers = Vec::new(); + if let Some(value) = headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + { + forwarded_headers.push(("content-type", value)); + } + if let Some(value) = headers + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + { + forwarded_headers.push(("content-encoding", value)); + } + if let Some(value) = headers + .get(header::ACCEPT) + .and_then(|value| value.to_str().ok()) + { + forwarded_headers.push(("accept", value)); + } + let url = format!("{}/{}", proxy.url.trim_end_matches('/'), path); + match proxy_forward_post( + &state.no_redirect_http_client, + &url, + Duration::from_secs(state.config.npm.proxy_timeout), + expose_opt(&proxy.auth), + &forwarded_headers, + &forward, + &state.circuit_breaker, + RegistryType::Npm, + MAX_NPM_PROXY_REDIRECTS, + |next_url| validated_proxy_url(&proxy, next_url.as_str()).is_some(), + ) + .await + { + Ok((status, response_body, content_type)) => { + state + .audit + .log(AuditEntry::new("proxy_fetch", "api", "", "npm", "audit")); + let status = StatusCode::from_u16(status).unwrap_or(StatusCode::BAD_GATEWAY); + let content_type = content_type + .as_deref() + .and_then(|value| HeaderValue::from_str(value).ok()) + .unwrap_or_else(|| HeaderValue::from_static("application/json")); + ( + status, + [(header::CONTENT_TYPE, content_type)], + response_body, + ) + .into_response() + } + Err(ProxyError::CircuitOpen(name)) => circuit_open_response(&name), + Err(error) => { + tracing::warn!(error = ?error, "npm audit upstream forward failed"); + StatusCode::BAD_GATEWAY.into_response() + } + } +} + +pub(crate) async fn named_post_request( + state: AppState, + repository: String, + path: String, + headers: HeaderMap, + body: Body, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_post(state, target, path, headers, body).await +} + +async fn alias_post( + State(state): State, + Path(path): Path, + headers: HeaderMap, + body: Body, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_post(state, target, path, headers, body).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn group_merge_is_member_ordered_and_latest_is_not_derived() { + let first = serde_json::json!({ + "versions": { + "1.0.0": {"name": "p", "version": "1.0.0"}, + "2.0.0": {"name": "hosted", "version": "2.0.0"} + }, + "dist-tags": {"latest": "1.0.0"} + }); + let second = serde_json::json!({ + "versions": { + "2.0.0": {"name": "proxy", "version": "2.0.0"}, + "9.0.0": {"name": "p", "version": "9.0.0"} + }, + "dist-tags": {"latest": "9.0.0", "next": "9.0.0"} + }); + let merged = merge_packuments("p", "https://nora/repository/group", vec![first, second]) + .expect("merge"); + assert_eq!(merged["dist-tags"]["latest"], "1.0.0"); + assert_eq!(merged["dist-tags"]["next"], "9.0.0"); + assert_eq!(merged["versions"]["2.0.0"]["name"], "hosted"); + } + + #[test] + fn attachment_validation_rejects_paths() { + assert!(is_valid_attachment_name("pkg-1.0.0.tgz")); + assert!(!is_valid_attachment_name("../pkg.tgz")); + assert!(!is_valid_attachment_name("scope/pkg.tgz")); + } + + #[test] + fn dist_tags_reject_semver_like_names() { + assert!(is_valid_dist_tag("latest")); + assert!(!is_valid_dist_tag("1.2.3")); + assert!(!is_valid_dist_tag("^1")); + } + + #[cfg(test)] + fn npm_tarball(package: &str, version: &str) -> Vec { + npm_tarball_with_marker(package, version, "") + } + + #[cfg(test)] + fn npm_tarball_with_marker(package: &str, version: &str, marker: &str) -> Vec { + use flate2::write::GzEncoder; + use flate2::Compression; + let encoder = GzEncoder::new(Vec::new(), Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let package_json = serde_json::to_vec(&serde_json::json!({ + "name": package, + "version": version + })) + .unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_size(package_json.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "package/package.json", package_json.as_slice()) + .unwrap(); + if !marker.is_empty() { + let mut marker_header = tar::Header::new_gnu(); + marker_header.set_size(marker.len() as u64); + marker_header.set_mode(0o644); + marker_header.set_cksum(); + archive + .append_data( + &mut marker_header, + "package/republish-marker.txt", + marker.as_bytes(), + ) + .unwrap(); + } + archive.into_inner().unwrap().finish().unwrap() + } + + fn named_config(config: &mut crate::config::Config) { + config.npm.proxy = None; + config.npm.repositories = vec![ + NpmRepository::Hosted { + name: "npm-private".into(), + write_policy: NpmWritePolicy::AllowOnce, + }, + NpmRepository::Proxy { + name: "npm-registry".into(), + url: "http://127.0.0.1:1".into(), + auth: None, + metadata_ttl: Some(300), + negative_ttl: 0, + }, + NpmRepository::Group { + name: "npm-group".into(), + members: vec!["npm-private".into(), "npm-registry".into()], + writable_member: Some("npm-private".into()), + }, + ]; + config.npm.default_repository = Some("npm-group".into()); + } + + #[test] + fn invalid_named_config_without_default_never_falls_into_legacy_layout() { + let ctx = crate::test_helpers::create_test_context_with_config(|config| { + config.npm.repositories = vec![NpmRepository::Hosted { + name: "packages".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }]; + config.npm.default_repository = None; + }); + assert!(alias_target(&ctx.state).is_none()); + } + + fn publish_payload(package: &str, version: &str, tag: &str) -> Vec { + let tgz = npm_tarball(package, version); + serde_json::to_vec(&serde_json::json!({ + "name": package, + "versions": { + (version): { + "name": package, + "version": version, + "dist": {} + } + }, + "_attachments": { + (canonical_tarball_filename(package, version)): { + "data": base64::engine::general_purpose::STANDARD.encode(&tgz), + "length": tgz.len() + } + }, + "dist-tags": {(tag): version} + })) + .unwrap() + } + + fn publish_payload_with_tarball( + package: &str, + version: &str, + tag: &str, + marker: &str, + ) -> Vec { + let mut payload: serde_json::Value = + serde_json::from_slice(&publish_payload(package, version, tag)).unwrap(); + let tarball = npm_tarball_with_marker(package, version, marker); + let filename = canonical_tarball_filename(package, version); + payload["_attachments"][&filename]["data"] = + serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(&tarball)); + payload["_attachments"][&filename]["length"] = + serde_json::Value::Number(tarball.len().into()); + serde_json::to_vec(&payload).unwrap() + } + + #[tokio::test] + async fn publish_through_group_commits_only_hosted_and_exact_retry_repairs() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + let ctx = create_test_context_with_config(named_config); + let body = publish_payload("pkg", "1.0.0", "latest"); + let first = send( + &ctx.app, + Method::PUT, + "/repository/npm-group/pkg", + body.clone(), + ) + .await; + assert_eq!(first.status(), StatusCode::CREATED); + // Model a crash after the version-manifest commit and before the + // durable post-commit marker. Only this incomplete state is repairable. + ctx.state + .storage + .delete(&hosted_tag_key("npm-private", "pkg", "latest")) + .await + .unwrap(); + ctx.state + .storage + .delete(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(); + let manifest = ctx + .state + .storage + .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(); + ctx.state + .storage + .put( + &hosted_publish_pending_key("npm-private", "pkg", "1.0.0"), + crate::npm_layout::hosted_manifest_digest(&manifest).as_bytes(), + ) + .await + .unwrap(); + let retry = send(&ctx.app, Method::PUT, "/repository/npm-group/pkg", body).await; + assert_eq!(retry.status(), StatusCode::CREATED); + assert!(ctx + .state + .storage + .stat("npm/repositories/npm-private/pkg/versions/1.0.0.json") + .await + .is_some()); + assert!(ctx + .state + .storage + .stat("npm/repositories/npm-group/pkg/versions/1.0.0.json") + .await + .is_none()); + assert_eq!( + ctx.state + .storage + .get(&hosted_tag_key("npm-private", "pkg", "latest")) + .await + .unwrap() + .as_ref(), + b"1.0.0" + ); + } + + #[tokio::test] + async fn fresh_publish_scans_only_pending_markers() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let backend = crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()); + let list_attempts = backend.list_attempts(); + let mut state = ctx.state.clone(); + state.storage = crate::storage::Storage::from_backend(std::sync::Arc::new(backend)); + + for version in ["1.0.0", "2.0.0"] { + let payload: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", version, "latest")).unwrap(); + assert_eq!( + publish( + &state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + ) + .await + .status(), + StatusCode::CREATED + ); + } + + assert_eq!( + list_attempts.lock().as_slice(), + [ + "npm/repositories/npm-private/pkg/publish-pending/", + "npm/repositories/npm-private/pkg/publish-pending/", + ] + ); + assert_eq!( + state + .storage + .get(&hosted_publish_pending_index_key("npm-private", "pkg")) + .await + .unwrap() + .as_ref(), + b"1" + ); + for version in ["1.0.0", "2.0.0"] { + assert!(matches!( + state + .storage + .get(&hosted_publish_pending_key("npm-private", "pkg", version)) + .await, + Err(StorageError::NotFound) + )); + } + } + + #[tokio::test] + async fn completed_pending_marker_is_cleaned_before_the_next_publish() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let first: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &first, + ) + .await + .status(), + StatusCode::CREATED + ); + + let manifest = ctx + .state + .storage + .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(); + let pending_key = hosted_publish_pending_key("npm-private", "pkg", "1.0.0"); + ctx.state + .storage + .put( + &pending_key, + crate::npm_layout::hosted_manifest_digest(&manifest).as_bytes(), + ) + .await + .unwrap(); + + let second: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "2.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(matches!( + ctx.state.storage.get(&pending_key).await, + Err(StorageError::NotFound) + )); + } + + #[tokio::test] + async fn pending_publish_accepts_only_the_exact_manifest_retry() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let exact: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( + "pkg", "1.0.0", "latest", "exact", + )) + .unwrap(); + let exact_publish = validate_publish("pkg", &exact).unwrap(); + let exact_digest = crate::npm_layout::hosted_manifest_digest(&exact_publish.manifest); + let pending_key = hosted_publish_pending_key("npm-private", "pkg", "1.0.0"); + ctx.state + .storage + .put( + &hosted_publish_pending_index_key("npm-private", "pkg"), + b"1", + ) + .await + .unwrap(); + ctx.state + .storage + .put(&pending_key, exact_digest.as_bytes()) + .await + .unwrap(); + + let different: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( + "pkg", + "1.0.0", + "latest", + "different", + )) + .unwrap(); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &different, + ) + .await + .status(), + StatusCode::CONFLICT + ); + assert_eq!( + ctx.state.storage.get(&pending_key).await.unwrap().as_ref(), + exact_digest.as_bytes() + ); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &exact, + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(matches!( + ctx.state.storage.get(&pending_key).await, + Err(StorageError::NotFound) + )); + } + + #[tokio::test] + async fn failed_pending_marker_cleanup_is_recovered_by_exact_retry() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let pending_key = hosted_publish_pending_key("npm-private", "pkg", "1.0.0"); + let backend = crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) + .fail_delete(&pending_key); + let mut failing_state = ctx.state.clone(); + failing_state.storage = crate::storage::Storage::from_backend(std::sync::Arc::new(backend)); + let payload: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + + assert_eq!( + publish( + &failing_state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx.state.storage.stat(&pending_key).await.is_some()); + + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(matches!( + ctx.state.storage.get(&pending_key).await, + Err(StorageError::NotFound) + )); + } + + #[tokio::test] + async fn completed_publish_retry_does_not_resurrect_deleted_tag() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + let ctx = create_test_context_with_config(named_config); + let body = publish_payload("pkg", "1.0.0", "next"); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + body.clone(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(ctx + .state + .storage + .get(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0",)) + .await + .is_ok()); + ctx.state + .storage + .delete(&hosted_tag_key("npm-private", "pkg", "next")) + .await + .unwrap(); + + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", body,) + .await + .status(), + StatusCode::CREATED + ); + assert!(matches!( + ctx.state + .storage + .get(&hosted_tag_key("npm-private", "pkg", "next")) + .await, + Err(StorageError::NotFound) + )); + } + + #[tokio::test] + async fn publish_deprecation_uses_overlay_and_completed_retry_does_not_resurrect_it() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + let mut payload: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + payload["versions"]["1.0.0"]["deprecated"] = + serde_json::Value::String("do not use".to_string()); + let body = serde_json::to_vec(&payload).unwrap(); + + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-group/pkg", + body.clone(), + ) + .await + .status(), + StatusCode::CREATED + ); + + let manifest_key = hosted_version_key("npm-private", "pkg", "1.0.0"); + let deprecation_key = hosted_deprecation_key("npm-private", "pkg", "1.0.0"); + let completion_key = hosted_publish_complete_key("npm-private", "pkg", "1.0.0"); + let manifest: serde_json::Value = + serde_json::from_slice(&ctx.state.storage.get(&manifest_key).await.unwrap()).unwrap(); + assert!(manifest.get("deprecated").is_none()); + assert_eq!( + ctx.state + .storage + .get(&deprecation_key) + .await + .unwrap() + .as_ref(), + b"do not use" + ); + + // An exact retry repairs a publish that reached the manifest commit + // but not the mutable overlay/completion phase. + ctx.state.storage.delete(&deprecation_key).await.unwrap(); + ctx.state.storage.delete(&completion_key).await.unwrap(); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-group/pkg", + body.clone(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + ctx.state + .storage + .get(&deprecation_key) + .await + .unwrap() + .as_ref(), + b"do not use" + ); + let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; + assert_eq!(response.status(), StatusCode::OK); + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(packument["versions"]["1.0.0"]["deprecated"], "do not use"); + + let clear = serde_json::json!({ + "name": "pkg", + "versions": { + "1.0.0": {"deprecated": ""} + } + }); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-group/pkg", + serde_json::to_vec(&clear).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(matches!( + ctx.state.storage.get(&deprecation_key).await, + Err(StorageError::NotFound) + )); + + let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; + assert_eq!(response.status(), StatusCode::OK); + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert!(packument["versions"]["1.0.0"].get("deprecated").is_none()); + + // Once the original publish completed, retrying that exact body must + // not rewind the later mutable undeprecation. + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-group/pkg", body) + .await + .status(), + StatusCode::CREATED + ); + assert!(matches!( + ctx.state.storage.get(&deprecation_key).await, + Err(StorageError::NotFound) + )); + let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; + assert_eq!(response.status(), StatusCode::OK); + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert!(packument["versions"]["1.0.0"].get("deprecated").is_none()); + } + + #[tokio::test] + async fn incomplete_publish_must_be_retried_before_later_mutable_operations() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + let mut first: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "next")).unwrap(); + first["description"] = serde_json::Value::String("old description".to_string()); + first["versions"]["1.0.0"]["deprecated"] = + serde_json::Value::String("old deprecation".to_string()); + let first = serde_json::to_vec(&first).unwrap(); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + first.clone(), + ) + .await + .status(), + StatusCode::CREATED + ); + + // Model a process loss after the manifest became visible but before + // the durable completion marker. No later mutable operation may race + // the exact retry that repairs this state. + ctx.state + .storage + .delete(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(); + let manifest = ctx + .state + .storage + .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(); + ctx.state + .storage + .put( + &hosted_publish_pending_key("npm-private", "pkg", "1.0.0"), + crate::npm_layout::hosted_manifest_digest(&manifest).as_bytes(), + ) + .await + .unwrap(); + let clear = serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": ""}} + }); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + serde_json::to_vec(&clear).unwrap(), + ) + .await + .status(), + StatusCode::CONFLICT + ); + assert_eq!( + send( + &ctx.app, + Method::DELETE, + "/repository/npm-private/-/package/pkg/dist-tags/next", + "", + ) + .await + .status(), + StatusCode::CONFLICT + ); + let second = publish_payload("pkg", "2.0.0", "latest"); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + second.clone(), + ) + .await + .status(), + StatusCode::CONFLICT + ); + + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + first.clone(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + serde_json::to_vec(&clear).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send( + &ctx.app, + Method::DELETE, + "/repository/npm-private/-/package/pkg/dist-tags/next", + "", + ) + .await + .status(), + StatusCode::NO_CONTENT + ); + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", second,) + .await + .status(), + StatusCode::CREATED + ); + + // A delayed exact retry is now read-only because the original publish + // completed before the newer mutations were accepted. + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", first,) + .await + .status(), + StatusCode::CREATED + ); + assert!(matches!( + ctx.state + .storage + .get(&hosted_tag_key("npm-private", "pkg", "next")) + .await, + Err(StorageError::NotFound) + )); + assert!(matches!( + ctx.state + .storage + .get(&hosted_deprecation_key("npm-private", "pkg", "1.0.0")) + .await, + Err(StorageError::NotFound) + )); + let package: serde_json::Value = serde_json::from_slice( + &ctx.state + .storage + .get(&hosted_package_key("npm-private", "pkg")) + .await + .unwrap(), + ) + .unwrap(); + assert!(package.get("description").is_none()); + } + + #[tokio::test] + async fn delayed_retry_cannot_rewind_newer_publish_mutable_state() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + let ctx = create_test_context_with_config(named_config); + let mut first: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + first["description"] = serde_json::Value::String("old".to_string()); + let first = serde_json::to_vec(&first).unwrap(); + let mut second: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "2.0.0", "latest")).unwrap(); + second["description"] = serde_json::Value::String("new".to_string()); + let second = serde_json::to_vec(&second).unwrap(); + + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + first.clone(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", second,) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", first,) + .await + .status(), + StatusCode::CREATED + ); + + assert_eq!( + ctx.state + .storage + .get(&hosted_tag_key("npm-private", "pkg", "latest")) + .await + .unwrap() + .as_ref(), + b"2.0.0" + ); + let package: serde_json::Value = serde_json::from_slice( + &ctx.state + .storage + .get(&hosted_package_key("npm-private", "pkg")) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(package["description"], "new"); + } + + #[tokio::test] + async fn delayed_retry_cannot_undo_explicit_dist_tag_move() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + let ctx = create_test_context_with_config(named_config); + let first = publish_payload("pkg", "1.0.0", "next"); + let second = publish_payload("pkg", "2.0.0", "latest"); + for body in [first.clone(), second] { + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", body,) + .await + .status(), + StatusCode::CREATED + ); + } + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/package/pkg/dist-tags/next", + br#""2.0.0""#.as_slice(), + ) + .await + .status(), + StatusCode::CREATED + ); + + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", first,) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + ctx.state + .storage + .get(&hosted_tag_key("npm-private", "pkg", "next")) + .await + .unwrap() + .as_ref(), + b"2.0.0" + ); + } + + #[tokio::test] + async fn group_tarball_does_not_fall_through_on_hosted_manifest_read_error() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let manifest = hosted_version_key("npm-private", "pkg", "1.0.0"); + ctx.state + .storage + .put(&manifest, br#"{"name":"pkg","version":"1.0.0"}"#) + .await + .unwrap(); + let mut state = ctx.state.clone(); + state.storage = crate::storage::Storage::from_backend(std::sync::Arc::new( + crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) + .fail_get(&manifest), + )); + + let response = group_tarball( + &state, + &["npm-private".to_string(), "npm-registry".to_string()], + "pkg", + "pkg-1.0.0.tgz", + None, + ) + .await; + + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + } + + #[tokio::test] + async fn hosted_optional_read_error_fails_group_packument_closed() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let manifest = hosted_version_key("npm-private", "pkg", "1.0.0"); + let package_key = hosted_package_key("npm-private", "pkg"); + ctx.state + .storage + .put(&manifest, br#"{"name":"pkg","version":"1.0.0"}"#) + .await + .unwrap(); + ctx.state + .storage + .put(&package_key, br#"{"name":"pkg"}"#) + .await + .unwrap(); + let mut state = ctx.state.clone(); + state.storage = crate::storage::Storage::from_backend(std::sync::Arc::new( + crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) + .fail_get(&package_key), + )); + + assert!(matches!( + group_packument( + &state, + &["npm-private".to_string(), "npm-registry".to_string()], + "pkg", + "http://localhost/repository/npm-group", + ) + .await, + Err(ReadError::Unavailable) + )); + } + + #[tokio::test] + async fn deprecation_and_dist_tag_delete_are_idempotent_but_not_error_blind() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let manifest = hosted_version_key("npm-private", "pkg", "1.0.0"); + let manifest_bytes = br#"{"name":"pkg","version":"1.0.0"}"#; + let completion = hosted_publish_complete_key("npm-private", "pkg", "1.0.0"); + let deprecation = hosted_deprecation_key("npm-private", "pkg", "1.0.0"); + let tag = hosted_tag_key("npm-private", "pkg", "next"); + ctx.state + .storage + .put(&manifest, manifest_bytes) + .await + .unwrap(); + ctx.state + .storage + .put( + &completion, + crate::npm_layout::hosted_manifest_digest(manifest_bytes).as_bytes(), + ) + .await + .unwrap(); + ctx.state.storage.put(&deprecation, b"old").await.unwrap(); + ctx.state.storage.put(&tag, b"1.0.0").await.unwrap(); + let backend = crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) + .fail_delete(&deprecation) + .fail_delete(&tag); + let mut state = ctx.state.clone(); + state.storage = crate::storage::Storage::from_backend(std::sync::Arc::new(backend)); + + let payload = serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": ""}} + }); + assert_eq!( + deprecate(&state, "npm-private", "pkg", &payload) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + handle_dist_tag_delete( + state, + RepositoryTarget::Named(NpmRepository::Hosted { + name: "npm-private".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }), + "pkg".to_string(), + "next".to_string(), + NamespaceAuthority::Unrestricted, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + + let clean = crate::test_helpers::create_test_context_with_config(named_config); + clean + .state + .storage + .put(&manifest, manifest_bytes) + .await + .unwrap(); + clean + .state + .storage + .put( + &completion, + crate::npm_layout::hosted_manifest_digest(manifest_bytes).as_bytes(), + ) + .await + .unwrap(); + assert_eq!( + deprecate(&clean.state, "npm-private", "pkg", &payload) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + handle_dist_tag_delete( + clean.state, + RepositoryTarget::Named(NpmRepository::Hosted { + name: "npm-private".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }), + "pkg".to_string(), + "next".to_string(), + NamespaceAuthority::Unrestricted, + ) + .await + .status(), + StatusCode::NO_CONTENT + ); + } + + #[tokio::test] + async fn retry_after_precommit_tarball_orphan_completes_publish() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + let body = publish_payload("pkg", "1.0.0", "latest"); + let payload: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let encoded = payload["_attachments"]["pkg-1.0.0.tgz"]["data"] + .as_str() + .unwrap(); + let tarball = base64::engine::general_purpose::STANDARD + .decode(encoded) + .unwrap(); + let blob_key = crate::npm_layout::hosted_blob_key_for_digest( + "npm-private", + "pkg", + &hex::encode(sha2::Sha512::digest(&tarball)), + ); + ctx.state.storage.put(&blob_key, &tarball).await.unwrap(); + + let response = send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", body).await; + assert_eq!(response.status(), StatusCode::CREATED); + assert!(ctx + .state + .storage + .stat(&hosted_version_key("npm-private", "pkg", "1.0.0")) + .await + .is_some()); + } + + #[tokio::test] + async fn conflicting_same_version_is_rejected() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + let ctx = create_test_context_with_config(named_config); + let first = publish_payload("pkg", "1.0.0", "latest"); + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", first) + .await + .status(), + StatusCode::CREATED + ); + let mut conflicting: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + conflicting["versions"]["1.0.0"]["description"] = serde_json::json!("different"); + let response = send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + serde_json::to_vec(&conflicting).unwrap(), + ) + .await; + assert_eq!(response.status(), StatusCode::CONFLICT); + } + + #[tokio::test] + async fn allow_policy_republishes_same_version_via_new_blob_and_digest_bound_manifest() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(|config| { + named_config(config); + let NpmRepository::Hosted { write_policy, .. } = &mut config.npm.repositories[0] else { + unreachable!() + }; + *write_policy = NpmWritePolicy::Allow; + }); + let mut first: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( + "pkg", "1.0.0", "latest", "first", + )) + .unwrap(); + first["versions"]["1.0.0"]["description"] = serde_json::json!("first"); + let first = serde_json::to_vec(&first).unwrap(); + let mut second: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( + "pkg", "1.0.0", "latest", "second", + )) + .unwrap(); + second["versions"]["1.0.0"]["description"] = serde_json::json!("second"); + let second_tarball = base64::engine::general_purpose::STANDARD + .decode( + second["_attachments"]["pkg-1.0.0.tgz"]["data"] + .as_str() + .unwrap(), + ) + .unwrap(); + let second = serde_json::to_vec(&second).unwrap(); + + for body in [first, second.clone(), second] { + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", body,) + .await + .status(), + StatusCode::CREATED + ); + } + let mut same_manifest_new_state: serde_json::Value = serde_json::from_slice( + &publish_payload_with_tarball("pkg", "1.0.0", "next", "second"), + ) + .unwrap(); + same_manifest_new_state["description"] = serde_json::json!("replacement package fields"); + same_manifest_new_state["versions"]["1.0.0"]["description"] = serde_json::json!("second"); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + serde_json::to_vec(&same_manifest_new_state).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + + let manifest = ctx + .state + .storage + .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(); + assert_eq!( + ctx.state + .storage + .get(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap() + .as_ref(), + crate::npm_layout::hosted_manifest_digest(&manifest).as_bytes() + ); + assert_eq!( + ctx.state + .storage + .get( + &crate::npm_layout::hosted_blob_key_from_manifest( + "npm-private", + "pkg", + &manifest + ) + .unwrap() + ) + .await + .unwrap() + .as_ref(), + second_tarball.as_slice() + ); + + let packument = send(&ctx.app, Method::GET, "/repository/npm-private/pkg", "").await; + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(packument).await).unwrap(); + assert_eq!(packument["versions"]["1.0.0"]["description"], "second"); + assert_eq!(packument["description"], "replacement package fields"); + assert_eq!(packument["dist-tags"]["latest"], "1.0.0"); + assert_eq!(packument["dist-tags"]["next"], "1.0.0"); + let tarball = send( + &ctx.app, + Method::GET, + "/repository/npm-private/pkg/-/pkg-1.0.0.tgz", + "", + ) + .await; + assert_eq!(tarball.status(), StatusCode::OK); + assert_eq!( + body_bytes(tarball).await.as_ref(), + second_tarball.as_slice() + ); + } #[tokio::test] - async fn test_npm_namespace_scope_enforced() { - use crate::auth::NamespaceAuthority; - use crate::config::ScopeEnforcement; - use axum::body::Bytes; - use axum::extract::{Path, State}; - use axum::http::StatusCode; - use axum::Extension; - - let ctx = create_test_context(); - let scoped = NamespaceAuthority::from_oidc_scope( - "ci", - &["@myorg/**".to_string()], - ScopeEnforcement::Enforce, - ); - - // Out of scope -> 403, decided before any payload parsing. - let resp = super::handle_publish( - State(ctx.state.clone()), - Path("@other/pkg".to_string()), - Extension(scoped.clone()), - Bytes::from_static(b"{}"), - ) - .await; - assert_eq!(resp.status(), StatusCode::FORBIDDEN); - - // In scope -> enforcement passes (then fails payload validation, not 403). - let resp = super::handle_publish( - State(ctx.state.clone()), - Path("@myorg/pkg".to_string()), - Extension(scoped), - Bytes::from_static(b"{}"), - ) - .await; - assert_ne!(resp.status(), StatusCode::FORBIDDEN); - } - use axum::body::Body; - use axum::http::{Method, StatusCode}; - use base64::Engine; - - #[tokio::test] - async fn test_npm_metadata_from_cache() { - let ctx = create_test_context(); + async fn allow_redeploy_stale_completion_blocks_later_mutations_until_retry() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(|config| { + named_config(config); + let NpmRepository::Hosted { write_policy, .. } = &mut config.npm.repositories[0] else { + unreachable!() + }; + *write_policy = NpmWritePolicy::Allow; + }); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload_with_tarball("pkg", "1.0.0", "latest", "first"), + ) + .await + .status(), + StatusCode::CREATED + ); + + let second = publish_payload_with_tarball("pkg", "1.0.0", "latest", "second"); + let second_value: serde_json::Value = serde_json::from_slice(&second).unwrap(); + let validated = validate_publish("pkg", &second_value).unwrap(); + let blob_key = crate::npm_layout::hosted_blob_key_for_digest( + "npm-private", + "pkg", + &validated.blob_digest, + ); + ctx.state + .storage + .put(&blob_key, &validated.tarball) + .await + .unwrap(); + ctx.state + .storage + .put( + &hosted_version_key("npm-private", "pkg", "1.0.0"), + &validated.manifest, + ) + .await + .unwrap(); + ctx.state + .storage + .put( + &hosted_publish_pending_key("npm-private", "pkg", "1.0.0"), + crate::npm_layout::hosted_manifest_digest(&validated.manifest).as_bytes(), + ) + .await + .unwrap(); + let stale_marker = ctx + .state + .storage + .get(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(); + assert_ne!( + stale_marker.as_ref(), + crate::npm_layout::hosted_manifest_digest(&validated.manifest).as_bytes() + ); + + let deprecation = serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": "later"}} + }); + for response in [ + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "2.0.0", "latest"), + ) + .await, + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/package/pkg/dist-tags/next", + serde_json::to_vec("1.0.0").unwrap(), + ) + .await, + send( + &ctx.app, + Method::DELETE, + "/repository/npm-private/-/package/pkg/dist-tags/next", + "", + ) + .await, + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + serde_json::to_vec(&deprecation).unwrap(), + ) + .await, + ] { + assert_eq!(response.status(), StatusCode::CONFLICT); + } + + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", second,) + .await + .status(), + StatusCode::CREATED + ); + let manifest = ctx + .state + .storage + .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(); + assert_eq!( + ctx.state + .storage + .get(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap() + .as_ref(), + crate::npm_layout::hosted_manifest_digest(&manifest).as_bytes() + ); + } + + #[tokio::test] + async fn group_package_put_routes_deprecation_to_hosted_overlay() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + let deprecation = serde_json::json!({ + "name": "pkg", + "versions": { + "1.0.0": {"deprecated": "do not use"} + } + }); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-group/pkg", + serde_json::to_vec(&deprecation).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(packument["versions"]["1.0.0"]["deprecated"], "do not use"); + } + + #[tokio::test] + async fn group_dist_tag_mutation_is_rejected_but_hosted_delete_is_idempotent() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + let ctx = create_test_context_with_config(named_config); + let response = send( + &ctx.app, + Method::PUT, + "/repository/npm-group/-/package/pkg/dist-tags/next", + r#""1.0.0""#, + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + for _ in 0..2 { + let response = send( + &ctx.app, + Method::DELETE, + "/repository/npm-private/-/package/pkg/dist-tags/next", + "", + ) + .await; + assert_eq!(response.status(), StatusCode::NO_CONTENT); + } + } + + #[tokio::test] + async fn group_packument_rewrites_tarballs_to_group_route() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + let ctx = create_test_context_with_config(named_config); + let body = publish_payload("pkg", "1.0.0", "latest"); + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", body) + .await + .status(), + StatusCode::CREATED + ); + let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; + assert_eq!(response.status(), StatusCode::OK); + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(packument["dist-tags"]["latest"], "1.0.0"); + assert_eq!( + packument["versions"]["1.0.0"]["dist"]["tarball"], + "http://127.0.0.1:0/repository/npm-group/pkg/-/pkg-1.0.0.tgz" + ); + } + + #[tokio::test] + async fn proxy_never_receives_internal_namespace_request() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + let ctx = create_test_context_with_config(|config| { + named_config(config); + config.curation.internal_namespaces = vec!["@internal/**".into()]; + }); + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-group/@internal%2Fpkg", + "", + ) + .await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + + // Axum and the npm path decoder each consume one encoding layer. A + // further residual `%` must be rejected instead of becoming a public + // proxy coordinate that an upstream could decode again. + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-group/%252540internal%25252Fpkg", + "", + ) + .await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + + #[tokio::test] + async fn deleting_hosted_tag_reveals_lower_priority_proxy_tag() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; - let metadata = serde_json::json!({ - "name": "lodash", + let ctx = create_test_context_with_config(named_config); + let proxy_packument = serde_json::json!({ + "name": "pkg", "versions": { - "4.17.21": { "dist": { "tarball": "http://example.com/lodash.tgz" } } - } + "9.0.0": { + "name": "pkg", + "version": "9.0.0", + "dist": { + "shasum": "0000000000000000000000000000000000000000", + "tarball": "https://registry.npmjs.org/pkg/-/pkg-9.0.0.tgz" + } + } + }, + "dist-tags": {"next": "9.0.0"} }); - let metadata_bytes = serde_json::to_vec(&metadata).unwrap(); - ctx.state .storage - .put("npm/lodash/metadata.json", &metadata_bytes) + .put( + &proxy_packument_key("npm-registry", "pkg"), + &serde_json::to_vec(&proxy_packument).unwrap(), + ) .await .unwrap(); - let response = send(&ctx.app, Method::GET, "/npm/lodash", "").await; + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "next"), + ) + .await + .status(), + StatusCode::CREATED + ); - assert_eq!(response.status(), StatusCode::OK); - let body = body_bytes(response).await; - let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(json["name"], "lodash"); + let before = send( + &ctx.app, + Method::GET, + "/repository/npm-group/-/package/pkg/dist-tags", + "", + ) + .await; + let before: serde_json::Value = serde_json::from_slice(&body_bytes(before).await).unwrap(); + assert_eq!(before["next"], "1.0.0"); + + assert_eq!( + send( + &ctx.app, + Method::DELETE, + "/repository/npm-private/-/package/pkg/dist-tags/next", + "", + ) + .await + .status(), + StatusCode::NO_CONTENT + ); + let after = send( + &ctx.app, + Method::GET, + "/repository/npm-group/-/package/pkg/dist-tags", + "", + ) + .await; + let after: serde_json::Value = serde_json::from_slice(&body_bytes(after).await).unwrap(); + assert_eq!(after["next"], "9.0.0"); } #[tokio::test] - async fn test_npm_tarball_from_cache() { - let ctx = create_test_context(); + async fn group_tarball_uses_the_same_first_member_as_version_metadata() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + let hosted = ctx + .state + .storage + .get( + &crate::npm_layout::hosted_blob_key_from_manifest( + "npm-private", + "pkg", + &ctx.state + .storage + .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) + .await + .unwrap(), + ) + .unwrap(), + ) + .await + .unwrap(); - let tarball_data = b"fake-tarball-bytes"; + let proxy_bytes = b"different-proxy-bytes"; + let proxy_packument = serde_json::json!({ + "name": "pkg", + "versions": { + "1.0.0": { + "name": "pkg", + "version": "1.0.0", + "dist": { + "shasum": hex::encode(sha1::Sha1::digest(proxy_bytes)), + "tarball": "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" + } + } + }, + "dist-tags": {"latest": "1.0.0"} + }); + ctx.state + .storage + .put( + &proxy_packument_key("npm-registry", "pkg"), + &serde_json::to_vec(&proxy_packument).unwrap(), + ) + .await + .unwrap(); ctx.state .storage - .put("npm/lodash/tarballs/lodash-4.17.21.tgz", tarball_data) + .put( + &proxy_tarball_key("npm-registry", "pkg", "pkg-1.0.0.tgz"), + proxy_bytes, + ) .await .unwrap(); let response = send( &ctx.app, Method::GET, - "/npm/lodash/-/lodash-4.17.21.tgz", + "/repository/npm-group/pkg/-/pkg-1.0.0.tgz", "", ) .await; - assert_eq!(response.status(), StatusCode::OK); - let body = body_bytes(response).await; - assert_eq!(&body[..], tarball_data); + assert_eq!(body_bytes(response).await.as_ref(), hosted.as_ref()); } #[tokio::test] - async fn test_npm_not_found_no_proxy() { - let ctx = create_test_context(); - - // No proxy configured, no local data - let response = send(&ctx.app, Method::GET, "/npm/nonexistent", "").await; + async fn curation_blocklist_gates_hosted_and_proxy_tarball_reads() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + + let policy_dir = tempfile::TempDir::new().unwrap(); + let policy_path = policy_dir.path().join("blocklist.json"); + std::fs::write( + &policy_path, + serde_json::to_vec(&serde_json::json!({ + "version": 1, + "rules": [{ + "registry": "npm", + "name": "blocked", + "version": "*", + "reason": "test policy" + }] + })) + .unwrap(), + ) + .unwrap(); + let policy_path = policy_path.to_string_lossy().into_owned(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + config.curation.mode = crate::config::CurationMode::Enforce; + config.curation.blocklist_path = Some(policy_path); + }); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/blocked", + publish_payload("blocked", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); - assert_eq!(response.status(), StatusCode::NOT_FOUND); + for repository in ["npm-private", "npm-registry"] { + let response = send( + &ctx.app, + Method::GET, + &format!("/repository/{repository}/blocked/-/blocked-1.0.0.tgz"), + "", + ) + .await; + assert_eq!(response.status(), StatusCode::FORBIDDEN, "{repository}"); + assert_eq!( + response + .headers() + .get("x-nora-rule") + .and_then(|value| value.to_str().ok()), + Some("blocklist") + ); + } } #[tokio::test] - async fn test_npm_publish_basic() { - let ctx = create_test_context(); - - let tarball_data = b"fake-tarball"; - let base64_data = base64::engine::general_purpose::STANDARD.encode(tarball_data); - - let payload = serde_json::json!({ - "name": "mypkg", + async fn curation_integrity_gates_hosted_and_cached_proxy_tarballs() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + + let policy_dir = tempfile::TempDir::new().unwrap(); + let policy_path = policy_dir.path().join("allowlist.json"); + std::fs::write( + &policy_path, + serde_json::to_vec(&serde_json::json!({ + "version": 1, + "entries": [{ + "registry": "npm", + "name": "pkg", + "version": "1.0.0", + "integrity": format!("sha256:{}", "0".repeat(64)), + "integrity_source": "test" + }] + })) + .unwrap(), + ) + .unwrap(); + let policy_path = policy_path.to_string_lossy().into_owned(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + config.curation.mode = crate::config::CurationMode::Enforce; + config.curation.allowlist_path = Some(policy_path); + config.curation.require_integrity = true; + }); + let tarball = npm_tarball("pkg", "1.0.0"); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + let proxy_packument = serde_json::json!({ + "name": "pkg", "versions": { - "1.0.0": { "dist": {} } - }, - "_attachments": { - "mypkg-1.0.0.tgz": { "data": base64_data } + "1.0.0": { + "name": "pkg", + "version": "1.0.0", + "dist": { + "shasum": hex::encode(sha1::Sha1::digest(&tarball)), + "integrity": format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD + .encode(sha2::Sha512::digest(&tarball)) + ), + "tarball": "http://127.0.0.1:1/pkg/-/pkg-1.0.0.tgz" + } + } }, - "dist-tags": { "latest": "1.0.0" } + "dist-tags": {"latest": "1.0.0"} }); - - let body_bytes = serde_json::to_vec(&payload).unwrap(); - let response = send(&ctx.app, Method::PUT, "/npm/mypkg", Body::from(body_bytes)).await; - - assert_eq!(response.status(), StatusCode::CREATED); - - // Verify tarball was stored - let stored_tarball = ctx - .state + ctx.state .storage - .get("npm/mypkg/tarballs/mypkg-1.0.0.tgz") + .put( + &proxy_packument_key("npm-registry", "pkg"), + &serde_json::to_vec(&proxy_packument).unwrap(), + ) + .await + .unwrap(); + ctx.state + .storage + .put( + &proxy_tarball_key("npm-registry", "pkg", "pkg-1.0.0.tgz"), + &tarball, + ) .await .unwrap(); - assert_eq!(&stored_tarball[..], tarball_data); - } - #[tokio::test] - async fn test_npm_publish_multi_version_scan_regenerate() { - // Two separate publishes of different versions must BOTH survive in the regenerated - // packument — the version data lives in immutable per-version keys, not a merged file - // (the #39 multi-replica lost-update fix). - let ctx = create_test_context(); - for v in ["1.0.0", "2.0.0"] { - let b64 = base64::engine::general_purpose::STANDARD.encode(b"tgz"); - let mut versions = serde_json::Map::new(); - versions.insert(v.to_string(), serde_json::json!({ "dist": {} })); - let mut atts = serde_json::Map::new(); - atts.insert( - format!("multi-{}.tgz", v), - serde_json::json!({ "data": b64 }), - ); - let payload = - serde_json::json!({ "name": "multi", "versions": versions, "_attachments": atts }); - let resp = send( + for repository in ["npm-private", "npm-registry"] { + let response = send( &ctx.app, - Method::PUT, - "/npm/multi", - Body::from(serde_json::to_vec(&payload).unwrap()), + Method::GET, + &format!("/repository/{repository}/pkg/-/pkg-1.0.0.tgz"), + "", ) .await; - assert_eq!(resp.status(), StatusCode::CREATED); + assert_eq!(response.status(), StatusCode::FORBIDDEN, "{repository}"); + assert_eq!( + response + .headers() + .get("x-nora-rule") + .and_then(|value| value.to_str().ok()), + Some("allowlist:integrity") + ); } - // both per-version keys exist (immutable, distinct — no merge) - assert!(ctx - .state - .storage - .get("npm/multi/versions/1.0.0.json") - .await - .is_ok()); - assert!(ctx - .state - .storage - .get("npm/multi/versions/2.0.0.json") - .await - .is_ok()); - // the regenerated packument lists BOTH - let meta = ctx - .state - .storage - .get("npm/multi/metadata.json") - .await - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&meta).unwrap(); - let versions = json["versions"].as_object().unwrap(); - assert!(versions.contains_key("1.0.0"), "v1 lost from packument"); - assert!(versions.contains_key("2.0.0"), "v2 lost from packument"); } #[tokio::test] - async fn test_npm_publish_migrates_embedded_packument() { - // An old-layout package (versions embedded in metadata.json, no per-version keys) is - // lazily migrated on the next publish, preserving the old versions. - let ctx = create_test_context(); - let old = serde_json::json!({ - "name": "legacy", - "versions": { "1.0.0": { "name": "legacy", "version": "1.0.0", "dist": {} } }, - "dist-tags": { "latest": "1.0.0" } + async fn concurrent_proxy_tarball_cold_miss_is_single_flight() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + use std::sync::Arc; + use tokio::sync::Barrier; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let upstream = MockServer::start().await; + let tarball = npm_tarball("pkg", "1.0.0"); + Mock::given(method("GET")) + .and(path("/pkg/-/pkg-1.0.0.tgz")) + .respond_with( + ResponseTemplate::new(200) + .set_delay(Duration::from_millis(100)) + .set_body_bytes(tarball.clone()), + ) + .expect(1) + .mount(&upstream) + .await; + + let upstream_url = upstream.uri(); + let configured_url = upstream_url.clone(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } + }); + let packument = serde_json::json!({ + "name": "pkg", + "versions": { + "1.0.0": { + "name": "pkg", + "version": "1.0.0", + "dist": { + "shasum": hex::encode(sha1::Sha1::digest(&tarball)), + "integrity": format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD + .encode(sha2::Sha512::digest(&tarball)) + ), + "tarball": format!("{upstream_url}/pkg/-/pkg-1.0.0.tgz") + } + } + }, + "dist-tags": {"latest": "1.0.0"} }); ctx.state .storage .put( - "npm/legacy/metadata.json", - &serde_json::to_vec(&old).unwrap(), + &proxy_packument_key("npm-registry", "pkg"), + &serde_json::to_vec(&packument).unwrap(), ) .await .unwrap(); - let b64 = base64::engine::general_purpose::STANDARD.encode(b"tgz"); - let payload = serde_json::json!({ - "name": "legacy", - "versions": { "2.0.0": { "dist": {} } }, - "_attachments": { "legacy-2.0.0.tgz": { "data": b64 } }, - }); - let resp = send( - &ctx.app, - Method::PUT, - "/npm/legacy", - Body::from(serde_json::to_vec(&payload).unwrap()), - ) - .await; - assert_eq!(resp.status(), StatusCode::CREATED); - // old version migrated to its own per-version key - assert!( - ctx.state - .storage - .get("npm/legacy/versions/1.0.0.json") + + const CLIENTS: usize = 16; + let barrier = Arc::new(Barrier::new(CLIENTS)); + let mut tasks = Vec::new(); + for _ in 0..CLIENTS { + let app = ctx.app.clone(); + let barrier = Arc::clone(&barrier); + tasks.push(tokio::spawn(async move { + barrier.wait().await; + send( + &app, + Method::GET, + "/repository/npm-registry/pkg/-/pkg-1.0.0.tgz", + "", + ) .await - .is_ok(), - "old embedded version not migrated to a per-version key" - ); - // packument has BOTH the migrated old and the new version - let meta = ctx - .state - .storage - .get("npm/legacy/metadata.json") - .await - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&meta).unwrap(); - let versions = json["versions"].as_object().unwrap(); - assert!(versions.contains_key("1.0.0"), "migrated v1 lost"); - assert!(versions.contains_key("2.0.0"), "new v2 lost"); + .status() + })); + } + for result in futures::future::join_all(tasks).await { + assert_eq!(result.unwrap(), StatusCode::OK); + } + upstream.verify().await; } #[tokio::test] - async fn test_npm_publish_scoped_with_prefixed_attachment() { - use crate::auth::NamespaceAuthority; - use axum::body::Bytes; - use axum::extract::{Path, State}; - use axum::http::StatusCode; - use axum::Extension; + async fn proxy_packument_validates_every_redirect_before_auth() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - let ctx = create_test_context(); + let upstream = MockServer::start().await; + let attacker = MockServer::start().await; + let credentials = "reader:secret"; + let authorization = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(credentials) + ); + let upstream_base = format!("{}/repository/npm", upstream.uri()); + let cross_origin_location = format!("{}/steal", attacker.uri()); - let tarball_data = b"fake-tarball"; - let base64_data = base64::engine::general_purpose::STANDARD.encode(tarball_data); + Mock::given(method("GET")) + .and(path("/repository/npm/cross")) + .and(header("authorization", authorization.as_str())) + .respond_with( + ResponseTemplate::new(302) + .insert_header("location", cross_origin_location.as_str()), + ) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/repository/npm/outside")) + .and(header("authorization", authorization.as_str())) + .respond_with(ResponseTemplate::new(302).insert_header("location", "/outside/steal")) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/outside/steal")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/repository/npm/allowed")) + .and(header("authorization", authorization.as_str())) + .respond_with( + ResponseTemplate::new(302) + .insert_header("location", "/repository/npm/allowed-final"), + ) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/repository/npm/allowed-final")) + .and(header("authorization", authorization.as_str())) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "name": "allowed", + "versions": {}, + "dist-tags": {} + }))) + .expect(1) + .mount(&upstream) + .await; - // Scoped package where the attachment filename includes the scope prefix - // (e.g. "@scope/pkg-1.0.0.tgz" instead of "pkg-1.0.0.tgz"). The handler - // must normalize this by stripping the scope prefix. - let payload = serde_json::json!({ - "name": "@scope/mypkg", - "versions": { - "1.0.0": { "dist": {} } - }, - "_attachments": { - "@scope/mypkg-1.0.0.tgz": { "data": base64_data } - }, - "dist-tags": { "latest": "1.0.0" } + let configured_url = upstream_base.clone(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { + url, + auth, + metadata_ttl, + .. + } = &mut config.npm.repositories[1] + { + *url = configured_url; + *auth = Some(crate::secrets::ProtectedString::new( + credentials.to_string(), + )); + *metadata_ttl = Some(0); + } }); - let body_bytes = serde_json::to_vec(&payload).unwrap(); + for package in ["cross", "outside"] { + let response = send( + &ctx.app, + Method::GET, + &format!("/repository/npm-registry/{package}"), + "", + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_GATEWAY, "{package}"); + } + assert!( + attacker.received_requests().await.unwrap().is_empty(), + "cross-origin redirect target must receive neither request nor auth" + ); + assert!( + upstream + .received_requests() + .await + .unwrap() + .iter() + .all(|request| request.url.path() != "/outside/steal"), + "same-origin redirect outside the configured base path must not be requested" + ); - let resp = super::handle_publish( - State(ctx.state.clone()), - Path("@scope/mypkg".to_string()), - Extension(NamespaceAuthority::Unrestricted), - Bytes::from(body_bytes), + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-registry/allowed", + "", ) .await; - assert_eq!(resp.status(), StatusCode::CREATED); + assert_eq!(response.status(), StatusCode::OK); + upstream.verify().await; + } + + #[tokio::test] + async fn proxy_tarball_validates_initial_url_and_every_redirect_before_auth() { + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - // Verify tarball was stored with the normalized (scope-stripped) filename - let stored_tarball = ctx + let upstream = MockServer::start().await; + let attacker = MockServer::start().await; + let tarball = npm_tarball("pkg", "1.0.0"); + let credentials = "reader:secret"; + let authorization = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(credentials) + ); + let upstream_base = format!("{}/repository/npm", upstream.uri()); + let cross_origin_location = format!("{}/steal", attacker.uri()); + + Mock::given(method("GET")) + .and(path("/repository/npm/redirect-cross-origin")) + .and(header("authorization", authorization.as_str())) + .respond_with( + ResponseTemplate::new(302) + .insert_header("location", cross_origin_location.as_str()), + ) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/repository/npm/redirect-outside-base")) + .and(header("authorization", authorization.as_str())) + .respond_with(ResponseTemplate::new(302).insert_header("location", "/outside/steal")) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/repository/npm/redirect-allowed")) + .and(header("authorization", authorization.as_str())) + .respond_with( + ResponseTemplate::new(302) + .insert_header("location", "/repository/npm/pkg/-/pkg-1.0.0.tgz"), + ) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/repository/npm/pkg/-/pkg-1.0.0.tgz")) + .and(header("authorization", authorization.as_str())) + .respond_with(ResponseTemplate::new(200).set_body_bytes(tarball.clone())) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/outside/steal")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&upstream) + .await; + + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let repository = ProxyRepository { + name: "npm-registry".to_string(), + url: upstream_base.clone(), + auth: Some(crate::secrets::ProtectedString::new( + credentials.to_string(), + )), + metadata_ttl: 300, + negative_ttl: 0, + }; + assert!(ctx .state + .repo_index + .get("npm", &ctx.state.storage) + .await + .is_empty()); + let packument = |tarball_url: String| { + serde_json::json!({ + "name": "pkg", + "versions": { + "1.0.0": { + "name": "pkg", + "version": "1.0.0", + "dist": { + "shasum": hex::encode(sha1::Sha1::digest(&tarball)), + "tarball": tarball_url + } + } + }, + "dist-tags": {"latest": "1.0.0"} + }) + }; + let key = proxy_packument_key("npm-registry", "pkg"); + let misses_before = ctx.state.metrics.cache_misses(); + let hits_before = ctx.state.metrics.cache_hits(); + ctx.state .storage - .get("npm/@scope/mypkg/tarballs/mypkg-1.0.0.tgz") + .put( + &key, + &serde_json::to_vec(&packument(format!("{}/steal", attacker.uri()))).unwrap(), + ) .await .unwrap(); - assert_eq!(&stored_tarball[..], tarball_data); - } - - #[tokio::test] - async fn test_npm_publish_name_mismatch() { - let ctx = create_test_context(); - - let tarball_data = b"fake-tarball"; - let base64_data = base64::engine::general_purpose::STANDARD.encode(tarball_data); - - let payload = serde_json::json!({ - "name": "other", - "versions": { - "1.0.0": { "dist": {} } - }, - "_attachments": { - "other-1.0.0.tgz": { "data": base64_data } - }, - "dist-tags": { "latest": "1.0.0" } - }); - - let body_bytes = serde_json::to_vec(&payload).unwrap(); - let response = send(&ctx.app, Method::PUT, "/npm/mypkg", Body::from(body_bytes)).await; - - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - } - /// Corrupt metadata in storage → publish returns 500 (#533). - #[tokio::test] - async fn test_publish_corrupt_metadata_returns_500() { - let ctx = create_test_context(); + let rejected = + serve_proxy_tarball(&ctx.state, &repository, "pkg", "pkg-1.0.0.tgz", None).await; + assert_eq!(rejected.status(), StatusCode::BAD_GATEWAY); + assert!( + attacker.received_requests().await.unwrap().is_empty(), + "cross-origin URL must be rejected without a request" + ); - // Plant corrupt (non-JSON) data in metadata key ctx.state .storage - .put("npm/mypkg/metadata.json", b"NOT VALID JSON{{{") + .put( + &key, + &serde_json::to_vec(&packument(format!("{upstream_base}/redirect-cross-origin"))) + .unwrap(), + ) .await .unwrap(); + let rejected_redirect = + serve_proxy_tarball(&ctx.state, &repository, "pkg", "pkg-1.0.0.tgz", None).await; + assert_eq!(rejected_redirect.status(), StatusCode::BAD_GATEWAY); + assert!( + attacker.received_requests().await.unwrap().is_empty(), + "cross-origin redirect target must receive neither request nor auth" + ); - let tarball_data = b"fake-tarball"; - let base64_data = base64::engine::general_purpose::STANDARD.encode(tarball_data); - - let payload = serde_json::json!({ - "name": "mypkg", - "versions": { - "2.0.0": { "dist": {} } - }, - "_attachments": { - "mypkg-2.0.0.tgz": { "data": base64_data } - }, - "dist-tags": { "latest": "2.0.0" } - }); - - let body_bytes = serde_json::to_vec(&payload).unwrap(); - let response = send(&ctx.app, Method::PUT, "/npm/mypkg", Body::from(body_bytes)).await; - - assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + ctx.state + .storage + .put( + &key, + &serde_json::to_vec(&packument(format!("{upstream_base}/redirect-outside-base"))) + .unwrap(), + ) + .await + .unwrap(); + let rejected_redirect = + serve_proxy_tarball(&ctx.state, &repository, "pkg", "pkg-1.0.0.tgz", None).await; + assert_eq!(rejected_redirect.status(), StatusCode::BAD_GATEWAY); + assert!( + upstream + .received_requests() + .await + .unwrap() + .iter() + .all(|request| request.url.path() != "/outside/steal"), + "same-origin redirect outside the configured base path must not be requested" + ); - // Corrupt data must NOT be overwritten — preserved for forensics - let stored = ctx - .state + ctx.state .storage - .get("npm/mypkg/metadata.json") + .put( + &key, + &serde_json::to_vec(&packument(format!("{upstream_base}/redirect-allowed"))) + .unwrap(), + ) .await .unwrap(); - assert_eq!(&stored[..], b"NOT VALID JSON{{{"); + let accepted = + serve_proxy_tarball(&ctx.state, &repository, "pkg", "pkg-1.0.0.tgz", None).await; + assert_eq!(accepted.status(), StatusCode::OK); + assert!(ctx.state.metrics.cache_misses() >= misses_before + 1); + let cached = + serve_proxy_tarball(&ctx.state, &repository, "pkg", "pkg-1.0.0.tgz", None).await; + assert_eq!(cached.status(), StatusCode::OK); + assert!(ctx.state.metrics.cache_hits() >= hits_before + 1); + assert!(ctx + .state + .repo_index + .get("npm", &ctx.state.storage) + .await + .iter() + .any(|entry| entry.name == "repositories/npm-registry/pkg")); + upstream.verify().await; } - /// First publish (no existing metadata) still works (#533 regression guard). #[tokio::test] - async fn test_first_publish_no_existing_metadata() { - let ctx = create_test_context(); - - let tarball_data = b"fake-tarball"; - let base64_data = base64::engine::general_purpose::STANDARD.encode(tarball_data); + async fn stale_proxy_packument_marks_response_and_rewrites_nested_upstream_urls() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - let payload = serde_json::json!({ - "name": "newpkg", + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/pkg")) + .respond_with(ResponseTemplate::new(500)) + .expect(1) + .mount(&upstream) + .await; + let upstream_url = upstream.uri(); + let configured_url = upstream_url.clone(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { + url, metadata_ttl, .. + } = &mut config.npm.repositories[1] + { + *url = configured_url; + *metadata_ttl = Some(0); + } + }); + let cached = serde_json::json!({ + "name": "pkg", + "custom": format!("{upstream_url}/custom"), + "nested": [{"docs": format!("{upstream_url}/docs")}], + "description": format!("do not replace embedded {upstream_url}/text"), "versions": { - "1.0.0": { "dist": {} } - }, - "_attachments": { - "newpkg-1.0.0.tgz": { "data": base64_data } + "1.0.0": { + "name": "pkg", + "version": "1.0.0", + "dist": { + "tarball": format!("{upstream_url}/pkg/-/pkg-1.0.0.tgz") + } + } }, - "dist-tags": { "latest": "1.0.0" } + "dist-tags": {"latest": "1.0.0"} }); + ctx.state + .storage + .put( + &proxy_packument_key("npm-registry", "pkg"), + &serde_json::to_vec(&cached).unwrap(), + ) + .await + .unwrap(); - let body_bytes = serde_json::to_vec(&payload).unwrap(); - let response = send(&ctx.app, Method::PUT, "/npm/newpkg", Body::from(body_bytes)).await; - - assert_eq!(response.status(), StatusCode::CREATED); - } - - #[tokio::test] - async fn test_npm_whoami_anonymous() { - use axum::http::StatusCode; - - let ctx = create_test_context(); - let resp = send(&ctx.app, axum::http::Method::GET, "/npm/-/whoami", "").await; - assert_eq!(resp.status(), StatusCode::OK); - let body = body_bytes(resp).await; - let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(json["username"], "anonymous"); + let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response + .headers() + .get("x-nora-stale") + .and_then(|value| value.to_str().ok()), + Some("true") + ); + let body: serde_json::Value = serde_json::from_slice(&body_bytes(response).await).unwrap(); + let public = public_base(&ctx.state, Some("npm-group")); + assert_eq!(body["custom"], format!("{public}/custom")); + assert_eq!(body["nested"][0]["docs"], format!("{public}/docs")); + assert_eq!( + body["versions"]["1.0.0"]["dist"]["tarball"], + format!("{public}/pkg/-/pkg-1.0.0.tgz") + ); + assert_eq!( + body["description"], + format!("do not replace embedded {upstream_url}/text") + ); + upstream.verify().await; } #[tokio::test] - async fn test_npm_whoami_authenticated() { - use axum::http::StatusCode; - use base64::Engine; - - let ctx = create_test_context_with_auth(&[("alice", "hunter2")]); + async fn proxy_packument_304_restores_revalidation_telemetry() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - let basic = format!( - "Basic {}", - base64::engine::general_purpose::STANDARD.encode("alice:hunter2") - ); - let resp = send_with_headers( - &ctx.app, - axum::http::Method::GET, - "/npm/-/whoami", - vec![("authorization", &basic)], - "", + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/pkg")) + .and(header("if-none-match", "\"v1\"")) + .respond_with(ResponseTemplate::new(304)) + .expect(1) + .mount(&upstream) + .await; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { + url, metadata_ttl, .. + } = &mut config.npm.repositories[1] + { + *url = configured_url; + *metadata_ttl = Some(0); + } + }); + let key = proxy_packument_key("npm-registry", "pkg"); + ctx.state + .storage + .put(&key, br#"{"name":"pkg","versions":{},"dist-tags":{}}"#) + .await + .unwrap(); + write_validators( + &ctx.state.storage, + &key, + &Validators { + etag: Some("\"v1\"".to_string()), + last_modified: None, + }, ) .await; - assert_eq!(resp.status(), StatusCode::OK); - let body = body_bytes(resp).await; - let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(json["username"], "alice"); - } - - #[tokio::test] - async fn test_npm_whoami_requires_auth() { - use axum::http::StatusCode; - - let ctx = create_test_context_with_auth(&[("alice", "hunter2")]); - - let resp = send(&ctx.app, axum::http::Method::GET, "/npm/-/whoami", "").await; - assert_eq!(resp.status(), StatusCode::UNAUTHORIZED); - } - - // A username containing `"` (reachable via the OIDC `sub` claim) must NOT - // break the JSON or inject extra fields — handle_whoami serializes via serde. - #[tokio::test] - async fn test_npm_whoami_escapes_username() { - use crate::auth::AuthenticatedUser; - - let evil = r#"a","admin":"x"#; - let resp = super::handle_whoami(&AuthenticatedUser(evil.to_string())).await; - let body = body_bytes(resp).await; - let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); - assert_eq!(json["username"], evil); - assert!(json.get("admin").is_none()); - } -} - -// ── Spec conformance tests (#390) ───────────────────────────────────── -// -// Invariant: after tarball URL rewriting, no upstream registry domains -// remain in the response. Uses golden fixtures from testdata/npm/. - -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod spec_conformance_tests { - use super::*; - - const NPM_UPSTREAM_DOMAIN: &str = "registry.npmjs.org"; - - /// Assert that no upstream URLs remain in rewritten response body. - fn assert_no_upstream_urls(body: &str, context: &str) { - assert!( - !body.contains(NPM_UPSTREAM_DOMAIN), - "upstream domain '{}' leaked in {}: {}", - NPM_UPSTREAM_DOMAIN, - context, - &body[..body.len().min(500)] - ); - } - - fn load_fixture(name: &str) -> Vec { - let path = format!("{}/testdata/npm/{}", env!("CARGO_MANIFEST_DIR"), name); - std::fs::read(&path).unwrap_or_else(|e| panic!("failed to load fixture {}: {}", path, e)) - } - - // ── Regular package rewrite ── - - #[test] - fn test_regular_package_golden_no_upstream_leak() { - let fixture = load_fixture("package-metadata.json"); - let result = - rewrite_tarball_urls(&fixture, "http://nora:4000", "https://registry.npmjs.org") - .unwrap(); - let body = String::from_utf8(result).unwrap(); - assert_no_upstream_urls(&body, "regular package rewrite"); - } - - #[test] - fn test_regular_package_golden_all_tarballs_rewritten() { - let fixture = load_fixture("package-metadata.json"); - let result = - rewrite_tarball_urls(&fixture, "http://nora:4000", "https://registry.npmjs.org") - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - - let versions = json["versions"].as_object().unwrap(); - for (ver, data) in versions { - let tarball = data["dist"]["tarball"].as_str().unwrap(); - assert!( - tarball.starts_with("http://nora:4000/npm/"), - "version {} tarball not rewritten: {}", - ver, - tarball - ); - } - } + let before_304 = crate::metrics::PROXY_UPSTREAM_304_TOTAL + .with_label_values(&["npm"]) + .get(); + let before_bytes = crate::metrics::PROXY_REVALIDATION_BYTES_SAVED_TOTAL + .with_label_values(&["npm"]) + .get(); - #[test] - fn test_regular_package_golden_preserves_integrity() { - let fixture = load_fixture("package-metadata.json"); - let result = - rewrite_tarball_urls(&fixture, "http://nora:4000", "https://registry.npmjs.org") - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); + let response = send(&ctx.app, Method::GET, "/repository/npm-registry/pkg", "").await; - // integrity and shasum must survive rewriting - let dist = &json["versions"]["4.17.21"]["dist"]; + assert_eq!(response.status(), StatusCode::OK); assert!( - dist["shasum"].as_str().is_some(), - "shasum must be preserved" + crate::metrics::PROXY_UPSTREAM_304_TOTAL + .with_label_values(&["npm"]) + .get() + > before_304 ); assert!( - dist["integrity"].as_str().is_some(), - "integrity must be preserved" + crate::metrics::PROXY_REVALIDATION_BYTES_SAVED_TOTAL + .with_label_values(&["npm"]) + .get() + > before_bytes ); + assert!(response.headers().get("x-nora-stale").is_none()); + upstream.verify().await; } - #[test] - fn test_regular_package_golden_snapshot() { - let fixture = load_fixture("package-metadata.json"); - let result = - rewrite_tarball_urls(&fixture, "http://nora:4000", "https://registry.npmjs.org") - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - - // Snapshot only tarball URLs (stable against other metadata changes) - let tarball_urls: Vec<&str> = json["versions"] - .as_object() - .unwrap() - .values() - .filter_map(|v| v["dist"]["tarball"].as_str()) - .collect(); - insta::assert_json_snapshot!("npm_regular_tarball_urls", tarball_urls); - } + #[tokio::test] + async fn direct_proxy_search_forwards_query_once_and_applies_pagination_once() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + use std::collections::HashMap; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - // ── Scoped package rewrite ── + let upstream = MockServer::start().await; + let objects = ["one", "two"] + .into_iter() + .map(|name| serde_json::json!({"package": {"name": name, "version": "1.0.0"}})) + .collect::>(); + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": objects, + "total": 3, + "time": "1ms" + }))) + .expect(1) + .mount(&upstream) + .await; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } + }); - #[test] - fn test_scoped_package_golden_no_upstream_leak() { - let fixture = load_fixture("scoped-package-metadata.json"); - let result = rewrite_tarball_urls( - &fixture, - "https://registry.airgap.local", - "https://registry.npmjs.org", + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-registry/-/v1/search?text=pkg&from=1&size=2", + "", ) - .unwrap(); - let body = String::from_utf8(result).unwrap(); - assert_no_upstream_urls(&body, "scoped package rewrite"); - } - - #[test] - fn test_scoped_package_golden_all_tarballs_rewritten() { - let fixture = load_fixture("scoped-package-metadata.json"); - let result = - rewrite_tarball_urls(&fixture, "http://nora:4000", "https://registry.npmjs.org") - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - - let versions = json["versions"].as_object().unwrap(); - for (ver, data) in versions { - let tarball = data["dist"]["tarball"].as_str().unwrap(); - assert!( - tarball.starts_with("http://nora:4000/npm/"), - "scoped version {} tarball not rewritten: {}", - ver, - tarball - ); - // Scoped packages must preserve the @scope prefix in the path - assert!( - tarball.contains("@babel/core"), - "scoped package path lost: {}", - tarball - ); - } - } - - #[test] - fn test_scoped_package_golden_snapshot() { - let fixture = load_fixture("scoped-package-metadata.json"); - let result = - rewrite_tarball_urls(&fixture, "http://nora:4000", "https://registry.npmjs.org") - .unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); - - let tarball_urls: Vec<&str> = json["versions"] - .as_object() - .unwrap() - .values() - .filter_map(|v| v["dist"]["tarball"].as_str()) - .collect(); - insta::assert_json_snapshot!("npm_scoped_tarball_urls", tarball_urls); + .await; + assert_eq!(response.status(), StatusCode::OK); + let response: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(response["objects"][0]["package"]["name"], "one"); + assert_eq!(response["objects"][1]["package"]["name"], "two"); + assert_eq!(response["total"], 3); + assert_eq!(response["totalIsApproximate"], false); + + let requests = upstream.received_requests().await.unwrap(); + let query: HashMap<_, _> = requests[0].url.query_pairs().into_owned().collect(); + assert_eq!(query.get("text").map(String::as_str), Some("pkg")); + assert_eq!(query.get("from").map(String::as_str), Some("1")); + assert_eq!(query.get("size").map(String::as_str), Some("2")); } - // ── Content-Type assertions ── + #[tokio::test] + async fn direct_proxy_search_preserves_offsets_beyond_upstream_page_size() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - #[test] - fn test_with_content_type_metadata_is_json() { - let data = Bytes::from(b"{}".to_vec()); - let (status, headers, _body) = with_content_type(false, data); - assert_eq!(status, StatusCode::OK); - assert_eq!(headers[0].1, "application/json"); - } + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .and(query_param("text", "pkg")) + .and(query_param("from", "300")) + .and(query_param("size", "2")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": [ + {"package": {"name": "item-300", "version": "1.0.0"}}, + {"package": {"name": "item-301", "version": "1.0.0"}} + ], + "total": 1000, + "time": "1ms" + }))) + .expect(1) + .mount(&upstream) + .await; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } + }); - #[test] - fn test_with_content_type_tarball_is_octet() { - let data = Bytes::from(b"\x1f\x8b".to_vec()); - let (status, headers, _body) = with_content_type(true, data); - assert_eq!(status, StatusCode::OK); - assert_eq!(headers[0].1, "application/octet-stream"); + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-registry/-/v1/search?text=pkg&from=300&size=2", + "", + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let response: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(response["objects"][0]["package"]["name"], "item-300"); + assert_eq!(response["objects"][1]["package"]["name"], "item-301"); + assert_eq!(response["total"], 1000); + assert_eq!(response["totalIsApproximate"], false); + upstream.verify().await; } - // ── Edge cases for URL rewriting ── + #[tokio::test] + async fn direct_proxy_search_uses_hot_reloaded_namespace_filter_before_applying_offset() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + use wiremock::matchers::{method, path, query_param}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - #[test] - fn test_rewrite_custom_upstream_no_leak() { - let metadata = serde_json::json!({ - "name": "pkg", - "versions": { - "1.0.0": { - "dist": { - "tarball": "https://private.npm.corp/pkg/-/pkg-1.0.0.tgz" - } - } + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .and(query_param("from", "0")) + .and(query_param("size", "250")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": [ + {"package": {"name": "@internal/pkg", "version": "1.0.0"}}, + {"package": {"name": "public-one", "version": "1.0.0"}} + ], + "total": 3 + }))) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .and(query_param("from", "2")) + .and(query_param("size", "250")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": [ + {"package": {"name": "public-two", "version": "1.0.0"}} + ], + "total": 3 + }))) + .expect(1) + .mount(&upstream) + .await; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; } }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:4000", "https://private.npm.corp").unwrap(); - let body = String::from_utf8(result).unwrap(); assert!( - !body.contains("private.npm.corp"), - "custom upstream domain leaked" + ctx.state.config.curation.internal_namespaces.is_empty(), + "the immutable startup snapshot must remain empty in this reload regression" ); - } - - #[test] - fn test_rewrite_preserves_non_dist_urls() { - let metadata = serde_json::json!({ - "name": "pkg", - "repository": {"url": "https://github.com/test/pkg.git"}, - "homepage": "https://pkg.example.com", - "versions": { - "1.0.0": { - "dist": { - "tarball": "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz" - }, - "repository": {"url": "https://github.com/test/pkg.git"} - } - } - }); - let data = serde_json::to_vec(&metadata).unwrap(); - let result = - rewrite_tarball_urls(&data, "http://nora:4000", "https://registry.npmjs.org").unwrap(); - let json: serde_json::Value = serde_json::from_slice(&result).unwrap(); + let mut reloaded_config = ctx.state.config.curation.clone(); + reloaded_config.internal_namespaces = vec!["@internal/**".into()]; + let mut reloaded_engine = crate::curation::CurationEngine::new(reloaded_config); + reloaded_engine.set_namespace_filter(Box::new(crate::curation::NamespaceFilter::new( + vec!["@internal/**".into()], + ))); + ctx.state + .reloadable + .store(std::sync::Arc::new(crate::ReloadableConfig { + curation_engine: reloaded_engine, + bypass_token: None, + })); - // Non-dist URLs must be untouched - assert_eq!( - json["repository"]["url"].as_str().unwrap(), - "https://github.com/test/pkg.git" - ); - assert_eq!( - json["homepage"].as_str().unwrap(), - "https://pkg.example.com" - ); + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-registry/-/v1/search?text=public&from=1&size=1", + "", + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let response: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(response["objects"][0]["package"]["name"], "public-two"); + assert_eq!(response["objects"].as_array().unwrap().len(), 1); + assert_eq!(response["total"], 3); + assert_eq!(response["totalIsApproximate"], true); + upstream.verify().await; } - /// #596 acceptance: with a cached metadata body + stored validators, a stale - /// request revalidates with `If-None-Match`; on upstream 304 the cached body - /// is served and NO 200-with-body is ever fetched. Drives the real handler. #[tokio::test] - async fn test_npm_revalidation_304_serves_cache_no_body_download() { - use crate::registry::{write_validators, Validators}; + async fn group_search_pages_upstream_until_large_offset_is_satisfied() { use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; - use wiremock::matchers::{header_exists, method}; + use axum::http::Method; + use wiremock::matchers::{method, path, query_param}; use wiremock::{Mock, MockServer, ResponseTemplate}; let upstream = MockServer::start().await; - // Conditional request (has If-None-Match) → 304. A request WITHOUT it - // would 404 here (no mount), so any full fetch would visibly fail — - // proving the 304 path served from cache. + let first = (0..250) + .map(|index| { + serde_json::json!({ + "package": {"name": format!("item-{index:03}"), "version": "1.0.0"} + }) + }) + .collect::>(); + let second = (250..302) + .map(|index| { + serde_json::json!({ + "package": {"name": format!("item-{index:03}"), "version": "1.0.0"} + }) + }) + .collect::>(); Mock::given(method("GET")) - .and(header_exists("if-none-match")) - .respond_with(ResponseTemplate::new(304)) + .and(path("/-/v1/search")) + .and(query_param("from", "0")) + .and(query_param("size", "250")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": first, + "total": 302 + }))) + .expect(1) .mount(&upstream) .await; - - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some(upstream.uri()); - cfg.npm.metadata_ttl = 0; // always stale → always revalidate - cfg.npm.revalidate = true; - cfg.npm.serve_stale = false; + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .and(query_param("from", "250")) + .and(query_param("size", "250")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": second, + "total": 302 + }))) + .expect(1) + .mount(&upstream) + .await; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } }); - // Pre-seed the cache body + validator sidecar (as a prior 200 would have). - let key = "npm/testpkg/metadata.json"; - ctx.state - .storage - .put(key, b"CACHED-PACKUMENT") - .await - .unwrap(); - write_validators( - &ctx.state.storage, - key, - &Validators { - etag: Some("\"v1\"".to_string()), - last_modified: None, - }, + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-group/-/v1/search?text=item&from=300&size=2", + "", ) .await; - - let before = crate::metrics::PROXY_UPSTREAM_304_TOTAL - .with_label_values(&["npm"]) - .get(); - - let resp = send(&ctx.app, Method::GET, "/npm/testpkg", "").await; - assert_eq!(resp.status(), StatusCode::OK); - let body = body_bytes(resp).await; - assert_eq!(&body[..], b"CACHED-PACKUMENT", "must serve the cached body"); - - let after = crate::metrics::PROXY_UPSTREAM_304_TOTAL - .with_label_values(&["npm"]) - .get(); - assert!(after > before, "a 304 revalidation must be recorded"); + assert_eq!(response.status(), StatusCode::OK); + let response: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(response["objects"][0]["package"]["name"], "item-300"); + assert_eq!(response["objects"][1]["package"]["name"], "item-301"); + assert_eq!(response["total"], 302); + assert_eq!(response["totalIsApproximate"], true); + upstream.verify().await; } - /// #595: a thundering herd of concurrent requests for the same expired - /// metadata key must collapse to a SINGLE upstream fetch. M clients race - /// `GET /npm/testpkg` while the key is stale; a counting mock upstream - /// (delayed so followers pile up) must observe exactly one request, and - /// every client must receive the leader's body. #[tokio::test] - async fn test_npm_concurrent_metadata_miss_coalesces_to_one_upstream_fetch() { - use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; - use std::sync::Arc; - use std::time::Duration; - use wiremock::matchers::method; + async fn group_search_rejects_windows_beyond_the_scan_budget_without_upstream_io() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; let upstream = MockServer::start().await; - // Delay the response so all followers reach the single-flight election - // while the leader is still fetching. Body is not valid JSON, so the - // handler's byte-level URL rewrite passes it through unchanged. Mock::given(method("GET")) - .respond_with( - ResponseTemplate::new(200) - .set_body_string("FRESH-PACKUMENT") - .set_delay(Duration::from_millis(400)), - ) + .and(path("/-/v1/search")) + .respond_with(ResponseTemplate::new(500)) + .expect(0) .mount(&upstream) .await; - - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some(upstream.uri()); - cfg.npm.metadata_ttl = 0; // always stale → always refetch - cfg.npm.revalidate = false; // plain 200, no conditional headers - cfg.npm.serve_stale = false; - // proxy_coalesce defaults to true. + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } }); - // Pre-seed a stale cached body so the stale-metadata refetch path runs. - let key = "npm/testpkg/metadata.json"; - ctx.state.storage.put(key, b"STALE").await.unwrap(); - - let before = crate::metrics::PROXY_COALESCED_TOTAL - .with_label_values(&["npm"]) - .get(); - - const M: usize = 16; - let app = Arc::new(ctx.app.clone()); - let mut handles = Vec::new(); - for _ in 0..M { - let app = Arc::clone(&app); - handles.push(tokio::spawn(async move { - let resp = send(&app, Method::GET, "/npm/testpkg", "").await; - let status = resp.status(); - let body = body_bytes(resp).await; - (status, body) - })); - } + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-group/-/v1/search?text=pkg&from=10000&size=1", + "", + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + upstream.verify().await; + } - for h in handles { - let (status, body) = h.await.unwrap(); - assert_eq!(status, StatusCode::OK); - assert_eq!(&body[..], b"FRESH-PACKUMENT", "every client gets the body"); - } + #[tokio::test] + async fn group_search_stops_repeating_upstream_pages_at_the_page_and_result_budget() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - let upstream_hits = upstream.received_requests().await.unwrap().len(); - assert_eq!( - upstream_hits, 1, - "M concurrent requests for one key must hit upstream exactly once" - ); + let upstream = MockServer::start().await; + let repeated = (0..250) + .map(|index| { + serde_json::json!({ + "package": {"name": format!("same-{index:03}"), "version": "1.0.0"} + }) + }) + .collect::>(); + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": repeated, + "total": 10_001 + }))) + .expect(NPM_SEARCH_SCAN_PAGE_CAP as u64) + .mount(&upstream) + .await; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } + }); - let after = crate::metrics::PROXY_COALESCED_TOTAL - .with_label_values(&["npm"]) - .get(); - assert_eq!( - after - before, - (M - 1) as u64, - "M-1 followers must be served without their own upstream fetch" - ); + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-group/-/v1/search?text=same&from=250&size=1", + "", + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + upstream.verify().await; } - /// #595 kill-switch: with `server.proxy_coalesce = false`, the coalescer is - /// bypassed and every concurrent request fetches independently — so the - /// counting upstream observes one hit per client (proves the gate works). #[tokio::test] - async fn test_npm_coalesce_disabled_lets_every_request_fetch() { - use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; - use std::sync::Arc; - use std::time::Duration; - use wiremock::matchers::method; + async fn proxy_search_scan_deadline_fails_before_upstream_io() { + use crate::test_helpers::create_test_context_with_config; + use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; let upstream = MockServer::start().await; Mock::given(method("GET")) - .respond_with( - ResponseTemplate::new(200) - .set_body_string("FRESH-PACKUMENT") - .set_delay(Duration::from_millis(200)), - ) + .and(path("/-/v1/search")) + .respond_with(ResponseTemplate::new(500)) + .expect(0) .mount(&upstream) .await; - - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some(upstream.uri()); - cfg.npm.metadata_ttl = 0; - cfg.npm.revalidate = false; - cfg.npm.serve_stale = false; - cfg.server.proxy_coalesce = false; // kill-switch off + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } }); + let repository = configured_proxy( + &ctx.state, + ctx.state.config.npm.repository("npm-registry").unwrap(), + ) + .unwrap(); - let key = "npm/testpkg/metadata.json"; - ctx.state.storage.put(key, b"STALE").await.unwrap(); - - const M: usize = 8; - let app = Arc::new(ctx.app.clone()); - let mut handles = Vec::new(); - for _ in 0..M { - let app = Arc::clone(&app); - handles.push(tokio::spawn(async move { - let resp = send(&app, Method::GET, "/npm/testpkg", "").await; - let status = resp.status(); - let _ = body_bytes(resp).await; - status - })); - } - for h in handles { - assert_eq!(h.await.unwrap(), StatusCode::OK); - } - - let upstream_hits = upstream.received_requests().await.unwrap().len(); - assert_eq!( - upstream_hits, M, - "with coalescing disabled every request fetches independently" - ); + assert!(matches!( + proxy_search_page_before( + &ctx.state, + &repository, + Some("text=pkg"), + Instant::now() - Duration::from_millis(1), + ) + .await, + Err(ReadError::SearchScanLimit) + )); + upstream.verify().await; } - // ── npm audit proxy (#597) ── - - /// bulk audit is forwarded to upstream and returned verbatim; internal-namespace - /// package names are stripped from the forwarded body (dependency-confusion). #[tokio::test] - async fn test_npm_audit_bulk_forwards_and_strips_internal() { + async fn group_search_returns_empty_success_and_never_proxies_mixed_internal_query() { use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; + use axum::http::Method; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; let upstream = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/-/npm/v1/security/advisories/bulk")) - .respond_with(ResponseTemplate::new(200).set_body_string(r#"{"lodash":[]}"#)) + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": [], + "total": 0, + "time": "1ms" + }))) + .expect(1) .mount(&upstream) .await; - - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some(upstream.uri()); - cfg.curation.mode = crate::config::CurationMode::Enforce; - cfg.curation.internal_namespaces = vec!["@internal/*".to_string()]; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + config.curation.internal_namespaces = vec!["@internal/**".into()]; + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } }); - let resp = send( + let empty = send( &ctx.app, - Method::POST, - "/npm/-/npm/v1/security/advisories/bulk", - r#"{"lodash":["4.17.0"],"@internal/secret":["1.0.0"]}"#, + Method::GET, + "/repository/npm-group/-/v1/search?text=no-match", + "", + ) + .await; + assert_eq!(empty.status(), StatusCode::OK); + let empty: serde_json::Value = serde_json::from_slice(&body_bytes(empty).await).unwrap(); + assert_eq!(empty["objects"], serde_json::json!([])); + assert_eq!(empty["total"], 0); + + let internal = send( + &ctx.app, + Method::GET, + "/repository/npm-group/-/v1/search?text=foo%20%40internal%2Fpkg", + "", ) .await; - assert_eq!(resp.status(), StatusCode::OK); - let body = body_bytes(resp).await; + assert_eq!(internal.status(), StatusCode::OK); + let internal: serde_json::Value = + serde_json::from_slice(&body_bytes(internal).await).unwrap(); + assert_eq!(internal["objects"], serde_json::json!([])); assert_eq!( - &body[..], - br#"{"lodash":[]}"#, - "upstream body returned verbatim" + upstream.received_requests().await.unwrap().len(), + 1, + "the mixed internal query must not be sent upstream" ); - // The forwarded request must NOT carry the internal package name. - let reqs = upstream.received_requests().await.unwrap(); - assert_eq!(reqs.len(), 1); - let fwd = String::from_utf8_lossy(&reqs[0].body); - assert!(fwd.contains("lodash"), "public pkg forwarded: {fwd}"); - assert!( - !fwd.contains("@internal/secret"), - "internal pkg name must be stripped before forwarding: {fwd}" + for query in ["text=not%3A%40internal%2Fpkg", "quality=%40internal%2Fpkg"] { + let response = send( + &ctx.app, + Method::GET, + &format!("/repository/npm-group/-/v1/search?{query}"), + "", + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let response: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(response["objects"], serde_json::json!([])); + } + assert_eq!( + upstream.received_requests().await.unwrap().len(), + 1, + "decoded internal package names in qualifiers or arbitrary parameters must not leak" ); } - /// Hosted-only repo (no upstream proxy) → npm-compatible empty result, not 405/500. #[tokio::test] - async fn test_npm_audit_no_proxy_returns_empty() { + async fn group_search_deduplicates_with_hosted_member_precedence() { use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; + use axum::http::Method; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = None; + let upstream = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "objects": [ + {"package": {"name": "pkg", "version": "9.0.0"}}, + {"package": {"name": "pkg-public", "version": "2.0.0"}} + ], + "total": 2, + "time": "1ms" + }))) + .expect(1) + .mount(&upstream) + .await; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } }); - let resp = send( + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + + let response = send( &ctx.app, - Method::POST, - "/npm/-/npm/v1/security/advisories/bulk", - r#"{"lodash":["4.17.0"]}"#, + Method::GET, + "/repository/npm-group/-/v1/search?text=pkg&size=20", + "", ) .await; - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(&body_bytes(resp).await[..], b"{}"); + assert_eq!(response.status(), StatusCode::OK); + let response: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(response["objects"].as_array().unwrap().len(), 2); + assert_eq!(response["objects"][0]["package"]["name"], "pkg"); + assert_eq!(response["objects"][0]["package"]["version"], "1.0.0"); + assert_eq!( + response["objects"][0]["package"]["maintainers"], + serde_json::json!([]) + ); + assert_eq!(response["objects"][1]["package"]["name"], "pkg-public"); } - /// quick audit (gzipped lockfile — cannot per-name strip) is refused with an - /// empty result when an internal-namespace filter is configured; upstream is - /// never contacted (no internal-name leak). #[tokio::test] - async fn test_npm_audit_quick_refused_under_filter() { + async fn group_search_marks_healthy_member_results_approximate_when_proxy_fails() { use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; - use wiremock::matchers::any; + use axum::http::Method; + use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; let upstream = MockServer::start().await; - Mock::given(any()) - .respond_with(ResponseTemplate::new(200).set_body_string("SHOULD-NOT-BE-CALLED")) + Mock::given(method("GET")) + .and(path("/-/v1/search")) + .respond_with(ResponseTemplate::new(503)) + .expect(2) .mount(&upstream) .await; - - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some(upstream.uri()); - cfg.curation.mode = crate::config::CurationMode::Enforce; - cfg.curation.internal_namespaces = vec!["@internal/*".to_string()]; + let configured_url = upstream.uri(); + let ctx = create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } }); - let resp = send( - &ctx.app, - Method::POST, - "/npm/-/npm/v1/security/audits/quick", - "lockfile-payload", - ) - .await; - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(&body_bytes(resp).await[..], b"{}"); assert_eq!( - upstream.received_requests().await.unwrap().len(), - 0, - "quick must not reach upstream under a namespace filter" + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED ); - } - - /// POST on a non-audit npm path → 405 (POST is only valid on the audit endpoints). - #[tokio::test] - async fn test_npm_audit_non_audit_post_405() { - use crate::test_helpers::{create_test_context, send}; - use axum::http::{Method, StatusCode}; - let ctx = create_test_context(); - let resp = send(&ctx.app, Method::POST, "/npm/lodash", "x").await; - assert_eq!(resp.status(), StatusCode::METHOD_NOT_ALLOWED); + let response = send( + &ctx.app, + Method::GET, + "/repository/npm-group/-/v1/search?text=pkg&size=20", + "", + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let response: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(response["objects"].as_array().unwrap().len(), 1); + assert_eq!(response["objects"][0]["package"]["name"], "pkg"); + assert_eq!(response["total"], 1); + assert_eq!(response["totalIsApproximate"], true); + upstream.verify().await; } - /// An oversized audit body is rejected with 413 before any upstream forward. #[tokio::test] - async fn test_npm_audit_body_too_large() { - use crate::test_helpers::{create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; - - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some("http://127.0.0.1:1".to_string()); + async fn hosted_only_group_search_marks_partial_results_approximate_on_member_read_error() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(|config| { + config.npm.proxy = None; + config.npm.repositories = vec![ + NpmRepository::Hosted { + name: "healthy-hosted".into(), + write_policy: NpmWritePolicy::AllowOnce, + }, + NpmRepository::Hosted { + name: "broken-hosted".into(), + write_policy: NpmWritePolicy::AllowOnce, + }, + NpmRepository::Group { + name: "npm-group".into(), + members: vec!["healthy-hosted".into(), "broken-hosted".into()], + writable_member: None, + }, + ]; + config.npm.default_repository = Some("npm-group".into()); }); - let big = "x".repeat(NPM_AUDIT_BODY_CAP + 1); - let resp = send( - &ctx.app, - Method::POST, - "/npm/-/npm/v1/security/advisories/bulk", - big, + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/healthy-hosted/healthy", + publish_payload("healthy", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/broken-hosted/broken", + publish_payload("broken", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + ctx.state + .repo_index + .get_strict("npm", &ctx.state.storage) + .await + .expect("prime a verified hosted package index before injecting a member read failure"); + + let broken_manifest = hosted_version_key("broken-hosted", "broken", "1.0.0"); + let mut state = ctx.state.clone(); + state.storage = crate::storage::Storage::from_backend(std::sync::Arc::new( + crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) + .fail_get(&broken_manifest), + )); + let target = named_target(&state, "npm-group").unwrap(); + let response = handle_search( + &state, + &target, + &public_base(&state, Some("npm-group")), + &HeaderMap::new(), + Some("text=&size=20"), ) .await; - assert_eq!(resp.status(), StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(response.status(), StatusCode::OK); + let response: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(response["objects"].as_array().unwrap().len(), 1); + assert_eq!(response["objects"][0]["package"]["name"], "healthy"); + assert_eq!(response["total"], 1); + assert_eq!(response["totalIsApproximate"], true); } - /// Review #1 regression: a Content-Encoded (e.g. gzipped) bulk body under an - /// active namespace filter cannot be name-verified → must fail CLOSED (refuse, - /// upstream never contacted), not forward internal names. #[tokio::test] - async fn test_npm_audit_bulk_encoded_refused_under_filter() { - use crate::test_helpers::{body_bytes, create_test_context_with_config, send_with_headers}; - use axum::http::{Method, StatusCode}; - use wiremock::matchers::any; + async fn successful_named_audit_forward_records_non_sensitive_audit_event() { + use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; let upstream = MockServer::start().await; - Mock::given(any()) - .respond_with(ResponseTemplate::new(200).set_body_string("LEAKED")) + Mock::given(method("POST")) + .and(path("/-/npm/v1/security/advisories/bulk")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({}))) + .expect(1) .mount(&upstream) .await; - - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some(upstream.uri()); - cfg.curation.mode = crate::config::CurationMode::Enforce; - cfg.curation.internal_namespaces = vec!["@internal/*".to_string()]; + let configured_url = upstream.uri(); + let ctx = crate::test_helpers::create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } }); - // Claims gzip encoding → NORA cannot see the names → must refuse. - let resp = send_with_headers( - &ctx.app, - Method::POST, - "/npm/-/npm/v1/security/advisories/bulk", - vec![("content-encoding", "gzip")], - "gzipped-body-not-inspected-because-refused", + let audit_dir = tempfile::tempdir().unwrap(); + let mut state = ctx.state.clone(); + state.audit = std::sync::Arc::new(crate::audit::AuditLog::new( + audit_dir.path().to_str().unwrap(), + crate::audit::AuditMode::File, + )); + let target = named_target(&state, "npm-group").unwrap(); + + let response = handle_post( + state.clone(), + target, + "-/npm/v1/security/advisories/bulk".to_string(), + HeaderMap::new(), + Body::from(br#"{"pkg":["1.0.0"]}"#.as_slice()), ) .await; - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!(&body_bytes(resp).await[..], b"{}"); - assert_eq!( - upstream.received_requests().await.unwrap().len(), - 0, - "encoded bulk under a filter must NOT be forwarded" + assert_eq!(response.status(), StatusCode::OK); + state.audit.shutdown().await; + let line = std::fs::read_to_string(audit_dir.path().join("audit.jsonl")).unwrap(); + let event: serde_json::Value = serde_json::from_str(line.trim()).unwrap(); + assert_eq!(event["action"], "proxy_fetch"); + assert_eq!(event["registry"], "npm"); + assert_eq!(event["detail"], "audit"); + assert!( + !line.contains("1.0.0"), + "audit event must not include request body contents" ); + upstream.verify().await; } - /// Review #3 regression: upstream 5xx must surface as 502 (audit-endpoint error), - /// NOT `200 {}` (which npm renders as a false "0 vulnerabilities"). #[tokio::test] - async fn test_npm_audit_upstream_5xx_returns_502() { - use crate::test_helpers::{create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; - use wiremock::matchers::any; + async fn gzip_npm6_audit_paths_strip_internal_dependencies_and_lockfile_packages() { + use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; let upstream = MockServer::start().await; - Mock::given(any()) - .respond_with(ResponseTemplate::new(500)) - .mount(&upstream) + for path_value in [ + "/-/npm/v1/security/audits", + "/-/npm/v1/security/audits/quick", + ] { + Mock::given(method("POST")) + .and(path(path_value)) + .respond_with( + ResponseTemplate::new(400) + .set_body_json(serde_json::json!({"error": "unsupported by upstream"})), + ) + .expect(1) + .mount(&upstream) + .await; + } + let configured_url = upstream.uri(); + let ctx = crate::test_helpers::create_test_context_with_config(move |config| { + named_config(config); + config.curation.internal_namespaces = vec!["@internal/**".into()]; + if let NpmRepository::Proxy { url, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + } + }); + let payload = serde_json::to_vec(&serde_json::json!({ + "name": "application", + "dependencies": { + "public-package": {"version": "1.0.0"}, + "@internal/pkg": {"version": "2.0.0"} + }, + "requires": { + "public-package": "^1", + "@internal/pkg": "^2" + }, + "packages": { + "": {"name": "application"}, + "node_modules/public-package": {"name": "public-package"}, + "node_modules/@internal/pkg": {"name": "@internal/pkg"} + } + })) + .unwrap(); + let compressed = gzip_audit_body(&payload).unwrap(); + let mut headers = HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + headers.insert(header::CONTENT_ENCODING, HeaderValue::from_static("gzip")); + + for audit_path in ["-/npm/v1/security/audits", "-/npm/v1/security/audits/quick"] { + let response = handle_post( + ctx.state.clone(), + named_target(&ctx.state, "npm-group").unwrap(), + audit_path.to_string(), + headers.clone(), + Body::from(compressed.clone()), + ) .await; + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "upstream status must be relayed for {audit_path}" + ); + } - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some(upstream.uri()); - }); - let resp = send( - &ctx.app, - Method::POST, - "/npm/-/npm/v1/security/advisories/bulk", - r#"{"lodash":["4.17.0"]}"#, + let requests = upstream.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2); + for request in requests { + let (decoded, was_gzip) = decode_audit_body(&headers, request.body.as_slice()).unwrap(); + assert!(was_gzip); + let forwarded: serde_json::Value = serde_json::from_slice(&decoded).unwrap(); + assert!( + !String::from_utf8_lossy(&decoded).contains("@internal"), + "internal package names must never leave Nora" + ); + assert!(forwarded["dependencies"].get("public-package").is_some()); + assert!(forwarded["requires"].get("public-package").is_some()); + assert!(forwarded["packages"] + .get("node_modules/public-package") + .is_some()); + assert!(forwarded["dependencies"].get("@internal/pkg").is_none()); + assert!(forwarded["packages"] + .get("node_modules/@internal/pkg") + .is_none()); + } + upstream.verify().await; + } + + #[tokio::test] + async fn audit_rejects_empty_unsafe_or_hosted_only_requests_instead_of_false_success() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let hosted = named_target(&ctx.state, "npm-private").unwrap(); + let response = handle_post( + ctx.state.clone(), + hosted, + "-/npm/v1/security/audits/quick".to_string(), + HeaderMap::new(), + Body::empty(), + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + + let mut headers = HeaderMap::new(); + headers.insert(header::CONTENT_ENCODING, HeaderValue::from_static("br")); + let response = handle_post( + ctx.state.clone(), + named_target(&ctx.state, "npm-group").unwrap(), + "-/npm/v1/security/audits".to_string(), + headers, + Body::from(br#"{"name":"app"}"#.as_slice()), ) .await; - assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); } - /// Review #5: the client's Authorization (its NORA token) must never be - /// forwarded upstream — only the allowlisted headers + configured proxy_auth. #[tokio::test] - async fn test_npm_audit_client_authorization_not_forwarded() { - use crate::test_helpers::{create_test_context_with_config, send_with_headers}; - use axum::http::{Method, StatusCode}; - use wiremock::matchers::any; + async fn audit_redirects_are_validated_before_replaying_auth_and_body() { + use wiremock::matchers::{header, method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; let upstream = MockServer::start().await; - Mock::given(any()) - .respond_with(ResponseTemplate::new(200).set_body_string("{}")) + let attacker = MockServer::start().await; + let credentials = "reader:secret"; + let authorization = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(credentials) + ); + let audit_path = "-/npm/v1/security/advisories/bulk"; + let request_body = br#"{"pkg":["1.0.0"]}"#; + let cross_location = format!("{}/steal", attacker.uri()); + + Mock::given(method("POST")) + .and(path( + "/repository/npm-cross/-/npm/v1/security/advisories/bulk", + )) + .and(header("authorization", authorization.as_str())) + .respond_with( + ResponseTemplate::new(307).insert_header("location", cross_location.as_str()), + ) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("POST")) + .and(path( + "/repository/npm-allowed/-/npm/v1/security/advisories/bulk", + )) + .and(header("authorization", authorization.as_str())) + .respond_with( + ResponseTemplate::new(307) + .insert_header("location", "/repository/npm-allowed/audit-final"), + ) + .expect(1) + .mount(&upstream) + .await; + Mock::given(method("POST")) + .and(path("/repository/npm-allowed/audit-final")) + .and(header("authorization", authorization.as_str())) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({}))) + .expect(1) .mount(&upstream) .await; - let ctx = create_test_context_with_config(|cfg| { - cfg.npm.proxy = Some(upstream.uri()); - }); - let resp = send_with_headers( - &ctx.app, - Method::POST, - "/npm/-/npm/v1/security/advisories/bulk", - vec![("authorization", "Bearer nora-client-token")], - r#"{"lodash":["4.17.0"]}"#, + let context = |base_path: &'static str| { + let configured_url = format!("{}{base_path}", upstream.uri()); + crate::test_helpers::create_test_context_with_config(move |config| { + named_config(config); + if let NpmRepository::Proxy { url, auth, .. } = &mut config.npm.repositories[1] { + *url = configured_url; + *auth = Some(crate::secrets::ProtectedString::new( + credentials.to_string(), + )); + } + }) + }; + + let cross = context("/repository/npm-cross"); + let cross_target = named_target(&cross.state, "npm-group").unwrap(); + let response = handle_post( + cross.state, + cross_target, + audit_path.to_string(), + HeaderMap::new(), + Body::from(request_body.as_slice()), ) .await; - assert_eq!(resp.status(), StatusCode::OK); - let reqs = upstream.received_requests().await.unwrap(); - assert_eq!(reqs.len(), 1); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); assert!( - reqs[0].headers.get("authorization").is_none(), - "client Authorization must not be forwarded upstream" + attacker.received_requests().await.unwrap().is_empty(), + "rejected audit redirect must receive neither request, auth nor body" ); - } - /// Under `anonymous_read`, an unauthenticated `npm audit` POST must pass the - /// auth gate (audit is a read-semantics query) and reach the handler — hosted- - /// only (no proxy) → 200 `{}`. A non-audit npm POST stays gated → 401. - #[tokio::test] - async fn test_npm_audit_anonymous_read_allows_post() { - use crate::test_helpers::{create_test_context_with_config, send}; - use axum::http::{Method, StatusCode}; - - let ctx = create_test_context_with_config(|cfg| { - cfg.auth.enabled = true; - cfg.auth.anonymous_read = true; - cfg.npm.proxy = None; // hosted-only → handler returns 200 {} - }); - let audit = send( - &ctx.app, - Method::POST, - "/npm/-/npm/v1/security/advisories/bulk", - r#"{"lodash":["4.17.0"]}"#, + let allowed = context("/repository/npm-allowed"); + let allowed_target = named_target(&allowed.state, "npm-group").unwrap(); + let response = handle_post( + allowed.state, + allowed_target, + audit_path.to_string(), + HeaderMap::new(), + Body::from(request_body.as_slice()), ) .await; - assert_eq!( - audit.status(), - StatusCode::OK, - "anonymous npm audit must pass the auth gate under anonymous_read" - ); - // A non-audit npm POST is not read-eligible → still requires auth. - let other = send(&ctx.app, Method::POST, "/npm/somepkg", "x").await; - assert_eq!(other.status(), StatusCode::UNAUTHORIZED); + assert_eq!(response.status(), StatusCode::OK); + let requests = upstream.received_requests().await.unwrap(); + let final_request = requests + .iter() + .find(|request| request.url.path() == "/repository/npm-allowed/audit-final") + .expect("allowed redirect target request"); + assert_eq!(final_request.method.as_str(), "POST"); + assert_eq!(final_request.body.as_slice(), request_body); + upstream.verify().await; } } diff --git a/nora-registry/src/registry/ns_isolation_metadata_tests.rs b/nora-registry/src/registry/ns_isolation_metadata_tests.rs index ea7c457d..6331f9d7 100644 --- a/nora-registry/src/registry/ns_isolation_metadata_tests.rs +++ b/nora-registry/src/registry/ns_isolation_metadata_tests.rs @@ -296,13 +296,13 @@ async fn npm_internal_stale_not_refetched() { c.npm.metadata_ttl = 0; // every pull is "stale" → would trigger refetch_metadata c.npm.serve_stale = false; // without the guard, the failed refetch → 502 }); - // A locally-published internal packument whose cache is "stale" must be served - // from the local copy, NOT re-fetched upstream (the #725 residual leak). + // A locally-published internal package in the fresh named hosted layout + // must be served from local state, NOT re-fetched upstream. ctx.state .storage .put( - "npm/internalpkg/metadata.json", - br#"{"name":"internalpkg","versions":{}}"#, + "npm/repositories/npm-private/internalpkg/pkg.json", + br#"{"name":"internalpkg"}"#, ) .await .unwrap(); diff --git a/nora-registry/src/repo_index.rs b/nora-registry/src/repo_index.rs index 91591f47..a950d11b 100644 --- a/nora-registry/src/repo_index.rs +++ b/nora-registry/src/repo_index.rs @@ -166,41 +166,7 @@ impl RepoIndex { // Double-check under lock (another thread may have rebuilt) if index.is_dirty() { - let data = match reg_type { - RegistryType::Docker => build_docker_index(storage).await, - RegistryType::Maven => build_maven_index(storage).await, - RegistryType::Npm => build_npm_index(storage).await, - RegistryType::Cargo => build_cargo_index(storage).await, - RegistryType::PyPI => build_pypi_index(storage).await, - RegistryType::Go => build_go_index(storage).await, - RegistryType::Raw => build_raw_index(storage).await, - RegistryType::Nuget => { - let (p, s) = crate::registry::nuget::INDEX_PATTERN; - build_generic_index(storage, p, s).await - } - RegistryType::Gems => build_gems_index(storage).await, - RegistryType::Terraform => { - let (p, s) = crate::registry::terraform::INDEX_PATTERN; - build_generic_index(storage, p, s).await - } - RegistryType::Ansible => { - let (p, s) = crate::registry::ansible::INDEX_PATTERN; - build_generic_index(storage, p, s).await - } - RegistryType::PubDart => { - let (p, s) = crate::registry::pub_dart::INDEX_PATTERN; - build_generic_index(storage, p, s).await - } - RegistryType::Conan => build_conan_index(storage).await, - RegistryType::Rpm => { - let (p, s) = crate::registry::rpm::INDEX_PATTERN; - build_generic_index(storage, p, s).await - } - RegistryType::Deb => { - let (p, s) = crate::registry::deb::INDEX_PATTERN; - build_generic_index(storage, p, s).await - } - }; + let data = build_index(reg_type, storage).await; match data { Some(data) => { info!(registry = registry, count = data.len(), "Index rebuilt"); @@ -222,6 +188,29 @@ impl RepoIndex { index.get_cached() } + /// Rebuild a dirty index and surface storage uncertainty instead of + /// serving stale data. Protocol handlers use this when an empty/partial + /// answer would be semantically different from a UI's stale snapshot. + pub async fn get_strict( + &self, + registry: &str, + storage: &Storage, + ) -> Result>, ()> { + let reg_type = RegistryType::from_str_opt(registry).ok_or(())?; + let index = self.indexes.get(®_type).ok_or(())?; + if !index.is_dirty() { + return Ok(index.get_cached()); + } + + let _guard = index.rebuild_lock.lock().await; + if index.is_dirty() { + let data = build_index(reg_type, storage).await.ok_or(())?; + info!(registry, count = data.len(), "Index rebuilt"); + index.set(data); + } + Ok(index.get_cached()) + } + /// Get counts for stats (no rebuild, just current state) pub fn counts(&self) -> HashMap { self.indexes @@ -239,6 +228,44 @@ impl RepoIndex { } } +async fn build_index(reg_type: RegistryType, storage: &Storage) -> Option> { + match reg_type { + RegistryType::Docker => build_docker_index(storage).await, + RegistryType::Maven => build_maven_index(storage).await, + RegistryType::Npm => build_npm_index(storage).await, + RegistryType::Cargo => build_cargo_index(storage).await, + RegistryType::PyPI => build_pypi_index(storage).await, + RegistryType::Go => build_go_index(storage).await, + RegistryType::Raw => build_raw_index(storage).await, + RegistryType::Nuget => { + let (prefix, suffix) = crate::registry::nuget::INDEX_PATTERN; + build_generic_index(storage, prefix, suffix).await + } + RegistryType::Gems => build_gems_index(storage).await, + RegistryType::Terraform => { + let (prefix, suffix) = crate::registry::terraform::INDEX_PATTERN; + build_generic_index(storage, prefix, suffix).await + } + RegistryType::Ansible => { + let (prefix, suffix) = crate::registry::ansible::INDEX_PATTERN; + build_generic_index(storage, prefix, suffix).await + } + RegistryType::PubDart => { + let (prefix, suffix) = crate::registry::pub_dart::INDEX_PATTERN; + build_generic_index(storage, prefix, suffix).await + } + RegistryType::Conan => build_conan_index(storage).await, + RegistryType::Rpm => { + let (prefix, suffix) = crate::registry::rpm::INDEX_PATTERN; + build_generic_index(storage, prefix, suffix).await + } + RegistryType::Deb => { + let (prefix, suffix) = crate::registry::deb::INDEX_PATTERN; + build_generic_index(storage, prefix, suffix).await + } + } +} + impl Default for RepoIndex { fn default() -> Self { Self::new() @@ -350,30 +377,60 @@ async fn build_maven_index(storage: &Storage) -> Option> { async fn build_npm_index(storage: &Storage) -> Option> { let keys = list_keys(storage, "npm/").await?; let mut packages: HashMap = HashMap::new(); + let by_key: HashMap<&str, &crate::storage::FileMeta> = keys + .iter() + .map(|(key, meta)| (key.as_str(), meta)) + .collect(); - // Count tarballs instead of parsing metadata.json (faster than parsing JSON) + // A hosted version manifest is the publish commit point: a staged tarball + // without it is an invisible orphan and must not become a UI artifact. + // Proxy cache has no hosted manifest, so its concrete tarballs remain the + // countable unit. Groups own no objects and never appear here. for (key, meta) in &keys { - if let Some(rest) = key.strip_prefix("npm/") { - // Pattern: npm/{package}/tarballs/{file}.tgz - // Scoped: npm/@scope/package/tarballs/{file}.tgz - if rest.contains("/tarballs/") && ends_with_ci(key, ".tgz") { - let parts: Vec<_> = rest.split('/').collect(); - if !parts.is_empty() { - // Scoped packages: @scope/package → parts[0]="@scope", parts[1]="package" - let name = if parts[0].starts_with('@') && parts.len() >= 4 { - format!("{}/{}", parts[0], parts[1]) - } else { - parts[0].to_string() - }; - let entry = packages.entry(name).or_insert((0, 0, 0)); - entry.0 += 1; - - entry.1 += meta.size; - if meta.modified > entry.2 { - entry.2 = meta.modified; + let Some(parsed) = crate::npm_layout::parse_npm_object_key(key) else { + continue; + }; + let name = format!("repositories/{}/{}", parsed.repository, parsed.package); + match parsed.kind { + crate::npm_layout::NpmObjectKind::HostedVersion(_) => { + let manifest = match storage.get(key).await { + Ok(manifest) => manifest, + Err(error) => { + tracing::warn!( + key, + error = %error, + "npm index: cannot read hosted version manifest" + ); + return None; } - } + }; + let Some(blob_key) = crate::npm_layout::hosted_blob_key_from_manifest( + &parsed.repository, + &parsed.package, + &manifest, + ) else { + tracing::warn!( + key, + "npm index: hosted manifest has no valid blob reference" + ); + return None; + }; + let blob = by_key.get(blob_key.as_str()).copied(); + let entry = packages.entry(name).or_insert((0, 0, 0)); + entry.0 += 1; + entry.1 += meta.size + blob.map(|value| value.size).unwrap_or(0); + entry.2 = entry + .2 + .max(meta.modified) + .max(blob.map(|value| value.modified).unwrap_or(0)); } + crate::npm_layout::NpmObjectKind::ProxyTarball(_) => { + let entry = packages.entry(name).or_insert((0, 0, 0)); + entry.0 += 1; + entry.1 += meta.size; + entry.2 = entry.2.max(meta.modified); + } + _ => {} } } @@ -928,6 +985,128 @@ mod tests { assert_eq!(size, 200 + 30, "metadata bytes still count toward size==du"); } + #[tokio::test] + async fn maven_index_keeps_named_repository_paths_isolated() { + let (_d, storage) = temp_storage(); + storage + .put( + "maven/repositories/releases/com/example/a/1.0/a-1.0.jar", + b"release", + ) + .await + .unwrap(); + storage + .put( + "maven/repositories/open/com/example/a/1.0/a-1.0.jar", + b"open", + ) + .await + .unwrap(); + + let repos = build_maven_index(&storage).await.expect("index built"); + + assert!(repos + .iter() + .any(|entry| entry.name == "repositories/releases/com/example/a/1.0")); + assert!(repos + .iter() + .any(|entry| entry.name == "repositories/open/com/example/a/1.0")); + assert_eq!(repos.iter().map(|entry| entry.versions).sum::(), 2); + } + + #[tokio::test] + async fn npm_index_keeps_hosted_and_proxy_repositories_isolated() { + use base64::Engine as _; + use sha2::Digest as _; + + let (_d, storage) = temp_storage(); + let hosted_manifest = |name: &str, version: &str, blob: &[u8]| { + serde_json::to_vec(&serde_json::json!({ + "name": name, + "version": version, + "dist": { + "integrity": format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD + .encode(sha2::Sha512::digest(blob)) + ) + } + })) + .unwrap() + }; + let scoped_manifest = hosted_manifest("@scope/pkg", "1.0.0", b"hosted"); + let scoped_blob = crate::npm_layout::hosted_blob_key_from_manifest( + "npm-private", + "@scope/pkg", + &scoped_manifest, + ) + .unwrap(); + storage.put(&scoped_blob, b"hosted").await.unwrap(); + storage + .put( + "npm/repositories/npm-private/@scope/pkg/versions/1.0.0.json", + &scoped_manifest, + ) + .await + .unwrap(); + storage + .put( + "npm/repositories/npm-registry/proxy/tarballs/@scope/pkg/pkg-1.0.0.tgz", + b"proxy", + ) + .await + .unwrap(); + let proxy_manifest = hosted_manifest("proxy", "1.0.0", b"hosted-package-named-proxy"); + let hosted_proxy_blob = crate::npm_layout::hosted_blob_key_from_manifest( + "npm-private", + "proxy", + &proxy_manifest, + ) + .unwrap(); + storage + .put(&hosted_proxy_blob, b"hosted-package-named-proxy") + .await + .unwrap(); + storage + .put( + "npm/repositories/npm-private/proxy/versions/1.0.0.json", + &proxy_manifest, + ) + .await + .unwrap(); + let orphan_manifest = hosted_manifest("orphan", "1.0.0", b"precommit-orphan"); + let orphan_blob = crate::npm_layout::hosted_blob_key_from_manifest( + "npm-private", + "orphan", + &orphan_manifest, + ) + .unwrap(); + storage + .put(&orphan_blob, b"precommit-orphan") + .await + .unwrap(); + + let repos = build_npm_index(&storage).await.expect("index built"); + + assert_eq!(repos.len(), 3); + assert!(repos + .iter() + .any(|entry| entry.name == "repositories/npm-private/@scope/pkg")); + assert!(repos + .iter() + .any(|entry| entry.name == "repositories/npm-private/proxy")); + assert!(repos + .iter() + .any(|entry| entry.name == "repositories/npm-registry/@scope/pkg")); + assert_eq!(repos.iter().map(|entry| entry.versions).sum::(), 3); + assert!( + !repos + .iter() + .any(|entry| entry.name == "repositories/npm-private/orphan"), + "hosted pre-commit tarballs must not enter the index" + ); + } + #[tokio::test] async fn docker_index_real_size_not_virtual_and_single_count() { let (_d, s) = temp_storage(); diff --git a/nora-registry/src/retention.rs b/nora-registry/src/retention.rs index dbc7e011..1c1a72b9 100644 --- a/nora-registry/src/retention.rs +++ b/nora-registry/src/retention.rs @@ -12,10 +12,11 @@ use std::time::{Instant, SystemTime, UNIX_EPOCH}; use prometheus::{ register_histogram, register_int_counter, register_int_gauge, Histogram, IntCounter, IntGauge, }; +use sha2::Digest as _; use tracing::info; -use crate::config::RetentionRule; -use crate::storage::Storage; +use crate::config::{MavenConfig, MavenRepository, RetentionRule}; +use crate::storage::{Storage, StorageError}; use crate::validation::ends_with_ci; use crate::PublishLocks; @@ -78,6 +79,9 @@ pub struct VersionEntry { pub struct DeletionPlan { pub version_name: String, pub keys: Vec, + /// Identity of the planned version directory at collection time. + /// Maven revalidates this under the GA publish lock before deletion. + pub modified: u64, pub size: u64, pub reason: String, } @@ -148,6 +152,7 @@ pub fn plan_deletions( deletions.push(DeletionPlan { version_name: version.name.clone(), keys: version.keys.clone(), + modified: version.modified, size: version.size, reason: reason_parts.join(", "), }); @@ -245,57 +250,158 @@ fn glob_match_inner(p: &[char], t: &[char]) -> bool { // Version collectors (per-registry) // ============================================================================ -/// Collect Maven versions for a given group/artifact. -async fn collect_maven_versions(storage: &Storage) -> Vec<(String, Vec)> { - let all_keys = storage.list("maven/").await.unwrap_or_else(|e| { - tracing::error!("Failed to list maven/ keys: {}", e); - Vec::new() - }); +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum MavenRetentionKind { + /// Authoritative hosted content. Retention must rewrite artifact-level + /// discovery metadata before removing a version directory. + Hosted, + /// Rebuildable upstream cache. Artifact-level metadata remains + /// upstream-owned and must never be filtered by local cache retention. + ProxyCache, + /// Pre-named layout where hosted/proxy provenance is unknowable. Keep the + /// old deletion behaviour but never rewrite discovery metadata. + Legacy, +} + +#[derive(Debug)] +struct MavenVersionGroup { + group_name: String, + versions: Vec, + kind: MavenRetentionKind, + storage_prefix: String, + group_path: String, + artifact_id: String, +} + +#[derive(Debug, Clone)] +struct MavenRetentionContext { + kind: MavenRetentionKind, + storage_prefix: String, + group_path: String, + artifact_id: String, +} + +#[derive(Default)] +struct MavenVersionDirectory { + keys: Vec, + modified: u64, + size: u64, + has_payload: bool, +} + +/// Collect Maven version directories while retaining repository provenance. +/// +/// A directory is considered a version only when it contains at least one +/// non-`maven-metadata.xml*` object. Once identified, every object in that +/// directory is retained in the plan, including V-level SNAPSHOT metadata and +/// all of its checksum sidecars. This avoids mistaking the artifact-level +/// `maven-metadata.xml` directory itself for a version. +async fn collect_maven_versions(storage: &Storage, config: &MavenConfig) -> Vec { + let all_entries = match storage.list_with_meta("maven/").await { + Ok(entries) => entries, + Err(error) => { + tracing::error!(%error, "retention: failed to list Maven keys"); + return Vec::new(); + } + }; let mut artifacts: std::collections::HashMap< - String, - std::collections::HashMap>, + (String, String, MavenRetentionKind), + std::collections::HashMap, > = std::collections::HashMap::new(); - for key in &all_keys { - let parts: Vec<&str> = key - .strip_prefix("maven/") - .unwrap_or("") + for (key, meta) in all_entries { + let Some(after_maven) = key.strip_prefix("maven/") else { + continue; + }; + let (repository, relative, kind) = + if let Some(named) = after_maven.strip_prefix("repositories/") { + let Some((repository, relative)) = named.split_once('/') else { + continue; + }; + let kind = match config.repository(repository) { + Some(MavenRepository::Hosted { .. }) => MavenRetentionKind::Hosted, + Some(MavenRepository::Proxy { .. }) => MavenRetentionKind::ProxyCache, + Some(MavenRepository::Group { .. }) => { + tracing::warn!( + repository, + key, + "retention: group repository unexpectedly owns Maven storage; key kept" + ); + continue; + } + None => { + tracing::warn!( + repository, + key, + "retention: Maven repository is not configured; key kept" + ); + continue; + } + }; + (Some(repository.to_string()), relative, kind) + } else { + (None, after_maven, MavenRetentionKind::Legacy) + }; + + let parts: Vec<&str> = relative .split('/') + .filter(|part| !part.is_empty()) .collect(); - // maven/{group...}/{artifact}/{version}/{file} - // Minimum: maven/g/a/v/f = 4+ segments after maven/ + // {group...}/{artifact}/{version}/{file} if parts.len() < 4 { continue; } - // Skip maven-metadata.xml at artifact level - if parts[parts.len() - 1].starts_with("maven-metadata") { - continue; - } + let filename = parts[parts.len() - 1]; let version = parts[parts.len() - 2]; let artifact_path = parts[..parts.len() - 2].join("/"); - artifacts - .entry(artifact_path) + let directory = artifacts + .entry((repository.unwrap_or_default(), artifact_path, kind)) .or_default() .entry(version.to_string()) - .or_default() - .push(key.clone()); + .or_default(); + directory.keys.push(key.clone()); + directory.modified = directory.modified.max(meta.modified); + directory.size += meta.size; + directory.has_payload |= !filename.starts_with("maven-metadata.xml"); } - let mut result = Vec::new(); - for (artifact, versions) in &artifacts { - let mut entries = Vec::new(); - for (version, keys) in versions { - let (modified, size) = aggregate_meta(storage, keys).await; - entries.push(VersionEntry { - name: version.clone(), - keys: keys.clone(), - modified, - size, - }); - } - result.push((format!("maven:{}", artifact), entries)); - } - result + artifacts + .into_iter() + .filter_map(|((repository, artifact_path, kind), versions)| { + let (group_path, artifact_id) = artifact_path.rsplit_once('/')?; + let versions: Vec = versions + .into_iter() + .filter(|(_, directory)| directory.has_payload) + .map(|(name, directory)| VersionEntry { + name, + keys: directory.keys, + modified: directory.modified, + size: directory.size, + }) + .collect(); + if versions.is_empty() { + return None; + } + let group_name = if repository.is_empty() { + format!("maven:{artifact_path}") + } else { + format!("maven:{repository}:{artifact_path}") + }; + let storage_prefix = if repository.is_empty() { + "maven/".to_string() + } else { + format!("maven/repositories/{repository}/") + }; + Some(MavenVersionGroup { + group_name, + versions, + kind, + storage_prefix, + group_path: group_path.to_string(), + artifact_id: artifact_id.to_string(), + }) + }) + .collect() } /// Collect rpm package versions per repository from the metadata sidecars @@ -529,742 +635,2717 @@ async fn collect_docker_versions(storage: &Storage) -> Vec<(String, Vec Vec<(String, Vec)> { - let all_keys = storage.list("npm/").await.unwrap_or_else(|e| { - tracing::error!("Failed to list npm/ keys: {}", e); - Vec::new() - }); - let mut packages: std::collections::HashMap> = - std::collections::HashMap::new(); +#[derive(Debug, Clone)] +struct NpmVersionGroup { + group_name: String, + repository: String, + package: String, + versions: Vec, + /// Digest of the complete authoritative hosted package state observed + /// while planning. Every plan in this package is validated against the + /// same digest under the exact npm publish lock. + snapshot_guard: String, +} - for key in &all_keys { - // npm/{package}/tarballs/{file} — each tarball is a "version" - // Skip metadata.json and checksum files — they are indexes, not versions. - if let Some(rest) = key.strip_prefix("npm/") { - if rest.contains("/tarballs/") - && !ends_with_ci(key, ".sha256") - && !ends_with_ci(key, "/metadata.json") - { - let pkg = rest.split("/tarballs/").next().unwrap_or(""); - if !pkg.is_empty() { - packages - .entry(pkg.to_string()) - .or_default() - .push(key.clone()); - } +#[derive(Debug)] +struct NpmPackageSnapshot { + versions: Vec, + guard: String, +} + +fn is_hosted_npm_object(kind: &crate::npm_layout::NpmObjectKind) -> bool { + matches!( + kind, + crate::npm_layout::NpmObjectKind::HostedPackage + | crate::npm_layout::NpmObjectKind::HostedVersion(_) + | crate::npm_layout::NpmObjectKind::HostedPublishComplete(_) + | crate::npm_layout::NpmObjectKind::HostedTarball(_) + | crate::npm_layout::NpmObjectKind::HostedBlob { .. } + | crate::npm_layout::NpmObjectKind::HostedDistTag(_) + | crate::npm_layout::NpmObjectKind::HostedDeprecation(_) + ) +} + +fn npm_guard_bytes(hasher: &mut sha2::Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +/// Read one exact hosted npm package snapshot. +/// +/// `LIST` establishes the relevant key set, `HEAD` supplies metadata for every +/// listed key, and mutable/visibility objects are read and hashed. Any +/// uncertainty fails closed for the whole package. Blob bodies are not read: +/// their content-addressed key plus metadata is sufficient identity, while +/// avoiding a retention-time download of every tarball. +async fn npm_package_snapshot( + storage: &Storage, + repository: &str, + package: &str, + listed_keys: Vec, +) -> Result { + use std::collections::BTreeMap; + + let mut objects = BTreeMap::new(); + for key in listed_keys { + let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { + continue; + }; + if parsed.repository != repository + || parsed.package != package + || !is_hosted_npm_object(&parsed.kind) + { + continue; + } + let meta = storage + .stat(&key) + .await + .ok_or_else(|| format!("metadata unavailable for {key}"))?; + let contents = if matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedPackage + | crate::npm_layout::NpmObjectKind::HostedVersion(_) + | crate::npm_layout::NpmObjectKind::HostedPublishComplete(_) + | crate::npm_layout::NpmObjectKind::HostedDistTag(_) + | crate::npm_layout::NpmObjectKind::HostedDeprecation(_) + ) { + let contents = storage + .get(&key) + .await + .map_err(|error| format!("cannot read {key}: {error}"))?; + if contents.len() as u64 != meta.size { + return Err(format!("size changed while reading {key}")); + } + let after = storage + .stat(&key) + .await + .ok_or_else(|| format!("metadata disappeared for {key}"))?; + if after.size != meta.size || after.modified != meta.modified { + return Err(format!("metadata changed while reading {key}")); } + Some(contents) + } else { + None + }; + if objects + .insert(key.clone(), (parsed.kind, meta, contents)) + .is_some() + { + return Err(format!("duplicate key in listing: {key}")); } } - let mut result = Vec::new(); - for (pkg, tarball_keys) in &packages { - let mut entries = Vec::new(); - for key in tarball_keys { - let filename = key.rsplit('/').next().unwrap_or(""); - let (modified, size) = aggregate_meta(storage, std::slice::from_ref(key)).await; - // Include associated .sha256 - let mut keys = vec![key.clone()]; - let hash_key = format!("{}.sha256", key); - if storage.stat(&hash_key).await.is_some() { - keys.push(hash_key); + let mut hasher = sha2::Sha256::new(); + npm_guard_bytes(&mut hasher, b"nora/npm-retention-package/v1"); + npm_guard_bytes(&mut hasher, repository.as_bytes()); + npm_guard_bytes(&mut hasher, package.as_bytes()); + for (key, (_, meta, contents)) in &objects { + npm_guard_bytes(&mut hasher, key.as_bytes()); + hasher.update(meta.size.to_be_bytes()); + hasher.update(meta.modified.to_be_bytes()); + match contents { + Some(contents) => { + hasher.update([1]); + npm_guard_bytes(&mut hasher, contents); } - entries.push(VersionEntry { - name: filename.to_string(), - keys, - modified, - size, - }); + None => hasher.update([0]), } - result.push((format!("npm:{}", pkg), entries)); } - result -} - -/// Collect PyPI package files. -async fn collect_pypi_versions(storage: &Storage) -> Vec<(String, Vec)> { - let all_keys = storage.list("pypi/").await.unwrap_or_else(|e| { - tracing::error!("Failed to list pypi/ keys: {}", e); - Vec::new() - }); - let mut packages: std::collections::HashMap> = - std::collections::HashMap::new(); - for key in &all_keys { - if let Some(rest) = key.strip_prefix("pypi/") { - // Skip checksums and metadata.json — metadata is the package index, - // not a version artifact. Deleting it makes the package undiscoverable. - if !ends_with_ci(key, ".sha256") - && !ends_with_ci(key, ".sha1") - && !ends_with_ci(key, ".md5") - && !ends_with_ci(key, ".sha512") - && !ends_with_ci(key, "/metadata.json") - { - let pkg = rest.split('/').next().unwrap_or(""); - if !pkg.is_empty() { - packages - .entry(pkg.to_string()) - .or_default() - .push(key.clone()); + let mut manifests = BTreeMap::new(); + let mut deprecations = BTreeMap::new(); + let mut completions = BTreeMap::new(); + let mut tags = Vec::new(); + for (key, (kind, _, contents)) in &objects { + match kind { + crate::npm_layout::NpmObjectKind::HostedVersion(version) => { + if manifests + .insert( + version.clone(), + ( + key.clone(), + contents + .as_ref() + .ok_or_else(|| format!("manifest unreadable: {key}"))? + .clone(), + ), + ) + .is_some() + { + return Err(format!("duplicate manifest for npm version {version}")); } } + crate::npm_layout::NpmObjectKind::HostedDeprecation(version) => { + deprecations.insert(version.clone(), key.clone()); + } + crate::npm_layout::NpmObjectKind::HostedPublishComplete(version) => { + completions.insert(version.clone(), key.clone()); + } + crate::npm_layout::NpmObjectKind::HostedDistTag(_) => { + tags.push(( + key.clone(), + contents + .as_ref() + .ok_or_else(|| format!("dist-tag unreadable: {key}"))? + .clone(), + )); + } + _ => {} } } - let mut result = Vec::new(); - for (pkg, file_keys) in &packages { - let mut entries = Vec::new(); - for key in file_keys { - let filename = key.rsplit('/').next().unwrap_or(""); - let (modified, size) = aggregate_meta(storage, std::slice::from_ref(key)).await; - let mut keys = vec![key.clone()]; - let hash_key = format!("{}.sha256", key); - if storage.stat(&hash_key).await.is_some() { - keys.push(hash_key); + let mut versions = Vec::with_capacity(manifests.len()); + for (version, (manifest_key, manifest)) in manifests { + let blob_key = + crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) + .ok_or_else(|| { + format!("npm version manifest has no valid blob reference: {manifest_key}") + })?; + let Some((blob_kind, _, _)) = objects.get(&blob_key) else { + return Err(format!("npm version blob is missing: {blob_key}")); + }; + if !matches!( + blob_kind, + crate::npm_layout::NpmObjectKind::HostedBlob { .. } + ) { + return Err(format!("npm version blob has invalid layout: {blob_key}")); + } + + let mut keys = vec![manifest_key, blob_key]; + if let Some(key) = completions.get(&version) { + keys.push(key.clone()); + } + if let Some(key) = deprecations.get(&version) { + keys.push(key.clone()); + } + for (tag_key, target) in &tags { + if target.as_ref() == version.as_bytes() { + keys.push(tag_key.clone()); } - entries.push(VersionEntry { - name: filename.to_string(), - keys, - modified, - size, - }); } - result.push((format!("pypi:{}", pkg), entries)); + keys.sort(); + keys.dedup(); + + let mut modified = 0u64; + let mut size = 0u64; + for key in &keys { + let (_, meta, _) = objects + .get(key) + .ok_or_else(|| format!("snapshot key disappeared: {key}"))?; + modified = modified.max(meta.modified); + size += meta.size; + } + versions.push(VersionEntry { + name: version, + keys, + modified, + size, + }); } - result + + Ok(NpmPackageSnapshot { + versions, + guard: hex::encode(hasher.finalize()), + }) } -/// Collect Cargo crate versions. -async fn collect_cargo_versions(storage: &Storage) -> Vec<(String, Vec)> { - let all_keys = storage.list("cargo/").await.unwrap_or_else(|e| { - tracing::error!("Failed to list cargo/ keys: {}", e); - Vec::new() - }); - let mut crates: std::collections::HashMap< - String, - std::collections::HashMap>, - > = std::collections::HashMap::new(); +async fn read_npm_package_snapshot( + storage: &Storage, + repository: &str, + package: &str, +) -> Result { + let prefix = format!("npm/repositories/{repository}/{package}/"); + let keys = storage + .list(&prefix) + .await + .map_err(|error| format!("cannot list {prefix}: {error}"))?; + npm_package_snapshot(storage, repository, package, keys).await +} - for key in &all_keys { - // cargo/{crate}/{version}/{crate}-{version}.crate - // Also: cargo/{crate}/metadata.json, cargo/index/... - if let Some(rest) = key.strip_prefix("cargo/") { - if rest.starts_with("index/") { - continue; // Skip sparse index - } - let parts: Vec<&str> = rest.split('/').collect(); - if parts.len() >= 3 { - let crate_name = parts[0]; - let version = parts[1]; - if crate_name != "index" && version != "metadata.json" { - crates - .entry(crate_name.to_string()) - .or_default() - .entry(version.to_string()) - .or_default() - .push(key.clone()); - } - } +/// Collect npm packages, including packages that only contain cleanup state. +/// Empty packages are retained in the result so a failed post-commit cleanup +/// is independently discoverable and retryable on the next retention run. +async fn collect_npm_versions(storage: &Storage) -> Vec { + let all_keys = match storage.list("npm/").await { + Ok(keys) => keys, + Err(error) => { + tracing::error!(%error, "retention: failed to list npm keys"); + return Vec::new(); + } + }; + let mut packages: std::collections::BTreeMap<(String, String), Vec> = + std::collections::BTreeMap::new(); + for key in all_keys { + let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { + continue; + }; + if is_hosted_npm_object(&parsed.kind) { + packages + .entry((parsed.repository, parsed.package)) + .or_default() + .push(key); } } - let mut result = Vec::new(); - for (crate_name, versions) in &crates { - let mut entries = Vec::new(); - for (version, keys) in versions { - let (modified, size) = aggregate_meta(storage, keys).await; - entries.push(VersionEntry { - name: version.clone(), - keys: keys.clone(), - modified, - size, - }); + let mut result = Vec::with_capacity(packages.len()); + for ((repository, package), keys) in packages { + match npm_package_snapshot(storage, &repository, &package, keys).await { + Ok(snapshot) => result.push(NpmVersionGroup { + group_name: format!("npm:{repository}:{package}"), + repository, + package, + versions: snapshot.versions, + snapshot_guard: snapshot.guard, + }), + Err(error) => { + tracing::warn!( + repository, + package, + %error, + "retention: npm package snapshot is uncertain; package skipped" + ); + } } - result.push((format!("cargo:{}", crate_name), entries)); } result } -async fn collect_go_versions(storage: &Storage) -> Vec<(String, Vec)> { - let all_keys = storage.list("go/").await.unwrap_or_else(|e| { - tracing::error!("Failed to list go/ keys: {}", e); - Vec::new() - }); - let mut modules: std::collections::HashMap< - String, - std::collections::HashMap>, - > = std::collections::HashMap::new(); +#[derive(Debug, Default)] +struct NpmCleanupOutcome { + complete: bool, + deleted_keys: usize, + bytes_freed: u64, +} - for key in &all_keys { - // go/{module}/@v/{version}.{info|mod|zip} - if let Some(at_v_pos) = key.find("/@v/") { - let module = &key["go/".len()..at_v_pos]; - let file = &key[at_v_pos + 4..]; // after "/@v/" - // Extract version: "v1.0.0.info" → "v1.0.0" - let version = file - .strip_suffix(".info") - .or_else(|| file.strip_suffix(".mod")) - .or_else(|| file.strip_suffix(".zip")); - if let Some(ver) = version { - modules - .entry(module.to_string()) - .or_default() - .entry(ver.to_string()) - .or_default() - .push(key.clone()); - } +/// Remove package-level hosted state after retention deleted the last version +/// manifest. Empty packages are collected independently, so a partial failure +/// leaves at least one discoverable key and is retried on a later run. +/// +/// The caller holds npm's exact package publish lock. +async fn clean_empty_npm_package( + storage: &Storage, + repository: &str, + package: &str, +) -> NpmCleanupOutcome { + let versions_prefix = format!("npm/repositories/{repository}/{package}/versions/"); + let versions = match storage.list(&versions_prefix).await { + Ok(keys) => keys, + Err(error) => { + tracing::warn!( + repository, + package, + error = %error, + "retention: cannot verify empty npm package; package metadata kept" + ); + return NpmCleanupOutcome::default(); } + }; + if !versions.is_empty() { + return NpmCleanupOutcome { + complete: true, + ..NpmCleanupOutcome::default() + }; } - let mut result = Vec::new(); - for (module, versions) in &modules { - let mut entries = Vec::new(); - for (version, keys) in versions { - let (modified, size) = aggregate_meta(storage, keys).await; - entries.push(VersionEntry { - name: version.clone(), - keys: keys.clone(), - modified, - size, - }); + let package_prefix = format!("npm/repositories/{repository}/{package}/"); + let mut keys = match storage.list(&package_prefix).await { + Ok(keys) => keys, + Err(error) => { + tracing::warn!( + repository, + package, + error = %error, + "retention: cannot list empty npm package state; metadata kept" + ); + return NpmCleanupOutcome::default(); + } + }; + keys.sort(); + let mut outcome = NpmCleanupOutcome::default(); + for key in keys { + let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { + continue; + }; + if parsed.repository != repository || parsed.package != package { + continue; + } + if !matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedPackage + | crate::npm_layout::NpmObjectKind::HostedDistTag(_) + | crate::npm_layout::NpmObjectKind::HostedDeprecation(_) + | crate::npm_layout::NpmObjectKind::HostedPublishComplete(_) + ) { + continue; + } + let Some(meta) = storage.stat(&key).await else { + tracing::warn!( + repository, + package, + key, + "retention: cannot stat empty npm package state; cleanup will retry" + ); + return outcome; + }; + match storage.delete(&key).await { + Ok(()) => { + outcome.deleted_keys += 1; + outcome.bytes_freed += meta.size; + } + Err(error) => { + tracing::warn!( + repository, + package, + key, + %error, + "retention: empty npm package cleanup failed; cleanup will retry" + ); + return outcome; + } } - result.push((format!("go:{}", module), entries)); } - result + outcome.complete = true; + outcome } -/// Get max modified time and total size across keys. -async fn aggregate_meta(storage: &Storage, keys: &[String]) -> (u64, u64) { - let mut max_modified = 0u64; - let mut total_size = 0u64; - for key in keys { - if let Some(meta) = storage.stat(key).await { - max_modified = max_modified.max(meta.modified); - total_size += meta.size; +async fn npm_blob_still_referenced( + storage: &Storage, + blob_key: &str, +) -> Result { + let Some(parsed) = crate::npm_layout::parse_npm_object_key(blob_key) else { + return Ok(false); + }; + if !matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedBlob { .. } + ) { + return Ok(false); + } + let prefix = format!( + "npm/repositories/{}/{}/versions/", + parsed.repository, parsed.package + ); + for manifest_key in storage.list(&prefix).await? { + let manifest = storage.get(&manifest_key).await?; + if crate::npm_layout::hosted_blob_key_from_manifest( + &parsed.repository, + &parsed.package, + &manifest, + ) + .as_deref() + == Some(blob_key) + { + return Ok(true); } } - (max_modified, total_size) + Ok(false) } -// ============================================================================ -// Retention execution -// ============================================================================ - -/// Result of a retention run. -pub struct RetentionResult { - pub planned: usize, - pub deleted_keys: usize, - pub bytes_freed: u64, - pub duration_secs: f64, - pub plans: Vec<(String, Vec)>, +#[derive(Debug, Default)] +struct NpmDeleteOutcome { + applied: bool, + deleted_keys: usize, + bytes_freed: u64, } -/// Run retention across all registries. -/// -/// `publish_locks` serializes deletions with concurrent publish operations -/// to prevent race conditions (e.g., deleting a blob while a manifest -/// referencing it is being written). -pub async fn run_retention( +/// Delete one hosted npm version with its version manifest as the visibility +/// commit point. Mutable dependants are removed first, with +/// `publish-complete` first of all: if a later pre-commit operation fails, an +/// exact publish retry observes the missing completion marker and can repair +/// the original publish state. Arbitrary tags or deprecations added later are +/// not reconstructible, so this phase is fail-safe rather than atomic. The +/// content-addressed blob is considered only after the manifest commit and may +/// safely remain as a GC-healable orphan. +async fn delete_npm_plan( storage: &Storage, - publish_locks: &PublishLocks, - signer: Option<&crate::signing::RepoSigner>, - rules: &[RetentionRule], - dry_run: bool, -) -> RetentionResult { - let start = Instant::now(); - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - - // Collect versions from all registries - let mut all_groups: Vec<(String, Vec)> = Vec::new(); - all_groups.extend(collect_maven_versions(storage).await); - all_groups.extend(collect_docker_versions(storage).await); - all_groups.extend(collect_npm_versions(storage).await); - all_groups.extend(collect_pypi_versions(storage).await); - all_groups.extend(collect_cargo_versions(storage).await); - all_groups.extend(collect_go_versions(storage).await); - all_groups.extend(collect_rpm_versions(storage).await); - all_groups.extend(collect_deb_versions(storage).await); - all_groups.extend(collect_raw_versions(storage).await); - - let mut all_plans: Vec<(String, Vec)> = Vec::new(); - let mut total_planned = 0usize; - let mut total_deleted_keys = 0usize; - let mut total_bytes = 0u64; - // rpm/deb repos whose packages were deleted — their indexes must be - // rebuilt (and re-signed) afterwards or they keep advertising ghosts. - let mut regen: std::collections::BTreeSet<(&'static str, String)> = - std::collections::BTreeSet::new(); - - for (group_name, versions) in all_groups { - // Find matching rule for this group - let registry = group_name.split(':').next().unwrap_or(""); - let rule = match find_matching_rule(rules, registry, &group_name) { - Some(r) => r, - None => continue, + group_name: &str, + plan: &DeletionPlan, +) -> NpmDeleteOutcome { + let mut manifest_key = None; + let mut completion_keys = Vec::new(); + let mut tag_keys = Vec::new(); + let mut deprecation_keys = Vec::new(); + let mut blob_keys = Vec::new(); + for key in &plan.keys { + let Some(parsed) = crate::npm_layout::parse_npm_object_key(key) else { + tracing::error!( + group = group_name, + version = %plan.version_name, + key, + "retention: npm plan contains an invalid key; deletion aborted" + ); + return NpmDeleteOutcome::default(); }; - - let plans = plan_deletions(versions, rule, now); - if plans.is_empty() { - continue; - } - - total_planned += plans.len(); - - if !dry_run { - if let Some(repo) = group_name - .strip_prefix("rpm:") - .map(|n| ("rpm", n)) - .or_else(|| group_name.strip_prefix("deb:").map(|n| ("deb", n))) - .and_then(|(fmt, n)| n.split('/').next().map(|r| (fmt, r.to_string()))) + match parsed.kind { + crate::npm_layout::NpmObjectKind::HostedVersion(version) + if version == plan.version_name => { - regen.insert(( - if group_name.starts_with("rpm:") { - "rpm" - } else { - "deb" - }, - repo.1, - )); - } - for plan in &plans { - for key in &plan.keys { - // Serialize with concurrent publish to prevent deleting - // an artifact that is being referenced by a new publish. - let lock = crate::acquire_publish_lock(publish_locks, key); - let _guard = lock.lock().await; - if storage.delete(key).await.is_ok() { - total_deleted_keys += 1; - } + if manifest_key.replace(key.clone()).is_some() { + tracing::error!( + group = group_name, + version = %plan.version_name, + "retention: npm plan has multiple version manifests; deletion aborted" + ); + return NpmDeleteOutcome::default(); } - total_bytes += plan.size; - info!( - group = %group_name, + } + crate::npm_layout::NpmObjectKind::HostedPublishComplete(version) + if version == plan.version_name => + { + completion_keys.push(key.clone()); + } + crate::npm_layout::NpmObjectKind::HostedDeprecation(version) + if version == plan.version_name => + { + deprecation_keys.push(key.clone()); + } + crate::npm_layout::NpmObjectKind::HostedDistTag(_) => tag_keys.push(key.clone()), + crate::npm_layout::NpmObjectKind::HostedBlob { .. } => blob_keys.push(key.clone()), + _ => { + tracing::error!( + group = group_name, version = %plan.version_name, - reason = %plan.reason, - "Retention: deleted" + key, + "retention: npm plan contains an unrelated key; deletion aborted" ); + return NpmDeleteOutcome::default(); } - } else { - for plan in &plans { - total_bytes += plan.size; - info!( - group = %group_name, + } + } + let Some(manifest_key) = manifest_key else { + tracing::error!( + group = group_name, + version = %plan.version_name, + "retention: npm plan has no version manifest; deletion aborted" + ); + return NpmDeleteOutcome::default(); + }; + completion_keys.sort(); + tag_keys.sort(); + deprecation_keys.sort(); + blob_keys.sort(); + + let mut outcome = NpmDeleteOutcome::default(); + + // Pre-commit phase. A failure may leave already-deleted mutable state, but + // never removes the version manifest. Deleting the completion marker first + // makes the original publish state explicitly retryable by npm's + // exact-publish repair path. Later operator mutations are not + // reconstructible; the manifest nevertheless remains a safe visibility + // boundary and no tag can dangle from a deleted version. + for key in completion_keys + .iter() + .chain(tag_keys.iter()) + .chain(deprecation_keys.iter()) + { + if tag_keys.binary_search(key).is_ok() { + match storage.get(key).await { + Ok(target) if target.as_ref() == plan.version_name.as_bytes() => {} + Ok(_) => { + tracing::warn!( + group = group_name, + version = %plan.version_name, + key, + "retention: npm dist-tag target changed; deletion aborted" + ); + return outcome; + } + Err(error) => { + tracing::warn!( + group = group_name, + version = %plan.version_name, + key, + %error, + "retention: cannot verify npm dist-tag target; deletion aborted" + ); + return outcome; + } + } + } + let Some(meta) = storage.stat(key).await else { + tracing::warn!( + group = group_name, + version = %plan.version_name, + key, + "retention: cannot stat dependent npm object; deletion aborted" + ); + return outcome; + }; + match storage.delete(key).await { + Ok(()) => { + outcome.deleted_keys += 1; + outcome.bytes_freed += meta.size; + } + Err(error) => { + tracing::warn!( + group = group_name, version = %plan.version_name, - keys = plan.keys.len(), - reason = %plan.reason, - "[dry-run] Retention: would delete" + key, + %error, + "retention: dependent npm object deletion failed; manifest kept" ); + return outcome; } } + } - all_plans.push((group_name, plans)); + let Some(manifest_meta) = storage.stat(&manifest_key).await else { + tracing::warn!( + group = group_name, + version = %plan.version_name, + key = %manifest_key, + "retention: cannot stat npm manifest; deletion aborted" + ); + return outcome; + }; + match storage.delete(&manifest_key).await { + Ok(()) => { + outcome.deleted_keys += 1; + outcome.bytes_freed += manifest_meta.size; + outcome.applied = true; + } + Err(error) => { + tracing::warn!( + group = group_name, + version = %plan.version_name, + key = %manifest_key, + %error, + "retention: npm manifest deletion failed; blob kept" + ); + return outcome; + } } - // Rebuild + re-sign the indexes of every rpm/deb repo retention touched, - // under the same per-repo publish lock the handlers use. Fail-open per - // repo: a failed rebuild logs loudly and the next publish/reindex heals - // it; the deletions themselves are already durable. - for (fmt, repo) in ®en { - let lock_key = match *fmt { - "rpm" => format!("rpm/{repo}/repodata/repomd.xml"), - _ => format!("deb/{repo}/Release"), - }; - let lock = crate::acquire_publish_lock(publish_locks, &lock_key); - let _guard = lock.lock().await; - let result = match *fmt { - "rpm" => crate::registry::rpm::regenerate_repodata(storage, signer, repo).await, - _ => crate::registry::deb::regenerate_indexes(storage, signer, repo).await, + // Post-commit phase: failures only leave content-addressed orphan blobs. + for key in blob_keys { + match npm_blob_still_referenced(storage, &key).await { + Ok(true) => continue, + Ok(false) => {} + Err(error) => { + tracing::warn!( + group = group_name, + version = %plan.version_name, + key, + %error, + "retention: cannot prove npm blob is unreferenced; blob kept" + ); + continue; + } + } + let Some(meta) = storage.stat(&key).await else { + tracing::warn!( + group = group_name, + version = %plan.version_name, + key, + "retention: cannot stat unreferenced npm blob; blob kept" + ); + continue; }; - if let Err(e) = result { - tracing::error!(registry = %fmt, repo = %repo, error = %e, "retention: index regeneration failed — run -/reindex to heal"); - } else { - info!(registry = %fmt, repo = %repo, "retention: indexes regenerated"); + match storage.delete(&key).await { + Ok(()) => { + outcome.deleted_keys += 1; + outcome.bytes_freed += meta.size; + } + Err(error) => { + tracing::warn!( + group = group_name, + version = %plan.version_name, + key, + %error, + "retention: unreferenced npm blob deletion failed; GC will retry" + ); + } } } - let duration = start.elapsed().as_secs_f64(); - RETENTION_DURATION.observe(duration); - RETENTION_LAST_RUN.set( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(0), - ); + outcome +} - if !dry_run { - RETENTION_VERSIONS_DELETED.inc_by(total_planned as u64); - RETENTION_BYTES_FREED.inc_by(total_bytes); - if total_planned > 0 { - info!( - versions = total_planned, - keys = total_deleted_keys, - bytes_freed = total_bytes, - "Retention complete" +#[derive(Debug, Default)] +struct NpmBatchOutcome { + applied_versions: usize, + deleted_keys: usize, + bytes_freed: u64, +} + +/// Validate and apply a complete npm package plan while holding the same lock +/// used by publish, dist-tag and deprecation mutations. +/// +/// A package-wide guard is essential: validating only the candidate version +/// would still allow another version to change retention ordering after +/// `keep_last` was planned. The guard is checked once before any mutation and +/// the lock is held across the complete package batch. +async fn apply_npm_plans( + storage: &Storage, + publish_locks: &PublishLocks, + group: &NpmVersionGroup, + plans: &[DeletionPlan], +) -> NpmBatchOutcome { + let lock = crate::acquire_publish_lock(publish_locks, &group.group_name); + let _guard = lock.lock().await; + + let current = match read_npm_package_snapshot(storage, &group.repository, &group.package).await + { + Ok(snapshot) => snapshot, + Err(error) => { + tracing::warn!( + repository = group.repository, + package = group.package, + %error, + "retention: cannot revalidate npm package snapshot; batch skipped" ); + return NpmBatchOutcome::default(); } + }; + if current.guard != group.snapshot_guard { + tracing::info!( + repository = group.repository, + package = group.package, + "retention: npm package changed after planning; batch skipped" + ); + return NpmBatchOutcome::default(); } - RetentionResult { - planned: total_planned, - deleted_keys: total_deleted_keys, - bytes_freed: total_bytes, - duration_secs: duration, - plans: all_plans, + let mut outcome = NpmBatchOutcome::default(); + for plan in plans { + let deleted = delete_npm_plan(storage, &group.group_name, plan).await; + outcome.deleted_keys += deleted.deleted_keys; + outcome.bytes_freed += deleted.bytes_freed; + if !deleted.applied { + // A pre-commit failure may already have removed mutable state. + // Stop the package batch. The manifest remains visible and no tag + // can dangle from a deleted version. Removing publish-complete + // first also lets an exact retry repair original publish state. + return outcome; + } + outcome.applied_versions += 1; + info!( + group = %group.group_name, + version = %plan.version_name, + reason = %plan.reason, + "Retention: deleted" + ); } -} -/// Find the first matching retention rule for a registry/group. -fn find_matching_rule<'a>( - rules: &'a [RetentionRule], - registry: &str, - group_name: &str, -) -> Option<&'a RetentionRule> { - // First rule whose registry matches (or "*") AND whose name_glob (if any) - // matches the group's name within the registry. - let name = group_name - .split_once(':') - .map(|(_, n)| n) - .unwrap_or(group_name); - rules.iter().find(|r| { - (r.registry == registry || r.registry == "*") - && r.name_glob.as_deref().is_none_or(|g| glob_match(g, name)) - }) + let cleanup = clean_empty_npm_package(storage, &group.repository, &group.package).await; + outcome.deleted_keys += cleanup.deleted_keys; + outcome.bytes_freed += cleanup.bytes_freed; + if !cleanup.complete { + tracing::warn!( + repository = group.repository, + package = group.package, + "retention: empty npm package cleanup remains pending" + ); + } + outcome } -// ============================================================================ -// Background scheduler -// ============================================================================ +/// Collect PyPI package files. +async fn collect_pypi_versions(storage: &Storage) -> Vec<(String, Vec)> { + let all_keys = storage.list("pypi/").await.unwrap_or_else(|e| { + tracing::error!("Failed to list pypi/ keys: {}", e); + Vec::new() + }); + let mut packages: std::collections::HashMap> = + std::collections::HashMap::new(); -/// Spawn a background retention task that runs periodically. -/// Accepts a shared cleanup lock to prevent concurrent runs with GC scheduler. -/// Returns a `JoinHandle` so the caller can await graceful completion on shutdown. -#[allow(clippy::too_many_arguments)] -pub fn spawn_retention_scheduler( - storage: Storage, - publish_locks: PublishLocks, - signer: Option>, - rules: Vec, - interval_secs: u64, - dry_run: bool, - audit: Option>, - cleanup_lock: Arc>, - cancel: tokio_util::sync::CancellationToken, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(interval_secs)); - // The interval's first tick fires immediately: retention runs once at - // boot, then every `interval_secs`. Waiting a full interval instead - // means a process that restarts more often than the interval NEVER - // runs retention — deploy-happy environments accumulated unbounded - // garbage exactly when the schedule looked configured. - let mut boot_run = true; - - loop { - // CANCEL-SAFETY: Same as GC — interval.tick() is stateless between polls, - // cancel.cancelled() is a CancellationToken. Retention work runs to - // completion within each tick iteration, no partial state on drop. - tokio::select! { - _ = cancel.cancelled() => { - info!("Retention scheduler: cancellation requested, stopping"); - break; + for key in &all_keys { + if let Some(rest) = key.strip_prefix("pypi/") { + // Skip checksums and metadata.json — metadata is the package index, + // not a version artifact. Deleting it makes the package undiscoverable. + if !ends_with_ci(key, ".sha256") + && !ends_with_ci(key, ".sha1") + && !ends_with_ci(key, ".md5") + && !ends_with_ci(key, ".sha512") + && !ends_with_ci(key, "/metadata.json") + { + let pkg = rest.split('/').next().unwrap_or(""); + if !pkg.is_empty() { + packages + .entry(pkg.to_string()) + .or_default() + .push(key.clone()); } - _ = interval.tick() => {} } + } + } - if cancel.is_cancelled() { - break; + let mut result = Vec::new(); + for (pkg, file_keys) in &packages { + let mut entries = Vec::new(); + for key in file_keys { + let filename = key.rsplit('/').next().unwrap_or(""); + let (modified, size) = aggregate_meta(storage, std::slice::from_ref(key)).await; + let mut keys = vec![key.clone()]; + let hash_key = format!("{}.sha256", key); + if storage.stat(&hash_key).await.is_some() { + keys.push(hash_key); } + entries.push(VersionEntry { + name: filename.to_string(), + keys, + modified, + size, + }); + } + result.push((format!("pypi:{}", pkg), entries)); + } + result +} - // Cross-scheduler lock: skip if GC or retention is already running. - // The boot run waits for the lock instead — GC's boot pass fires at - // the same instant, and skipping here would silently postpone the - // first retention by a whole interval again. - let guard = if boot_run { - boot_run = false; - // CANCEL-SAFETY: the boot pass waits on the lock (vs skip-if-held) so it - // can't forfeit its first run to GC's simultaneous boot pass — but race - // the wait against cancellation, so a SIGTERM during boot contention - // breaks promptly instead of blocking behind the sibling's whole pass. - // Dropping the not-yet-acquired lock() future only removes this waiter. - tokio::select! { - _ = cancel.cancelled() => break, - g = cleanup_lock.lock() => Ok(g), - } - } else { - cleanup_lock.try_lock() - }; - let Ok(guard) = guard else { - info!("Retention: cleanup lock held (GC or retention running), skipping"); - continue; - }; - - info!( - dry_run = dry_run, - "Retention scheduler: starting periodic run" - ); - let result = - run_retention(&storage, &publish_locks, signer.as_deref(), &rules, dry_run).await; - info!( - "Retention scheduler: done in {:.1}s — {} versions, {} keys, {} bytes freed", - result.duration_secs, result.planned, result.deleted_keys, result.bytes_freed - ); +/// Collect Cargo crate versions. +async fn collect_cargo_versions(storage: &Storage) -> Vec<(String, Vec)> { + let all_keys = storage.list("cargo/").await.unwrap_or_else(|e| { + tracing::error!("Failed to list cargo/ keys: {}", e); + Vec::new() + }); + let mut crates: std::collections::HashMap< + String, + std::collections::HashMap>, + > = std::collections::HashMap::new(); - if let Some(ref audit_log) = audit { - if result.planned > 0 { - audit_log.log(crate::audit::AuditEntry::new( - "retention-apply", - "scheduler", - &format!("{} versions", result.planned), - "*", - &format!( - "keys={} bytes_freed={} duration={:.1}s", - result.deleted_keys, result.bytes_freed, result.duration_secs - ), - )); + for key in &all_keys { + // cargo/{crate}/{version}/{crate}-{version}.crate + // Also: cargo/{crate}/metadata.json, cargo/index/... + if let Some(rest) = key.strip_prefix("cargo/") { + if rest.starts_with("index/") { + continue; // Skip sparse index + } + let parts: Vec<&str> = rest.split('/').collect(); + if parts.len() >= 3 { + let crate_name = parts[0]; + let version = parts[1]; + if crate_name != "index" && version != "metadata.json" { + crates + .entry(crate_name.to_string()) + .or_default() + .entry(version.to_string()) + .or_default() + .push(key.clone()); } } + } + } - drop(guard); + let mut result = Vec::new(); + for (crate_name, versions) in &crates { + let mut entries = Vec::new(); + for (version, keys) in versions { + let (modified, size) = aggregate_meta(storage, keys).await; + entries.push(VersionEntry { + name: version.clone(), + keys: keys.clone(), + modified, + size, + }); } - }) + result.push((format!("cargo:{}", crate_name), entries)); + } + result } -// ============================================================================ -// Tests -// ============================================================================ +async fn collect_go_versions(storage: &Storage) -> Vec<(String, Vec)> { + let all_keys = storage.list("go/").await.unwrap_or_else(|e| { + tracing::error!("Failed to list go/ keys: {}", e); + Vec::new() + }); + let mut modules: std::collections::HashMap< + String, + std::collections::HashMap>, + > = std::collections::HashMap::new(); -#[cfg(test)] -#[allow(clippy::unwrap_used)] -mod tests { - use super::*; + for key in &all_keys { + // go/{module}/@v/{version}.{info|mod|zip} + if let Some(at_v_pos) = key.find("/@v/") { + let module = &key["go/".len()..at_v_pos]; + let file = &key[at_v_pos + 4..]; // after "/@v/" + // Extract version: "v1.0.0.info" → "v1.0.0" + let version = file + .strip_suffix(".info") + .or_else(|| file.strip_suffix(".mod")) + .or_else(|| file.strip_suffix(".zip")); + if let Some(ver) = version { + modules + .entry(module.to_string()) + .or_default() + .entry(ver.to_string()) + .or_default() + .push(key.clone()); + } + } + } - fn test_publish_locks() -> PublishLocks { - Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())) + let mut result = Vec::new(); + for (module, versions) in &modules { + let mut entries = Vec::new(); + for (version, keys) in versions { + let (modified, size) = aggregate_meta(storage, keys).await; + entries.push(VersionEntry { + name: version.clone(), + keys: keys.clone(), + modified, + size, + }); + } + result.push((format!("go:{}", module), entries)); } + result +} - fn make_rule( - keep_last: Option, - older_than_days: Option, - exclude_tags: Vec<&str>, - ) -> RetentionRule { - RetentionRule { - registry: "*".to_string(), - name_glob: None, - keep_last, - older_than_days, - exclude_tags: exclude_tags.into_iter().map(String::from).collect(), +/// Get max modified time and total size across keys. +async fn aggregate_meta(storage: &Storage, keys: &[String]) -> (u64, u64) { + let mut max_modified = 0u64; + let mut total_size = 0u64; + for key in keys { + if let Some(meta) = storage.stat(key).await { + max_modified = max_modified.max(meta.modified); + total_size += meta.size; } } + (max_modified, total_size) +} - fn make_version(name: &str, modified: u64, size: u64) -> VersionEntry { - VersionEntry { - name: name.to_string(), - keys: vec![format!("test/{}", name)], - modified, - size, +#[derive(Default)] +struct MavenDeletionOutcome { + applied: bool, + deleted_keys: usize, + bytes_freed: u64, +} + +fn is_maven_metadata_base(key: &str) -> bool { + key.rsplit('/').next() == Some("maven-metadata.xml") +} + +fn is_maven_metadata_sidecar(key: &str) -> bool { + matches!( + key.rsplit('/').next(), + Some( + "maven-metadata.xml.md5" + | "maven-metadata.xml.sha1" + | "maven-metadata.xml.sha256" + | "maven-metadata.xml.sha512" + ) + ) +} + +fn is_checksum_sidecar(key: &str) -> bool { + [".md5", ".sha1", ".sha256", ".sha512"] + .iter() + .any(|suffix| key.ends_with(suffix)) +} + +async fn delete_retention_key(storage: &Storage, key: &str) -> Result<(usize, u64), StorageError> { + let size = storage.stat(key).await.map(|meta| meta.size).unwrap_or(0); + match storage.delete(key).await { + Ok(()) => Ok((1, size)), + Err(StorageError::NotFound) => Ok((0, 0)), + Err(error) => Err(error), + } +} + +async fn maven_candidate_matches_plan( + storage: &Storage, + context: &MavenRetentionContext, + plan: &DeletionPlan, +) -> Result { + let prefix = format!( + "{}{}/{}/{}/", + context.storage_prefix, context.group_path, context.artifact_id, plan.version_name + ); + let current = storage.list_with_meta(&prefix).await?; + let current_keys: std::collections::BTreeSet<&str> = + current.iter().map(|(key, _)| key.as_str()).collect(); + let planned_keys: std::collections::BTreeSet<&str> = + plan.keys.iter().map(String::as_str).collect(); + let current_size = current.iter().map(|(_, meta)| meta.size).sum::(); + let current_modified = current + .iter() + .map(|(_, meta)| meta.modified) + .max() + .unwrap_or(0); + + Ok(current_keys == planned_keys + && current_size == plan.size + && current_modified == plan.modified) +} + +/// Delete one Maven version under the exact artifact metadata lock shared with +/// publish/proxy metadata writes. +/// +/// Hosted discovery is hidden first: A-level metadata is regenerated (or +/// removed) and V-level metadata sidecars/base are deleted before any payload. +/// A later payload failure therefore leaves only hidden orphan bytes. Proxy and +/// legacy metadata remain untouched because their A-level version list is +/// upstream-owned or has unknowable provenance. +async fn delete_maven_plan( + storage: &Storage, + publish_locks: &PublishLocks, + context: &MavenRetentionContext, + plan: &DeletionPlan, +) -> MavenDeletionOutcome { + let metadata_key = format!( + "{}{}/{}/maven-metadata.xml", + context.storage_prefix, context.group_path, context.artifact_id + ); + let lock = crate::acquire_publish_lock(publish_locks, &metadata_key); + let _guard = lock.lock().await; + let mut outcome = MavenDeletionOutcome::default(); + + match maven_candidate_matches_plan(storage, context, plan).await { + Ok(true) => {} + Ok(false) => { + tracing::warn!( + group = %context.group_path, + artifact = %context.artifact_id, + version = %plan.version_name, + "retention: Maven version changed after planning; candidate skipped" + ); + return outcome; } + Err(error) => { + tracing::error!( + group = %context.group_path, + artifact = %context.artifact_id, + version = %plan.version_name, + error = %error, + "retention: cannot revalidate Maven version under publish lock; candidate skipped" + ); + return outcome; + } + } + + if context.kind == MavenRetentionKind::Hosted { + match crate::registry::update_hosted_metadata_after_retention( + storage, + &context.storage_prefix, + &context.group_path, + &context.artifact_id, + &plan.version_name, + ) + .await + { + Ok((deleted_keys, bytes_freed)) => { + outcome.deleted_keys += deleted_keys; + outcome.bytes_freed += bytes_freed; + } + Err(error) => { + tracing::error!( + group = %context.group_path, + artifact = %context.artifact_id, + version = %plan.version_name, + error = %error, + "retention: Maven metadata update failed; version payload kept" + ); + return outcome; + } + } + } + + // V-level SNAPSHOT discovery must disappear before version payloads. + for key in plan + .keys + .iter() + .filter(|key| is_maven_metadata_sidecar(key)) + { + match delete_retention_key(storage, key).await { + Ok((deleted, bytes)) => { + outcome.deleted_keys += deleted; + outcome.bytes_freed += bytes; + } + Err(error) => { + tracing::error!( + key, + version = %plan.version_name, + error = %error, + "retention: Maven version metadata sidecar deletion failed; payload kept" + ); + return outcome; + } + } + } + for key in plan.keys.iter().filter(|key| is_maven_metadata_base(key)) { + match delete_retention_key(storage, key).await { + Ok((deleted, bytes)) => { + outcome.deleted_keys += deleted; + outcome.bytes_freed += bytes; + } + Err(error) => { + tracing::error!( + key, + version = %plan.version_name, + error = %error, + "retention: Maven version metadata deletion failed; payload kept" + ); + return outcome; + } + } + } + + let mut payload_keys: Vec<&String> = plan + .keys + .iter() + .filter(|key| !is_maven_metadata_base(key) && !is_maven_metadata_sidecar(key)) + .collect(); + // Remove derived checksum sidecars before their base object. Once A/V + // discovery is hidden, an interruption can only leave invisible orphans. + payload_keys.sort_by_key(|key| !is_checksum_sidecar(key)); + for key in payload_keys { + match delete_retention_key(storage, key).await { + Ok((deleted, bytes)) => { + outcome.deleted_keys += deleted; + outcome.bytes_freed += bytes; + } + Err(error) => { + tracing::error!( + key, + version = %plan.version_name, + error = %error, + "retention: Maven version payload deletion failed; remaining objects kept" + ); + return outcome; + } + } + } + + outcome.applied = true; + outcome +} + +// ============================================================================ +// Retention execution +// ============================================================================ + +/// Result of a retention run. +pub struct RetentionResult { + pub planned: usize, + pub deleted_keys: usize, + pub bytes_freed: u64, + pub duration_secs: f64, + pub plans: Vec<(String, Vec)>, +} + +/// Run retention across all registries. +/// +/// `publish_locks` serializes deletions with concurrent publish operations +/// to prevent race conditions (e.g., deleting a blob while a manifest +/// referencing it is being written). +#[cfg(test)] +pub async fn run_retention( + storage: &Storage, + publish_locks: &PublishLocks, + signer: Option<&crate::signing::RepoSigner>, + rules: &[RetentionRule], + dry_run: bool, +) -> RetentionResult { + let maven = MavenConfig::default(); + run_retention_configured(storage, publish_locks, signer, rules, dry_run, &maven, None).await +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_retention_configured( + storage: &Storage, + publish_locks: &PublishLocks, + signer: Option<&crate::signing::RepoSigner>, + rules: &[RetentionRule], + dry_run: bool, + maven_config: &MavenConfig, + repo_index: Option<&crate::repo_index::RepoIndex>, +) -> RetentionResult { + let start = Instant::now(); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + // Collect versions from all registries + let mut all_groups: Vec<(String, Vec)> = Vec::new(); + let mut maven_contexts = std::collections::HashMap::new(); + for group in collect_maven_versions(storage, maven_config).await { + maven_contexts.insert( + group.group_name.clone(), + MavenRetentionContext { + kind: group.kind, + storage_prefix: group.storage_prefix, + group_path: group.group_path, + artifact_id: group.artifact_id, + }, + ); + all_groups.push((group.group_name, group.versions)); + } + all_groups.extend(collect_docker_versions(storage).await); + let mut npm_groups = std::collections::HashMap::new(); + for group in collect_npm_versions(storage).await { + all_groups.push((group.group_name.clone(), group.versions.clone())); + npm_groups.insert(group.group_name.clone(), group); + } + all_groups.extend(collect_pypi_versions(storage).await); + all_groups.extend(collect_cargo_versions(storage).await); + all_groups.extend(collect_go_versions(storage).await); + all_groups.extend(collect_rpm_versions(storage).await); + all_groups.extend(collect_deb_versions(storage).await); + all_groups.extend(collect_raw_versions(storage).await); + + let mut all_plans: Vec<(String, Vec)> = Vec::new(); + let mut total_planned = 0usize; + let mut total_applied = 0usize; + let mut total_deleted_keys = 0usize; + let mut total_bytes = 0u64; + let mut mutated_registries: std::collections::BTreeSet = + std::collections::BTreeSet::new(); + // rpm/deb repos whose packages were deleted — their indexes must be + // rebuilt (and re-signed) afterwards or they keep advertising ghosts. + let mut regen: std::collections::BTreeSet<(&'static str, String)> = + std::collections::BTreeSet::new(); + + for (group_name, versions) in all_groups { + // Find matching rule for this group + let registry = group_name.split(':').next().unwrap_or(""); + let rule = match find_matching_rule(rules, registry, &group_name) { + Some(r) => r, + None => continue, + }; + + let plans = plan_deletions(versions, rule, now); + + // npm is validated and applied as one package batch under its exact + // publish lock. Empty groups are intentionally retained so cleanup + // failures after the last manifest commit are retried independently. + if let Some(group) = npm_groups.get(&group_name) { + if plans.is_empty() { + if !dry_run && group.versions.is_empty() { + let outcome = apply_npm_plans(storage, publish_locks, group, &[]).await; + total_deleted_keys += outcome.deleted_keys; + total_bytes += outcome.bytes_freed; + if outcome.deleted_keys > 0 { + mutated_registries.insert("npm".to_string()); + } + } + continue; + } + + total_planned += plans.len(); + if dry_run { + for plan in &plans { + total_bytes += plan.size; + info!( + group = %group_name, + version = %plan.version_name, + keys = plan.keys.len(), + reason = %plan.reason, + "[dry-run] Retention: would delete" + ); + } + } else { + let outcome = apply_npm_plans(storage, publish_locks, group, &plans).await; + total_applied += outcome.applied_versions; + total_deleted_keys += outcome.deleted_keys; + total_bytes += outcome.bytes_freed; + if outcome.deleted_keys > 0 { + mutated_registries.insert("npm".to_string()); + } + } + all_plans.push((group_name, plans)); + continue; + } + + if plans.is_empty() { + continue; + } + + total_planned += plans.len(); + + if !dry_run { + if let Some(repo) = group_name + .strip_prefix("rpm:") + .map(|n| ("rpm", n)) + .or_else(|| group_name.strip_prefix("deb:").map(|n| ("deb", n))) + .and_then(|(fmt, n)| n.split('/').next().map(|r| (fmt, r.to_string()))) + { + regen.insert(( + if group_name.starts_with("rpm:") { + "rpm" + } else { + "deb" + }, + repo.1, + )); + } + for plan in &plans { + let applied = if let Some(context) = maven_contexts.get(&group_name) { + // The helper may have changed A-level sidecars before + // returning an error. Invalidate conservatively whenever a + // hosted metadata mutation is attempted. + if context.kind == MavenRetentionKind::Hosted { + mutated_registries.insert("maven".to_string()); + } + let outcome = delete_maven_plan(storage, publish_locks, context, plan).await; + total_deleted_keys += outcome.deleted_keys; + total_bytes += outcome.bytes_freed; + if outcome.deleted_keys > 0 { + mutated_registries.insert("maven".to_string()); + } + if !outcome.applied { + // A storage failure on one version is evidence that + // later deletions in the same GA cannot be trusted. + // Stop this artifact group instead of compounding a + // partial cleanup with more mutations. + break; + } + total_applied += 1; + true + } else { + let mut deleted_any = false; + for key in &plan.keys { + // Serialize with concurrent publish to prevent deleting + // an artifact that is being referenced by a new publish. + let lock = crate::acquire_publish_lock(publish_locks, key); + let _guard = lock.lock().await; + if storage.delete(key).await.is_ok() { + total_deleted_keys += 1; + deleted_any = true; + } + } + total_applied += 1; + total_bytes += plan.size; + if deleted_any { + mutated_registries.insert(registry.to_string()); + } + true + }; + if applied { + info!( + group = %group_name, + version = %plan.version_name, + reason = %plan.reason, + "Retention: deleted" + ); + } + } + } else { + for plan in &plans { + total_bytes += plan.size; + info!( + group = %group_name, + version = %plan.version_name, + keys = plan.keys.len(), + reason = %plan.reason, + "[dry-run] Retention: would delete" + ); + } + } + + all_plans.push((group_name, plans)); + } + + // Rebuild + re-sign the indexes of every rpm/deb repo retention touched, + // under the same per-repo publish lock the handlers use. Fail-open per + // repo: a failed rebuild logs loudly and the next publish/reindex heals + // it; the deletions themselves are already durable. + for (fmt, repo) in ®en { + let lock_key = match *fmt { + "rpm" => format!("rpm/{repo}/repodata/repomd.xml"), + _ => format!("deb/{repo}/Release"), + }; + let lock = crate::acquire_publish_lock(publish_locks, &lock_key); + let _guard = lock.lock().await; + let result = match *fmt { + "rpm" => crate::registry::rpm::regenerate_repodata(storage, signer, repo).await, + _ => crate::registry::deb::regenerate_indexes(storage, signer, repo).await, + }; + if let Err(e) = result { + tracing::error!(registry = %fmt, repo = %repo, error = %e, "retention: index regeneration failed — run -/reindex to heal"); + } else { + info!(registry = %fmt, repo = %repo, "retention: indexes regenerated"); + } + } + + let duration = start.elapsed().as_secs_f64(); + RETENTION_DURATION.observe(duration); + RETENTION_LAST_RUN.set( + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0), + ); + + if !dry_run { + if let Some(repo_index) = repo_index { + for registry in mutated_registries { + repo_index.invalidate(®istry); + } + } + RETENTION_VERSIONS_DELETED.inc_by(total_applied as u64); + RETENTION_BYTES_FREED.inc_by(total_bytes); + if total_applied > 0 { + info!( + versions = total_applied, + keys = total_deleted_keys, + bytes_freed = total_bytes, + "Retention complete" + ); + } + } + + RetentionResult { + planned: total_planned, + deleted_keys: total_deleted_keys, + bytes_freed: total_bytes, + duration_secs: duration, + plans: all_plans, + } +} + +/// Find the first matching retention rule for a registry/group. +fn find_matching_rule<'a>( + rules: &'a [RetentionRule], + registry: &str, + group_name: &str, +) -> Option<&'a RetentionRule> { + // First rule whose registry matches (or "*") AND whose name_glob (if any) + // matches the group's name within the registry. + let name = group_name + .split_once(':') + .map(|(_, n)| n) + .unwrap_or(group_name); + let npm_package = (registry == "npm") + .then(|| name.split_once(':').map(|(_, package)| package)) + .flatten(); + rules.iter().find(|r| { + (r.registry == registry || r.registry == "*") + && r.name_glob.as_deref().is_none_or(|glob| { + // Preserve the historical npm package selector across named + // hosted repositories. A glob containing ':' opts into the + // qualified `{repository}:{package}` identity. + if registry == "npm" && !glob.contains(':') { + glob_match(glob, npm_package.unwrap_or(name)) + } else { + glob_match(glob, name) + } + }) + }) +} + +// ============================================================================ +// Background scheduler +// ============================================================================ + +/// Spawn a background retention task that runs periodically. +/// Accepts a shared cleanup lock to prevent concurrent runs with GC scheduler. +/// Returns a `JoinHandle` so the caller can await graceful completion on shutdown. +#[allow(clippy::too_many_arguments)] +pub fn spawn_retention_scheduler( + storage: Storage, + publish_locks: PublishLocks, + signer: Option>, + maven_config: MavenConfig, + repo_index: Arc, + rules: Vec, + interval_secs: u64, + dry_run: bool, + audit: Option>, + cleanup_lock: Arc>, + cancel: tokio_util::sync::CancellationToken, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(interval_secs)); + // The interval's first tick fires immediately: retention runs once at + // boot, then every `interval_secs`. Waiting a full interval instead + // means a process that restarts more often than the interval NEVER + // runs retention — deploy-happy environments accumulated unbounded + // garbage exactly when the schedule looked configured. + let mut boot_run = true; + + loop { + // CANCEL-SAFETY: Same as GC — interval.tick() is stateless between polls, + // cancel.cancelled() is a CancellationToken. Retention work runs to + // completion within each tick iteration, no partial state on drop. + tokio::select! { + _ = cancel.cancelled() => { + info!("Retention scheduler: cancellation requested, stopping"); + break; + } + _ = interval.tick() => {} + } + + if cancel.is_cancelled() { + break; + } + + // Cross-scheduler lock: skip if GC or retention is already running. + // The boot run waits for the lock instead — GC's boot pass fires at + // the same instant, and skipping here would silently postpone the + // first retention by a whole interval again. + let guard = if boot_run { + boot_run = false; + // CANCEL-SAFETY: the boot pass waits on the lock (vs skip-if-held) so it + // can't forfeit its first run to GC's simultaneous boot pass — but race + // the wait against cancellation, so a SIGTERM during boot contention + // breaks promptly instead of blocking behind the sibling's whole pass. + // Dropping the not-yet-acquired lock() future only removes this waiter. + tokio::select! { + _ = cancel.cancelled() => break, + g = cleanup_lock.lock() => Ok(g), + } + } else { + cleanup_lock.try_lock() + }; + let Ok(guard) = guard else { + info!("Retention: cleanup lock held (GC or retention running), skipping"); + continue; + }; + + info!( + dry_run = dry_run, + "Retention scheduler: starting periodic run" + ); + let result = run_retention_configured( + &storage, + &publish_locks, + signer.as_deref(), + &rules, + dry_run, + &maven_config, + Some(&repo_index), + ) + .await; + info!( + "Retention scheduler: done in {:.1}s — {} versions, {} keys, {} bytes freed", + result.duration_secs, result.planned, result.deleted_keys, result.bytes_freed + ); + + if let Some(ref audit_log) = audit { + if result.planned > 0 { + audit_log.log(crate::audit::AuditEntry::new( + "retention-apply", + "scheduler", + &format!("{} versions", result.planned), + "*", + &format!( + "keys={} bytes_freed={} duration={:.1}s", + result.deleted_keys, result.bytes_freed, result.duration_secs + ), + )); + } + } + + drop(guard); + } + }) +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + async fn seed_npm_version( + storage: &Storage, + prefix: &str, + package: &str, + version: &str, + blob: &[u8], + ) -> (String, String) { + use base64::Engine as _; + let integrity = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(blob)) + ); + let manifest = serde_json::to_vec(&serde_json::json!({ + "name": package, + "version": version, + "dist": {"integrity": integrity} + })) + .unwrap(); + let manifest_key = format!("{prefix}/versions/{version}.json"); + let repository = prefix + .strip_prefix("npm/repositories/") + .and_then(|value| value.split_once('/')) + .map(|(repository, _)| repository) + .unwrap(); + let blob_key = + crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) + .unwrap(); + storage.put(&blob_key, blob).await.unwrap(); + storage.put(&manifest_key, &manifest).await.unwrap(); + (manifest_key, blob_key) + } + + fn test_publish_locks() -> PublishLocks { + Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())) + } + + fn make_rule( + keep_last: Option, + older_than_days: Option, + exclude_tags: Vec<&str>, + ) -> RetentionRule { + RetentionRule { + registry: "*".to_string(), + name_glob: None, + keep_last, + older_than_days, + exclude_tags: exclude_tags.into_iter().map(String::from).collect(), + } + } + + fn make_version(name: &str, modified: u64, size: u64) -> VersionEntry { + VersionEntry { + name: name.to_string(), + keys: vec![format!("test/{}", name)], + modified, + size, + } + } + + const NOW: u64 = 1_776_000_000; + const DAY: u64 = 86400; + + // -- Glob matching -- + + #[test] + fn test_glob_exact() { + assert!(glob_match("latest", "latest")); + assert!(!glob_match("latest", "latest2")); + } + + #[test] + fn test_glob_star() { + assert!(glob_match("v*", "v1.0.0")); + assert!(glob_match("v*", "v")); + assert!(!glob_match("v*", "1.0.0")); + assert!(glob_match("*-SNAPSHOT", "1.0.0-SNAPSHOT")); + assert!(!glob_match("*-SNAPSHOT", "1.0.0")); + } + + #[test] + fn test_glob_question() { + assert!(glob_match("v?.0", "v1.0")); + assert!(!glob_match("v?.0", "v10.0")); + } + + #[test] + fn test_glob_complex() { + assert!(glob_match("release-*", "release-1.0")); + assert!(glob_match("release-*", "release-")); + assert!(!glob_match("release-*", "dev-1.0")); + } + + // -- plan_deletions -- + + #[test] + fn test_keep_last_basic() { + let versions = vec![ + make_version("1.0", NOW - 3 * DAY, 100), + make_version("2.0", NOW - 2 * DAY, 200), + make_version("3.0", NOW - DAY, 300), + ]; + let rule = make_rule(Some(2), None, vec![]); + let plans = plan_deletions(versions, &rule, NOW); + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].version_name, "1.0"); + } + + #[test] + fn test_keep_last_keeps_all_if_under_limit() { + let versions = vec![ + make_version("1.0", NOW - DAY, 100), + make_version("2.0", NOW, 200), + ]; + let rule = make_rule(Some(5), None, vec![]); + let plans = plan_deletions(versions, &rule, NOW); + assert!(plans.is_empty()); + } + + #[test] + fn test_older_than_days() { + let versions = vec![ + make_version("old", NOW - 31 * DAY, 100), + make_version("new", NOW - DAY, 200), + ]; + let rule = make_rule(None, Some(30), vec![]); + let plans = plan_deletions(versions, &rule, NOW); + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].version_name, "old"); + } + + #[test] + fn test_keep_last_and_older_than() { + // AND logic: both must agree + let versions = vec![ + make_version("1.0", NOW - 60 * DAY, 100), // old + beyond keep_last + make_version("2.0", NOW - 2 * DAY, 200), // recent + beyond keep_last + make_version("3.0", NOW - DAY, 300), // newest, kept + ]; + let rule = make_rule(Some(1), Some(30), vec![]); + let plans = plan_deletions(versions, &rule, NOW); + // 2.0 is beyond keep_last=1 but NOT older than 30 days → NOT deleted + // 1.0 is beyond keep_last=1 AND older than 30 days → deleted + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].version_name, "1.0"); + } + + #[test] + fn test_exclude_tags() { + let versions = vec![ + make_version("latest", NOW - 100 * DAY, 100), + make_version("1.0", NOW - 100 * DAY, 200), + make_version("2.0", NOW, 300), + ]; + let rule = make_rule(Some(1), None, vec!["latest"]); + let plans = plan_deletions(versions, &rule, NOW); + // "latest" excluded, "2.0" kept (newest), "1.0" deleted + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].version_name, "1.0"); + } + + #[test] + fn test_exclude_glob_pattern() { + let versions = vec![ + make_version("release-1.0", NOW - 100 * DAY, 100), + make_version("release-2.0", NOW - 50 * DAY, 200), + make_version("dev-build", NOW - 100 * DAY, 300), + ]; + let rule = make_rule(Some(1), None, vec!["release-*"]); + let plans = plan_deletions(versions, &rule, NOW); + // Both release-* excluded, only dev-build is candidate (and it's beyond keep_last=1) + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].version_name, "dev-build"); + } + + #[test] + fn test_version_name_tiebreak_is_numeric_aware() { + assert_eq!( + cmp_version_names("1.10_amd64", "1.9_amd64"), + std::cmp::Ordering::Greater + ); + assert_eq!( + cmp_version_names("1.0~rc1", "1.0"), + std::cmp::Ordering::Less + ); + assert_eq!(cmp_version_names("2.0", "2.0"), std::cmp::Ordering::Equal); + assert_eq!( + cmp_version_names("1.2.3-4", "1.2.3-10"), + std::cmp::Ordering::Less + ); + + // Tied mtimes (bulk-imported sidecars): the newer version survives. + let versions = vec![ + make_version("1.9_amd64", NOW, 100), + make_version("1.10_amd64", NOW, 100), + ]; + let rule = make_rule(Some(1), None, vec![]); + let plans = plan_deletions(versions, &rule, NOW); + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].version_name, "1.9_amd64"); + } + + #[test] + fn test_empty_versions() { + let rule = make_rule(Some(1), None, vec![]); + let plans = plan_deletions(vec![], &rule, NOW); + assert!(plans.is_empty()); + } + + #[test] + fn test_deletion_reason_format() { + let versions = vec![ + make_version("old", NOW - 100 * DAY, 100), + make_version("new", NOW, 200), + ]; + let rule = make_rule(Some(1), Some(30), vec![]); + let plans = plan_deletions(versions, &rule, NOW); + assert_eq!(plans.len(), 1); + assert!(plans[0].reason.contains("keep_last")); + assert!(plans[0].reason.contains("older than")); + } + + // -- Integration tests with storage -- + + #[tokio::test] + async fn test_retention_maven_keep_last() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + + // Create 3 Maven versions (same mtime is fine — tiebreaker is name desc) + storage + .put("maven/com/example/lib/1.0/lib-1.0.jar", b"v1") + .await + .unwrap(); + storage + .put("maven/com/example/lib/2.0/lib-2.0.jar", b"v2") + .await + .unwrap(); + storage + .put("maven/com/example/lib/3.0/lib-3.0.jar", b"v3") + .await + .unwrap(); + + let rules = vec![RetentionRule { + registry: "maven".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + assert_eq!(result.planned, 2); // 1.0 and 2.0 deleted, 3.0 kept + assert!(storage + .get("maven/com/example/lib/3.0/lib-3.0.jar") + .await + .is_ok()); + } + + #[tokio::test] + async fn test_retention_keeps_named_maven_repositories_isolated() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + + for repository in ["releases", "open"] { + for version in ["1.0", "2.0"] { + storage + .put( + &format!( + "maven/repositories/{repository}/com/example/lib/{version}/lib-{version}.jar" + ), + version.as_bytes(), + ) + .await + .unwrap(); + } + } + + let rules = vec![RetentionRule { + registry: "maven".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + let mut maven = MavenConfig::default(); + maven.proxies.clear(); + maven.repositories = vec![ + MavenRepository::Hosted { + name: "releases".to_string(), + version_policy: crate::config::MavenVersionPolicy::Mixed, + write_policy: crate::config::MavenWritePolicy::AllowOnce, + }, + MavenRepository::Proxy { + name: "open".to_string(), + url: "https://repo1.maven.org/maven2".to_string(), + auth: None, + version_policy: crate::config::MavenVersionPolicy::Mixed, + metadata_ttl: None, + negative_ttl: 60, + }, + ]; + + let result = run_retention_configured( + &storage, + &test_publish_locks(), + None, + &rules, + false, + &maven, + None, + ) + .await; + + assert_eq!(result.planned, 2); + for repository in ["releases", "open"] { + assert!(storage + .get(&format!( + "maven/repositories/{repository}/com/example/lib/1.0/lib-1.0.jar" + )) + .await + .is_err()); + assert!(storage + .get(&format!( + "maven/repositories/{repository}/com/example/lib/2.0/lib-2.0.jar" + )) + .await + .is_ok()); + } + } + + fn single_named_maven(repository: MavenRepository) -> MavenConfig { + let mut config = MavenConfig::default(); + config.proxies.clear(); + config.repositories = vec![repository]; + config + } + + async fn seed_maven_metadata(storage: &Storage, key: &str, document: &[u8]) { + storage.put(key, document).await.unwrap(); + for suffix in ["md5", "sha1", "sha256", "sha512"] { + storage + .put(&format!("{key}.{suffix}"), b"old-checksum") + .await + .unwrap(); + } + } + + #[tokio::test] + async fn hosted_maven_retention_hides_discovery_then_removes_v_level_and_rebuilds_index() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "maven/repositories/releases/com/example/lib"; + for version in ["1.0-SNAPSHOT", "2.0"] { + storage + .put( + &format!("{prefix}/{version}/lib-{version}.jar"), + version.as_bytes(), + ) + .await + .unwrap(); + } + let v_metadata = format!("{prefix}/1.0-SNAPSHOT/maven-metadata.xml"); + seed_maven_metadata( + &storage, + &v_metadata, + br#"com.examplelib1.0-SNAPSHOT20260730.0100001"#, + ) + .await; + let a_metadata = format!("{prefix}/maven-metadata.xml"); + seed_maven_metadata( + &storage, + &a_metadata, + br#"com.examplelib2.02.01.0-SNAPSHOT2.020260730010000Retainedretainedretained-plugin"#, + ) + .await; + + let repo_index = crate::repo_index::RepoIndex::new(); + let before = repo_index.get("maven", &storage).await; + assert!(before + .iter() + .any(|entry| entry.name.contains("/1.0-SNAPSHOT"))); + + let rules = vec![RetentionRule { + registry: "maven".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + let config = single_named_maven(MavenRepository::Hosted { + name: "releases".to_string(), + version_policy: crate::config::MavenVersionPolicy::Mixed, + write_policy: crate::config::MavenWritePolicy::AllowOnce, + }); + let result = run_retention_configured( + &storage, + &test_publish_locks(), + None, + &rules, + false, + &config, + Some(&repo_index), + ) + .await; + + assert_eq!(result.planned, 1); + assert!(storage + .get(&format!("{prefix}/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar")) + .await + .is_err()); + for suffix in ["", ".md5", ".sha1", ".sha256", ".sha512"] { + assert!(storage.get(&format!("{v_metadata}{suffix}")).await.is_err()); + } + let metadata = storage.get(&a_metadata).await.unwrap(); + let metadata = String::from_utf8_lossy(&metadata); + assert!(!metadata.contains("1.0-SNAPSHOT")); + assert!(metadata.contains("2.0")); + assert!(metadata.contains("retained")); + for suffix in ["md5", "sha1", "sha256", "sha512"] { + assert!(storage.get(&format!("{a_metadata}.{suffix}")).await.is_ok()); + } + + let after = repo_index.get("maven", &storage).await; + assert!( + !after + .iter() + .any(|entry| entry.name.contains("/1.0-SNAPSHOT")), + "retention must invalidate and rebuild the non-TTL repository index" + ); + } + + #[tokio::test] + async fn hosted_maven_snapshot_changed_after_plan_is_skipped_under_ga_lock() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "maven/repositories/snapshots/com/example/lib"; + let snapshot_payload = format!("{prefix}/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar"); + let snapshot_metadata = format!("{prefix}/1.0-SNAPSHOT/maven-metadata.xml"); + storage.put(&snapshot_payload, b"old").await.unwrap(); + storage + .put(&format!("{prefix}/2.0/lib-2.0.jar"), b"newer-version") + .await + .unwrap(); + storage + .put( + &snapshot_metadata, + br#"1.0-SNAPSHOT1"#, + ) + .await + .unwrap(); + let a_metadata = format!("{prefix}/maven-metadata.xml"); + seed_maven_metadata( + &storage, + &a_metadata, + br#"com.examplelib2.02.01.0-SNAPSHOT2.020260730010000"#, + ) + .await; + let config = single_named_maven(MavenRepository::Hosted { + name: "snapshots".to_string(), + version_policy: crate::config::MavenVersionPolicy::Mixed, + write_policy: crate::config::MavenWritePolicy::Allow, + }); + let group = collect_maven_versions(&storage, &config) + .await + .into_iter() + .find(|group| group.group_name == "maven:snapshots:com/example/lib") + .unwrap(); + let rule = RetentionRule { + registry: "maven".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }; + let plan = plan_deletions(group.versions, &rule, NOW) + .into_iter() + .find(|plan| plan.version_name == "1.0-SNAPSHOT") + .unwrap(); + let context = MavenRetentionContext { + kind: group.kind, + storage_prefix: group.storage_prefix, + group_path: group.group_path, + artifact_id: group.artifact_id, + }; + + // Model a mutable SNAPSHOT publish completing after the retention scan + // but before retention acquires the exact GA metadata lock. + let republished_payload = b"new-snapshot-bytes-after-plan"; + let republished_v_metadata = br#"1.0-SNAPSHOT2"#; + let republished_a_metadata = br#"com.examplelib1.0-SNAPSHOT2.01.0-SNAPSHOT2.020260730020000"#; + storage + .put(&snapshot_payload, republished_payload) + .await + .unwrap(); + storage + .put(&snapshot_metadata, republished_v_metadata) + .await + .unwrap(); + storage + .put(&a_metadata, republished_a_metadata) + .await + .unwrap(); + + let outcome = delete_maven_plan(&storage, &test_publish_locks(), &context, &plan).await; + + assert!(!outcome.applied); + assert_eq!(outcome.deleted_keys, 0); + assert_eq!( + storage.get(&snapshot_payload).await.unwrap().as_ref(), + republished_payload + ); + assert_eq!( + storage.get(&snapshot_metadata).await.unwrap().as_ref(), + republished_v_metadata + ); + assert_eq!( + storage.get(&a_metadata).await.unwrap().as_ref(), + republished_a_metadata + ); + } + + #[tokio::test] + async fn proxy_maven_retention_keeps_upstream_a_level_discovery_unchanged() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "maven/repositories/central/com/example/lib"; + for version in ["1.0", "2.0"] { + storage + .put( + &format!("{prefix}/{version}/lib-{version}.jar"), + version.as_bytes(), + ) + .await + .unwrap(); + } + let a_metadata = format!("{prefix}/maven-metadata.xml"); + let document = br#"com.examplelib2.02.01.02.020260730010000"#; + seed_maven_metadata(&storage, &a_metadata, document).await; + let before: Vec<_> = ["", ".md5", ".sha1", ".sha256", ".sha512"] + .iter() + .map(|suffix| format!("{a_metadata}{suffix}")) + .collect(); + let mut before_bytes = Vec::new(); + for key in &before { + before_bytes.push(storage.get(key).await.unwrap()); + } + + let rules = vec![RetentionRule { + registry: "maven".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + let config = single_named_maven(MavenRepository::Proxy { + name: "central".to_string(), + url: "https://repo1.maven.org/maven2".to_string(), + auth: None, + version_policy: crate::config::MavenVersionPolicy::Mixed, + metadata_ttl: Some(300), + negative_ttl: 60, + }); + let result = run_retention_configured( + &storage, + &test_publish_locks(), + None, + &rules, + false, + &config, + None, + ) + .await; + + assert_eq!(result.planned, 1); + assert!(storage + .get(&format!("{prefix}/1.0/lib-1.0.jar")) + .await + .is_err()); + for (key, expected) in before.iter().zip(before_bytes) { + assert_eq!(storage.get(key).await.unwrap(), expected); + } + let metadata = + String::from_utf8_lossy(&storage.get(&a_metadata).await.unwrap()).to_string(); + assert!( + metadata.contains("1.0"), + "cache eviction must not edit upstream-owned discovery" + ); + } + + #[tokio::test] + async fn hosted_maven_v_metadata_delete_failure_keeps_payload_hidden_as_orphan() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "maven/repositories/releases/com/example/lib"; + for version in ["1.0-SNAPSHOT", "2.0"] { + inner + .put( + &format!("{prefix}/{version}/lib-{version}.jar"), + version.as_bytes(), + ) + .await + .unwrap(); + } + let v_metadata = format!("{prefix}/1.0-SNAPSHOT/maven-metadata.xml"); + seed_maven_metadata( + &inner, + &v_metadata, + br#"com.examplelib1.0-SNAPSHOT"#, + ) + .await; + let a_metadata = format!("{prefix}/maven-metadata.xml"); + seed_maven_metadata( + &inner, + &a_metadata, + br#"com.examplelib2.02.01.0-SNAPSHOT2.0"#, + ) + .await; + let failed_sidecar = format!("{v_metadata}.md5"); + let backend = crate::test_helpers::FaultInjectBackend::new(inner.clone()) + .fail_delete(&failed_sidecar); + let attempts = backend.delete_attempts(); + let storage = Storage::from_backend(Arc::new(backend)); + let rules = vec![RetentionRule { + registry: "maven".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + let config = single_named_maven(MavenRepository::Hosted { + name: "releases".to_string(), + version_policy: crate::config::MavenVersionPolicy::Mixed, + write_policy: crate::config::MavenWritePolicy::AllowOnce, + }); + + let result = run_retention_configured( + &storage, + &test_publish_locks(), + None, + &rules, + false, + &config, + None, + ) + .await; + + assert_eq!(result.planned, 1); + let payload = format!("{prefix}/1.0-SNAPSHOT/lib-1.0-SNAPSHOT.jar"); + assert!(inner.get(&payload).await.is_ok()); + assert!( + !attempts.lock().contains(&payload), + "payload deletion must fail-stop after V-level metadata failure" + ); + let metadata = String::from_utf8_lossy(&inner.get(&a_metadata).await.unwrap()).to_string(); + assert!( + !metadata.contains("1.0-SNAPSHOT"), + "A-level discovery must be hidden before any version deletion attempt" + ); + } + + #[tokio::test] + async fn hosted_maven_a_metadata_failure_deletes_no_version_objects() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "maven/repositories/releases/com/example/lib"; + for version in ["1.0", "2.0"] { + inner + .put( + &format!("{prefix}/{version}/lib-{version}.jar"), + version.as_bytes(), + ) + .await + .unwrap(); + } + let a_metadata = format!("{prefix}/maven-metadata.xml"); + seed_maven_metadata( + &inner, + &a_metadata, + br#"com.examplelib2.02.01.02.0"#, + ) + .await; + let failed_sidecar = format!("{a_metadata}.md5"); + let backend = crate::test_helpers::FaultInjectBackend::new(inner.clone()) + .fail_delete(&failed_sidecar); + let attempts = backend.delete_attempts(); + let storage = Storage::from_backend(Arc::new(backend)); + let rules = vec![RetentionRule { + registry: "maven".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + let config = single_named_maven(MavenRepository::Hosted { + name: "releases".to_string(), + version_policy: crate::config::MavenVersionPolicy::Mixed, + write_policy: crate::config::MavenWritePolicy::AllowOnce, + }); + + let result = run_retention_configured( + &storage, + &test_publish_locks(), + None, + &rules, + false, + &config, + None, + ) + .await; + + assert_eq!(result.planned, 1); + let payload = format!("{prefix}/1.0/lib-1.0.jar"); + assert!(inner.get(&payload).await.is_ok()); + let attempts = attempts.lock(); + assert!(attempts.contains(&failed_sidecar)); + assert!( + !attempts.contains(&payload), + "A-level metadata failure must abort before every version-object delete" + ); + } + + #[tokio::test] + async fn hosted_maven_last_version_retention_removes_a_level_metadata_and_sidecars() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "maven/repositories/releases/com/example/lib"; + storage + .put(&format!("{prefix}/1.0/lib-1.0.jar"), b"one") + .await + .unwrap(); + let a_metadata = format!("{prefix}/maven-metadata.xml"); + seed_maven_metadata( + &storage, + &a_metadata, + br#"com.examplelib1.01.01.0"#, + ) + .await; + let rules = vec![RetentionRule { + registry: "maven".to_string(), + name_glob: None, + keep_last: Some(0), + older_than_days: None, + exclude_tags: vec![], + }]; + let config = single_named_maven(MavenRepository::Hosted { + name: "releases".to_string(), + version_policy: crate::config::MavenVersionPolicy::Mixed, + write_policy: crate::config::MavenWritePolicy::AllowOnce, + }); + + let result = run_retention_configured( + &storage, + &test_publish_locks(), + None, + &rules, + false, + &config, + None, + ) + .await; + + assert_eq!(result.planned, 1); + for suffix in ["", ".md5", ".sha1", ".sha256", ".sha512"] { + assert!(storage.get(&format!("{a_metadata}{suffix}")).await.is_err()); + } + assert!(storage + .get(&format!("{prefix}/1.0/lib-1.0.jar")) + .await + .is_err()); + } + + #[tokio::test] + async fn test_retention_keeps_named_npm_repositories_and_proxy_cache_isolated() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + + for repository in ["npm-private-a", "npm-private-b"] { + for version in ["1.0.0", "2.0.0"] { + seed_npm_version( + &storage, + &format!("npm/repositories/{repository}/pkg"), + "pkg", + version, + version.as_bytes(), + ) + .await; + } + storage + .put( + &format!("npm/repositories/{repository}/pkg/dist-tags/old"), + b"1.0.0", + ) + .await + .unwrap(); + } + let proxy_key = "npm/repositories/npm-registry/proxy/tarballs/pkg/pkg-1.0.0.tgz"; + storage.put(proxy_key, b"cache").await.unwrap(); + + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 2); + for repository in ["npm-private-a", "npm-private-b"] { + assert!(storage + .stat(&format!( + "npm/repositories/{repository}/pkg/versions/1.0.0.json" + )) + .await + .is_none()); + assert!(storage + .stat(&format!( + "npm/repositories/{repository}/pkg/blobs/sha512/{}.tgz", + hex::encode(sha2::Sha512::digest(b"1.0.0")) + )) + .await + .is_none()); + assert!(storage + .stat(&format!("npm/repositories/{repository}/pkg/dist-tags/old")) + .await + .is_none()); + assert!(storage + .stat(&format!( + "npm/repositories/{repository}/pkg/versions/2.0.0.json" + )) + .await + .is_some()); + } + assert!(storage.get(proxy_key).await.is_ok()); + } + + #[tokio::test] + async fn test_retention_keeps_blob_referenced_by_remaining_npm_manifest() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (_, first_blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"shared").await; + let (_, second_blob) = seed_npm_version(&storage, prefix, "pkg", "2.0.0", b"shared").await; + assert_eq!(first_blob, second_blob); + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 1); + assert!(storage + .stat(&format!("{prefix}/versions/1.0.0.json")) + .await + .is_none()); + assert!(storage + .stat(&format!("{prefix}/versions/2.0.0.json")) + .await + .is_some()); + assert!(storage.get(&first_blob).await.is_ok()); + } + + #[tokio::test] + async fn test_retention_handles_hosted_package_named_proxy() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + for version in ["1.0.0", "2.0.0"] { + seed_npm_version( + &storage, + "npm/repositories/npm-private/proxy", + "proxy", + version, + version.as_bytes(), + ) + .await; + } + + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 1); + assert!(storage + .stat("npm/repositories/npm-private/proxy/versions/1.0.0.json") + .await + .is_none()); + assert!(storage + .stat("npm/repositories/npm-private/proxy/versions/2.0.0.json") + .await + .is_some()); + } + + #[tokio::test] + async fn test_retention_removes_package_state_after_last_npm_version() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"tarball").await; + storage + .put(&format!("{prefix}/pkg.json"), br#"{"name":"pkg"}"#) + .await + .unwrap(); + storage + .put(&format!("{prefix}/dist-tags/latest"), b"1.0.0") + .await + .unwrap(); + storage + .put(&format!("{prefix}/deprecations/1.0.0"), b"old") + .await + .unwrap(); + storage + .put(&format!("{prefix}/publish-complete/1.0.0"), b"complete") + .await + .unwrap(); + + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(0), + older_than_days: None, + exclude_tags: vec![], + }]; + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 1); + assert!( + storage + .list(&format!("{prefix}/")) + .await + .unwrap() + .is_empty(), + "last-version retention must not leave a 200-but-empty package shadow" + ); } - const NOW: u64 = 1_776_000_000; - const DAY: u64 = 86400; + #[tokio::test] + async fn test_retention_npm_manifest_delete_failure_keeps_tarball() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, tarball) = + seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"tarball").await; + + let backend = + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(&manifest); + let attempts = backend.delete_attempts(); + let storage = Storage::from_backend(Arc::new(backend)); + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(0), + older_than_days: None, + exclude_tags: vec![], + }]; - // -- Glob matching -- + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; - #[test] - fn test_glob_exact() { - assert!(glob_match("latest", "latest")); - assert!(!glob_match("latest", "latest2")); + assert_eq!(result.planned, 1); + assert!(inner.get(&manifest).await.is_ok()); + assert!(inner.get(&tarball).await.is_ok()); + let attempts = attempts.lock(); + assert!(attempts.contains(&manifest)); + assert!( + !attempts.contains(&tarball), + "tarball deletion must not be attempted after manifest failure" + ); } - #[test] - fn test_glob_star() { - assert!(glob_match("v*", "v1.0.0")); - assert!(glob_match("v*", "v")); - assert!(!glob_match("v*", "1.0.0")); - assert!(glob_match("*-SNAPSHOT", "1.0.0-SNAPSHOT")); - assert!(!glob_match("*-SNAPSHOT", "1.0.0")); - } + #[tokio::test] + async fn stale_npm_redeploy_plan_cannot_delete_replacement_manifest_or_blob() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"old-tarball").await; + seed_npm_version(&storage, prefix, "pkg", "2.0.0", b"newer-version").await; + let group = collect_npm_versions(&storage) + .await + .into_iter() + .find(|group| group.group_name == "npm:npm-private:pkg") + .unwrap(); + let rule = RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }; + let plans = plan_deletions(group.versions.clone(), &rule, NOW); + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].version_name, "1.0.0"); - #[test] - fn test_glob_question() { - assert!(glob_match("v?.0", "v1.0")); - assert!(!glob_match("v?.0", "v10.0")); - } + // Model write_policy=allow replacing the same version after scan and + // before retention acquires the package publish lock. + let (replacement_manifest, replacement_blob) = + seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"replacement-tarball").await; + let replacement_manifest_bytes = storage.get(&replacement_manifest).await.unwrap(); - #[test] - fn test_glob_complex() { - assert!(glob_match("release-*", "release-1.0")); - assert!(glob_match("release-*", "release-")); - assert!(!glob_match("release-*", "dev-1.0")); + let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; + + assert_eq!(outcome.applied_versions, 0); + assert_eq!(outcome.deleted_keys, 0); + assert_eq!( + storage.get(&replacement_manifest).await.unwrap(), + replacement_manifest_bytes + ); + assert!(storage.get(&replacement_blob).await.is_ok()); } - // -- plan_deletions -- + #[tokio::test] + async fn npm_tag_introduced_after_plan_skips_whole_package_batch() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"old").await; + seed_npm_version(&storage, prefix, "pkg", "2.0.0", b"new").await; + let group = collect_npm_versions(&storage) + .await + .into_iter() + .find(|group| group.group_name == "npm:npm-private:pkg") + .unwrap(); + let rule = RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }; + let plans = plan_deletions(group.versions.clone(), &rule, NOW); + let tag = format!("{prefix}/dist-tags/stable"); + storage.put(&tag, b"1.0.0").await.unwrap(); - #[test] - fn test_keep_last_basic() { - let versions = vec![ - make_version("1.0", NOW - 3 * DAY, 100), - make_version("2.0", NOW - 2 * DAY, 200), - make_version("3.0", NOW - DAY, 300), - ]; - let rule = make_rule(Some(2), None, vec![]); - let plans = plan_deletions(versions, &rule, NOW); - assert_eq!(plans.len(), 1); - assert_eq!(plans[0].version_name, "1.0"); - } + let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; - #[test] - fn test_keep_last_keeps_all_if_under_limit() { - let versions = vec![ - make_version("1.0", NOW - DAY, 100), - make_version("2.0", NOW, 200), - ]; - let rule = make_rule(Some(5), None, vec![]); - let plans = plan_deletions(versions, &rule, NOW); - assert!(plans.is_empty()); + assert_eq!(outcome.applied_versions, 0); + assert_eq!(outcome.deleted_keys, 0); + assert!(storage.get(&manifest).await.is_ok()); + assert!(storage.get(&blob).await.is_ok()); + assert_eq!(storage.get(&tag).await.unwrap().as_ref(), b"1.0.0"); } - #[test] - fn test_older_than_days() { - let versions = vec![ - make_version("old", NOW - 31 * DAY, 100), - make_version("new", NOW - DAY, 200), - ]; - let rule = make_rule(None, Some(30), vec![]); - let plans = plan_deletions(versions, &rule, NOW); - assert_eq!(plans.len(), 1); - assert_eq!(plans[0].version_name, "old"); - } + #[tokio::test] + async fn npm_new_version_after_plan_invalidates_package_wide_keep_last_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (old_manifest, old_blob) = + seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"old").await; + seed_npm_version(&storage, prefix, "pkg", "2.0.0", b"new").await; + let group = collect_npm_versions(&storage) + .await + .into_iter() + .find(|group| group.group_name == "npm:npm-private:pkg") + .unwrap(); + let rule = RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }; + let plans = plan_deletions(group.versions.clone(), &rule, NOW); + seed_npm_version(&storage, prefix, "pkg", "3.0.0", b"newest").await; - #[test] - fn test_keep_last_and_older_than() { - // AND logic: both must agree - let versions = vec![ - make_version("1.0", NOW - 60 * DAY, 100), // old + beyond keep_last - make_version("2.0", NOW - 2 * DAY, 200), // recent + beyond keep_last - make_version("3.0", NOW - DAY, 300), // newest, kept - ]; - let rule = make_rule(Some(1), Some(30), vec![]); - let plans = plan_deletions(versions, &rule, NOW); - // 2.0 is beyond keep_last=1 but NOT older than 30 days → NOT deleted - // 1.0 is beyond keep_last=1 AND older than 30 days → deleted - assert_eq!(plans.len(), 1); - assert_eq!(plans[0].version_name, "1.0"); - } + let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; - #[test] - fn test_exclude_tags() { - let versions = vec![ - make_version("latest", NOW - 100 * DAY, 100), - make_version("1.0", NOW - 100 * DAY, 200), - make_version("2.0", NOW, 300), - ]; - let rule = make_rule(Some(1), None, vec!["latest"]); - let plans = plan_deletions(versions, &rule, NOW); - // "latest" excluded, "2.0" kept (newest), "1.0" deleted - assert_eq!(plans.len(), 1); - assert_eq!(plans[0].version_name, "1.0"); + assert_eq!(outcome.applied_versions, 0); + assert_eq!(outcome.deleted_keys, 0); + assert!(storage.get(&old_manifest).await.is_ok()); + assert!(storage.get(&old_blob).await.is_ok()); } - #[test] - fn test_exclude_glob_pattern() { - let versions = vec![ - make_version("release-1.0", NOW - 100 * DAY, 100), - make_version("release-2.0", NOW - 50 * DAY, 200), - make_version("dev-build", NOW - 100 * DAY, 300), - ]; - let rule = make_rule(Some(1), None, vec!["release-*"]); - let plans = plan_deletions(versions, &rule, NOW); - // Both release-* excluded, only dev-build is candidate (and it's beyond keep_last=1) - assert_eq!(plans.len(), 1); - assert_eq!(plans[0].version_name, "dev-build"); - } + #[tokio::test] + async fn npm_target_tag_delete_failure_keeps_manifest_tag_and_blob() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, blob) = seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"old").await; + seed_npm_version(&inner, prefix, "pkg", "2.0.0", b"new").await; + let completion = format!("{prefix}/publish-complete/1.0.0"); + let tag = format!("{prefix}/dist-tags/stable"); + inner.put(&completion, b"completed").await.unwrap(); + inner.put(&tag, b"1.0.0").await.unwrap(); + let backend = crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(&tag); + let attempts = backend.delete_attempts(); + let storage = Storage::from_backend(Arc::new(backend)); + let group = collect_npm_versions(&storage) + .await + .into_iter() + .find(|group| group.group_name == "npm:npm-private:pkg") + .unwrap(); + let rule = RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }; + let plans = plan_deletions(group.versions.clone(), &rule, NOW); - #[test] - fn test_version_name_tiebreak_is_numeric_aware() { - assert_eq!( - cmp_version_names("1.10_amd64", "1.9_amd64"), - std::cmp::Ordering::Greater - ); - assert_eq!( - cmp_version_names("1.0~rc1", "1.0"), - std::cmp::Ordering::Less - ); - assert_eq!(cmp_version_names("2.0", "2.0"), std::cmp::Ordering::Equal); - assert_eq!( - cmp_version_names("1.2.3-4", "1.2.3-10"), - std::cmp::Ordering::Less - ); + let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; - // Tied mtimes (bulk-imported sidecars): the newer version survives. - let versions = vec![ - make_version("1.9_amd64", NOW, 100), - make_version("1.10_amd64", NOW, 100), - ]; - let rule = make_rule(Some(1), None, vec![]); - let plans = plan_deletions(versions, &rule, NOW); - assert_eq!(plans.len(), 1); - assert_eq!(plans[0].version_name, "1.9_amd64"); + assert_eq!(outcome.applied_versions, 0); + assert_eq!(outcome.deleted_keys, 1); + assert!( + inner.get(&completion).await.is_err(), + "publish-complete must be removed first so an exact retry repairs partial state" + ); + assert_eq!(inner.get(&tag).await.unwrap().as_ref(), b"1.0.0"); + assert!(inner.get(&manifest).await.is_ok()); + assert!(inner.get(&blob).await.is_ok()); + let attempts = attempts.lock(); + assert_eq!(attempts.first(), Some(&completion)); + assert_eq!(attempts.get(1), Some(&tag)); + assert!(!attempts.contains(&manifest)); + assert!(!attempts.contains(&blob)); } - #[test] - fn test_empty_versions() { - let rule = make_rule(Some(1), None, vec![]); - let plans = plan_deletions(vec![], &rule, NOW); - assert!(plans.is_empty()); - } + #[tokio::test] + async fn empty_npm_package_cleanup_is_discoverable_and_retryable() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let package_key = "npm/repositories/npm-private/pkg/pkg.json"; + inner.put(package_key, br#"{"name":"pkg"}"#).await.unwrap(); + let failing = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(package_key), + )); + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(0), + older_than_days: None, + exclude_tags: vec![], + }]; - #[test] - fn test_deletion_reason_format() { - let versions = vec![ - make_version("old", NOW - 100 * DAY, 100), - make_version("new", NOW, 200), - ]; - let rule = make_rule(Some(1), Some(30), vec![]); - let plans = plan_deletions(versions, &rule, NOW); - assert_eq!(plans.len(), 1); - assert!(plans[0].reason.contains("keep_last")); - assert!(plans[0].reason.contains("older than")); - } + let first = run_retention(&failing, &test_publish_locks(), None, &rules, false).await; + assert_eq!(first.planned, 0); + assert!(inner.get(package_key).await.is_ok()); - // -- Integration tests with storage -- + let second = run_retention(&inner, &test_publish_locks(), None, &rules, false).await; + assert_eq!(second.planned, 0); + assert!(inner.get(package_key).await.is_err()); + assert_eq!(second.deleted_keys, 1); + } #[tokio::test] - async fn test_retention_maven_keep_last() { + async fn test_retention_npm_missing_listing_metadata_skips_age_deletion() { let dir = tempfile::tempdir().unwrap(); - let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, tarball) = + seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"tarball").await; + let storage = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).stat_none(&manifest), + )); + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: None, + older_than_days: Some(0), + exclude_tags: vec![], + }]; - // Create 3 Maven versions (same mtime is fine — tiebreaker is name desc) - storage - .put("maven/com/example/lib/1.0/lib-1.0.jar", b"v1") - .await - .unwrap(); - storage - .put("maven/com/example/lib/2.0/lib-2.0.jar", b"v2") - .await - .unwrap(); - storage - .put("maven/com/example/lib/3.0/lib-3.0.jar", b"v3") - .await - .unwrap(); + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 0); + assert!(inner.get(&manifest).await.is_ok()); + assert!(inner.get(&tarball).await.is_ok()); + } + #[tokio::test] + async fn test_retention_npm_tag_read_failure_skips_whole_package() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, tarball) = + seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"tarball").await; + let tag = format!("{prefix}/dist-tags/stable"); + inner.put(&tag, b"1.0.0").await.unwrap(); + let storage = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_get(&tag), + )); let rules = vec![RetentionRule { - registry: "maven".to_string(), + registry: "npm".to_string(), name_glob: None, - keep_last: Some(1), + keep_last: Some(0), older_than_days: None, exclude_tags: vec![], }]; let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; - assert_eq!(result.planned, 2); // 1.0 and 2.0 deleted, 3.0 kept - assert!(storage - .get("maven/com/example/lib/3.0/lib-3.0.jar") - .await - .is_ok()); + + assert_eq!(result.planned, 0); + assert!(inner.get(&manifest).await.is_ok()); + assert!(inner.get(&tarball).await.is_ok()); + assert!(inner.get(&tag).await.is_ok()); } /// The scheduler must run once at boot, not a full interval later — a @@ -1295,6 +3376,8 @@ mod tests { storage.clone(), test_publish_locks(), None, + MavenConfig::default(), + Arc::new(crate::repo_index::RepoIndex::new()), rules, 86400, // the boot run must not wait for this false, @@ -1344,6 +3427,8 @@ mod tests { storage.clone(), test_publish_locks(), None, + MavenConfig::default(), + Arc::new(crate::repo_index::RepoIndex::new()), rules, 86400, false, @@ -1400,6 +3485,8 @@ mod tests { storage.clone(), test_publish_locks(), None, + MavenConfig::default(), + Arc::new(crate::repo_index::RepoIndex::new()), rules, 86400, false, @@ -1599,6 +3686,37 @@ mod format_retention_tests { ); } + #[test] + fn test_npm_name_glob_preserves_package_match_and_can_qualify_repository() { + let package_rule = rule("npm", Some("@scope/*"), Some(5), None); + assert!(find_matching_rule( + std::slice::from_ref(&package_rule), + "npm", + "npm:npm-private:@scope/pkg" + ) + .is_some()); + assert!(find_matching_rule( + std::slice::from_ref(&package_rule), + "npm", + "npm:other-hosted:@scope/pkg" + ) + .is_some()); + + let repository_rule = rule("npm", Some("npm-private:@scope/*"), Some(1), None); + assert!(find_matching_rule( + std::slice::from_ref(&repository_rule), + "npm", + "npm:npm-private:@scope/pkg" + ) + .is_some()); + assert!(find_matching_rule( + std::slice::from_ref(&repository_rule), + "npm", + "npm:other-hosted:@scope/pkg" + ) + .is_none()); + } + fn build_rpm(name: &str, version: &str) -> Vec { build_rpm_arch(name, version, "x86_64") } diff --git a/nora-registry/src/storage/local.rs b/nora-registry/src/storage/local.rs index e7a7b846..67122f57 100644 --- a/nora-registry/src/storage/local.rs +++ b/nora-registry/src/storage/local.rs @@ -13,6 +13,33 @@ use super::{FileMeta, Result, StorageBackend, StorageError}; /// Monotonic counter for unique temp file names (atomic — no collisions). static TMP_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); +/// Best-effort cancellation cleanup for an atomic-write staging file. +/// +/// Normal paths remove the file asynchronously. If the future is dropped +/// between staging-file creation and that cleanup, `Drop` makes one synchronous +/// removal attempt so cancelled requests do not routinely leak temp files. +struct TempFileGuard { + path: Option, +} + +impl TempFileGuard { + fn new(path: PathBuf) -> Self { + Self { path: Some(path) } + } + + fn disarm(&mut self) { + self.path = None; + } +} + +impl Drop for TempFileGuard { + fn drop(&mut self) { + if let Some(path) = self.path.take() { + let _ = std::fs::remove_file(path); + } + } +} + /// fsync the parent directory of `path` so the directory entry written by a /// just-completed `rename` is durable across power-loss. The file's own data is /// fsync'd (`sync_all`) before the rename; the rename only becomes crash-durable @@ -140,6 +167,79 @@ impl StorageBackend for LocalStorage { write_result } + async fn put_if_absent(&self, key: &str, data: &[u8]) -> Result<()> { + let path = self.key_to_path(key); + + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).await?; + } + + // Stage complete bytes beside the destination, then publish them with + // an atomic hard-link create. Exactly one contender can create `path`; + // readers never observe the staging write or a partial destination. + let open_path = path.clone(); + let opened = tokio::task::spawn_blocking(move || loop { + let seq = TMP_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let tmp = open_path.with_file_name(format!(".nora-tmp.{}.{}", std::process::id(), seq)); + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + { + Ok(file) => { + let cleanup = TempFileGuard::new(tmp.clone()); + return Ok::<_, std::io::Error>((tmp, file, cleanup)); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + } + }) + .await + .map_err(|error| { + StorageError::Io(std::io::Error::other(format!( + "atomic-create staging task failed: {error}" + ))) + })? + .map_err(StorageError::Io)?; + let (tmp, file, mut cleanup) = opened; + let mut file = fs::File::from_std(file); + + let publish_result: Result<()> = async { + file.write_all(data).await?; + file.flush().await?; + file.sync_all().await?; + fs::hard_link(&tmp, &path).await.map_err(|e| { + if e.kind() == std::io::ErrorKind::AlreadyExists { + StorageError::AlreadyExists + } else { + StorageError::Io(e) + } + })?; + Ok(()) + } + .await; + drop(file); + + match fs::remove_file(&tmp).await { + Ok(()) => cleanup.disarm(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => cleanup.disarm(), + Err(e) => { + tracing::warn!( + error = %e, + path = %tmp.display(), + "failed to clean up atomic-create staging file" + ); + } + } + + if publish_result.is_ok() { + // Persist both the destination link and staging-name removal before + // returning success. + sync_parent_dir(&path).await?; + } + publish_result + } + async fn get(&self, key: &str) -> Result { let path = self.key_to_path(key); @@ -374,6 +474,73 @@ mod tests { assert_eq!(&*data, b"test data"); } + #[tokio::test] + async fn put_if_absent_preserves_existing_bytes_and_cleans_staging() { + let temp_dir = TempDir::new().unwrap(); + let storage = LocalStorage::new(temp_dir.path().to_str().unwrap()); + + storage.put_if_absent("nested/key", b"first").await.unwrap(); + assert!(matches!( + storage.put_if_absent("nested/key", b"second").await, + Err(StorageError::AlreadyExists) + )); + assert_eq!(&storage.get("nested/key").await.unwrap()[..], b"first"); + assert_eq!( + storage.list("").await.unwrap(), + vec!["nested/key".to_string()], + "atomic-create staging files must be cleaned on success and conflict" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn concurrent_put_if_absent_has_exactly_one_winner() { + const CONTENDERS: usize = 16; + const LEN: usize = 32 * 1024; + + let temp_dir = TempDir::new().unwrap(); + let storage = std::sync::Arc::new(LocalStorage::new(temp_dir.path().to_str().unwrap())); + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(CONTENDERS)); + + let mut handles = Vec::new(); + for i in 0..CONTENDERS { + let storage = std::sync::Arc::clone(&storage); + let barrier = std::sync::Arc::clone(&barrier); + handles.push(tokio::spawn(async move { + let payload = vec![i as u8; LEN]; + barrier.wait().await; + (i as u8, storage.put_if_absent("shared/key", &payload).await) + })); + } + + let mut winner = None; + let mut conflicts = 0; + for handle in handles { + let (candidate, result) = handle.await.expect("task panicked"); + match result { + Ok(()) => assert!( + winner.replace(candidate).is_none(), + "more than one atomic create succeeded" + ), + Err(StorageError::AlreadyExists) => conflicts += 1, + Err(e) => panic!("unexpected atomic-create error: {e}"), + } + } + + let winner = winner.expect("one contender must create the key"); + assert_eq!(conflicts, CONTENDERS - 1); + let stored = storage.get("shared/key").await.unwrap(); + assert_eq!(stored.len(), LEN); + assert!( + stored.iter().all(|byte| *byte == winner), + "published bytes must be the complete winning payload" + ); + assert_eq!( + storage.list("").await.unwrap(), + vec!["shared/key".to_string()], + "no contender may leave a staging file behind" + ); + } + #[tokio::test] async fn test_get_not_found() { let temp_dir = TempDir::new().unwrap(); diff --git a/nora-registry/src/storage/mod.rs b/nora-registry/src/storage/mod.rs index 03c7a522..9574e9c6 100644 --- a/nora-registry/src/storage/mod.rs +++ b/nora-registry/src/storage/mod.rs @@ -34,6 +34,9 @@ pub enum StorageError { #[error("Object not found")] NotFound, + #[error("Object already exists")] + AlreadyExists, + #[error("IO error: {0}")] Io(#[from] std::io::Error), @@ -87,6 +90,17 @@ pub enum RepinOutcome { #[async_trait] pub trait StorageBackend: Send + Sync { async fn put(&self, key: &str, data: &[u8]) -> Result<()>; + /// Atomically create `key`, returning [`StorageError::AlreadyExists`] if a + /// concurrent writer or a pre-existing object already owns the key. + /// + /// The default fails closed instead of emulating this with `stat` + `put`, + /// which would allow two callers to win the race. Production backends must + /// override this method with a backend-native conditional create. + async fn put_if_absent(&self, _key: &str, _data: &[u8]) -> Result<()> { + Err(StorageError::Network( + "atomic create is not supported by this storage backend".to_string(), + )) + } async fn get(&self, key: &str) -> Result; async fn delete(&self, key: &str) -> Result<()>; async fn list(&self, prefix: &str) -> Result>; @@ -299,6 +313,103 @@ impl Storage { } } + /// Atomically create `key` without replacing an existing object. + /// + /// Exactly one concurrent caller can succeed. Only that winner records a + /// hash pin; losing callers leave both the stored bytes and existing pin + /// untouched. + pub async fn put_if_absent(&self, key: &str, data: &[u8]) -> Result<()> { + validate_storage_key(key)?; + match self.inner.put_if_absent(key, data).await { + Ok(()) => { + STORAGE_OPERATIONS + .with_label_values(&["put_if_absent", "ok"]) + .inc(); + if let Some(ref pins) = self.pin_store { + let pins = Arc::clone(pins); + let key_owned = key.to_string(); + let data_owned = data.to_vec(); + // Match put(): the winning create is not reported complete + // until its integrity pin is durable. + match tokio::task::spawn_blocking(move || pins.record(&key_owned, &data_owned)) + .await + { + Ok(Ok(())) => {} + Ok(Err(e)) => { + STORAGE_OPERATIONS + .with_label_values(&["put_if_absent", "pin_error"]) + .inc(); + tracing::error!(error = %e, key = %key, "hash-pin record failed"); + return Err(StorageError::Io(std::io::Error::other(format!( + "hash-pin record failed: {e}" + )))); + } + Err(e) => { + STORAGE_OPERATIONS + .with_label_values(&["put_if_absent", "pin_error"]) + .inc(); + tracing::error!(error = %e, key = %key, "hash-pin record task panicked"); + return Err(StorageError::Io(std::io::Error::other(format!( + "hash-pin record failed: {e}" + )))); + } + } + } + Ok(()) + } + Err(StorageError::AlreadyExists) => { + STORAGE_OPERATIONS + .with_label_values(&["put_if_absent", "already_exists"]) + .inc(); + // The create may have published the body and then failed while + // durably recording its local hash pin. An exact client retry + // is the only safe automatic repair: read the already-existing + // bytes without overwriting them, compare to the candidate, + // and fill the missing pin only on byte identity. + if let Some(ref pins) = self.pin_store { + if pins.get(key).is_none() { + let existing = self.inner.get(key).await?; + if existing.as_ref() == data { + let pins = Arc::clone(pins); + let key_owned = key.to_string(); + let data_owned = data.to_vec(); + match tokio::task::spawn_blocking(move || { + pins.record(&key_owned, &data_owned) + }) + .await + { + Ok(Ok(())) => {} + Ok(Err(e)) => { + STORAGE_OPERATIONS + .with_label_values(&["put_if_absent", "pin_error"]) + .inc(); + return Err(StorageError::Io(std::io::Error::other(format!( + "hash-pin retry repair failed: {e}" + )))); + } + Err(e) => { + STORAGE_OPERATIONS + .with_label_values(&["put_if_absent", "pin_error"]) + .inc(); + return Err(StorageError::Io(std::io::Error::other(format!( + "hash-pin retry repair task panicked: {e}" + )))); + } + } + } + } + } + Err(StorageError::AlreadyExists) + } + Err(e) => { + STORAGE_OPERATIONS + .with_label_values(&["put_if_absent", "error"]) + .inc(); + Err(e) + } + } + } + pub async fn get(&self, key: &str) -> Result { validate_storage_key(key)?; match self.inner.get(key).await { @@ -730,6 +841,70 @@ mod tests { assert_eq!(&storage.get("raw/x/app.bin").await.unwrap()[..], b"payload"); } + #[tokio::test] + async fn put_if_absent_pins_only_the_winning_bytes() { + let dir = TempDir::new().unwrap(); + let storage = Storage::new_local(dir.path().to_str().unwrap()); + let key = "raw/x/immutable.bin"; + + storage.put_if_absent(key, b"winner").await.unwrap(); + let winner_pin = hex::encode(Sha256::digest(b"winner")); + assert_eq!( + storage.get_pin_hash(key).as_deref(), + Some(winner_pin.as_str()), + "the successful create must persist its pin before returning" + ); + + assert!(matches!( + storage.put_if_absent(key, b"loser").await, + Err(StorageError::AlreadyExists) + )); + assert_eq!( + storage.get_pin_hash(key).as_deref(), + Some(winner_pin.as_str()), + "a losing create must not replace the existing pin" + ); + assert_eq!(&storage.get(key).await.unwrap()[..], b"winner"); + } + + #[tokio::test] + async fn put_if_absent_exact_retry_repairs_missing_pin_without_overwrite() { + let dir = TempDir::new().unwrap(); + let storage = Storage::new_local(dir.path().join("store").to_str().unwrap()); + let key = "raw/x/immutable.bin"; + let src = dir.path().join("published-before-pin.bin"); + std::fs::write(&src, b"winner").unwrap(); + storage.put_from_path(key, &src, None).await.unwrap(); + assert_eq!(storage.get_pin_hash(key), None); + + assert!(matches!( + storage.put_if_absent(key, b"winner").await, + Err(StorageError::AlreadyExists) + )); + assert_eq!( + storage.get_pin_hash(key).as_deref(), + Some(hex::encode(Sha256::digest(b"winner")).as_str()) + ); + assert_eq!(&storage.get(key).await.unwrap()[..], b"winner"); + } + + #[tokio::test] + async fn put_if_absent_different_retry_never_pins_candidate_or_overwrites() { + let dir = TempDir::new().unwrap(); + let storage = Storage::new_local(dir.path().join("store").to_str().unwrap()); + let key = "raw/x/immutable.bin"; + let src = dir.path().join("published-before-pin.bin"); + std::fs::write(&src, b"winner").unwrap(); + storage.put_from_path(key, &src, None).await.unwrap(); + + assert!(matches!( + storage.put_if_absent(key, b"loser").await, + Err(StorageError::AlreadyExists) + )); + assert_eq!(storage.get_pin_hash(key), None); + assert_eq!(&storage.get(key).await.unwrap()[..], b"winner"); + } + #[test] fn registry_label_extracts_prefix_and_bounds_cardinality() { assert_eq!(registry_label("npm/lodash/metadata.json"), "npm"); diff --git a/nora-registry/src/storage/object.rs b/nora-registry/src/storage/object.rs index a8abd68a..f3efbe3e 100644 --- a/nora-registry/src/storage/object.rs +++ b/nora-registry/src/storage/object.rs @@ -7,7 +7,7 @@ use futures::TryStreamExt; use object_store::aws::AmazonS3Builder; use object_store::gcp::GoogleCloudStorageBuilder; use object_store::path::Path; -use object_store::{ObjectStore, ObjectStoreExt, PutPayload, WriteMultipart}; +use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload, WriteMultipart}; use std::pin::Pin; use tokio::io::{AsyncRead, AsyncReadExt}; @@ -173,6 +173,7 @@ fn decode_object_key(key: &str) -> String { fn map_err(e: object_store::Error) -> StorageError { match e { object_store::Error::NotFound { .. } => StorageError::NotFound, + object_store::Error::AlreadyExists { .. } => StorageError::AlreadyExists, other => StorageError::Network(other.to_string()), } } @@ -187,6 +188,17 @@ impl StorageBackend for ObjectStorage { Ok(()) } + async fn put_if_absent(&self, key: &str, data: &[u8]) -> Result<()> { + let encoded = encode_object_key(key); + let path = Path::from(encoded); + let payload = PutPayload::from(data.to_vec()); + self.store + .put_opts(&path, payload, PutMode::Create.into()) + .await + .map_err(map_err)?; + Ok(()) + } + async fn get(&self, key: &str) -> Result { let encoded = encode_object_key(key); let path = Path::from(encoded); @@ -526,6 +538,51 @@ mod tests { assert!(storage.get(&listed_plain[0]).await.is_ok()); } + #[tokio::test(flavor = "multi_thread")] + async fn concurrent_put_if_absent_has_exactly_one_winner() { + const CONTENDERS: usize = 16; + let storage = std::sync::Arc::new(ObjectStorage { + store: Box::new(object_store::memory::InMemory::new()), + name: "s3", + cached_total_size: std::sync::atomic::AtomicU64::new(0), + size_cache_initialized: std::sync::atomic::AtomicBool::new(false), + cached_reachable: std::sync::atomic::AtomicBool::new(true), + last_refresh_unix: std::sync::atomic::AtomicU64::new(0), + }); + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(CONTENDERS)); + + let mut handles = Vec::new(); + for i in 0..CONTENDERS { + let storage = std::sync::Arc::clone(&storage); + let barrier = std::sync::Arc::clone(&barrier); + handles.push(tokio::spawn(async move { + let payload = vec![i as u8; 4096]; + barrier.wait().await; + (i as u8, storage.put_if_absent("raw/shared", &payload).await) + })); + } + + let mut winner = None; + let mut conflicts = 0; + for handle in handles { + let (candidate, result) = handle.await.expect("task panicked"); + match result { + Ok(()) => assert!( + winner.replace(candidate).is_none(), + "more than one conditional object put succeeded" + ), + Err(StorageError::AlreadyExists) => conflicts += 1, + Err(e) => panic!("unexpected conditional-put error: {e}"), + } + } + + let winner = winner.expect("one contender must create the object"); + assert_eq!(conflicts, CONTENDERS - 1); + let stored = storage.get("raw/shared").await.unwrap(); + assert_eq!(stored.len(), 4096); + assert!(stored.iter().all(|byte| *byte == winner)); + } + #[test] fn test_s3_storage_creation_anonymous() { let storage = ObjectStorage::new( @@ -628,6 +685,15 @@ mod tests { } } + #[test] + fn test_error_mapping_already_exists() { + let err = object_store::Error::AlreadyExists { + path: "test/key".to_string(), + source: "exists".into(), + }; + assert!(matches!(map_err(err), StorageError::AlreadyExists)); + } + #[test] fn test_error_mapping_network() { let err = object_store::Error::Generic { diff --git a/nora-registry/src/test_helpers.rs b/nora-registry/src/test_helpers.rs index fe4a2b63..67868fcd 100644 --- a/nora-registry/src/test_helpers.rs +++ b/nora-registry/src/test_helpers.rs @@ -8,6 +8,8 @@ #![allow(clippy::unwrap_used)] // tests may use .unwrap() freely +use async_trait::async_trait; +use axum::body::Bytes; use axum::{ body::Body, extract::{ConnectInfo, DefaultBodyLimit}, @@ -15,11 +17,14 @@ use axum::{ middleware, Router, }; use http_body_util::BodyExt; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::net::SocketAddr; +use std::path::Path; +use std::pin::Pin; use std::sync::Arc; use std::time::Instant; use tempfile::TempDir; +use tokio::io::AsyncRead; use crate::activity_log::ActivityLog; use crate::audit::AuditLog; @@ -29,12 +34,134 @@ use crate::curation::CurationEngine; use crate::dashboard_metrics::DashboardMetrics; use crate::registry; use crate::repo_index::RepoIndex; -use crate::storage::Storage; +use crate::storage::{FileMeta, Storage, StorageBackend, StorageError}; use crate::tokens::TokenStore; use crate::AppState; use parking_lot::RwLock; +/// Test-only storage wrapper for exercising fail-closed behavior without +/// teaching production backends about synthetic failures. +pub struct FaultInjectBackend { + inner: Storage, + get_failures: HashSet, + delete_failures: HashSet, + stat_none: HashSet, + delete_attempts: Arc>>, + list_attempts: Arc>>, +} + +impl FaultInjectBackend { + pub fn new(inner: Storage) -> Self { + Self { + inner, + get_failures: HashSet::new(), + delete_failures: HashSet::new(), + stat_none: HashSet::new(), + delete_attempts: Arc::new(parking_lot::Mutex::new(Vec::new())), + list_attempts: Arc::new(parking_lot::Mutex::new(Vec::new())), + } + } + + pub fn fail_get(mut self, key: impl Into) -> Self { + self.get_failures.insert(key.into()); + self + } + + pub fn fail_delete(mut self, key: impl Into) -> Self { + self.delete_failures.insert(key.into()); + self + } + + #[allow(dead_code)] // consumed by binary-only cleanup tests, not lib test target + pub fn stat_none(mut self, key: impl Into) -> Self { + self.stat_none.insert(key.into()); + self + } + + pub fn delete_attempts(&self) -> Arc>> { + Arc::clone(&self.delete_attempts) + } + + pub fn list_attempts(&self) -> Arc>> { + Arc::clone(&self.list_attempts) + } +} + +#[async_trait] +impl StorageBackend for FaultInjectBackend { + async fn put(&self, key: &str, data: &[u8]) -> crate::storage::Result<()> { + self.inner.put(key, data).await + } + + async fn put_if_absent(&self, key: &str, data: &[u8]) -> crate::storage::Result<()> { + self.inner.put_if_absent(key, data).await + } + + async fn get(&self, key: &str) -> crate::storage::Result { + if self.get_failures.contains(key) { + return Err(StorageError::Network("injected get failure".to_string())); + } + self.inner.get(key).await + } + + async fn delete(&self, key: &str) -> crate::storage::Result<()> { + self.delete_attempts.lock().push(key.to_string()); + if self.delete_failures.contains(key) { + return Err(StorageError::Network("injected delete failure".to_string())); + } + self.inner.delete(key).await + } + + async fn list(&self, prefix: &str) -> crate::storage::Result> { + self.list_attempts.lock().push(prefix.to_string()); + self.inner.list(prefix).await + } + + async fn stat(&self, key: &str) -> Option { + if self.stat_none.contains(key) { + return None; + } + self.inner.stat(key).await + } + + async fn list_with_meta( + &self, + prefix: &str, + ) -> crate::storage::Result> { + let mut entries = self.inner.list_with_meta(prefix).await?; + entries.retain(|(key, _)| !self.stat_none.contains(key)); + Ok(entries) + } + + async fn health_check(&self) -> bool { + self.inner.health_check().await + } + + async fn total_size(&self) -> u64 { + self.inner.total_size().await + } + + fn backend_name(&self) -> &'static str { + "fault-inject" + } + + async fn refresh_total_size(&self) { + self.inner.refresh_total_size_cache().await; + } + + async fn put_from_path(&self, key: &str, src: &Path) -> crate::storage::Result<()> { + self.inner.put_from_path(key, src, None).await + } + + async fn get_reader( + &self, + key: &str, + ) -> crate::storage::Result<(u64, Pin>)> { + self.inner.get_reader(key).await + } +} + /// Everything a test needs: tempdir (must stay alive), shared state, and the router. pub struct TestContext { pub state: AppState, @@ -60,6 +187,14 @@ pub fn create_test_context_with_anonymous_read(users: &[(&str, &str)]) -> TestCo build_context(true, users, true, |_| {}) } +/// Build a test context with auth + anonymous_read and custom registry config. +pub fn create_test_context_with_anonymous_read_config( + users: &[(&str, &str)], + customize: impl FnOnce(&mut Config), +) -> TestContext { + build_context(true, users, true, customize) +} + /// Build a test context with auth + `docker_anon_pull` (general /// `anonymous_read` left OFF, to prove Docker is governed by its own switch). pub fn create_test_context_with_docker_anon_pull(users: &[(&str, &str)]) -> TestContext { @@ -115,9 +250,10 @@ fn build_context( enabled: true, proxies: vec![], proxy_timeout: 5, - checksum_verify: true, immutable_releases: true, metadata_ttl: 300, + repositories: Vec::new(), + default_repository: None, }, npm: NpmConfig { enabled: true, @@ -127,6 +263,8 @@ fn build_context( metadata_ttl: -1, serve_stale: true, revalidate: true, + repositories: Vec::new(), + default_repository: None, }, pypi: PypiConfig { enabled: true, @@ -290,8 +428,13 @@ fn build_context( docker_auth: Arc::new(docker_auth), repo_index: Arc::new(RepoIndex::new()), http_client: reqwest::Client::new(), + no_redirect_http_client: reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .expect("test no-redirect HTTP client"), upload_sessions: Arc::new(RwLock::new(HashMap::new())), publish_locks: Arc::new(parking_lot::Mutex::new(HashMap::new())), + maven_negative_cache: Arc::new(parking_lot::Mutex::new(HashMap::new())), reloadable, auth_failures: Arc::new(crate::auth::AuthFailureTracker::new(5, 900)), oidc: None, @@ -357,6 +500,11 @@ fn build_context( } } } + if enabled_registries.contains(&crate::registry_type::RegistryType::Maven) + || enabled_registries.contains(&crate::registry_type::RegistryType::Npm) + { + registry_routes = registry_routes.merge(registry::named_repository_routes()); + } let public_routes = Router::new() .merge(crate::health::routes()) diff --git a/nora-registry/src/ui/api.rs b/nora-registry/src/ui/api.rs index 7e84b964..445f3d20 100644 --- a/nora-registry/src/ui/api.rs +++ b/nora-registry/src/ui/api.rs @@ -206,45 +206,76 @@ pub async fn api_dashboard(State(state): State) -> Json = match reg { - RegistryType::Docker => state - .config - .docker - .upstreams - .iter() - .map(|u| u.url.clone()) - .collect(), - RegistryType::Maven => state - .config - .maven - .proxies - .iter() - .map(|p| p.url().to_string()) - .collect(), - RegistryType::Npm => state.config.npm.proxy.clone().into_iter().collect(), - RegistryType::Cargo => state.config.cargo.proxy.clone().into_iter().collect(), - RegistryType::PyPI => state - .config - .pypi - .upstreams() - .iter() - .map(|u| u.url().to_string()) - .collect(), - RegistryType::Go => state.config.go.proxy.clone().into_iter().collect(), - RegistryType::Raw => vec![], - RegistryType::Gems => state.config.gems.proxy.clone().into_iter().collect(), - RegistryType::Terraform => state.config.terraform.proxy.clone().into_iter().collect(), - RegistryType::Ansible => state.config.ansible.proxy.clone().into_iter().collect(), - RegistryType::Nuget => state.config.nuget.proxy.clone().into_iter().collect(), - RegistryType::PubDart => state.config.pub_dart.proxy.clone().into_iter().collect(), - RegistryType::Conan => state.config.conan.proxy.clone().into_iter().collect(), - RegistryType::Rpm => vec![], - RegistryType::Deb => vec![], - }; + let proxy_upstreams: Vec = + match reg { + RegistryType::Docker => state + .config + .docker + .upstreams + .iter() + .map(|u| u.url.clone()) + .collect(), + RegistryType::Maven => { + let mut upstreams: std::collections::BTreeSet = state + .config + .maven + .proxies + .iter() + .map(|proxy| proxy.url().to_string()) + .collect(); + upstreams.extend(state.config.maven.repositories.iter().filter_map( + |repository| match repository { + crate::config::MavenRepository::Proxy { url, .. } => Some(url.clone()), + _ => None, + }, + )); + upstreams.into_iter().collect() + } + RegistryType::Npm => { + let mut upstreams: std::collections::BTreeSet = + state.config.npm.proxy.clone().into_iter().collect(); + upstreams.extend(state.config.npm.repositories.iter().filter_map( + |repository| match repository { + crate::config::NpmRepository::Proxy { url, .. } => Some(url.clone()), + _ => None, + }, + )); + upstreams.into_iter().collect() + } + RegistryType::Cargo => state.config.cargo.proxy.clone().into_iter().collect(), + RegistryType::PyPI => state + .config + .pypi + .upstreams() + .iter() + .map(|u| u.url().to_string()) + .collect(), + RegistryType::Go => state.config.go.proxy.clone().into_iter().collect(), + RegistryType::Raw => vec![], + RegistryType::Gems => state.config.gems.proxy.clone().into_iter().collect(), + RegistryType::Terraform => { + state.config.terraform.proxy.clone().into_iter().collect() + } + RegistryType::Ansible => state.config.ansible.proxy.clone().into_iter().collect(), + RegistryType::Nuget => state.config.nuget.proxy.clone().into_iter().collect(), + RegistryType::PubDart => state.config.pub_dart.proxy.clone().into_iter().collect(), + RegistryType::Conan => state.config.conan.proxy.clone().into_iter().collect(), + RegistryType::Rpm => vec![], + RegistryType::Deb => vec![], + }; + let mount_path = match reg { + RegistryType::Maven if !state.config.maven.repositories.is_empty() => { + "/repository/{repository}/".to_string() + } + RegistryType::Npm if !state.config.npm.repositories.is_empty() => { + "/repository/{repository}/".to_string() + } + _ => reg.mount_point().to_string(), + }; mount_points.push(MountPoint { registry: reg.display_name().to_string(), - mount_path: reg.mount_point().to_string(), + mount_path, proxy_upstreams, }); } @@ -573,64 +604,110 @@ pub async fn get_npm_detail( show_prerelease: bool, show_all: bool, ) -> PackageDetail { - let metadata_key = format!("npm/{}/metadata.json", name); - - let mut stable_versions = Vec::new(); - let mut prerelease_count: usize = 0; + let Some((repository, package)) = name + .strip_prefix("repositories/") + .and_then(|rest| rest.split_once('/')) + else { + return PackageDetail { + versions: vec![], + prerelease_count: 0, + total_stable: 0, + metadata: PackageMetadata::default(), + }; + }; - // Parse metadata.json for version info - if let Ok(data) = storage.get(&metadata_key).await { - if let Ok(metadata) = serde_json::from_slice::(&data) { - if let Some(versions_obj) = metadata.get("versions").and_then(|v| v.as_object()) { - let time_obj = metadata.get("time").and_then(|t| t.as_object()); - - for (version, info) in versions_obj { - let is_prerelease = version.contains('-'); - - let meta_size = info - .get("dist") - .and_then(|d| d.get("unpackedSize")) - .and_then(|s| s.as_u64()) - .unwrap_or(0); - - let published = time_obj - .and_then(|t| t.get(version)) - .and_then(|p| p.as_str()) - .map(|s| s.get(..10).unwrap_or(s).to_string()) + let package_leaf = package.split('/').next_back().unwrap_or(package); + let hosted_versions_prefix = format!("npm/repositories/{repository}/{package}/versions/"); + let hosted_version_keys = storage + .list(&hosted_versions_prefix) + .await + .unwrap_or_default(); + let mut version_rows: Vec<(String, serde_json::Value, String, String)> = Vec::new(); + let metadata_json = if hosted_version_keys.is_empty() { + let packument_key = + format!("npm/repositories/{repository}/proxy/packuments/{package}.json"); + storage.get(&packument_key).await.ok().and_then(|data| { + let metadata = serde_json::from_slice::(&data).ok()?; + let time = metadata.get("time").and_then(|value| value.as_object()); + if let Some(versions) = metadata.get("versions").and_then(|value| value.as_object()) { + for (version, info) in versions { + let published = time + .and_then(|times| times.get(version)) + .and_then(|value| value.as_str()) + .map(|value| value.get(..10).unwrap_or(value).to_string()) .unwrap_or_else(|| "N/A".to_string()); - - // Count pre-release, skip unless toggled - if is_prerelease { - prerelease_count += 1; - if !show_prerelease { - continue; - } - } - - // Check if tarball is actually cached on disk - // For scoped packages (@scope/name), tarball uses just the "name" part - let name_part = if name.contains('/') { - name.rsplit('/').next().unwrap_or(name) - } else { - name - }; - let tarball_key = - format!("npm/{}/tarballs/{}-{}.tgz", name, name_part, version); - let (size, cached) = if let Some(meta) = storage.stat(&tarball_key).await { - (meta.size, true) - } else { - (meta_size, false) - }; - - stable_versions.push(VersionInfo { - version: version.clone(), - size, + version_rows.push(( + version.clone(), + info.clone(), published, - cached, - }); + format!( + "npm/repositories/{repository}/proxy/tarballs/{package}/{package_leaf}-{version}.tgz" + ), + )); } } + Some(metadata) + }) + } else { + for key in hosted_version_keys { + let Some(version) = key + .rsplit('/') + .next() + .and_then(|part| part.strip_suffix(".json")) + else { + continue; + }; + let Ok(data) = storage.get(&key).await else { + continue; + }; + let Ok(info) = serde_json::from_slice::(&data) else { + continue; + }; + let Some(blob_key) = + crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &data) + else { + continue; + }; + let published = storage + .stat(&key) + .await + .map(|meta| format_timestamp(meta.modified)) + .unwrap_or_else(|| "N/A".to_string()); + version_rows.push((version.to_string(), info, published, blob_key)); + } + let package_key = format!("npm/repositories/{repository}/{package}/pkg.json"); + storage + .get(&package_key) + .await + .ok() + .and_then(|data| serde_json::from_slice::(&data).ok()) + }; + + let mut stable_versions = Vec::new(); + let mut prerelease_count: usize = 0; + for (version, info, published, tarball_key) in version_rows { + let is_prerelease = version.contains('-'); + if is_prerelease { + prerelease_count += 1; + if !show_prerelease { + continue; + } } + let declared_size = info + .get("dist") + .and_then(|dist| dist.get("unpackedSize")) + .and_then(|size| size.as_u64()) + .unwrap_or(0); + let (size, cached) = storage + .stat(&tarball_key) + .await + .map_or((declared_size, false), |meta| (meta.size, true)); + stable_versions.push(VersionInfo { + version, + size, + published, + cached, + }); } // Sort by version (semver-like, newest first) @@ -658,56 +735,51 @@ pub async fn get_npm_detail( stable_versions.truncate(20); } - // Extract package-level metadata from the same JSON let mut metadata = PackageMetadata::default(); - if let Ok(data) = storage.get(&metadata_key).await { - if let Ok(meta_json) = serde_json::from_slice::(&data) { - metadata.description = meta_json - .get("description") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - metadata.license = meta_json - .get("license") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - metadata.author = meta_json.get("author").and_then(|v| { - // author can be a string or { name: "..." } - v.as_str().map(|s| s.to_string()).or_else(|| { - v.get("name") - .and_then(|n| n.as_str()) - .map(|s| s.to_string()) - }) - }); - metadata.homepage = meta_json - .get("homepage") - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .filter(|s| sanitize_href(s).is_some()) - .map(|s| s.to_string()); - metadata.repository = meta_json - .get("repository") - .and_then(|v| { - // repository can be a string or { url: "..." } - v.as_str() - .map(|s| s.to_string()) - .or_else(|| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string())) - }) - .map(|s| { - s.trim_start_matches("git+") - .trim_end_matches(".git") - .to_string() - }) - .filter(|s| sanitize_href(s).is_some()); - metadata.keywords = meta_json - .get("keywords") - .and_then(|v| v.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|v| v.as_str().map(|s| s.to_string())) - .collect() - }) - .unwrap_or_default(); - } + if let Some(meta_json) = metadata_json { + metadata.description = meta_json + .get("description") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + metadata.license = meta_json + .get("license") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + metadata.author = meta_json.get("author").and_then(|v| { + v.as_str().map(|s| s.to_string()).or_else(|| { + v.get("name") + .and_then(|n| n.as_str()) + .map(|s| s.to_string()) + }) + }); + metadata.homepage = meta_json + .get("homepage") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .filter(|s| sanitize_href(s).is_some()) + .map(|s| s.to_string()); + metadata.repository = meta_json + .get("repository") + .and_then(|v| { + v.as_str() + .map(|s| s.to_string()) + .or_else(|| v.get("url").and_then(|u| u.as_str()).map(|s| s.to_string())) + }) + .map(|s| { + s.trim_start_matches("git+") + .trim_end_matches(".git") + .to_string() + }) + .filter(|s| sanitize_href(s).is_some()); + metadata.keywords = meta_json + .get("keywords") + .and_then(|v| v.as_array()) + .map(|arr| { + arr.iter() + .filter_map(|v| v.as_str().map(|s| s.to_string())) + .collect() + }) + .unwrap_or_default(); } PackageDetail { @@ -1836,3 +1908,66 @@ pub async fn get_raw_dir_listing(storage: &Storage, path: &str) -> (Vec Router { .route("/ui/maven", get(maven_list)) .route("/ui/maven/{*path}", get(maven_detail)) .route("/ui/npm", get(npm_list)) - .route("/ui/npm/{name}", get(npm_detail)) + .route("/ui/npm/{*name}", get(npm_detail)) .route("/ui/cargo", get(cargo_list)) .route("/ui/cargo/{name}", get(cargo_detail)) .route("/ui/pypi", get(pypi_list)) @@ -167,7 +167,7 @@ pub fn routes() -> Router { .route("/api/ui/stats", get(api_stats)) .route("/api/ui/dashboard", get(api_dashboard)) .route("/api/ui/{registry_type}/list", get(api_list)) - .route("/api/ui/{registry_type}/{name}", get(api_detail)) + .route("/api/ui/{registry_type}/{*name}", get(api_detail)) .route("/api/ui/{registry_type}/search", get(api_search)) } @@ -1083,3 +1083,74 @@ mod base_path_tests { assert_eq!(apply_base_path(html, "/nora"), html); } } + +#[cfg(test)] +mod named_npm_route_tests { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::{Method, StatusCode}; + + #[tokio::test] + async fn encoded_repository_qualified_npm_link_opens_detail_page() { + use base64::Engine as _; + use sha2::Digest as _; + + let context = create_test_context_with_config(|config| { + config.npm.repositories = vec![crate::config::NpmRepository::Hosted { + name: "npm-private".to_string(), + write_policy: crate::config::NpmWritePolicy::AllowOnce, + }]; + config.npm.default_repository = Some("npm-private".to_string()); + }); + let blob = b"tarball"; + let manifest = serde_json::to_vec(&serde_json::json!({ + "name": "@scope/pkg", + "version": "1.0.0", + "dist": { + "integrity": format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD + .encode(sha2::Sha512::digest(blob)) + ) + } + })) + .unwrap(); + context + .state + .storage + .put( + "npm/repositories/npm-private/@scope/pkg/versions/1.0.0.json", + &manifest, + ) + .await + .unwrap(); + context + .state + .storage + .put( + &crate::npm_layout::hosted_blob_key_from_manifest( + "npm-private", + "@scope/pkg", + &manifest, + ) + .unwrap(), + blob, + ) + .await + .unwrap(); + context.state.repo_index.invalidate("npm"); + + let list = send(&context.app, Method::GET, "/ui/npm", "").await; + assert_eq!(list.status(), StatusCode::OK); + let list_html = String::from_utf8(body_bytes(list).await.to_vec()).unwrap(); + let detail_path = "/ui/npm/repositories%2Fnpm-private%2F%40scope%2Fpkg"; + assert!(list_html.contains(detail_path)); + + let detail = send(&context.app, Method::GET, detail_path, "").await; + assert_eq!(detail.status(), StatusCode::OK); + let detail_html = String::from_utf8(body_bytes(detail).await.to_vec()).unwrap(); + assert!(detail_html.contains(&format!( + "npm install @scope/pkg --registry {}/repository/npm-private", + context.state.config.server.public_base_url() + ))); + } +} diff --git a/nora-registry/src/ui/templates.rs b/nora-registry/src/ui/templates.rs index e18d6f31..287d38d1 100644 --- a/nora-registry/src/ui/templates.rs +++ b/nora-registry/src/ui/templates.rs @@ -1153,7 +1153,18 @@ pub fn render_package_detail( }; let install_cmd = match registry_type { - "npm" => format!("npm install {} --registry {}/npm", name, base_url), + "npm" => name + .strip_prefix("repositories/") + .and_then(|rest| rest.split_once('/')) + .map_or_else( + || format!("npm install {} --registry {}/npm", name, base_url), + |(repository, package)| { + format!( + "npm install {} --registry {}/repository/{}", + package, base_url, repository + ) + }, + ), "cargo" => format!("cargo add {}", name), "pypi" => format!("pip install {} --index-url {}/simple", name, base_url), "go" => format!("GOPROXY={}/go go get {}", base_url, name), @@ -1195,7 +1206,18 @@ pub fn render_package_detail( }; // Build breadcrumbs — make each path segment clickable for hierarchical names - let breadcrumb_html = if registry_type == "ansible" && name.contains('.') { + let named_npm = (registry_type == "npm") + .then(|| name.strip_prefix("repositories/")) + .flatten() + .and_then(|rest| rest.split_once('/')); + let breadcrumb_html = if let Some((repository, package)) = named_npm { + format!( + r#"{title}/{repository}/{package}"#, + title = registry_title, + repository = html_escape(repository), + package = html_escape(package), + ) + } else if registry_type == "ansible" && name.contains('.') { // Ansible: community.general → Ansible Galaxy / community / general let parts: Vec<&str> = name.splitn(2, '.').collect(); let mut crumbs = format!( @@ -1255,7 +1277,11 @@ pub fn render_package_detail( ) }; - let detail_title = if registry_type == "raw" && name.contains('/') { + let detail_title = if registry_type == "npm" { + named_npm + .map(|(_, package)| html_escape(package)) + .unwrap_or_else(|| html_escape(name)) + } else if registry_type == "raw" && name.contains('/') { html_escape(name.rsplit('/').next().unwrap_or(name)) } else { html_escape(name) @@ -1393,11 +1419,25 @@ pub fn render_maven_detail( auth_enabled: bool, ) -> String { let _t = get_translations(lang); + let (repository, coordinate_path) = path + .strip_prefix("repositories/") + .and_then(|rest| rest.split_once('/')) + .map_or((None, path), |(repository, coordinate_path)| { + (Some(repository), coordinate_path) + }); let artifact_rows = if detail.artifacts.is_empty() { r##"No artifacts found"##.to_string() } else { detail.artifacts.iter().map(|a| { - let download_url = format!("/maven2/{}/{}", path, a.filename); + let download_url = repository.map_or_else( + || format!("/maven2/{coordinate_path}/{}", a.filename), + |repository| { + format!( + "/repository/{repository}/{coordinate_path}/{}", + a.filename + ) + }, + ); format!(r##" @@ -1410,7 +1450,7 @@ pub fn render_maven_detail( }; // Extract artifact name from path (last component before version) - let parts: Vec<&str> = path.split('/').collect(); + let parts: Vec<&str> = coordinate_path.split('/').collect(); let artifact_name = if parts.len() >= 2 { parts[parts.len() - 2] } else { @@ -1873,7 +1913,7 @@ pub fn encode_uri_component(s: &str) -> String { #[cfg(test)] mod tests { use super::*; - use crate::ui::api::PackageDetail; + use crate::ui::api::{MavenArtifact, PackageDetail}; fn empty_detail() -> PackageDetail { PackageDetail { @@ -1945,6 +1985,24 @@ mod tests { html.contains("https://registry.example.com/npm"), "npm install command must use public_url" ); + let html = render_package_detail( + "npm", + "repositories/npm-group/@scope/pkg", + &empty_detail(), + Lang::En, + base_url, + false, + ); + assert!( + html.contains( + "npm install @scope/pkg --registry https://registry.example.com/repository/npm-group" + ), + "named npm install command must use the selected repository and real package name" + ); + assert!( + !html.contains("npm install repositories/"), + "storage-index qualification must not leak into the npm package argument" + ); let html = render_package_detail( "go", @@ -2159,4 +2217,27 @@ mod tests { "Docker template must not use inline JS string interpolation" ); } + + #[test] + fn named_maven_detail_uses_repository_route_and_coordinate() { + let detail = MavenDetail { + artifacts: vec![MavenArtifact { + filename: "library-1.2.3.jar".to_string(), + size: 42, + }], + }; + + let html = render_maven_detail( + "repositories/maven-releases/com/example/library/1.2.3", + &detail, + Lang::En, + false, + ); + + assert!( + html.contains("/repository/maven-releases/com/example/library/1.2.3/library-1.2.3.jar") + ); + assert!(html.contains("<groupId>com.example</groupId>")); + assert!(!html.contains("<groupId>repositories.maven-releases")); + } } From 5d069260eeffd3ff2a4af845d81ebb1c2268209a Mon Sep 17 00:00:00 2001 From: ivaseeq <126263471+ivaseeq@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:48:24 +0300 Subject: [PATCH 2/3] fix(npm): materialize hosted packuments --- nora-registry/src/npm_layout.rs | 30 +- nora-registry/src/registry/npm.rs | 442 +++++++++++++++++++++++++++--- nora-registry/src/retention.rs | 53 +++- 3 files changed, 487 insertions(+), 38 deletions(-) diff --git a/nora-registry/src/npm_layout.rs b/nora-registry/src/npm_layout.rs index 32f351ab..be474e2d 100644 --- a/nora-registry/src/npm_layout.rs +++ b/nora-registry/src/npm_layout.rs @@ -14,10 +14,16 @@ use sha2::Digest; #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum NpmObjectKind { HostedPackage, + /// Rebuildable materialized hosted packument. Authoritative hosted state + /// remains in the package/version/tag/deprecation objects. + HostedPackumentCache, HostedVersion(String), HostedPublishComplete(String), HostedTarball(String), - HostedBlob { algorithm: String, digest: String }, + HostedBlob { + algorithm: String, + digest: String, + }, HostedDistTag(String), HostedDeprecation(String), ProxyPackument, @@ -25,6 +31,10 @@ pub(crate) enum NpmObjectKind { ProxyNegative, } +pub(crate) fn hosted_packument_cache_key(repository: &str, package: &str) -> String { + format!("npm/repositories/{repository}/{package}/packument-cache.json") +} + pub(crate) fn hosted_blob_key_for_digest(repository: &str, package: &str, digest: &str) -> String { format!("npm/repositories/{repository}/{package}/blobs/sha512/{digest}.tgz") } @@ -131,6 +141,18 @@ pub(crate) fn parse_npm_object_key(key: &str) -> Option { }); } + if tail.len() >= 2 && tail.last() == Some(&"packument-cache.json") { + let package = tail[..tail.len() - 1].join("/"); + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedPackumentCache, + }); + } + if let Some(marker) = tail.iter().rposition(|part| *part == "blobs") { if marker > 0 && marker + 3 == tail.len() @@ -239,6 +261,12 @@ mod tests { NpmObjectKind::HostedPublishComplete("1.0.0".to_string()) ); + let packument_cache = + parse_npm_object_key("npm/repositories/npm-private/@scope/pkg/packument-cache.json") + .expect("hosted packument cache"); + assert_eq!(packument_cache.package, "@scope/pkg"); + assert_eq!(packument_cache.kind, NpmObjectKind::HostedPackumentCache); + let digest = "a".repeat(128); let blob = parse_npm_object_key(&format!( "npm/repositories/npm-private/@scope/pkg/blobs/sha512/{digest}.tgz" diff --git a/nora-registry/src/registry/npm.rs b/nora-registry/src/registry/npm.rs index 053eb854..98b53653 100644 --- a/nora-registry/src/registry/npm.rs +++ b/nora-registry/src/registry/npm.rs @@ -4,8 +4,10 @@ //! npm registry with explicit hosted, proxy and group repositories. //! //! Hosted and proxy state deliberately live in different namespaces. A group -//! owns no package metadata: its packument is synthesized for every request -//! using member order as the conflict-resolution rule. +//! owns no package metadata: its packument is synthesized using member order +//! as the conflict-resolution rule. Hosted authoritative state stays split by +//! responsibility; a rebuildable materialized packument avoids object-store +//! fan-out on every client metadata request. use crate::activity_log::{ActionType, ActivityEntry}; use crate::audit::AuditEntry; @@ -31,7 +33,7 @@ use axum::{ use base64::Engine; use futures::{stream, StreamExt}; use sha2::Digest; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::io::{Read, Write}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -136,6 +138,18 @@ async fn optional_storage_get(state: &AppState, key: &str) -> Result Result<(), StorageError> { + let key = crate::npm_layout::hosted_packument_cache_key(repository, package); + match state.storage.delete(&key).await { + Ok(()) | Err(StorageError::NotFound) => Ok(()), + Err(error) => Err(error), + } +} + fn named_target(state: &AppState, repository: &str) -> Option { state .config @@ -738,11 +752,62 @@ fn curated_tarball_response( tarball_response(data) } -async fn hosted_packument( +fn valid_cached_hosted_packument(bytes: &[u8], package: &str) -> Option { + let value = serde_json::from_slice::(bytes).ok()?; + let object = value.as_object()?; + (object.get("name").and_then(|value| value.as_str()) == Some(package) + && object + .get("versions") + .is_some_and(|value| value.is_object()) + && object + .get("dist-tags") + .is_some_and(|value| value.is_object())) + .then_some(value) +} + +fn hosted_packument_response( + mut packument: serde_json::Value, + package: &str, + response_base: &str, +) -> serde_json::Value { + if let Some(versions) = packument + .get_mut("versions") + .and_then(|value| value.as_object_mut()) + { + for (version, manifest) in versions { + set_tarball_url(manifest, response_base, package, version); + } + } + packument +} + +async fn read_hosted_packument_cache( + state: &AppState, + repository: &str, + package: &str, +) -> Result, ReadError> { + let key = crate::npm_layout::hosted_packument_cache_key(repository, package); + let Some(bytes) = optional_storage_get(state, &key).await? else { + return Ok(None); + }; + match valid_cached_hosted_packument(&bytes, package) { + Some(packument) => Ok(Some(packument)), + None => { + tracing::warn!( + repository, + package, + key, + "ignoring invalid rebuildable npm hosted packument cache" + ); + Ok(None) + } + } +} + +async fn build_hosted_packument( state: &AppState, repository: &str, package: &str, - response_base: &str, ) -> Result { let prefix = package_prefix(repository, package); let version_prefix = format!("{prefix}/versions/"); @@ -751,6 +816,18 @@ async fn hosted_packument( .list(&version_prefix) .await .map_err(|_| ReadError::Unavailable)?; + let deprecation_prefix = format!("{prefix}/deprecations/"); + let deprecation_keys = state + .storage + .list(&deprecation_prefix) + .await + .map_err(|_| ReadError::Unavailable)? + .into_iter() + .filter_map(|key| { + let version = key.strip_prefix(&deprecation_prefix)?.to_string(); + (!version.is_empty() && !version.contains('/')).then_some((version, key)) + }) + .collect::>(); let version_reads = stream::iter(version_keys.into_iter().filter_map(|key| { let version = key @@ -759,38 +836,40 @@ async fn hosted_packument( .to_string(); (!version.is_empty() && !version.contains('/')).then_some((version, key)) })) - .map(|(version, key)| async move { - let bytes = state.storage.get(&key).await.map_err(|error| { - match storage_read_error(error) { - // LIST advertised this committed manifest. Disappearing between - // LIST and GET is an incomplete hosted read, not permission to - // let a lower-priority group member shadow the version. - ReadError::NotFound => ReadError::Unavailable, - other => other, + .map(|(version, key)| { + let deprecation_key = deprecation_keys.get(&version).cloned(); + async move { + let bytes = state.storage.get(&key).await.map_err(|error| { + match storage_read_error(error) { + // LIST advertised this committed manifest. Disappearing between + // LIST and GET is an incomplete hosted read, not permission to + // let a lower-priority group member shadow the version. + ReadError::NotFound => ReadError::Unavailable, + other => other, + } + })?; + let mut value = serde_json::from_slice::(&bytes).map_err(|_| { + crate::metrics::METADATA_CORRUPT_TOTAL + .with_label_values(&["npm"]) + .inc(); + ReadError::Corrupt + })?; + if let Some(deprecation_key) = deprecation_key { + let deprecated = state + .storage + .get(&deprecation_key) + .await + .map_err(storage_read_error)?; + let Some(message) = read_string(deprecated) else { + return Err(ReadError::Corrupt); + }; + let Some(object) = value.as_object_mut() else { + return Err(ReadError::Corrupt); + }; + object.insert("deprecated".to_string(), serde_json::Value::String(message)); } - })?; - let mut value = serde_json::from_slice::(&bytes).map_err(|_| { - crate::metrics::METADATA_CORRUPT_TOTAL - .with_label_values(&["npm"]) - .inc(); - ReadError::Corrupt - })?; - if let Some(deprecated) = optional_storage_get( - state, - &hosted_deprecation_key(repository, package, &version), - ) - .await? - { - let Some(message) = read_string(deprecated) else { - return Err(ReadError::Corrupt); - }; - let Some(object) = value.as_object_mut() else { - return Err(ReadError::Corrupt); - }; - object.insert("deprecated".to_string(), serde_json::Value::String(message)); + Ok::<_, ReadError>((version, value)) } - set_tarball_url(&mut value, response_base, package, &version); - Ok::<_, ReadError>((version, value)) }) .buffered(HOSTED_PACKUMENT_READ_CONCURRENCY) .collect::>() @@ -858,6 +937,54 @@ async fn hosted_packument( Ok(packument) } +async fn hosted_packument( + state: &AppState, + repository: &str, + package: &str, + response_base: &str, +) -> Result { + if let Some(packument) = read_hosted_packument_cache(state, repository, package).await? { + return Ok(hosted_packument_response(packument, package, response_base)); + } + + // A cold read shares the exact package lock used by hosted mutations. + // This prevents a cache miss from materializing stale state after a + // concurrent mutation invalidated the previous cache. + let lock = state.publish_lock(&format!("npm:{repository}:{package}")); + let _guard = lock.lock().await; + if let Some(packument) = read_hosted_packument_cache(state, repository, package).await? { + return Ok(hosted_packument_response(packument, package, response_base)); + } + + let packument = build_hosted_packument(state, repository, package).await?; + let key = crate::npm_layout::hosted_packument_cache_key(repository, package); + match serde_json::to_vec(&packument) { + Ok(bytes) => { + if let Err(error) = state.storage.put(&key, &bytes).await { + // The cache is derived. A failed cache write must not turn a + // complete authoritative read into an unavailable response. + tracing::warn!( + repository, + package, + key, + ?error, + "failed to persist rebuildable npm hosted packument cache" + ); + } + } + Err(error) => { + tracing::error!( + repository, + package, + ?error, + "failed to serialize npm packument" + ); + return Err(ReadError::Corrupt); + } + } + Ok(hosted_packument_response(packument, package, response_base)) +} + fn negative_fresh(modified: u64, ttl: i64) -> bool { if ttl <= 0 { return false; @@ -2807,6 +2934,12 @@ async fn publish( Ok(_) => {} Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } + if invalidate_hosted_packument_cache(state, repository, package) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } let pending_key = hosted_publish_pending_key(repository, package, &validated.version); let completion_digest = crate::npm_layout::hosted_manifest_digest(&validated.manifest); @@ -3142,6 +3275,12 @@ async fn deprecate( Ok(_) => {} Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } + if invalidate_hosted_packument_cache(state, repository, package) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } let mut changed = 0usize; for (version, data) in versions { match hosted_has_version(state, repository, package, version).await { @@ -3334,6 +3473,12 @@ async fn handle_dist_tag_put( Ok(false) => return StatusCode::NOT_FOUND.into_response(), Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } + if invalidate_hosted_packument_cache(&state, &repository, &package) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } if state .storage .put( @@ -3407,6 +3552,12 @@ async fn handle_dist_tag_delete( Ok(_) => {} Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } + if invalidate_hosted_packument_cache(&state, &repository, &package) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } let key = hosted_tag_key(&repository, &package, &tag); match state.storage.delete(&key).await { Ok(()) | Err(StorageError::NotFound) => {} @@ -4895,6 +5046,227 @@ mod tests { assert_eq!(response.status(), StatusCode::CONFLICT); } + #[tokio::test] + async fn hosted_packument_cache_removes_storage_fanout_and_rewrites_per_route() { + use crate::test_helpers::{create_test_context_with_config, send, FaultInjectBackend}; + use axum::http::Method; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + + let backend = FaultInjectBackend::new(ctx.state.storage.clone()) + .fail_get(hosted_deprecation_key("npm-private", "pkg", "1.0.0")); + let list_attempts = backend.list_attempts(); + let mut state = ctx.state.clone(); + state.storage = crate::storage::Storage::from_backend(Arc::new(backend)); + + let hosted = hosted_packument( + &state, + "npm-private", + "pkg", + "https://nora.example/repository/npm-private", + ) + .await + .expect("cold hosted packument"); + assert_eq!( + hosted["versions"]["1.0.0"]["dist"]["tarball"], + "https://nora.example/repository/npm-private/pkg/-/pkg-1.0.0.tgz" + ); + assert_eq!(list_attempts.lock().len(), 3); + + let cache_key = crate::npm_layout::hosted_packument_cache_key("npm-private", "pkg"); + let cache: serde_json::Value = serde_json::from_slice( + &state + .storage + .get(&cache_key) + .await + .expect("persisted cache"), + ) + .unwrap(); + assert!(cache["versions"]["1.0.0"]["dist"].get("tarball").is_none()); + + let grouped = hosted_packument( + &state, + "npm-private", + "pkg", + "https://nora.example/repository/npm-group", + ) + .await + .expect("warm hosted packument"); + assert_eq!(list_attempts.lock().len(), 3); + assert_eq!( + grouped["versions"]["1.0.0"]["dist"]["tarball"], + "https://nora.example/repository/npm-group/pkg/-/pkg-1.0.0.tgz" + ); + + let second: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "2.0.0", "next")).unwrap(); + assert_eq!( + publish( + &state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(state.storage.stat(&cache_key).await.is_none()); + let rebuilt = hosted_packument( + &state, + "npm-private", + "pkg", + "https://nora.example/repository/npm-private", + ) + .await + .expect("rebuilt hosted packument"); + assert_eq!(rebuilt["versions"].as_object().unwrap().len(), 2); + assert_eq!(rebuilt["dist-tags"]["next"], "2.0.0"); + } + + #[tokio::test] + async fn hosted_mutation_fails_closed_when_packument_cache_cannot_be_invalidated() { + use crate::test_helpers::{create_test_context_with_config, send, FaultInjectBackend}; + use axum::http::Method; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + hosted_packument( + &ctx.state, + "npm-private", + "pkg", + "https://nora.example/repository/npm-private", + ) + .await + .expect("materialize hosted cache"); + + let cache_key = crate::npm_layout::hosted_packument_cache_key("npm-private", "pkg"); + let mut failing_state = ctx.state.clone(); + failing_state.storage = crate::storage::Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_delete(&cache_key), + )); + let payload: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "2.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &failing_state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx + .state + .storage + .stat(&hosted_version_key("npm-private", "pkg", "2.0.0")) + .await + .is_none()); + assert!(ctx.state.storage.stat(&cache_key).await.is_some()); + } + + #[tokio::test] + async fn hosted_overlays_invalidate_and_rebuild_packument_cache() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + let package_uri = "/repository/npm-private/pkg"; + assert_eq!( + send( + &ctx.app, + Method::PUT, + package_uri, + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + let cache_key = crate::npm_layout::hosted_packument_cache_key("npm-private", "pkg"); + assert_eq!( + send(&ctx.app, Method::GET, package_uri, "").await.status(), + StatusCode::OK + ); + assert!(ctx.state.storage.stat(&cache_key).await.is_some()); + + let deprecation = serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": "do not use"}} + }); + assert_eq!( + send( + &ctx.app, + Method::PUT, + package_uri, + serde_json::to_vec(&deprecation).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(ctx.state.storage.stat(&cache_key).await.is_none()); + let response = send(&ctx.app, Method::GET, package_uri, "").await; + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(packument["versions"]["1.0.0"]["deprecated"], "do not use"); + + let tag_uri = "/repository/npm-private/-/package/pkg/dist-tags/next"; + assert_eq!( + send( + &ctx.app, + Method::PUT, + tag_uri, + serde_json::to_vec("1.0.0").unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(ctx.state.storage.stat(&cache_key).await.is_none()); + let response = send(&ctx.app, Method::GET, package_uri, "").await; + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert_eq!(packument["dist-tags"]["next"], "1.0.0"); + + assert_eq!( + send(&ctx.app, Method::DELETE, tag_uri, "").await.status(), + StatusCode::NO_CONTENT + ); + assert!(ctx.state.storage.stat(&cache_key).await.is_none()); + let response = send(&ctx.app, Method::GET, package_uri, "").await; + let packument: serde_json::Value = + serde_json::from_slice(&body_bytes(response).await).unwrap(); + assert!(packument["dist-tags"].get("next").is_none()); + } + #[tokio::test] async fn allow_policy_republishes_same_version_via_new_blob_and_digest_bound_manifest() { use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; diff --git a/nora-registry/src/retention.rs b/nora-registry/src/retention.rs index 1c1a72b9..3624afb3 100644 --- a/nora-registry/src/retention.rs +++ b/nora-registry/src/retention.rs @@ -1307,6 +1307,34 @@ async fn apply_npm_plans( } let mut outcome = NpmBatchOutcome::default(); + // The hosted packument cache is derived and deliberately excluded from + // the authoritative retention snapshot. Remove it before the first + // mutation while holding the exact publish lock; a later read will rebuild + // from whatever authoritative state the batch commits. + let cache_key = + crate::npm_layout::hosted_packument_cache_key(&group.repository, &group.package); + let cache_size = storage + .stat(&cache_key) + .await + .map(|meta| meta.size) + .unwrap_or(0); + match storage.delete(&cache_key).await { + Ok(()) => { + outcome.deleted_keys += 1; + outcome.bytes_freed += cache_size; + } + Err(StorageError::NotFound) => {} + Err(error) => { + tracing::warn!( + repository = group.repository, + package = group.package, + key = cache_key, + %error, + "retention: cannot invalidate npm hosted packument cache; batch skipped" + ); + return outcome; + } + } for plan in plans { let deleted = delete_npm_plan(storage, &group.group_name, plan).await; outcome.deleted_keys += deleted.deleted_keys; @@ -2936,6 +2964,13 @@ mod tests { ) .await .unwrap(); + storage + .put( + &crate::npm_layout::hosted_packument_cache_key(repository, "pkg"), + br#"{"name":"pkg","versions":{},"dist-tags":{}}"#, + ) + .await + .unwrap(); } let proxy_key = "npm/repositories/npm-registry/proxy/tarballs/pkg/pkg-1.0.0.tgz"; storage.put(proxy_key, b"cache").await.unwrap(); @@ -2974,6 +3009,12 @@ mod tests { )) .await .is_some()); + assert!(storage + .stat(&crate::npm_layout::hosted_packument_cache_key( + repository, "pkg" + )) + .await + .is_none()); } assert!(storage.get(proxy_key).await.is_ok()); } @@ -3262,8 +3303,16 @@ mod tests { assert!(inner.get(&manifest).await.is_ok()); assert!(inner.get(&blob).await.is_ok()); let attempts = attempts.lock(); - assert_eq!(attempts.first(), Some(&completion)); - assert_eq!(attempts.get(1), Some(&tag)); + assert_eq!( + attempts.first(), + Some(&crate::npm_layout::hosted_packument_cache_key( + "npm-private", + "pkg" + )), + "derived cache must be invalidated before authoritative state changes" + ); + assert_eq!(attempts.get(1), Some(&completion)); + assert_eq!(attempts.get(2), Some(&tag)); assert!(!attempts.contains(&manifest)); assert!(!attempts.contains(&blob)); } From 45a35a7547844a16f4377cca979f43711b5dabdd Mon Sep 17 00:00:00 2001 From: ivaseeq <126263471+ivaseeq@users.noreply.github.com> Date: Mon, 3 Aug 2026 02:45:48 +0300 Subject: [PATCH 3/3] fix(registry): make hosted metadata transactions crash-safe --- nora-registry/src/gc.rs | 1728 +++- nora-registry/src/main.rs | 6 +- nora-registry/src/npm_layout.rs | 456 +- nora-registry/src/registry/maven.rs | 38 +- nora-registry/src/registry/mod.rs | 6 + nora-registry/src/registry/named.rs | 142 +- nora-registry/src/registry/npm.rs | 7885 +++++++++++++---- .../registry/ns_isolation_metadata_tests.rs | 19 +- nora-registry/src/retention.rs | 2062 +++-- nora-registry/src/test_helpers.rs | 144 +- 10 files changed, 10047 insertions(+), 2439 deletions(-) diff --git a/nora-registry/src/gc.rs b/nora-registry/src/gc.rs index 33bb4ce1..5cd77c72 100644 --- a/nora-registry/src/gc.rs +++ b/nora-registry/src/gc.rs @@ -20,6 +20,7 @@ use std::time::Instant; use prometheus::{ register_histogram, register_int_counter, register_int_gauge, Histogram, IntCounter, IntGauge, }; +use sha2::Digest as _; use tracing::{info, warn}; use crate::storage::{Storage, StorageError}; @@ -71,7 +72,7 @@ pub static GC_METADATA_PHANTOMS: LazyLock = LazyLock::new(|| { pub static GC_STAT_FAILURES: LazyLock = LazyLock::new(|| { register_int_counter!( "nora_gc_stat_failures_total", - "Orphans GC could not stat (kept, age unknown) — nonzero means GC may be unable to reclaim space; alert on it" + "Orphans GC could not safely stat or revalidate (kept) — nonzero means GC may be unable to reclaim space; alert on it" ) .expect("gc_stat_failures metric") }); @@ -94,10 +95,10 @@ pub struct GcResult { /// Orphans skipped because they were younger than the grace period — /// protected from the write-vs-GC race (#584). Benign: collected next pass. pub skipped_recent: usize, - /// Orphans kept because their age could not be determined (stat failed). - /// Nonzero is a warning sign: GC may be unable to make progress (disk grows - /// silently). Tracked separately from `skipped_recent` and metered via - /// `nora_gc_stat_failures_total` so it can be alerted on. + /// Orphans kept because age/reachability could not be safely determined + /// (stat/read validation failed). Nonzero is a warning sign: GC may be + /// unable to make progress. Tracked separately from `skipped_recent` and + /// metered via `nora_gc_stat_failures_total` so it can be alerted on. pub stat_failures: usize, } @@ -148,6 +149,52 @@ pub async fn run_gc( all_orphans.extend(npm_result.orphans); let npm_read_failures = npm_result.read_failures; + // A durable npm maintenance marker owns the complete package transition. + // Do not GC any object in that package, including generic checksum + // candidates detected above. A corrupt/unreadable marker is equally + // blocking: inability to understand recovery state is never permission to + // delete around it. + let mut maintenance_states = HashMap::new(); + let mut maintenance_failures = HashSet::new(); + let mut maintenance_skips = HashSet::new(); + let mut filtered_orphans = Vec::with_capacity(all_orphans.len()); + for key in all_orphans { + let Some((repository, package)) = npm_hosted_package_identity(&key) else { + filtered_orphans.push(key); + continue; + }; + let identity = (repository, package); + let state = if let Some(state) = maintenance_states.get(&identity) { + *state + } else { + let state = npm_maintenance_state(storage, &identity.0, &identity.1).await; + maintenance_states.insert(identity.clone(), state); + state + }; + match state { + NpmMaintenanceState::Inactive => filtered_orphans.push(key), + NpmMaintenanceState::Active => { + if maintenance_skips.insert(identity.clone()) { + info!( + repository = identity.0, + package = identity.1, + "GC: active npm maintenance; whole package skipped" + ); + } + } + NpmMaintenanceState::Unreadable => { + if maintenance_failures.insert(identity.clone()) { + warn!( + repository = identity.0, + package = identity.1, + "GC: npm maintenance marker unreadable; whole package skipped" + ); + } + } + } + } + all_orphans = filtered_orphans; + // Go incomplete version detection let go_result = detect_go_incomplete_versions(storage).await; total_candidates += go_result.total; @@ -166,7 +213,8 @@ pub async fn run_gc( // Sort orphans: delete blobs before manifests so that if GC is interrupted // mid-run, we only leave harmless orphan blobs — never broken manifests - // pointing to already-deleted blobs (#305). + // pointing to already-deleted blobs (#305). The permanent npm retirement + // tombstone is never an orphan candidate. all_orphans.sort_by(|a, b| { let a_is_manifest = a.contains("/manifests/"); let b_is_manifest = b.contains("/manifests/"); @@ -178,6 +226,7 @@ pub async fn run_gc( let mut skipped_recent = 0usize; let mut stat_failures = npm_read_failures; let now = now_unix_secs(); + let mut npm_current_validation_cache = HashMap::new(); for key in &all_orphans { // Grace period (#584): never reap an orphan whose backing file is @@ -204,6 +253,31 @@ pub async fn run_gc( } if dry_run { + // Keep the preview faithful to apply for npm read-model objects: + // the pointer may have switched since the initial LIST snapshot. + if npm_read_model_identity(key).is_some() { + match recheck_npm_read_model_candidate( + storage, + publish_locks, + key, + now, + grace_secs, + &mut npm_current_validation_cache, + ) + .await + { + NpmReadModelRecheckOutcome::Obsolete => {} + NpmReadModelRecheckOutcome::GraceProtected => { + skipped_recent += 1; + continue; + } + NpmReadModelRecheckOutcome::Kept => continue, + NpmReadModelRecheckOutcome::ReadFailure => { + stat_failures += 1; + continue; + } + } + } bytes_freed += meta.size; info!("[dry-run] Would delete: {} ({} bytes)", key, meta.size); continue; @@ -212,8 +286,21 @@ pub async fn run_gc( // npm staged blobs use the package lock plus a commit-manifest // readback; other formats retain the exact-key lock. let removed = if npm_hosted_orphan_candidate(key) { - match delete_npm_orphan_if_uncommitted(storage, publish_locks, key).await { + match delete_npm_orphan_if_uncommitted( + storage, + publish_locks, + key, + now, + grace_secs, + &mut npm_current_validation_cache, + ) + .await + { NpmOrphanDeleteOutcome::Removed => true, + NpmOrphanDeleteOutcome::GraceProtected => { + skipped_recent += 1; + false + } NpmOrphanDeleteOutcome::Kept => false, NpmOrphanDeleteOutcome::ReadFailure => { stat_failures += 1; @@ -240,7 +327,7 @@ pub async fn run_gc( } if stat_failures > 0 { warn!( - "GC could not stat {} orphan(s); kept them (age unknown). GC may be unable to reclaim space", + "GC could not safely inspect {} orphan(s); kept them. GC may be unable to reclaim space", stat_failures ); GC_STAT_FAILURES.inc_by(stat_failures as u64); @@ -327,6 +414,192 @@ struct DetectionResult { read_failures: usize, } +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct GcHostedPackumentPointer { + generation: String, + full_sha256: String, + install_v1_sha256: String, +} + +struct GcCurrentPackument { + generation: String, + install_v1_sha256: String, + root_modified: u64, + referenced_blobs: HashSet, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct GcCurrentValidationKey { + repository: String, + package: String, + generation: String, + install_v1_sha256: String, + root_modified: u64, +} + +struct GcRetiredPackument { + root_modified: u64, +} + +enum GcPackumentRoot { + Current(GcCurrentPackument), + Retired(GcRetiredPackument), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum NpmReadModelObject { + LegacyCache, + Retired, + Generation(String), +} + +fn valid_lower_sha256(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) +} + +/// Read and validate the hosted read-model commit point. +/// +/// This validates the small pointer and both immutable documents through exact +/// GETs. Blob reachability is derived only from the exact full document; LIST +/// is never an authority for a destructive decision. +async fn npm_current_packument_generation( + storage: &Storage, + repository: &str, + package: &str, +) -> Result { + let key = crate::npm_layout::hosted_packument_current_key(repository, package); + let bytes = storage.get(&key).await?; + let pointer = serde_json::from_slice::(&bytes) + .map_err(|_| StorageError::IntegrityViolation)?; + if !valid_lower_sha256(&pointer.generation) + || !valid_lower_sha256(&pointer.full_sha256) + || !valid_lower_sha256(&pointer.install_v1_sha256) + || pointer.generation != pointer.full_sha256 + { + return Err(StorageError::IntegrityViolation); + } + let pointer_modified = storage + .stat(&key) + .await + .ok_or(StorageError::IntegrityViolation)? + .modified; + let full_key = + crate::npm_layout::hosted_packument_full_key(repository, package, &pointer.generation); + let install_v1_key = crate::npm_layout::hosted_packument_install_v1_key( + repository, + package, + &pointer.generation, + ); + let (full, install_v1) = tokio::join!(storage.get(&full_key), storage.get(&install_v1_key)); + let (full, install_v1) = (full?, install_v1?); + if hex::encode(sha2::Sha256::digest(&full)) != pointer.full_sha256 + || hex::encode(sha2::Sha256::digest(&install_v1)) != pointer.install_v1_sha256 + { + return Err(StorageError::IntegrityViolation); + } + let packument: serde_json::Value = + serde_json::from_slice(&full).map_err(|_| StorageError::IntegrityViolation)?; + if packument.get("name").and_then(serde_json::Value::as_str) != Some(package) + || !packument + .get("dist-tags") + .is_some_and(serde_json::Value::is_object) + { + return Err(StorageError::IntegrityViolation); + } + let versions = packument + .get("versions") + .and_then(serde_json::Value::as_object) + .ok_or(StorageError::IntegrityViolation)?; + let mut referenced_blobs = HashSet::with_capacity(versions.len()); + for manifest in versions.values() { + let manifest = + serde_json::to_vec(manifest).map_err(|_| StorageError::IntegrityViolation)?; + let blob = crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) + .ok_or(StorageError::IntegrityViolation)?; + referenced_blobs.insert(blob); + } + match storage.get(&key).await { + Ok(after) if after == bytes => {} + Ok(_) | Err(StorageError::NotFound) => return Err(StorageError::AlreadyExists), + Err(error) => return Err(error), + } + Ok(GcCurrentPackument { + generation: pointer.generation, + install_v1_sha256: pointer.install_v1_sha256, + root_modified: pointer_modified, + referenced_blobs, + }) +} + +async fn npm_key_is_absent(storage: &Storage, key: &str) -> Result { + match storage.get_reader(key).await { + Ok((_size, reader)) => { + drop(reader); + Ok(false) + } + Err(StorageError::NotFound) => Ok(true), + Err(error) => Err(error), + } +} + +async fn npm_retired_packument_root( + storage: &Storage, + repository: &str, + package: &str, +) -> Result, StorageError> { + let marker_key = crate::npm_layout::hosted_packument_retired_key(repository, package); + let marker = storage.get(&marker_key).await?; + if marker.as_ref() != crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 { + return Err(StorageError::IntegrityViolation); + } + let root_modified = storage + .stat(&marker_key) + .await + .ok_or(StorageError::IntegrityViolation)? + .modified; + + // The fixed package root must be absent through an exact probe. Active + // import/publish/maintenance journals are checked by + // `npm_maintenance_state` before this root is used. + let package_key = crate::npm_layout::hosted_package_key(repository, package); + if !npm_key_is_absent(storage, &package_key).await? { + return Ok(None); + } + Ok(Some(GcRetiredPackument { root_modified })) +} + +async fn npm_packument_root( + storage: &Storage, + repository: &str, + package: &str, +) -> Result, StorageError> { + match npm_current_packument_generation(storage, repository, package).await { + Ok(current) => Ok(Some(GcPackumentRoot::Current(current))), + Err(StorageError::NotFound) => npm_retired_packument_root(storage, repository, package) + .await + .map(|retired| retired.map(GcPackumentRoot::Retired)), + Err(error) => Err(error), + } +} + +fn npm_read_model_identity(key: &str) -> Option<(String, String, NpmReadModelObject)> { + let parsed = crate::npm_layout::parse_npm_object_key(key)?; + let object = match parsed.kind { + crate::npm_layout::NpmObjectKind::HostedPackumentCache => NpmReadModelObject::LegacyCache, + crate::npm_layout::NpmObjectKind::HostedPackumentRetired => NpmReadModelObject::Retired, + crate::npm_layout::NpmObjectKind::HostedPackumentFull(generation) + | crate::npm_layout::NpmObjectKind::HostedPackumentInstallV1(generation) => { + NpmReadModelObject::Generation(generation) + } + _ => return None, + }; + Some((parsed.repository, parsed.package, object)) +} + /// Parse a named hosted npm tarball key. /// /// Returns `(repository, package, manifest_key)`. Proxy-cache tarballs live @@ -349,26 +622,212 @@ fn npm_manifest_for_tarball(key: &str) -> Option { npm_tarball_identity(key).map(|(_, _, manifest)| manifest) } -fn npm_package_lock_for_key(key: &str) -> Option { - if let Some((repository, package, _)) = npm_tarball_identity(key) { - return Some(format!("npm:{repository}:{package}")); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum NpmMaintenanceState { + Inactive, + Active, + Unreadable, +} + +async fn npm_maintenance_state( + storage: &Storage, + repository: &str, + package: &str, +) -> NpmMaintenanceState { + match crate::registry::read_hosted_maintenance_marker(storage, repository, package).await { + Ok(Some(_)) => return NpmMaintenanceState::Active, + Ok(None) => {} + Err(_) => return NpmMaintenanceState::Unreadable, + } + match crate::registry::read_hosted_active_transactions(storage, repository, package).await { + Ok(transactions) if transactions.import.is_some() || transactions.publish.is_some() => { + NpmMaintenanceState::Active + } + Ok(_) => NpmMaintenanceState::Inactive, + Err(_) => NpmMaintenanceState::Unreadable, } - let parsed = crate::npm_layout::parse_npm_object_key(key)?; - matches!( +} + +fn npm_hosted_package_identity(key: &str) -> Option<(String, String)> { + let primary = primary_key_for_sidecar(key).unwrap_or(key); + let parsed = crate::npm_layout::parse_npm_object_key(primary)?; + if matches!( parsed.kind, - crate::npm_layout::NpmObjectKind::HostedBlob { .. } - ) - .then(|| format!("npm:{}:{}", parsed.repository, parsed.package)) + crate::npm_layout::NpmObjectKind::ProxyPackument + | crate::npm_layout::NpmObjectKind::ProxyTarball(_) + | crate::npm_layout::NpmObjectKind::ProxyNegative + ) { + return None; + } + Some((parsed.repository, parsed.package)) +} + +fn npm_package_lock_for_key(key: &str) -> Option { + npm_hosted_package_identity(key) + .map(|(repository, package)| format!("npm:{repository}:{package}")) } fn npm_hosted_orphan_candidate(key: &str) -> bool { npm_manifest_for_tarball(key).is_some() + || npm_read_model_identity(key).is_some() || crate::npm_layout::parse_npm_object_key(key).is_some_and(|parsed| { matches!( parsed.kind, crate::npm_layout::NpmObjectKind::HostedBlob { .. } ) }) + || (is_orphanable_sidecar(key) && npm_hosted_package_identity(key).is_some()) +} + +#[derive(Debug, PartialEq, Eq)] +enum NpmReadModelRecheckOutcome { + Obsolete, + GraceProtected, + Kept, + ReadFailure, +} + +fn npm_root_is_grace_protected(root_modified: u64, now: u64, grace_secs: u64) -> bool { + grace_secs > 0 && now.saturating_sub(root_modified) < grace_secs +} + +async fn npm_current_documents_match_pointer( + storage: &Storage, + repository: &str, + package: &str, + current: &GcCurrentPackument, + cache: &mut HashMap, +) -> bool { + let cache_key = GcCurrentValidationKey { + repository: repository.to_string(), + package: package.to_string(), + generation: current.generation.clone(), + install_v1_sha256: current.install_v1_sha256.clone(), + root_modified: current.root_modified, + }; + if let Some(valid) = cache.get(&cache_key) { + return *valid; + } + let full_key = + crate::npm_layout::hosted_packument_full_key(repository, package, ¤t.generation); + let install_v1_key = crate::npm_layout::hosted_packument_install_v1_key( + repository, + package, + ¤t.generation, + ); + let (full, install_v1) = tokio::join!(storage.get(&full_key), storage.get(&install_v1_key)); + let valid = match (full, install_v1) { + (Ok(full), Ok(install_v1)) => { + hex::encode(sha2::Sha256::digest(&full)) == current.generation + && hex::encode(sha2::Sha256::digest(&install_v1)) == current.install_v1_sha256 + } + _ => false, + }; + cache.insert(cache_key, valid); + if !valid { + warn!( + repository, + package, + generation = current.generation, + "GC: current npm packument documents are unreadable or fail pointer hashes; obsolete read model kept" + ); + } + valid +} + +async fn recheck_npm_read_model_candidate_locked( + storage: &Storage, + key: &str, + now: u64, + grace_secs: u64, + current_validation_cache: &mut HashMap, +) -> NpmReadModelRecheckOutcome { + let Some((repository, package, candidate)) = npm_read_model_identity(key) else { + return NpmReadModelRecheckOutcome::Kept; + }; + let root = match npm_packument_root(storage, &repository, &package).await { + Ok(Some(root)) => root, + Ok(None) => return NpmReadModelRecheckOutcome::Kept, + Err(error) => { + warn!( + candidate = key, + error = %error, + "GC: cannot validate npm packument reachability root; read-model object kept" + ); + return NpmReadModelRecheckOutcome::ReadFailure; + } + }; + match (root, candidate) { + (GcPackumentRoot::Current(current), NpmReadModelObject::Generation(generation)) + if generation == current.generation => + { + NpmReadModelRecheckOutcome::Kept + } + // Retirement is a permanent exact-key tombstone. A future writer owns + // removing it as part of committing a new live pointer; GC never turns + // an empty LIST into permission to erase this authority root. + (GcPackumentRoot::Retired(_), NpmReadModelObject::Retired) + | (GcPackumentRoot::Current(_), NpmReadModelObject::Retired) => { + NpmReadModelRecheckOutcome::Kept + } + (GcPackumentRoot::Current(current), candidate) => { + if npm_root_is_grace_protected(current.root_modified, now, grace_secs) { + NpmReadModelRecheckOutcome::GraceProtected + } else if matches!( + candidate, + NpmReadModelObject::Generation(_) | NpmReadModelObject::LegacyCache + ) && !npm_current_documents_match_pointer( + storage, + &repository, + &package, + ¤t, + current_validation_cache, + ) + .await + { + NpmReadModelRecheckOutcome::ReadFailure + } else { + NpmReadModelRecheckOutcome::Obsolete + } + } + (GcPackumentRoot::Retired(retired), _) => { + if npm_root_is_grace_protected(retired.root_modified, now, grace_secs) { + NpmReadModelRecheckOutcome::GraceProtected + } else { + NpmReadModelRecheckOutcome::Obsolete + } + } + } +} + +/// Recheck read-model reachability under the exact package publish lock. +/// +/// Detection happens from an earlier LIST snapshot. A mutation may switch the +/// pointer while GC waits for this lock, so only this readback is authoritative +/// for a destructive decision. Legacy caches are collectible only after a +/// complete current pointer or a quiescent durable retirement root exists; +/// this also keeps mixed/failed rollouts fail-closed. +async fn recheck_npm_read_model_candidate( + storage: &Storage, + publish_locks: &PublishLocks, + key: &str, + now: u64, + grace_secs: u64, + current_validation_cache: &mut HashMap, +) -> NpmReadModelRecheckOutcome { + let Some((repository, package, _)) = npm_read_model_identity(key) else { + return NpmReadModelRecheckOutcome::Kept; + }; + let lock_key = format!("npm:{repository}:{package}"); + let lock = crate::acquire_publish_lock(publish_locks, &lock_key); + let _guard = lock.lock().await; + match npm_maintenance_state(storage, &repository, &package).await { + NpmMaintenanceState::Inactive => {} + NpmMaintenanceState::Active => return NpmReadModelRecheckOutcome::Kept, + NpmMaintenanceState::Unreadable => return NpmReadModelRecheckOutcome::ReadFailure, + } + recheck_npm_read_model_candidate_locked(storage, key, now, grace_secs, current_validation_cache) + .await } async fn npm_blob_is_referenced(storage: &Storage, key: &str) -> Result { @@ -381,38 +840,58 @@ async fn npm_blob_is_referenced(storage: &Storage, key: &str) -> Result Ok(current.referenced_blobs.contains(key)), + Some(GcPackumentRoot::Retired(_)) => Ok(false), + // Without a valid exact pointer or permanent retirement tombstone + // there is no authoritative negative reachability proof. + None => Err(StorageError::IntegrityViolation), } - Ok(false) } async fn delete_npm_orphan_if_uncommitted( storage: &Storage, publish_locks: &PublishLocks, key: &str, + now: u64, + grace_secs: u64, + current_validation_cache: &mut HashMap, ) -> NpmOrphanDeleteOutcome { let Some(lock_key) = npm_package_lock_for_key(key) else { return NpmOrphanDeleteOutcome::Kept; }; let lock = crate::acquire_publish_lock(publish_locks, &lock_key); let _guard = lock.lock().await; + let Some((repository, package)) = npm_hosted_package_identity(key) else { + return NpmOrphanDeleteOutcome::Kept; + }; + match npm_maintenance_state(storage, &repository, &package).await { + NpmMaintenanceState::Inactive => {} + NpmMaintenanceState::Active => return NpmOrphanDeleteOutcome::Kept, + NpmMaintenanceState::Unreadable => return NpmOrphanDeleteOutcome::ReadFailure, + } + let parsed_kind = crate::npm_layout::parse_npm_object_key(key).map(|parsed| parsed.kind); + if npm_read_model_identity(key).is_some() { + return match recheck_npm_read_model_candidate_locked( + storage, + key, + now, + grace_secs, + current_validation_cache, + ) + .await + { + NpmReadModelRecheckOutcome::Obsolete if storage.delete(key).await.is_ok() => { + NpmOrphanDeleteOutcome::Removed + } + NpmReadModelRecheckOutcome::Obsolete => NpmOrphanDeleteOutcome::Kept, + NpmReadModelRecheckOutcome::GraceProtected => NpmOrphanDeleteOutcome::GraceProtected, + NpmReadModelRecheckOutcome::Kept => NpmOrphanDeleteOutcome::Kept, + NpmReadModelRecheckOutcome::ReadFailure => NpmOrphanDeleteOutcome::ReadFailure, + }; + } if matches!( - crate::npm_layout::parse_npm_object_key(key).map(|parsed| parsed.kind), + parsed_kind, Some(crate::npm_layout::NpmObjectKind::HostedBlob { .. }) ) { return match npm_blob_is_referenced(storage, key).await { @@ -429,6 +908,24 @@ async fn delete_npm_orphan_if_uncommitted( } }; } + if let Some(primary) = primary_key_for_sidecar(key) { + return match storage.get(primary).await { + Ok(_) => NpmOrphanDeleteOutcome::Kept, + Err(StorageError::NotFound) if storage.delete(key).await.is_ok() => { + NpmOrphanDeleteOutcome::Removed + } + Err(StorageError::NotFound) => NpmOrphanDeleteOutcome::Kept, + Err(error) => { + warn!( + sidecar = key, + primary, + error = %error, + "GC: cannot revalidate npm sidecar primary; sidecar kept" + ); + NpmOrphanDeleteOutcome::ReadFailure + } + }; + } let Some(manifest_key) = npm_manifest_for_tarball(key) else { return NpmOrphanDeleteOutcome::Kept; }; @@ -456,6 +953,7 @@ async fn delete_npm_orphan_if_uncommitted( #[derive(Debug, PartialEq, Eq)] enum NpmOrphanDeleteOutcome { Removed, + GraceProtected, Kept, ReadFailure, } @@ -470,62 +968,110 @@ async fn detect_npm_hosted_orphans(storage: &Storage) -> DetectionResult { }); let mut legacy_tarballs = Vec::new(); let mut blobs = Vec::new(); - let mut manifests = Vec::new(); + let mut legacy_packument_caches = HashMap::<(String, String), Vec>::new(); + let mut retired_packuments = HashMap::<(String, String), Vec>::new(); + let mut packument_generations = HashMap::<(String, String), Vec<(String, String)>>::new(); + let mut discovered_packages = HashSet::new(); for key in keys { let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { continue; }; + discovered_packages.insert((parsed.repository.clone(), parsed.package.clone())); match parsed.kind { crate::npm_layout::NpmObjectKind::HostedBlob { .. } => { blobs.push((key, parsed.repository, parsed.package)); } - crate::npm_layout::NpmObjectKind::HostedVersion(_) => { - manifests.push((key, parsed.repository, parsed.package)); - } + crate::npm_layout::NpmObjectKind::HostedVersion(_) => {} crate::npm_layout::NpmObjectKind::HostedTarball(_) => { if let Some(manifest) = npm_manifest_for_tarball(&key) { legacy_tarballs.push((key, manifest)); } } + crate::npm_layout::NpmObjectKind::HostedPackumentCache => { + legacy_packument_caches + .entry((parsed.repository, parsed.package)) + .or_default() + .push(key); + } + crate::npm_layout::NpmObjectKind::HostedPackumentRetired => { + retired_packuments + .entry((parsed.repository, parsed.package)) + .or_default() + .push(key); + } + crate::npm_layout::NpmObjectKind::HostedPackumentFull(generation) + | crate::npm_layout::NpmObjectKind::HostedPackumentInstallV1(generation) => { + packument_generations + .entry((parsed.repository, parsed.package)) + .or_default() + .push((key, generation)); + } + crate::npm_layout::NpmObjectKind::HostedMaintenanceActive => {} _ => {} } } - let total = blobs.len() + legacy_tarballs.len(); + let total = blobs.len() + + legacy_tarballs.len() + + legacy_packument_caches + .values() + .map(Vec::len) + .sum::() + + retired_packuments.values().map(Vec::len).sum::() + + packument_generations.values().map(Vec::len).sum::(); let mut orphans = Vec::new(); let mut read_failures = 0usize; - let mut reachable = HashSet::new(); + let mut maintenance_packages = HashSet::new(); + for (repository, package) in discovered_packages { + match npm_maintenance_state(storage, &repository, &package).await { + NpmMaintenanceState::Inactive => {} + NpmMaintenanceState::Active => { + maintenance_packages.insert((repository, package)); + } + NpmMaintenanceState::Unreadable => { + read_failures += 1; + warn!( + repository, + package, "GC: npm transaction journal is unreadable; whole package kept" + ); + maintenance_packages.insert((repository, package)); + } + } + } let mut uncertain_packages = HashSet::new(); - for (manifest_key, repository, package) in manifests { - match storage.get(&manifest_key).await { - Ok(manifest) => { - if let Some(blob_key) = crate::npm_layout::hosted_blob_key_from_manifest( - &repository, - &package, - &manifest, - ) { - reachable.insert(blob_key); - } else { - uncertain_packages.insert((repository, package)); - read_failures += 1; + for (blob, repository, package) in blobs { + let identity = (repository.clone(), package.clone()); + if maintenance_packages.contains(&identity) || uncertain_packages.contains(&identity) { + continue; + } + match npm_packument_root(storage, &repository, &package).await { + Ok(Some(GcPackumentRoot::Current(current))) => { + if !current.referenced_blobs.contains(&blob) { + orphans.push(blob); } } + Ok(Some(GcPackumentRoot::Retired(_))) => orphans.push(blob), + Ok(None) => { + uncertain_packages.insert(identity); + read_failures += 1; + } Err(error) => { warn!( - manifest = manifest_key, + repository, + package, error = %error, - "GC: cannot inspect npm manifest blob reference; package blobs kept" + "GC: cannot validate exact npm blob reachability root; package blobs kept" ); - uncertain_packages.insert((repository, package)); + uncertain_packages.insert(identity); read_failures += 1; } } } - for (blob, repository, package) in blobs { - if !reachable.contains(&blob) && !uncertain_packages.contains(&(repository, package)) { - orphans.push(blob); - } - } for (tarball, manifest) in legacy_tarballs { + if npm_hosted_package_identity(&tarball) + .is_some_and(|identity| maintenance_packages.contains(&identity)) + { + continue; + } match storage.get(&manifest).await { Ok(_) => {} Err(StorageError::NotFound) => orphans.push(tarball), @@ -539,6 +1085,62 @@ async fn detect_npm_hosted_orphans(storage: &Storage) -> DetectionResult { } } } + + // A live package's current pointer is the reachability root for both + // immutable packument documents. A fully deleted package uses the durable + // retirement marker as the root/grace clock until the old read model has + // drained. Any corrupt/unreadable root fails closed per package. + let mut read_model_packages = HashSet::new(); + read_model_packages.extend(packument_generations.keys().cloned()); + read_model_packages.extend(legacy_packument_caches.keys().cloned()); + read_model_packages.extend(retired_packuments.keys().cloned()); + for (repository, package) in read_model_packages { + let identity = (repository.clone(), package.clone()); + if maintenance_packages.contains(&identity) || uncertain_packages.contains(&identity) { + continue; + } + match npm_packument_root(storage, &repository, &package).await { + Ok(Some(GcPackumentRoot::Current(current))) => { + if let Some(keys) = packument_generations.get(&identity) { + orphans.extend( + keys.iter() + .filter(|(_, generation)| generation != ¤t.generation) + .map(|(key, _)| key.clone()), + ); + } + if let Some(keys) = legacy_packument_caches.get(&identity) { + orphans.extend(keys.iter().cloned()); + } + } + Ok(Some(GcPackumentRoot::Retired(_))) => { + let generations = packument_generations.get(&identity); + let caches = legacy_packument_caches.get(&identity); + if let Some(keys) = generations { + orphans.extend(keys.iter().map(|(key, _)| key.clone())); + } + if let Some(keys) = caches { + orphans.extend(keys.iter().cloned()); + } + // The exact retirement tombstone is permanent. A future live + // pointer commit, not GC, owns removing it. + } + Ok(None) => { + info!( + repository, + package, "GC: npm package retirement is not quiescent; read-model objects kept" + ); + } + Err(error) => { + warn!( + repository, + package, + error = %error, + "GC: cannot validate npm packument reachability root; all read-model objects kept" + ); + read_failures += 1; + } + } + } DetectionResult { total, orphans, @@ -1062,33 +1664,364 @@ pub fn spawn_gc_scheduler( mod tests { use super::*; use base64::Engine as _; - use sha2::Digest as _; fn test_publish_locks() -> PublishLocks { Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())) } - fn npm_blob_fixture( - repository: &str, - package: &str, - version: &str, - blob: &[u8], - ) -> (String, Vec) { - let integrity = format!( - "sha512-{}", - base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(blob)) - ); - let manifest = serde_json::to_vec(&serde_json::json!({ - "name": package, - "version": version, - "dist": {"integrity": integrity} - })) - .unwrap(); - ( - crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) - .unwrap(), - manifest, - ) + fn npm_blob_fixture( + repository: &str, + package: &str, + version: &str, + blob: &[u8], + ) -> (String, Vec) { + let integrity = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(blob)) + ); + let manifest = serde_json::to_vec(&serde_json::json!({ + "name": package, + "version": version, + "dist": {"integrity": integrity} + })) + .unwrap(); + ( + crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) + .unwrap(), + manifest, + ) + } + + struct NpmPackumentFixture { + full_key: String, + install_v1_key: String, + full: Vec, + install_v1: Vec, + pointer: Vec, + } + + fn npm_packument_fixture( + repository: &str, + package: &str, + version: &str, + ) -> NpmPackumentFixture { + let integrity = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest( + format!("{package}:{version}").as_bytes() + )) + ); + let full = serde_json::to_vec(&serde_json::json!({ + "name": package, + "versions": { + (version): { + "name": package, + "version": version, + "description": "full-only", + "dist": {"integrity": integrity} + } + }, + "dist-tags": {"latest": version} + })) + .unwrap(); + let install_v1 = serde_json::to_vec(&serde_json::json!({ + "name": package, + "versions": { + (version): { + "name": package, + "version": version, + "dist": {"integrity": integrity} + } + }, + "dist-tags": {"latest": version} + })) + .unwrap(); + let generation = hex::encode(sha2::Sha256::digest(&full)); + let install_v1_sha256 = hex::encode(sha2::Sha256::digest(&install_v1)); + let pointer = serde_json::to_vec(&serde_json::json!({ + "generation": generation, + "full_sha256": generation, + "install_v1_sha256": install_v1_sha256, + })) + .unwrap(); + NpmPackumentFixture { + full_key: crate::npm_layout::hosted_packument_full_key( + repository, + package, + &generation, + ), + install_v1_key: crate::npm_layout::hosted_packument_install_v1_key( + repository, + package, + &generation, + ), + full, + install_v1, + pointer, + } + } + + fn npm_packument_fixture_with_manifest( + repository: &str, + package: &str, + version: &str, + manifest: &[u8], + ) -> NpmPackumentFixture { + let manifest: serde_json::Value = serde_json::from_slice(manifest).unwrap(); + let full = serde_json::to_vec(&serde_json::json!({ + "name": package, + "versions": {(version): manifest}, + "dist-tags": {} + })) + .unwrap(); + let install_v1 = full.clone(); + let generation = hex::encode(sha2::Sha256::digest(&full)); + let install_v1_sha256 = hex::encode(sha2::Sha256::digest(&install_v1)); + let pointer = serde_json::to_vec(&serde_json::json!({ + "generation": generation, + "full_sha256": generation, + "install_v1_sha256": install_v1_sha256, + })) + .unwrap(); + NpmPackumentFixture { + full_key: crate::npm_layout::hosted_packument_full_key( + repository, + package, + &generation, + ), + install_v1_key: crate::npm_layout::hosted_packument_install_v1_key( + repository, + package, + &generation, + ), + full, + install_v1, + pointer, + } + } + + async fn put_npm_retired(storage: &Storage, repository: &str, package: &str) -> String { + let key = crate::npm_layout::hosted_packument_retired_key(repository, package); + storage + .put(&key, crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1) + .await + .unwrap(); + key + } + + async fn put_npm_packument_generation(storage: &Storage, fixture: &NpmPackumentFixture) { + storage.put(&fixture.full_key, &fixture.full).await.unwrap(); + storage + .put(&fixture.install_v1_key, &fixture.install_v1) + .await + .unwrap(); + } + + async fn put_npm_packument_pointer( + storage: &Storage, + repository: &str, + package: &str, + fixture: &NpmPackumentFixture, + ) { + storage + .put( + &crate::npm_layout::hosted_packument_current_key(repository, package), + &fixture.pointer, + ) + .await + .unwrap(); + } + + async fn put_npm_active_maintenance( + storage: &Storage, + repository: &str, + package: &str, + ) -> String { + let generation = "a".repeat(64); + let operation = crate::npm_layout::HostedMaintenanceOperation { + schema: crate::npm_layout::HOSTED_MAINTENANCE_SCHEMA_V1, + repository: repository.to_string(), + package: package.to_string(), + base: crate::npm_layout::HostedPackumentPointer { + generation: generation.clone(), + full_sha256: generation.clone(), + install_v1_sha256: "b".repeat(64), + }, + target: crate::npm_layout::HostedMaintenanceTarget::Live { + pointer: crate::npm_layout::HostedPackumentPointer { + generation: generation.clone(), + full_sha256: generation, + install_v1_sha256: "b".repeat(64), + }, + }, + action: crate::npm_layout::HostedMaintenanceAction::DistTag { + tag: "latest".to_string(), + value: Some("1.0.0".to_string()), + }, + }; + crate::registry::create_hosted_maintenance_marker(storage, &operation) + .await + .unwrap(); + crate::npm_layout::hosted_maintenance_active_key(repository, package) + } + + async fn put_npm_active_import(storage: &Storage, repository: &str, package: &str) -> String { + let key = crate::npm_layout::hosted_import_pending_key(repository, package); + let session = crate::npm_layout::HostedImportSession { + schema: crate::npm_layout::HOSTED_IMPORT_SESSION_SCHEMA_V1, + repository: repository.to_string(), + package: package.to_string(), + packument_sha256: "a".repeat(64), + base: None, + versions: std::collections::BTreeMap::from([("1.0.0".to_string(), "b".repeat(64))]), + }; + storage + .put(&key, &serde_json::to_vec(&session).unwrap()) + .await + .unwrap(); + key + } + + struct StatNotifyBackend { + inner: Storage, + watched_keys: HashSet, + stat_seen: Arc, + } + + struct ListOmittingBackend { + inner: Storage, + omitted: HashSet, + } + + #[async_trait::async_trait] + impl crate::storage::StorageBackend for ListOmittingBackend { + async fn put(&self, key: &str, data: &[u8]) -> crate::storage::Result<()> { + self.inner.put(key, data).await + } + + async fn put_if_absent(&self, key: &str, data: &[u8]) -> crate::storage::Result<()> { + self.inner.put_if_absent(key, data).await + } + + async fn get(&self, key: &str) -> crate::storage::Result { + self.inner.get(key).await + } + + async fn delete(&self, key: &str) -> crate::storage::Result<()> { + self.inner.delete(key).await + } + + async fn list(&self, prefix: &str) -> crate::storage::Result> { + Ok(self + .inner + .list(prefix) + .await? + .into_iter() + .filter(|key| !self.omitted.contains(key)) + .collect()) + } + + async fn stat(&self, key: &str) -> Option { + self.inner.stat(key).await + } + + async fn health_check(&self) -> bool { + self.inner.health_check().await + } + + async fn total_size(&self) -> u64 { + self.inner.total_size().await + } + + fn backend_name(&self) -> &'static str { + "list-omitting-test" + } + + async fn refresh_total_size(&self) { + self.inner.refresh_total_size_cache().await; + } + + async fn put_from_path( + &self, + key: &str, + src: &std::path::Path, + ) -> crate::storage::Result<()> { + self.inner.put_from_path(key, src, None).await + } + + async fn get_reader( + &self, + key: &str, + ) -> crate::storage::Result<( + u64, + std::pin::Pin>, + )> { + self.inner.get_reader(key).await + } + } + + #[async_trait::async_trait] + impl crate::storage::StorageBackend for StatNotifyBackend { + async fn put(&self, key: &str, data: &[u8]) -> crate::storage::Result<()> { + self.inner.put(key, data).await + } + + async fn put_if_absent(&self, key: &str, data: &[u8]) -> crate::storage::Result<()> { + self.inner.put_if_absent(key, data).await + } + + async fn get(&self, key: &str) -> crate::storage::Result { + self.inner.get(key).await + } + + async fn delete(&self, key: &str) -> crate::storage::Result<()> { + self.inner.delete(key).await + } + + async fn list(&self, prefix: &str) -> crate::storage::Result> { + self.inner.list(prefix).await + } + + async fn stat(&self, key: &str) -> Option { + if self.watched_keys.contains(key) { + self.stat_seen.notify_one(); + } + self.inner.stat(key).await + } + + async fn health_check(&self) -> bool { + self.inner.health_check().await + } + + async fn total_size(&self) -> u64 { + self.inner.total_size().await + } + + fn backend_name(&self) -> &'static str { + "stat-notify-test" + } + + async fn refresh_total_size(&self) { + self.inner.refresh_total_size_cache().await; + } + + async fn put_from_path( + &self, + key: &str, + src: &std::path::Path, + ) -> crate::storage::Result<()> { + self.inner.put_from_path(key, src, None).await + } + + async fn get_reader( + &self, + key: &str, + ) -> crate::storage::Result<( + u64, + std::pin::Pin>, + )> { + self.inner.get_reader(key).await + } } #[test] @@ -1779,6 +2712,42 @@ mod tests { .is_ok()); } + #[tokio::test] + async fn test_gc_npm_active_maintenance_skips_whole_package() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (blob, _) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"orphan"); + let sidecar = "npm/repositories/npm-private/pkg/versions/ghost.json.sha256"; + storage.put(&blob, b"orphan").await.unwrap(); + storage.put(sidecar, b"hash").await.unwrap(); + let active = put_npm_active_maintenance(&storage, "npm-private", "pkg").await; + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.deleted, 0); + assert_eq!(result.stat_failures, 0); + assert!(storage.get(&blob).await.is_ok()); + assert!(storage.get(sidecar).await.is_ok()); + assert!(storage.get(&active).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_corrupt_maintenance_marker_fails_closed_for_package() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (blob, _) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"orphan"); + storage.put(&blob, b"orphan").await.unwrap(); + let active = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + storage.put(&active, b"not-json").await.unwrap(); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.deleted, 0); + assert_eq!(result.stat_failures, 1); + assert!(storage.get(&blob).await.is_ok()); + assert!(storage.get(&active).await.is_ok()); + } + #[tokio::test] async fn test_gc_pypi_checksum_orphan() { let dir = tempfile::tempdir().unwrap(); @@ -1914,12 +2883,69 @@ mod tests { .await .unwrap(); storage.put(&blob, b"tarball").await.unwrap(); + let current = + npm_packument_fixture_with_manifest("npm-private", "lodash", "1.0.0", &manifest); + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "lodash", ¤t).await; let result = run_gc(&storage, &test_publish_locks(), false, 0).await; assert_eq!(result.orphaned, 0); assert!(storage.get(&blob).await.is_ok()); } + #[tokio::test] + async fn test_gc_npm_list_omission_cannot_hide_live_blob_reachability() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (blob, manifest) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"live"); + let manifest_key = "npm/repositories/npm-private/pkg/versions/1.0.0.json"; + inner.put(manifest_key, &manifest).await.unwrap(); + inner.put(&blob, b"live").await.unwrap(); + let current = npm_packument_fixture_with_manifest("npm-private", "pkg", "1.0.0", &manifest); + put_npm_packument_generation(&inner, ¤t).await; + put_npm_packument_pointer(&inner, "npm-private", "pkg", ¤t).await; + let current_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let storage = Storage::from_backend(Arc::new(ListOmittingBackend { + inner: inner.clone(), + omitted: HashSet::from([ + manifest_key.to_string(), + current_key, + current.full_key.clone(), + current.install_v1_key.clone(), + ]), + })); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + assert_eq!(result.stat_failures, 0); + assert!(inner.get(&blob).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_list_omission_cannot_hide_active_import() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (blob, _) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"in-flight"); + inner.put(&blob, b"in-flight").await.unwrap(); + let retired = put_npm_retired(&inner, "npm-private", "pkg").await; + let import = put_npm_active_import(&inner, "npm-private", "pkg").await; + let storage = Storage::from_backend(Arc::new(ListOmittingBackend { + inner: inner.clone(), + omitted: HashSet::from([import.clone()]), + })); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + assert_eq!(result.stat_failures, 0); + assert!(inner.get(&blob).await.is_ok()); + assert!(inner.get(&retired).await.is_ok()); + assert!(inner.get(&import).await.is_ok()); + } + #[tokio::test] async fn test_gc_npm_removes_superseded_blob_but_keeps_manifest_reachable_blob() { let dir = tempfile::tempdir().unwrap(); @@ -1936,6 +2962,10 @@ mod tests { ) .await .unwrap(); + let current = + npm_packument_fixture_with_manifest("npm-private", "pkg", "1.0.0", ¤t_manifest); + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "pkg", ¤t).await; let result = run_gc(&storage, &test_publish_locks(), false, 0).await; @@ -1951,6 +2981,7 @@ mod tests { let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); let (blob, _) = npm_blob_fixture("npm-private", "lodash", "1.0.0", b"orphan"); storage.put(&blob, b"orphan").await.unwrap(); + put_npm_retired(&storage, "npm-private", "lodash").await; let result = run_gc(&storage, &test_publish_locks(), true, 0).await; assert_eq!(result.orphaned, 1); @@ -1964,6 +2995,7 @@ mod tests { let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); let (blob, _) = npm_blob_fixture("npm-private", "@scope/pkg", "2.0.0", b"orphan"); storage.put(&blob, b"orphan").await.unwrap(); + put_npm_retired(&storage, "npm-private", "@scope/pkg").await; let result = run_gc(&storage, &test_publish_locks(), false, 0).await; assert_eq!(result.orphaned, 1); @@ -1977,6 +3009,7 @@ mod tests { let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); let (key, _) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"in-flight"); storage.put(&key, b"in-flight").await.unwrap(); + put_npm_retired(&storage, "npm-private", "pkg").await; let result = run_gc(&storage, &test_publish_locks(), false, 3600).await; @@ -2004,6 +3037,7 @@ mod tests { let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); let (key, _) = npm_blob_fixture("npm-private", "proxy", "1.0.0", b"orphan"); storage.put(&key, b"orphan").await.unwrap(); + put_npm_retired(&storage, "npm-private", "proxy").await; let result = run_gc(&storage, &test_publish_locks(), false, 0).await; @@ -2019,6 +3053,7 @@ mod tests { let (tarball, manifest_body) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"staged"); let manifest = "npm/repositories/npm-private/pkg/versions/1.0.0.json"; storage.put(&tarball, b"staged").await.unwrap(); + put_npm_retired(&storage, "npm-private", "pkg").await; let snapshot = detect_npm_hosted_orphans(&storage).await; assert_eq!(snapshot.orphans, vec![tarball.clone()]); @@ -2026,8 +3061,20 @@ mod tests { // Model publish committing after GC's initial LIST but before its // destructive package-lock section. storage.put(manifest, &manifest_body).await.unwrap(); - let removed = - delete_npm_orphan_if_uncommitted(&storage, &test_publish_locks(), &tarball).await; + let current = + npm_packument_fixture_with_manifest("npm-private", "pkg", "1.0.0", &manifest_body); + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "pkg", ¤t).await; + let mut validation_cache = HashMap::new(); + let removed = delete_npm_orphan_if_uncommitted( + &storage, + &test_publish_locks(), + &tarball, + now_unix_secs(), + 0, + &mut validation_cache, + ) + .await; assert_eq!( removed, @@ -2037,6 +3084,34 @@ mod tests { assert!(storage.get(&tarball).await.is_ok()); } + #[tokio::test] + async fn test_gc_npm_rechecks_active_maintenance_under_package_lock() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let (blob, _) = npm_blob_fixture("npm-private", "pkg", "1.0.0", b"orphan"); + storage.put(&blob, b"orphan").await.unwrap(); + put_npm_retired(&storage, "npm-private", "pkg").await; + let snapshot = detect_npm_hosted_orphans(&storage).await; + assert_eq!(snapshot.orphans, vec![blob.clone()]); + + // The marker appeared after detection. The destructive path must + // discover it only after acquiring the exact package lock. + put_npm_active_maintenance(&storage, "npm-private", "pkg").await; + let mut validation_cache = HashMap::new(); + let removed = delete_npm_orphan_if_uncommitted( + &storage, + &test_publish_locks(), + &blob, + now_unix_secs(), + 0, + &mut validation_cache, + ) + .await; + + assert_eq!(removed, NpmOrphanDeleteOutcome::Kept); + assert!(storage.get(&blob).await.is_ok()); + } + #[tokio::test] async fn test_gc_npm_manifest_read_failure_keeps_tarball_and_counts_failure() { let dir = tempfile::tempdir().unwrap(); @@ -2046,8 +3121,12 @@ mod tests { let manifest = "npm/repositories/npm-private/pkg/versions/1.0.0.json"; inner.put(&tarball, b"committed").await.unwrap(); inner.put(manifest, &manifest_body).await.unwrap(); + let current = + npm_packument_fixture_with_manifest("npm-private", "pkg", "1.0.0", &manifest_body); + put_npm_packument_generation(&inner, ¤t).await; + put_npm_packument_pointer(&inner, "npm-private", "pkg", ¤t).await; let backend = - crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_get(manifest); + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_get(¤t.full_key); let storage = Storage::from_backend(Arc::new(backend)); let result = run_gc(&storage, &test_publish_locks(), false, 0).await; @@ -2058,6 +3137,485 @@ mod tests { assert!(inner.get(&tarball).await.is_ok()); } + #[tokio::test] + async fn test_gc_npm_packument_current_generation_is_kept() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let current = npm_packument_fixture("npm-private", "pkg", "2.0.0"); + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "pkg", ¤t).await; + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.total_candidates, 2); + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + assert!(storage.get(¤t.full_key).await.is_ok()); + assert!(storage.get(¤t.install_v1_key).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_hosted_package_named_proxy_uses_hosted_packument_layout() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let old = npm_packument_fixture("npm-private", "proxy", "1.0.0"); + let current = npm_packument_fixture("npm-private", "proxy", "2.0.0"); + put_npm_packument_generation(&storage, &old).await; + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "proxy", ¤t).await; + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 2); + assert_eq!(result.deleted, 2); + assert!(storage.stat(&old.full_key).await.is_none()); + assert!(storage.get(¤t.full_key).await.is_ok()); + assert!(storage.get(¤t.install_v1_key).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_obsolete_generation_honors_grace_then_is_removed() { + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("data"); + let storage = Storage::new_local(data.to_str().unwrap()); + let old = npm_packument_fixture("npm-private", "pkg", "1.0.0"); + let current = npm_packument_fixture("npm-private", "pkg", "2.0.0"); + put_npm_packument_generation(&storage, &old).await; + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "pkg", ¤t).await; + + // The documents themselves are old, but the pointer switched only now. + // A lock-free GET may still be serving the previous pointer, so grace + // must start from the pointer switch, not object creation. + let old_mtime = std::time::SystemTime::now() - std::time::Duration::from_secs(7200); + for key in [&old.full_key, &old.install_v1_key] { + std::fs::File::options() + .write(true) + .open(data.join(key)) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + } + let recent = run_gc(&storage, &test_publish_locks(), false, 3600).await; + assert_eq!(recent.orphaned, 2); + assert_eq!(recent.skipped_recent, 2); + assert_eq!(recent.deleted, 0); + + let pointer = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + std::fs::File::options() + .write(true) + .open(data.join(pointer)) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + + let preview = run_gc(&storage, &test_publish_locks(), true, 3600).await; + assert_eq!(preview.orphaned, 2); + assert_eq!(preview.deleted, 0); + assert_eq!( + preview.bytes_freed, + (old.full.len() + old.install_v1.len()) as u64 + ); + assert!(storage.get(&old.full_key).await.is_ok()); + + let collected = run_gc(&storage, &test_publish_locks(), false, 3600).await; + assert_eq!(collected.orphaned, 2); + assert_eq!(collected.skipped_recent, 0); + assert_eq!(collected.deleted, 2); + assert!(storage.stat(&old.full_key).await.is_none()); + assert!(storage.stat(&old.install_v1_key).await.is_none()); + assert!(storage.get(¤t.full_key).await.is_ok()); + assert!(storage.get(¤t.install_v1_key).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_pointer_switch_while_waiting_keeps_new_current_generation() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let newly_current = npm_packument_fixture("npm-private", "pkg", "1.0.0"); + let initially_current = npm_packument_fixture("npm-private", "pkg", "2.0.0"); + put_npm_packument_generation(&inner, &newly_current).await; + put_npm_packument_generation(&inner, &initially_current).await; + put_npm_packument_pointer(&inner, "npm-private", "pkg", &initially_current).await; + + let stat_seen = Arc::new(tokio::sync::Notify::new()); + let storage = Storage::from_backend(Arc::new(StatNotifyBackend { + inner: inner.clone(), + watched_keys: HashSet::from([ + newly_current.full_key.clone(), + newly_current.install_v1_key.clone(), + ]), + stat_seen: Arc::clone(&stat_seen), + })); + let locks = test_publish_locks(); + let package_lock = crate::acquire_publish_lock(&locks, "npm:npm-private:pkg"); + let guard = package_lock.lock().await; + let gc = tokio::spawn({ + let storage = storage.clone(); + let locks = locks.clone(); + async move { run_gc(&storage, &locks, false, 0).await } + }); + + tokio::time::timeout(std::time::Duration::from_secs(5), stat_seen.notified()) + .await + .expect("GC reached the stale candidate before taking the package lock"); + // This test task owns the package lock, so it models the mutation that + // atomically makes the formerly-obsolete generation current. + put_npm_packument_pointer(&inner, "npm-private", "pkg", &newly_current).await; + drop(guard); + + let result = gc.await.unwrap(); + assert_eq!(result.orphaned, 2, "both old-snapshot docs were candidates"); + assert_eq!(result.deleted, 0, "lock-time pointer readback must win"); + assert!(inner.get(&newly_current.full_key).await.is_ok()); + assert!(inner.get(&newly_current.install_v1_key).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_missing_corrupt_or_unreadable_pointer_keeps_read_models() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + + let missing_generation = npm_packument_fixture("npm-private", "missing", "1.0.0"); + put_npm_packument_generation(&inner, &missing_generation).await; + + let invalid_retired_generation = + npm_packument_fixture("npm-private", "invalid-retired", "1.0.0"); + put_npm_packument_generation(&inner, &invalid_retired_generation).await; + inner + .put( + &crate::npm_layout::hosted_packument_retired_key("npm-private", "invalid-retired"), + b"unknown-retirement-protocol", + ) + .await + .unwrap(); + + let corrupt_generation = npm_packument_fixture("npm-private", "corrupt", "1.0.0"); + put_npm_packument_generation(&inner, &corrupt_generation).await; + let corrupt_pointer = + crate::npm_layout::hosted_packument_current_key("npm-private", "corrupt"); + inner.put(&corrupt_pointer, b"{not-json").await.unwrap(); + let corrupt_cache = crate::npm_layout::hosted_packument_cache_key("npm-private", "corrupt"); + inner.put(&corrupt_cache, b"legacy").await.unwrap(); + + let unreadable_generation = npm_packument_fixture("npm-private", "unreadable", "1.0.0"); + put_npm_packument_generation(&inner, &unreadable_generation).await; + put_npm_packument_pointer(&inner, "npm-private", "unreadable", &unreadable_generation) + .await; + let unreadable_pointer = + crate::npm_layout::hosted_packument_current_key("npm-private", "unreadable"); + let backend = crate::test_helpers::FaultInjectBackend::new(inner.clone()) + .fail_get(&unreadable_pointer); + let storage = Storage::from_backend(Arc::new(backend)); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + assert_eq!(result.stat_failures, 4); + for key in [ + &missing_generation.full_key, + &missing_generation.install_v1_key, + &invalid_retired_generation.full_key, + &invalid_retired_generation.install_v1_key, + &corrupt_generation.full_key, + &corrupt_generation.install_v1_key, + &corrupt_cache, + &unreadable_generation.full_key, + &unreadable_generation.install_v1_key, + ] { + assert!(inner.get(key).await.is_ok(), "{key} must be kept"); + } + } + + #[tokio::test] + async fn test_gc_npm_missing_current_document_keeps_all_generations() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let old = npm_packument_fixture("npm-private", "pkg", "1.0.0"); + let current = npm_packument_fixture("npm-private", "pkg", "2.0.0"); + put_npm_packument_generation(&storage, &old).await; + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "pkg", ¤t).await; + storage.delete(¤t.install_v1_key).await.unwrap(); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + assert_eq!(result.stat_failures, 1); + assert!(storage.get(&old.full_key).await.is_ok()); + assert!(storage.get(&old.install_v1_key).await.is_ok()); + assert!(storage.get(¤t.full_key).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_corrupt_or_unreadable_current_document_keeps_old_generations() { + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("data"); + let inner = Storage::new_local(data.to_str().unwrap()); + + let corrupt_old = npm_packument_fixture("npm-private", "corrupt-doc", "1.0.0"); + let corrupt_current = npm_packument_fixture("npm-private", "corrupt-doc", "2.0.0"); + put_npm_packument_generation(&inner, &corrupt_old).await; + put_npm_packument_generation(&inner, &corrupt_current).await; + put_npm_packument_pointer(&inner, "npm-private", "corrupt-doc", &corrupt_current).await; + inner + .put(&corrupt_current.full_key, b"corrupt-current-full") + .await + .unwrap(); + + let unreadable_old = npm_packument_fixture("npm-private", "unreadable-doc", "1.0.0"); + let unreadable_current = npm_packument_fixture("npm-private", "unreadable-doc", "2.0.0"); + put_npm_packument_generation(&inner, &unreadable_old).await; + put_npm_packument_generation(&inner, &unreadable_current).await; + put_npm_packument_pointer(&inner, "npm-private", "unreadable-doc", &unreadable_current) + .await; + + let old_mtime = std::time::SystemTime::now() - std::time::Duration::from_secs(7200); + let corrupt_pointer = + crate::npm_layout::hosted_packument_current_key("npm-private", "corrupt-doc"); + let unreadable_pointer = + crate::npm_layout::hosted_packument_current_key("npm-private", "unreadable-doc"); + for key in [ + &corrupt_old.full_key, + &corrupt_old.install_v1_key, + &unreadable_old.full_key, + &unreadable_old.install_v1_key, + &corrupt_pointer, + &unreadable_pointer, + ] { + std::fs::File::options() + .write(true) + .open(data.join(key)) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + } + let storage = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()) + .fail_get(&unreadable_current.install_v1_key), + )); + + let result = run_gc(&storage, &test_publish_locks(), false, 3600).await; + + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + assert_eq!(result.stat_failures, 2); + for key in [ + &corrupt_old.full_key, + &corrupt_old.install_v1_key, + &unreadable_old.full_key, + &unreadable_old.install_v1_key, + ] { + assert!(inner.get(key).await.is_ok(), "{key} must be kept"); + } + } + + #[tokio::test] + async fn test_gc_npm_legacy_packument_cache_requires_current_and_honors_grace() { + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("data"); + let storage = Storage::new_local(data.to_str().unwrap()); + let current = npm_packument_fixture("npm-private", "pkg", "2.0.0"); + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "pkg", ¤t).await; + let cache = crate::npm_layout::hosted_packument_cache_key("npm-private", "pkg"); + storage.put(&cache, b"legacy-cache").await.unwrap(); + + let old_mtime = std::time::SystemTime::now() - std::time::Duration::from_secs(7200); + std::fs::File::options() + .write(true) + .open(data.join(&cache)) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + let recent = run_gc(&storage, &test_publish_locks(), false, 3600).await; + assert_eq!(recent.orphaned, 1); + assert_eq!(recent.skipped_recent, 1); + assert_eq!(recent.deleted, 0); + assert!(storage.get(&cache).await.is_ok()); + + let pointer = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + std::fs::File::options() + .write(true) + .open(data.join(pointer)) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + let collected = run_gc(&storage, &test_publish_locks(), false, 3600).await; + assert_eq!(collected.orphaned, 1); + assert_eq!(collected.deleted, 1); + assert!(storage.stat(&cache).await.is_none()); + + let no_pointer_cache = + crate::npm_layout::hosted_packument_cache_key("npm-private", "no-pointer"); + storage + .put(&no_pointer_cache, b"only-readable-copy") + .await + .unwrap(); + let fail_closed = run_gc(&storage, &test_publish_locks(), false, 0).await; + assert_eq!(fail_closed.deleted, 0); + assert!(storage.get(&no_pointer_cache).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_retired_read_model_uses_marker_grace_and_keeps_tombstone() { + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("data"); + let storage = Storage::new_local(data.to_str().unwrap()); + let generation = npm_packument_fixture("npm-private", "retired", "1.0.0"); + put_npm_packument_generation(&storage, &generation).await; + let cache = crate::npm_layout::hosted_packument_cache_key("npm-private", "retired"); + storage.put(&cache, b"legacy-cache").await.unwrap(); + let marker = crate::npm_layout::hosted_packument_retired_key("npm-private", "retired"); + storage + .put(&marker, crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1) + .await + .unwrap(); + + let old_mtime = std::time::SystemTime::now() - std::time::Duration::from_secs(7200); + for key in [&generation.full_key, &generation.install_v1_key, &cache] { + std::fs::File::options() + .write(true) + .open(data.join(key)) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + } + + let draining = run_gc(&storage, &test_publish_locks(), false, 3600).await; + assert_eq!(draining.orphaned, 3); + assert_eq!(draining.skipped_recent, 3); + assert_eq!(draining.deleted, 0); + assert!(storage.get(&generation.full_key).await.is_ok()); + assert!(storage.get(&marker).await.is_ok()); + + std::fs::File::options() + .write(true) + .open(data.join(&marker)) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + let collected = run_gc(&storage, &test_publish_locks(), false, 3600).await; + assert_eq!(collected.orphaned, 3); + assert_eq!(collected.deleted, 3); + assert!(storage.stat(&generation.full_key).await.is_none()); + assert!(storage.stat(&generation.install_v1_key).await.is_none()); + assert!(storage.stat(&cache).await.is_none()); + assert!( + storage.get(&marker).await.is_ok(), + "retirement root survives" + ); + + let marker_pass = run_gc(&storage, &test_publish_locks(), false, 3600).await; + assert_eq!(marker_pass.orphaned, 0); + assert_eq!(marker_pass.deleted, 0); + assert!(storage.get(&marker).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_list_omission_never_removes_retirement_tombstone() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let generation = npm_packument_fixture("npm-private", "retired", "1.0.0"); + put_npm_packument_generation(&inner, &generation).await; + let marker = put_npm_retired(&inner, "npm-private", "retired").await; + let storage = Storage::from_backend(Arc::new(ListOmittingBackend { + inner: inner.clone(), + omitted: HashSet::from([ + generation.full_key.clone(), + generation.install_v1_key.clone(), + ]), + })); + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + assert!(inner.get(&marker).await.is_ok()); + assert!(inner.get(&generation.full_key).await.is_ok()); + assert!(inner.get(&generation.install_v1_key).await.is_ok()); + } + + #[tokio::test] + async fn test_gc_npm_retired_root_requires_quiescent_authoritative_state() { + let dir = tempfile::tempdir().unwrap(); + let data = dir.path().join("data"); + let storage = Storage::new_local(data.to_str().unwrap()); + let generation = npm_packument_fixture("npm-private", "retired", "1.0.0"); + put_npm_packument_generation(&storage, &generation).await; + let marker = crate::npm_layout::hosted_packument_retired_key("npm-private", "retired"); + storage + .put(&marker, crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1) + .await + .unwrap(); + let old_mtime = std::time::SystemTime::now() - std::time::Duration::from_secs(7200); + for key in [&generation.full_key, &generation.install_v1_key, &marker] { + std::fs::File::options() + .write(true) + .open(data.join(key)) + .unwrap() + .set_modified(old_mtime) + .unwrap(); + } + + let active_keys = [ + crate::npm_layout::hosted_package_key("npm-private", "retired"), + crate::npm_layout::hosted_import_pending_key("npm-private", "retired"), + crate::npm_layout::hosted_publish_pending_index_key("npm-private", "retired"), + ]; + for (index, active) in active_keys.iter().enumerate() { + storage.put(active, b"active").await.unwrap(); + let blocked = run_gc(&storage, &test_publish_locks(), false, 3600).await; + assert_eq!(blocked.orphaned, 0, "{active} must block retirement GC"); + assert_eq!(blocked.deleted, 0); + assert_eq!( + blocked.stat_failures, + usize::from(index > 0), + "malformed transaction journals must fail closed and be counted" + ); + assert!(storage.get(&generation.full_key).await.is_ok()); + storage.delete(active).await.unwrap(); + } + } + + #[tokio::test] + async fn test_gc_npm_import_lifecycle_objects_are_not_candidates() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let current = npm_packument_fixture("npm-private", "pkg", "2.0.0"); + put_npm_packument_generation(&storage, ¤t).await; + put_npm_packument_pointer(&storage, "npm-private", "pkg", ¤t).await; + let digest = "a".repeat(64); + let lifecycle_keys = [ + crate::npm_layout::hosted_import_pending_key("npm-private", "pkg"), + crate::npm_layout::hosted_import_evidence_key( + "npm-private", + "pkg", + &digest, + "2.0.0", + &digest, + ), + crate::npm_layout::hosted_import_receipt_key("npm-private", "pkg", &digest), + ]; + for key in &lifecycle_keys { + storage.put(key, b"lifecycle-state").await.unwrap(); + } + + let result = run_gc(&storage, &test_publish_locks(), false, 0).await; + + assert_eq!( + result.total_candidates, 2, + "only generation docs are scanned" + ); + assert_eq!(result.orphaned, 0); + assert_eq!(result.deleted, 0); + for key in &lifecycle_keys { + assert!(storage.get(key).await.is_ok(), "{key} must be kept"); + } + } + #[tokio::test] async fn test_gc_pypi_no_phantoms() { let dir = tempfile::tempdir().unwrap(); diff --git a/nora-registry/src/main.rs b/nora-registry/src/main.rs index a41c969f..77f7e2f4 100644 --- a/nora-registry/src/main.rs +++ b/nora-registry/src/main.rs @@ -1663,7 +1663,11 @@ async fn run_server(mut config: Config, storage: Storage) { } // Spawn background retention scheduler if enabled - if state.config.retention.enabled && !state.config.retention.rules.is_empty() { + // An enabled retention scheduler must also recover durable npm + // maintenance operations left by an interrupted earlier run. Recovery is + // independent of the current rule set, so an empty/changed rule list must + // not prevent the boot pass from resuming those operations. + if state.config.retention.enabled { let handle = retention::spawn_retention_scheduler( state.storage.clone(), state.publish_locks.clone(), diff --git a/nora-registry/src/npm_layout.rs b/nora-registry/src/npm_layout.rs index be474e2d..f20a14c4 100644 --- a/nora-registry/src/npm_layout.rs +++ b/nora-registry/src/npm_layout.rs @@ -10,6 +10,138 @@ use base64::Engine; use sha2::Digest; +use std::collections::BTreeMap; + +pub(crate) const HOSTED_PACKUMENT_RETIRED_V1: &[u8] = b"retired-v1"; +pub(crate) const HOSTED_MAINTENANCE_SCHEMA_V1: u8 = 1; +pub(crate) const HOSTED_IMPORT_SESSION_SCHEMA_V1: u8 = 1; +pub(crate) const HOSTED_PUBLISH_PENDING_SCHEMA_V1: u8 = 1; +const HOSTED_MAINTENANCE_OPERATION_ID_DOMAIN: &[u8] = + b"nora:npm:hosted-maintenance:operation-id:v1\0"; + +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct HostedPackumentPointer { + pub(crate) generation: String, + pub(crate) full_sha256: String, + pub(crate) install_v1_sha256: String, +} + +/// Exact package-wide journal for a bulk import. The version roster is +/// extended before any object for that version is written, so finalize can +/// prove completeness without relying on object-store LIST consistency. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct HostedImportSession { + pub(crate) schema: u8, + pub(crate) repository: String, + pub(crate) package: String, + pub(crate) packument_sha256: String, + pub(crate) base: Option, + pub(crate) versions: BTreeMap, +} + +/// A normal publish can derive its immutable target from the exact base and +/// retry payload. An import version is instead bound to its package-wide +/// import journal. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub(crate) enum HostedPublishPendingTarget { + Publish { + base: Option, + target: HostedPackumentPointer, + }, + Import { + packument_sha256: String, + }, +} + +/// Singleton exact-key transaction record. A deployment owns one package +/// mutation lock, so a package has at most one active version publish. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct HostedPublishPending { + pub(crate) schema: u8, + pub(crate) repository: String, + pub(crate) package: String, + pub(crate) version: String, + pub(crate) manifest_sha256: String, + pub(crate) blob_sha512: String, + pub(crate) target: HostedPublishPendingTarget, +} + +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(tag = "state", rename_all = "kebab-case", deny_unknown_fields)] +pub(crate) enum HostedMaintenanceTarget { + Live { pointer: HostedPackumentPointer }, + Retired, +} + +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub(crate) enum HostedMaintenanceAction { + DistTag { + tag: String, + value: Option, + }, + Deprecations { + values: BTreeMap>, + }, + Retention { + snapshot_guard: String, + removed_versions: BTreeMap, + expected_authority: BTreeMap, + }, +} + +/// Deterministic operation payload used to derive `operation_id` without a +/// self-reference. Every map in the schema is ordered so serde's struct field +/// order plus canonical map iteration produces stable bytes. +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct HostedMaintenanceOperation { + pub(crate) schema: u8, + pub(crate) repository: String, + pub(crate) package: String, + pub(crate) base: HostedPackumentPointer, + pub(crate) target: HostedMaintenanceTarget, + pub(crate) action: HostedMaintenanceAction, +} + +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct HostedMaintenanceMarker { + pub(crate) schema: u8, + pub(crate) repository: String, + pub(crate) package: String, + pub(crate) operation_id: String, + pub(crate) base: HostedPackumentPointer, + pub(crate) target: HostedMaintenanceTarget, + pub(crate) action: HostedMaintenanceAction, +} + +impl HostedMaintenanceMarker { + pub(crate) fn operation(&self) -> HostedMaintenanceOperation { + HostedMaintenanceOperation { + schema: self.schema, + repository: self.repository.clone(), + package: self.package.clone(), + base: self.base.clone(), + target: self.target.clone(), + action: self.action.clone(), + } + } +} + +pub(crate) fn hosted_maintenance_operation_id( + operation: &HostedMaintenanceOperation, +) -> Result { + let encoded = serde_json::to_vec(operation)?; + let mut digest = sha2::Sha256::new(); + digest.update(HOSTED_MAINTENANCE_OPERATION_ID_DOMAIN); + digest.update(encoded); + Ok(hex::encode(digest.finalize())) +} #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) enum NpmObjectKind { @@ -17,7 +149,27 @@ pub(crate) enum NpmObjectKind { /// Rebuildable materialized hosted packument. Authoritative hosted state /// remains in the package/version/tag/deprecation objects. HostedPackumentCache, + /// Mutable pointer to a complete immutable hosted packument generation. + HostedPackumentCurrent, + HostedPackumentRetired, + /// Single immutable package-wide maintenance lease and recovery record. + HostedMaintenanceActive, + /// Immutable full hosted packument read model. + HostedPackumentFull(String), + /// Immutable abbreviated (`install-v1`) hosted packument read model. + HostedPackumentInstallV1(String), + /// Content-bound marker used while a package is imported in bulk. + HostedImportPending, + HostedImportEvidence { + packument_sha256: String, + version: String, + manifest_sha256: String, + }, + /// Immutable receipt for a completed package import. + HostedImportReceipt(String), HostedVersion(String), + HostedPublishPending(String), + HostedPublishPendingIndex, HostedPublishComplete(String), HostedTarball(String), HostedBlob { @@ -31,10 +183,87 @@ pub(crate) enum NpmObjectKind { ProxyNegative, } +#[cfg(test)] pub(crate) fn hosted_packument_cache_key(repository: &str, package: &str) -> String { format!("npm/repositories/{repository}/{package}/packument-cache.json") } +pub(crate) fn hosted_package_key(repository: &str, package: &str) -> String { + format!("npm/repositories/{repository}/{package}/pkg.json") +} + +pub(crate) fn hosted_packuments_prefix(repository: &str, package: &str) -> String { + format!("npm/repositories/{repository}/{package}/hosted-packuments/") +} + +pub(crate) fn hosted_packument_current_key(repository: &str, package: &str) -> String { + format!( + "{}current.json", + hosted_packuments_prefix(repository, package) + ) +} + +pub(crate) fn hosted_maintenance_active_key(repository: &str, package: &str) -> String { + format!("npm/repositories/{repository}/{package}/maintenance/active-v1.json") +} + +pub(crate) fn hosted_packument_retired_key(repository: &str, package: &str) -> String { + format!( + "{}retired-v1", + hosted_packuments_prefix(repository, package) + ) +} + +pub(crate) fn hosted_packument_full_key( + repository: &str, + package: &str, + generation: &str, +) -> String { + format!( + "{}{generation}/full.json", + hosted_packuments_prefix(repository, package) + ) +} + +pub(crate) fn hosted_packument_install_v1_key( + repository: &str, + package: &str, + generation: &str, +) -> String { + format!( + "{}{generation}/install-v1.json", + hosted_packuments_prefix(repository, package) + ) +} + +pub(crate) fn hosted_import_pending_key(repository: &str, package: &str) -> String { + format!("npm/repositories/{repository}/{package}/import/pending-v1") +} + +pub(crate) fn hosted_publish_pending_index_key(repository: &str, package: &str) -> String { + format!("npm/repositories/{repository}/{package}/publish-pending-index-v1") +} + +pub(crate) fn hosted_import_receipt_key( + repository: &str, + package: &str, + generation: &str, +) -> String { + format!("npm/repositories/{repository}/{package}/import/receipts/{generation}.json") +} + +pub(crate) fn hosted_import_evidence_key( + repository: &str, + package: &str, + packument_sha256: &str, + version: &str, + manifest_sha256: &str, +) -> String { + format!( + "npm/repositories/{repository}/{package}/import/generations/{packument_sha256}/versions/{version}/{manifest_sha256}" + ) +} + pub(crate) fn hosted_blob_key_for_digest(repository: &str, package: &str, digest: &str) -> String { format!("npm/repositories/{repository}/{package}/blobs/sha512/{digest}.tgz") } @@ -153,6 +382,153 @@ pub(crate) fn parse_npm_object_key(key: &str) -> Option { }); } + if tail.len() >= 2 && tail.last() == Some(&"publish-pending-index-v1") { + let package = tail[..tail.len() - 1].join("/"); + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedPublishPendingIndex, + }); + } + + if tail.len() >= 3 + && tail[tail.len() - 2] == "hosted-packuments" + && tail.last() == Some(&"current.json") + { + let package = tail[..tail.len() - 2].join("/"); + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedPackumentCurrent, + }); + } + + if tail.len() >= 3 + && tail[tail.len() - 2] == "maintenance" + && tail.last() == Some(&"active-v1.json") + { + let package = tail[..tail.len() - 2].join("/"); + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedMaintenanceActive, + }); + } + + if tail.len() >= 3 + && tail[tail.len() - 2] == "hosted-packuments" + && tail.last() == Some(&"retired-v1") + { + let package = tail[..tail.len() - 2].join("/"); + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedPackumentRetired, + }); + } + + if let Some(marker) = tail.iter().rposition(|part| *part == "hosted-packuments") { + if marker > 0 && marker + 3 == tail.len() { + let package = tail[..marker].join("/"); + let generation = tail[marker + 1]; + let kind = match tail[marker + 2] { + "full.json" => NpmObjectKind::HostedPackumentFull(generation.to_string()), + "install-v1.json" => { + NpmObjectKind::HostedPackumentInstallV1(generation.to_string()) + } + _ => return None, + }; + if package.is_empty() + || generation.len() != 64 + || !generation + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind, + }); + } + } + + if tail.len() >= 3 && tail[tail.len() - 2] == "import" && tail.last() == Some(&"pending-v1") { + let package = tail[..tail.len() - 2].join("/"); + if package.is_empty() { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedImportPending, + }); + } + + if let Some(marker) = tail.iter().rposition(|part| *part == "import") { + if marker > 0 + && marker + 6 == tail.len() + && tail[marker + 1] == "generations" + && tail[marker + 3] == "versions" + { + let package = tail[..marker].join("/"); + let packument_sha256 = tail[marker + 2]; + let version = tail[marker + 4]; + let manifest_sha256 = tail[marker + 5]; + if package.is_empty() + || version.is_empty() + || version.contains('/') + || ![packument_sha256, manifest_sha256].iter().all(|digest| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + }) + { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedImportEvidence { + packument_sha256: packument_sha256.to_string(), + version: version.to_string(), + manifest_sha256: manifest_sha256.to_string(), + }, + }); + } + if marker > 0 && marker + 3 == tail.len() && tail[marker + 1] == "receipts" { + let package = tail[..marker].join("/"); + let generation = tail[marker + 2].strip_suffix(".json")?; + if package.is_empty() + || generation.len() != 64 + || !generation + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + { + return None; + } + return Some(NpmObjectPath { + repository: repository.to_string(), + package, + kind: NpmObjectKind::HostedImportReceipt(generation.to_string()), + }); + } + } + if let Some(marker) = tail.iter().rposition(|part| *part == "blobs") { if marker > 0 && marker + 3 == tail.len() @@ -180,7 +556,12 @@ pub(crate) fn parse_npm_object_key(key: &str) -> Option { let marker = tail.iter().rposition(|part| { matches!( *part, - "versions" | "publish-complete" | "tarballs" | "dist-tags" | "deprecations" + "versions" + | "publish-pending" + | "publish-complete" + | "tarballs" + | "dist-tags" + | "deprecations" ) })?; if marker == 0 || marker + 2 != tail.len() { @@ -193,6 +574,7 @@ pub(crate) fn parse_npm_object_key(key: &str) -> Option { } let kind = match tail[marker] { "versions" => NpmObjectKind::HostedVersion(object.strip_suffix(".json")?.to_string()), + "publish-pending" => NpmObjectKind::HostedPublishPending(object.to_string()), "publish-complete" => NpmObjectKind::HostedPublishComplete(object.to_string()), "tarballs" => NpmObjectKind::HostedTarball(object.to_string()), "dist-tags" => NpmObjectKind::HostedDistTag(object.to_string()), @@ -282,6 +664,78 @@ mod tests { ); } + #[test] + fn hosted_packument_layout_never_collides_with_package_named_proxy() { + let digest = "a".repeat(64); + let cases = [ + ( + hosted_packument_current_key("npm-private", "proxy"), + NpmObjectKind::HostedPackumentCurrent, + ), + ( + hosted_packument_retired_key("npm-private", "proxy"), + NpmObjectKind::HostedPackumentRetired, + ), + ( + hosted_packument_full_key("npm-private", "proxy", &digest), + NpmObjectKind::HostedPackumentFull(digest.clone()), + ), + ( + hosted_packument_install_v1_key("npm-private", "proxy", &digest), + NpmObjectKind::HostedPackumentInstallV1(digest.clone()), + ), + ( + hosted_maintenance_active_key("npm-private", "proxy"), + NpmObjectKind::HostedMaintenanceActive, + ), + ]; + for (key, kind) in cases { + let parsed = parse_npm_object_key(&key).expect("hosted read-model key"); + assert_eq!(parsed.repository, "npm-private"); + assert_eq!(parsed.package, "proxy"); + assert_eq!(parsed.kind, kind); + } + + let cached = + parse_npm_object_key("npm/repositories/npm-private/proxy/packuments/current.json") + .expect("proxy cache packument"); + assert_eq!(cached.package, "current"); + assert_eq!(cached.kind, NpmObjectKind::ProxyPackument); + } + + #[test] + fn maintenance_operation_id_is_deterministic_and_content_bound() { + let pointer = HostedPackumentPointer { + generation: "a".repeat(64), + full_sha256: "a".repeat(64), + install_v1_sha256: "b".repeat(64), + }; + let operation = HostedMaintenanceOperation { + schema: HOSTED_MAINTENANCE_SCHEMA_V1, + repository: "npm-private".to_string(), + package: "@scope/pkg".to_string(), + base: pointer.clone(), + target: HostedMaintenanceTarget::Live { + pointer: pointer.clone(), + }, + action: HostedMaintenanceAction::DistTag { + tag: "next".to_string(), + value: Some("1.0.0".to_string()), + }, + }; + let first = hosted_maintenance_operation_id(&operation).unwrap(); + let second = hosted_maintenance_operation_id(&operation).unwrap(); + assert_eq!(first, second); + assert_eq!(first.len(), 64); + + let mut changed = operation; + changed.action = HostedMaintenanceAction::DistTag { + tag: "next".to_string(), + value: None, + }; + assert_ne!(first, hosted_maintenance_operation_id(&changed).unwrap()); + } + #[test] fn derives_hosted_blob_and_completion_digests_from_manifest() { let digest = [7u8; 64]; diff --git a/nora-registry/src/registry/maven.rs b/nora-registry/src/registry/maven.rs index 4fd580b3..fcee0700 100644 --- a/nora-registry/src/registry/maven.rs +++ b/nora-registry/src/registry/maven.rs @@ -1092,29 +1092,36 @@ async fn upload_legacy( Extension(authority): Extension, body: Bytes, ) -> Response { + let authorize = move |namespace: &str| enforce_namespace_scope(&authority, namespace).is_ok(); if let Some(repository) = state.config.maven.default_repository.clone() { - return upload_configured(state, &repository, path, authority, body).await; + return upload_configured(state, &repository, path, body, authorize).await; } let repository = DirectRepository::legacy(&state); - upload_direct(state, repository, path, authority, body).await + upload_direct(state, repository, path, body, authorize).await } -pub(crate) async fn upload_named( +pub(crate) async fn upload_named( State(state): State, Path((repository, path)): Path<(String, String)>, - Extension(authority): Extension, body: Bytes, -) -> Response { - upload_configured(state, &repository, path, authority, body).await + authorize: F, +) -> Response +where + F: FnOnce(&str) -> bool, +{ + upload_configured(state, &repository, path, body, authorize).await } -async fn upload_configured( +async fn upload_configured( state: AppState, repository: &str, path: String, - authority: NamespaceAuthority, body: Bytes, -) -> Response { + authorize: F, +) -> Response +where + F: FnOnce(&str) -> bool, +{ let Some(config) = state.config.maven.repository(repository).cloned() else { return StatusCode::NOT_FOUND.into_response(); }; @@ -1124,16 +1131,19 @@ async fn upload_configured( if repository.is_proxy() || repository.write_policy == MavenWritePolicy::Deny { return method_not_allowed("GET"); } - upload_direct(state, repository, path, authority, body).await + upload_direct(state, repository, path, body, authorize).await } -async fn upload_direct( +async fn upload_direct( state: AppState, repository: DirectRepository, path: String, - authority: NamespaceAuthority, body: Bytes, -) -> Response { + authorize: F, +) -> Response +where + F: FnOnce(&str) -> bool, +{ if !path.is_ascii() || path.contains("..") || path.contains('\0') || path.starts_with('/') { return (StatusCode::BAD_REQUEST, "Invalid path").into_response(); } @@ -1151,7 +1161,7 @@ async fn upload_direct( } => format!("{}/{}", group_path, artifact_id), MavenPathKind::Opaque => String::new(), }); - if enforce_namespace_scope(&authority, &maven_namespace).is_err() { + if !authorize(&maven_namespace) { return StatusCode::FORBIDDEN.into_response(); } diff --git a/nora-registry/src/registry/mod.rs b/nora-registry/src/registry/mod.rs index 7a1be66f..6c858b51 100644 --- a/nora-registry/src/registry/mod.rs +++ b/nora-registry/src/registry/mod.rs @@ -40,6 +40,12 @@ pub use npm::routes as npm_routes; // keys as strings (review R7, contract `import-key-format-equals-handler-key-format`). pub(crate) use maven::storage_key as maven_storage_key; pub(crate) use maven::update_hosted_metadata_after_retention; +pub(crate) use npm::{ + commit_hosted_packument_pointer, create_hosted_maintenance_marker, + prepare_hosted_packument_after_retention, read_hosted_active_transactions, + read_hosted_maintenance_marker, read_hosted_packument_pointer, + resume_hosted_maintenance_operation, validate_hosted_packument_pointer, +}; pub use nuget::alias_routes as nuget_alias_routes; pub use nuget::routes as nuget_routes; pub use pub_dart::routes as pub_dart_routes; diff --git a/nora-registry/src/registry/named.rs b/nora-registry/src/registry/named.rs index 7c127a08..2cbcc1d4 100644 --- a/nora-registry/src/registry/named.rs +++ b/nora-registry/src/registry/named.rs @@ -10,7 +10,7 @@ //! matching protocol handler without guessing from naming conventions. use super::method_not_allowed; -use crate::auth::{AuthenticatedUser, NamespaceAuthority}; +use crate::auth::{enforce_namespace_scope, AuthenticatedUser, NamespaceAuthority}; use crate::AppState; use axum::{ body::{Body, Bytes}, @@ -50,20 +50,20 @@ async fn download( async fn upload( State(state): State, Path((repository, path)): Path<(String, String)>, + headers: HeaderMap, Extension(authority): Extension, body: Bytes, ) -> Response { if state.config.maven.enabled && state.config.maven.repository(&repository).is_some() { - return super::maven::upload_named( - State(state), - Path((repository, path)), - Extension(authority), - body, - ) - .await; + let authorize = + move |namespace: &str| enforce_namespace_scope(&authority, namespace).is_ok(); + return super::maven::upload_named(State(state), Path((repository, path)), body, authorize) + .await; } if state.config.npm.enabled && state.config.npm.repository(&repository).is_some() { - return super::npm::named_put_request(state, repository, path, authority, body).await; + let authorize = move |package: &str| enforce_namespace_scope(&authority, package).is_ok(); + return super::npm::named_put_request(state, repository, path, headers, body, authorize) + .await; } StatusCode::NOT_FOUND.into_response() } @@ -89,7 +89,9 @@ mod tests { use crate::config::{ MavenRepository, MavenVersionPolicy, MavenWritePolicy, NpmRepository, NpmWritePolicy, }; - use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use crate::test_helpers::{ + body_bytes, create_test_context_with_config, npm_publish_payload, send, TestContext, + }; use axum::http::Method; fn combined_named_config(config: &mut crate::config::Config) { @@ -106,6 +108,20 @@ mod tests { config.npm.default_repository = Some("npm-private".to_string()); } + async fn seed_npm_package(context: &TestContext, package: &str) { + assert_eq!( + send( + &context.app, + Method::PUT, + &format!("/repository/npm-private/{package}"), + npm_publish_payload(package, "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + } + #[tokio::test] async fn shared_dispatcher_reaches_maven_and_npm_without_route_conflict() { // Constructing the combined test router is itself a regression for the @@ -125,26 +141,7 @@ mod tests { assert_eq!(downloaded.status(), StatusCode::OK); assert_eq!(body_bytes(downloaded).await.as_ref(), b"maven-bytes"); - let manifest = br#"{"name":"pkg","version":"1.0.0","dist":{}}"#; - context - .state - .storage - .put( - "npm/repositories/npm-private/pkg/versions/1.0.0.json", - manifest, - ) - .await - .unwrap(); - let completion = crate::npm_layout::hosted_manifest_digest(manifest); - context - .state - .storage - .put( - "npm/repositories/npm-private/pkg/publish-complete/1.0.0", - completion.as_bytes(), - ) - .await - .unwrap(); + seed_npm_package(&context, "pkg").await; let npm_path = "/repository/npm-private/pkg"; let deprecated = send( &context.app, @@ -169,12 +166,17 @@ mod tests { let npm_json: serde_json::Value = serde_json::from_slice(&body_bytes(npm).await).unwrap(); assert_eq!(npm_json["versions"]["1.0.0"]["name"], "pkg"); - context - .state - .storage - .put("npm/repositories/npm-private/pkg/dist-tags/next", b"1.0.0") + assert_eq!( + send( + &context.app, + Method::PUT, + "/repository/npm-private/-/package/pkg/dist-tags/next", + serde_json::to_vec("1.0.0").unwrap(), + ) .await - .unwrap(); + .status(), + StatusCode::CREATED + ); let deleted = send( &context.app, Method::DELETE, @@ -207,6 +209,66 @@ mod tests { assert_eq!(unknown.status(), StatusCode::NOT_FOUND); } + #[tokio::test] + async fn shared_dispatcher_enforces_protocol_canonical_namespace_once() { + use crate::config::ScopeEnforcement; + + let context = create_test_context_with_config(combined_named_config); + let maven_scope = NamespaceAuthority::from_oidc_scope( + "ci", + &["com/myorg/**".to_string()], + ScopeEnforcement::Enforce, + ); + let denied_maven = upload( + State(context.state.clone()), + Path(( + "maven-releases".to_string(), + "com/other/app/1.0/app-1.0.jar".to_string(), + )), + HeaderMap::new(), + Extension(maven_scope.clone()), + Bytes::from_static(b"denied"), + ) + .await; + assert_eq!(denied_maven.status(), StatusCode::FORBIDDEN); + let allowed_maven = upload( + State(context.state.clone()), + Path(( + "maven-releases".to_string(), + "com/myorg/app/1.0/app-1.0.jar".to_string(), + )), + HeaderMap::new(), + Extension(maven_scope), + Bytes::from_static(b"allowed"), + ) + .await; + assert_eq!(allowed_maven.status(), StatusCode::CREATED); + + let npm_scope = NamespaceAuthority::from_oidc_scope( + "ci", + &["@myorg/**".to_string()], + ScopeEnforcement::Enforce, + ); + let denied_npm = upload( + State(context.state.clone()), + Path(("npm-private".to_string(), "other-package".to_string())), + HeaderMap::new(), + Extension(npm_scope.clone()), + Bytes::from(npm_publish_payload("other-package", "1.0.0", "latest")), + ) + .await; + assert_eq!(denied_npm.status(), StatusCode::FORBIDDEN); + let allowed_npm = upload( + State(context.state.clone()), + Path(("npm-private".to_string(), "%40myorg%2Fpackage".to_string())), + HeaderMap::new(), + Extension(npm_scope), + Bytes::from(npm_publish_payload("@myorg/package", "1.0.0", "latest")), + ) + .await; + assert_eq!(allowed_npm.status(), StatusCode::CREATED); + } + #[tokio::test] async fn shared_dispatcher_blocks_configured_npm_when_protocol_is_disabled() { let context = create_test_context_with_config(|config| { @@ -240,15 +302,7 @@ mod tests { combined_named_config(config); config.maven.enabled = false; }); - context - .state - .storage - .put( - "npm/repositories/npm-private/pkg/versions/1.0.0.json", - br#"{"name":"pkg","version":"1.0.0","dist":{}}"#, - ) - .await - .unwrap(); + seed_npm_package(&context, "pkg").await; assert_eq!( send(&context.app, Method::GET, "/repository/npm-private/pkg", "",) diff --git a/nora-registry/src/registry/npm.rs b/nora-registry/src/registry/npm.rs index 98b53653..1d4ebccd 100644 --- a/nora-registry/src/registry/npm.rs +++ b/nora-registry/src/registry/npm.rs @@ -13,6 +13,11 @@ use crate::activity_log::{ActionType, ActivityEntry}; use crate::audit::AuditEntry; use crate::auth::{enforce_namespace_scope, AuthenticatedUser, NamespaceAuthority}; use crate::config::{NpmRepository, NpmWritePolicy}; +use crate::npm_layout::{ + HostedImportSession, HostedMaintenanceAction, HostedMaintenanceMarker, + HostedMaintenanceOperation, HostedMaintenanceTarget, HostedPackumentPointer, + HostedPublishPending, HostedPublishPendingTarget, +}; use crate::registry::{ circuit_open_response, method_not_allowed, proxy_fetch_conditional_with_validated_redirects, proxy_fetch_with_validated_redirects, proxy_fetch_with_validated_redirects_bounded, @@ -20,7 +25,7 @@ use crate::registry::{ }; use crate::registry_type::RegistryType; use crate::secrets::expose_opt; -use crate::storage::StorageError; +use crate::storage::{Storage, StorageError}; use crate::AppState; use axum::{ body::{Body, Bytes}, @@ -33,7 +38,7 @@ use axum::{ use base64::Engine; use futures::{stream, StreamExt}; use sha2::Digest; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::{Read, Write}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -45,12 +50,18 @@ const NPM_SEARCH_SCAN_TIMEOUT: Duration = Duration::from_secs(30); const PACKAGE_JSON_CAP: u64 = 2 * 1024 * 1024; const TAR_SCAN_CAP: u64 = 64 * 1024 * 1024; const MAX_NPM_PROXY_REDIRECTS: usize = 3; -const HOSTED_PACKUMENT_READ_CONCURRENCY: usize = 8; +const NPM_IMPORT_MUTATION_CONCURRENCY: usize = 32; +const NPM_IMPORT_PACKUMENT_HEADER: &str = "x-nora-import-packument-sha256"; const LEGACY_HOSTED: &str = "npm-private"; const LEGACY_PROXY: &str = "npm-registry"; pub fn routes() -> Router { Router::new() + .route( + "/repository/{repository}/-/nora/import/{package}", + axum::routing::put(named_import_finalize) + .fallback(|| async { method_not_allowed("PUT") }), + ) .route( "/repository/{repository}/-/package/{package}/dist-tags", get(named_dist_tags_get).fallback(|| async { method_not_allowed("GET") }), @@ -103,6 +114,7 @@ struct ProxyRepository { enum ReadError { NotFound, Unavailable, + MaterializationUnavailable, CircuitOpen(String), Corrupt, SearchScanLimit, @@ -138,15 +150,14 @@ async fn optional_storage_get(state: &AppState, key: &str) -> Result Result<(), StorageError> { - let key = crate::npm_layout::hosted_packument_cache_key(repository, package); - match state.storage.delete(&key).await { - Ok(()) | Err(StorageError::NotFound) => Ok(()), - Err(error) => Err(error), + key: &str, +) -> Result, ReadError> { + match state.storage.get(key).await { + Ok(bytes) => Ok(Some(bytes)), + Err(StorageError::NotFound) => Ok(None), + Err(_) => Err(ReadError::MaterializationUnavailable), } } @@ -197,22 +208,8 @@ fn hosted_publish_complete_key(repository: &str, package: &str, version: &str) - ) } -fn hosted_publish_pending_prefix(repository: &str, package: &str) -> String { - format!("{}/publish-pending/", package_prefix(repository, package)) -} - -fn hosted_publish_pending_key(repository: &str, package: &str, version: &str) -> String { - format!( - "{}{version}", - hosted_publish_pending_prefix(repository, package) - ) -} - fn hosted_publish_pending_index_key(repository: &str, package: &str) -> String { - format!( - "{}/publish-pending-index-v1", - package_prefix(repository, package) - ) + crate::npm_layout::hosted_publish_pending_index_key(repository, package) } fn hosted_tag_key(repository: &str, package: &str, tag: &str) -> String { @@ -227,69 +224,122 @@ fn hosted_deprecation_key(repository: &str, package: &str, version: &str) -> Str } fn hosted_package_key(repository: &str, package: &str) -> String { - format!("{}/pkg.json", package_prefix(repository, package)) + crate::npm_layout::hosted_package_key(repository, package) } -async fn pending_publish_versions( - state: &AppState, +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct HostedActiveTransactions { + pub(crate) import: Option, + pub(crate) publish: Option, +} + +fn parse_hosted_import_session( + bytes: &[u8], repository: &str, package: &str, -) -> Result, StorageError> { - let prefix = hosted_publish_pending_prefix(repository, package); - let mut incomplete = HashSet::new(); - for pending_key in state.storage.list(&prefix).await? { - let Some(version) = pending_key - .strip_prefix(&prefix) - .filter(|relative| !relative.contains('/')) - .filter(|version| !version.is_empty()) - else { - return Err(StorageError::IntegrityViolation); - }; - let expected = state.storage.get(&pending_key).await?; - let expected = std::str::from_utf8(&expected) - .ok() - .filter(|digest| { - digest.len() == 64 - && digest - .bytes() - .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) - }) - .ok_or(StorageError::IntegrityViolation)?; - let manifest = match state - .storage - .get(&hosted_version_key(repository, package, version)) - .await - { - Ok(manifest) => manifest, - Err(StorageError::NotFound) => { - incomplete.insert(version.to_string()); - continue; - } - Err(error) => return Err(error), - }; - let manifest_digest = crate::npm_layout::hosted_manifest_digest(&manifest); - if manifest_digest != expected { - incomplete.insert(version.to_string()); - continue; - } - match state - .storage - .get(&hosted_publish_complete_key(repository, package, version)) - .await - { - Ok(completion) if completion.as_ref() == expected.as_bytes() => { - match state.storage.delete(&pending_key).await { - Ok(()) | Err(StorageError::NotFound) => {} - Err(error) => return Err(error), - } - } - Ok(_) | Err(StorageError::NotFound) => { - incomplete.insert(version.to_string()); - } - Err(error) => return Err(error), +) -> Result { + let session: HostedImportSession = + serde_json::from_slice(bytes).map_err(|_| StorageError::IntegrityViolation)?; + if session.schema != crate::npm_layout::HOSTED_IMPORT_SESSION_SCHEMA_V1 + || session.repository != repository + || session.package != package + || !valid_sha256(&session.packument_sha256) + || session + .base + .as_ref() + .is_some_and(|base| !valid_hosted_packument_pointer(base)) + || session.versions.is_empty() + || !session + .versions + .iter() + .all(|(version, digest)| is_valid_npm_version(version) && valid_sha256(digest)) + { + return Err(StorageError::IntegrityViolation); + } + Ok(session) +} + +fn valid_sha512_hex(value: &str) -> bool { + value.len() == 128 + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) +} + +fn parse_hosted_publish_pending( + bytes: &[u8], + repository: &str, + package: &str, +) -> Result { + let pending: HostedPublishPending = + serde_json::from_slice(bytes).map_err(|_| StorageError::IntegrityViolation)?; + let valid_target = match &pending.target { + HostedPublishPendingTarget::Publish { base, target } => { + base.as_ref().is_none_or(valid_hosted_packument_pointer) + && valid_hosted_packument_pointer(target) } + HostedPublishPendingTarget::Import { packument_sha256 } => valid_sha256(packument_sha256), + }; + if pending.schema != crate::npm_layout::HOSTED_PUBLISH_PENDING_SCHEMA_V1 + || pending.repository != repository + || pending.package != package + || !is_valid_npm_version(&pending.version) + || !valid_sha256(&pending.manifest_sha256) + || !valid_sha512_hex(&pending.blob_sha512) + || !valid_target + { + return Err(StorageError::IntegrityViolation); + } + Ok(pending) +} + +async fn read_optional_import_session( + storage: &Storage, + repository: &str, + package: &str, +) -> Result, StorageError> { + match storage + .get(&crate::npm_layout::hosted_import_pending_key( + repository, package, + )) + .await + { + Ok(bytes) => parse_hosted_import_session(&bytes, repository, package).map(Some), + Err(StorageError::NotFound) => Ok(None), + Err(error) => Err(error), + } +} + +async fn read_optional_publish_pending( + storage: &Storage, + repository: &str, + package: &str, +) -> Result, StorageError> { + match storage + .get(&hosted_publish_pending_index_key(repository, package)) + .await + { + Ok(bytes) => parse_hosted_publish_pending(&bytes, repository, package).map(Some), + Err(StorageError::NotFound) => Ok(None), + Err(error) => Err(error), } - Ok(incomplete) +} + +/// Exact package transaction probe shared by mutation, retention and GC. +/// Malformed records are integrity errors rather than apparent quiescence. +pub(crate) async fn read_hosted_active_transactions( + storage: &Storage, + repository: &str, + package: &str, +) -> Result { + let (import, publish) = tokio::join!( + read_optional_import_session(storage, repository, package), + read_optional_publish_pending(storage, repository, package), + ); + Ok(HostedActiveTransactions { + import: import?, + publish: publish?, + }) } async fn incomplete_publish_versions( @@ -297,28 +347,39 @@ async fn incomplete_publish_versions( repository: &str, package: &str, ) -> Result, StorageError> { - let index_key = hosted_publish_pending_index_key(repository, package); - match state.storage.get(&index_key).await { - Ok(index) if index.as_ref() == b"1" => {} - Ok(_) => return Err(StorageError::IntegrityViolation), - Err(StorageError::NotFound) => state.storage.put(&index_key, b"1").await?, - Err(error) => return Err(error), - } - pending_publish_versions(state, repository, package).await + Ok( + read_optional_publish_pending(&state.storage, repository, package) + .await? + .into_iter() + .map(|pending| pending.version) + .collect(), + ) } -async fn restore_publish_pending( - state: &AppState, - pending_key: &str, - previous: Option<&[u8]>, +async fn create_hosted_publish_pending( + storage: &Storage, + pending: &HostedPublishPending, ) -> Result<(), StorageError> { - match previous { - Some(previous) => state.storage.put(pending_key, previous).await, - None => match state.storage.delete(pending_key).await { - Ok(()) | Err(StorageError::NotFound) => Ok(()), - Err(error) => Err(error), - }, - } + let bytes = serde_json::to_vec(pending).map_err(|_| StorageError::IntegrityViolation)?; + put_immutable_storage( + storage, + &hosted_publish_pending_index_key(&pending.repository, &pending.package), + &bytes, + ) + .await +} + +async fn clear_hosted_publish_pending( + storage: &Storage, + pending: &HostedPublishPending, +) -> Result<(), StorageError> { + let bytes = serde_json::to_vec(pending).map_err(|_| StorageError::IntegrityViolation)?; + delete_exact_with_readback( + storage, + &hosted_publish_pending_index_key(&pending.repository, &pending.package), + &bytes, + ) + .await } fn incomplete_publish_response() -> Response { @@ -329,6 +390,201 @@ fn incomplete_publish_response() -> Response { .into_response() } +async fn current_full_for_mutation( + state: &AppState, + repository: &str, + package: &str, +) -> Result, StorageError> { + let pointer_key = crate::npm_layout::hosted_packument_current_key(repository, package); + let pointer = match state.storage.get(&pointer_key).await { + Ok(pointer) => pointer, + Err(StorageError::NotFound) => { + let package_exists = match state + .storage + .get(&hosted_package_key(repository, package)) + .await + { + Ok(_) => true, + Err(StorageError::NotFound) => false, + Err(error) => return Err(error), + }; + if package_exists { + return Err(StorageError::IntegrityViolation); + } + return Ok(None); + } + Err(error) => return Err(error), + }; + let pointer: HostedPackumentPointer = + serde_json::from_slice(&pointer).map_err(|_| StorageError::IntegrityViolation)?; + if !valid_sha256(&pointer.generation) + || pointer.generation != pointer.full_sha256 + || !valid_sha256(&pointer.install_v1_sha256) + { + return Err(StorageError::IntegrityViolation); + } + let full = state + .storage + .get(&crate::npm_layout::hosted_packument_full_key( + repository, + package, + &pointer.generation, + )) + .await?; + if hex::encode(sha2::Sha256::digest(&full)) != pointer.full_sha256 { + return Err(StorageError::IntegrityViolation); + } + valid_hosted_packument(&full, package) + .map(Some) + .ok_or(StorageError::IntegrityViolation) +} + +async fn hosted_import_active( + storage: &Storage, + repository: &str, + package: &str, +) -> Result { + read_optional_import_session(storage, repository, package) + .await + .map(|session| session.is_some()) +} + +fn packument_after_publish( + package: &str, + previous: Option, + validated: &ValidatedPublish, +) -> Result { + let mut packument = previous + .unwrap_or_else(|| serde_json::json!({"name": package, "versions": {}, "dist-tags": {}})); + let object = packument + .as_object_mut() + .ok_or(StorageError::IntegrityViolation)?; + for field in ["name", "_id", "description", "readme", "license"] { + object.remove(field); + } + let package_fields: serde_json::Value = serde_json::from_slice(&validated.package_fields) + .map_err(|_| StorageError::IntegrityViolation)?; + for (field, value) in package_fields + .as_object() + .ok_or(StorageError::IntegrityViolation)? + { + object.insert(field.clone(), value.clone()); + } + object.insert( + "name".to_string(), + serde_json::Value::String(package.to_string()), + ); + + let mut manifest: serde_json::Value = serde_json::from_slice(&validated.manifest) + .map_err(|_| StorageError::IntegrityViolation)?; + let previous_deprecation = object + .get("versions") + .and_then(serde_json::Value::as_object) + .and_then(|versions| versions.get(&validated.version)) + .and_then(|manifest| manifest.get("deprecated")) + .cloned(); + let manifest_object = manifest + .as_object_mut() + .ok_or(StorageError::IntegrityViolation)?; + match validated.deprecation.as_deref() { + Some("") => { + manifest_object.remove("deprecated"); + } + Some(message) => { + manifest_object.insert( + "deprecated".to_string(), + serde_json::Value::String(message.to_string()), + ); + } + None => { + if let Some(message) = previous_deprecation { + manifest_object.insert("deprecated".to_string(), message); + } + } + } + object + .entry("versions") + .or_insert_with(|| serde_json::json!({})) + .as_object_mut() + .ok_or(StorageError::IntegrityViolation)? + .insert(validated.version.clone(), manifest); + let tags = object + .entry("dist-tags") + .or_insert_with(|| serde_json::json!({})) + .as_object_mut() + .ok_or(StorageError::IntegrityViolation)?; + for (tag, target) in &validated.tags { + tags.insert(tag.clone(), serde_json::Value::String(target.clone())); + } + Ok(packument) +} + +fn hosted_packument_pointer_for_value( + packument: &serde_json::Value, +) -> Result<(Vec, HostedPackumentPointer), StorageError> { + let full = serde_json::to_vec(packument).map_err(|_| StorageError::IntegrityViolation)?; + let install_v1 = install_v1_packument(packument) + .map_err(|_| StorageError::IntegrityViolation) + .and_then(|value| { + serde_json::to_vec(&value).map_err(|_| StorageError::IntegrityViolation) + })?; + let full_sha256 = hex::encode(sha2::Sha256::digest(&full)); + Ok(( + full, + HostedPackumentPointer { + generation: full_sha256.clone(), + full_sha256, + install_v1_sha256: hex::encode(sha2::Sha256::digest(&install_v1)), + }, + )) +} + +async fn ensure_completed_publish_materialized( + state: &AppState, + repository: &str, + package: &str, + version: &str, +) -> Result<(), StorageError> { + if current_full_for_mutation(state, repository, package) + .await + .ok() + .flatten() + .and_then(|packument| { + packument + .get("versions") + .and_then(serde_json::Value::as_object) + .map(|versions| versions.contains_key(version)) + }) + == Some(true) + { + return Ok(()); + } + Err(StorageError::IntegrityViolation) +} + +pub(crate) async fn read_hosted_packument_pointer( + storage: &Storage, + repository: &str, + package: &str, +) -> Result, StorageError> { + let bytes = match storage + .get(&crate::npm_layout::hosted_packument_current_key( + repository, package, + )) + .await + { + Ok(bytes) => bytes, + Err(StorageError::NotFound) => return Ok(None), + Err(error) => return Err(error), + }; + let pointer: HostedPackumentPointer = + serde_json::from_slice(&bytes).map_err(|_| StorageError::IntegrityViolation)?; + if !valid_hosted_packument_pointer(&pointer) { + return Err(StorageError::IntegrityViolation); + } + Ok(Some(pointer)) +} + fn proxy_packument_key(repository: &str, package: &str) -> String { format!( "{}/proxy/packuments/{package}.json", @@ -531,6 +787,14 @@ fn set_tarball_url( } fn json_response(headers: &HeaderMap, value: &serde_json::Value) -> Response { + json_response_with_content_type(headers, value, "application/json") +} + +fn json_response_with_content_type( + headers: &HeaderMap, + value: &serde_json::Value, + content_type: &'static str, +) -> Response { let Ok(bytes) = serde_json::to_vec(value) else { return StatusCode::INTERNAL_SERVER_ERROR.into_response(); }; @@ -545,7 +809,7 @@ fn json_response(headers: &HeaderMap, value: &serde_json::Value) -> Response { ( StatusCode::OK, [ - (header::CONTENT_TYPE, "application/json".to_string()), + (header::CONTENT_TYPE, content_type.to_string()), (header::CACHE_CONTROL, "no-cache".to_string()), (header::ETAG, etag), ], @@ -568,6 +832,24 @@ fn json_response_with_stale( response } +fn packument_response( + headers: &HeaderMap, + value: &serde_json::Value, + stale: bool, + flavor: PackumentFlavor, +) -> Response { + let mut response = json_response_with_content_type(headers, value, flavor.content_type()); + response + .headers_mut() + .insert(header::VARY, HeaderValue::from_static("Accept")); + if stale { + response + .headers_mut() + .insert("x-nora-stale", HeaderValue::from_static("true")); + } + response +} + fn tarball_response(data: Bytes) -> Response { ( StatusCode::OK, @@ -626,11 +908,13 @@ async fn hosted_blob_key_for_version( package: &str, version: &str, ) -> Result { - let manifest = state - .storage - .get(&hosted_version_key(repository, package, version)) - .await - .map_err(storage_read_error)?; + let manifest = match hosted_version_from_current(state, repository, package, version).await? { + HostedVersionResolution::Visible(manifest) => manifest, + HostedVersionResolution::Absent | HostedVersionResolution::AuthoritativelyAbsent => { + return Err(ReadError::NotFound) + } + }; + let manifest = serde_json::to_vec(&manifest).map_err(|_| ReadError::Corrupt)?; crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) .ok_or(ReadError::Corrupt) } @@ -657,14 +941,21 @@ async fn target_publish_date( }; match repository { NpmRepository::Hosted { name, .. } => { - if hosted_has_version(state, name, package, version).await? { - let blob_key = - hosted_blob_key_for_version(state, name, package, version).await?; - return Ok(crate::curation::extract_mtime_as_publish_date( - &state.storage, - &blob_key, - ) - .await); + match hosted_version_from_current(state, name, package, version).await? { + HostedVersionResolution::Visible(_) => { + let blob_key = + hosted_blob_key_for_version(state, name, package, version) + .await?; + return Ok(crate::curation::extract_mtime_as_publish_date( + &state.storage, + &blob_key, + ) + .await); + } + HostedVersionResolution::AuthoritativelyAbsent => { + return Err(ReadError::NotFound) + } + HostedVersionResolution::Absent => {} } } NpmRepository::Proxy { name, .. } if !is_internal(state, package) => { @@ -694,15 +985,22 @@ async fn target_publish_date( Ok(None) } RepositoryTarget::Legacy => { - if hosted_has_version(state, LEGACY_HOSTED, package, version).await? { - let blob_key = - hosted_blob_key_for_version(state, LEGACY_HOSTED, package, version).await?; - Ok(crate::curation::extract_mtime_as_publish_date(&state.storage, &blob_key).await) - } else { - Ok( - cached_proxy_publish_date(state, LEGACY_PROXY, package, version, filename) - .await, - ) + match hosted_version_from_current(state, LEGACY_HOSTED, package, version).await? { + HostedVersionResolution::Visible(_) => { + let blob_key = + hosted_blob_key_for_version(state, LEGACY_HOSTED, package, version).await?; + Ok( + crate::curation::extract_mtime_as_publish_date(&state.storage, &blob_key) + .await, + ) + } + HostedVersionResolution::AuthoritativelyAbsent => Err(ReadError::NotFound), + HostedVersionResolution::Absent => { + Ok( + cached_proxy_publish_date(state, LEGACY_PROXY, package, version, filename) + .await, + ) + } } } } @@ -752,299 +1050,1007 @@ fn curated_tarball_response( tarball_response(data) } -fn valid_cached_hosted_packument(bytes: &[u8], package: &str) -> Option { - let value = serde_json::from_slice::(bytes).ok()?; - let object = value.as_object()?; - (object.get("name").and_then(|value| value.as_str()) == Some(package) - && object - .get("versions") - .is_some_and(|value| value.is_object()) - && object - .get("dist-tags") - .is_some_and(|value| value.is_object())) - .then_some(value) +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PackumentFlavor { + Full, + InstallV1, } -fn hosted_packument_response( - mut packument: serde_json::Value, - package: &str, - response_base: &str, -) -> serde_json::Value { - if let Some(versions) = packument - .get_mut("versions") - .and_then(|value| value.as_object_mut()) - { - for (version, manifest) in versions { - set_tarball_url(manifest, response_base, package, version); +impl PackumentFlavor { + fn from_headers(headers: &HeaderMap) -> Self { + // Store the quality selected by the most-specific media range for + // each representation. An exact q=0 must therefore override a more + // permissive wildcard instead of being resurrected by it. + let mut install_match = None::<(u8, f32)>; + let mut full_match = None::<(u8, f32)>; + let update_match = |current: &mut Option<(u8, f32)>, specificity, quality| match current { + Some((current_specificity, current_quality)) if *current_specificity > specificity => {} + Some((current_specificity, current_quality)) if *current_specificity == specificity => { + *current_quality = current_quality.max(quality); + } + _ => *current = Some((specificity, quality)), + }; + for range in headers + .get_all(header::ACCEPT) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + { + let mut parts = range.split(';'); + let media_type = parts.next().unwrap_or_default().trim(); + let mut quality = 1.0f32; + for parameter in parts { + let Some((name, value)) = parameter.trim().split_once('=') else { + continue; + }; + if name.trim().eq_ignore_ascii_case("q") { + quality = value + .trim() + .parse::() + .ok() + .filter(|quality| (0.0..=1.0).contains(quality)) + .unwrap_or(0.0); + } + } + if media_type.eq_ignore_ascii_case("application/vnd.npm.install-v1+json") { + update_match(&mut install_match, 2, quality); + } else if media_type.eq_ignore_ascii_case("application/json") { + update_match(&mut full_match, 2, quality); + } else if media_type.eq_ignore_ascii_case("application/*") { + update_match(&mut install_match, 1, quality); + update_match(&mut full_match, 1, quality); + } else if media_type == "*/*" { + update_match(&mut install_match, 0, quality); + update_match(&mut full_match, 0, quality); + } + } + let full_quality = full_match.map(|(_, quality)| quality).unwrap_or(0.0); + if install_match.is_some_and(|(specificity, quality)| { + quality > 0.0 + && (quality > full_quality || (quality == full_quality && specificity == 2)) + }) { + Self::InstallV1 + } else { + Self::Full } } - packument -} -async fn read_hosted_packument_cache( - state: &AppState, - repository: &str, - package: &str, -) -> Result, ReadError> { - let key = crate::npm_layout::hosted_packument_cache_key(repository, package); - let Some(bytes) = optional_storage_get(state, &key).await? else { - return Ok(None); - }; - match valid_cached_hosted_packument(&bytes, package) { - Some(packument) => Ok(Some(packument)), - None => { - tracing::warn!( - repository, - package, - key, - "ignoring invalid rebuildable npm hosted packument cache" - ); - Ok(None) + const fn content_type(self) -> &'static str { + match self { + Self::Full => "application/json", + Self::InstallV1 => "application/vnd.npm.install-v1+json", } } } -async fn build_hosted_packument( - state: &AppState, - repository: &str, - package: &str, -) -> Result { - let prefix = package_prefix(repository, package); - let version_prefix = format!("{prefix}/versions/"); - let version_keys = state - .storage - .list(&version_prefix) - .await - .map_err(|_| ReadError::Unavailable)?; - let deprecation_prefix = format!("{prefix}/deprecations/"); - let deprecation_keys = state - .storage - .list(&deprecation_prefix) - .await - .map_err(|_| ReadError::Unavailable)? - .into_iter() - .filter_map(|key| { - let version = key.strip_prefix(&deprecation_prefix)?.to_string(); - (!version.is_empty() && !version.contains('/')).then_some((version, key)) - }) - .collect::>(); - - let version_reads = stream::iter(version_keys.into_iter().filter_map(|key| { - let version = key - .strip_prefix(&version_prefix)? - .strip_suffix(".json")? - .to_string(); - (!version.is_empty() && !version.contains('/')).then_some((version, key)) - })) - .map(|(version, key)| { - let deprecation_key = deprecation_keys.get(&version).cloned(); - async move { - let bytes = state.storage.get(&key).await.map_err(|error| { - match storage_read_error(error) { - // LIST advertised this committed manifest. Disappearing between - // LIST and GET is an incomplete hosted read, not permission to - // let a lower-priority group member shadow the version. - ReadError::NotFound => ReadError::Unavailable, - other => other, +#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct HostedImportReceipt { + package: String, + version_count: usize, + full_sha256: String, + install_v1_sha256: String, + generation: String, +} + +type WrittenPackumentGeneration = HostedPackumentPointer; + +fn valid_sha256(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) +} + +fn valid_hosted_packument_pointer(pointer: &HostedPackumentPointer) -> bool { + valid_sha256(&pointer.generation) + && pointer.generation == pointer.full_sha256 + && valid_sha256(&pointer.install_v1_sha256) +} + +fn valid_hosted_maintenance_operation(operation: &HostedMaintenanceOperation) -> bool { + if operation.schema != crate::npm_layout::HOSTED_MAINTENANCE_SCHEMA_V1 + || operation.repository.is_empty() + || operation.package.is_empty() + || !valid_hosted_packument_pointer(&operation.base) + { + return false; + } + if let HostedMaintenanceTarget::Live { pointer } = &operation.target { + if !valid_hosted_packument_pointer(pointer) { + return false; + } + } + match &operation.action { + HostedMaintenanceAction::DistTag { tag, value } => { + matches!(operation.target, HostedMaintenanceTarget::Live { .. }) + && is_valid_dist_tag(tag) + && value.as_deref().is_none_or(is_valid_npm_version) + } + HostedMaintenanceAction::Deprecations { values } => { + matches!(operation.target, HostedMaintenanceTarget::Live { .. }) + && !values.is_empty() + && values.iter().all(|(version, message)| { + is_valid_npm_version(version) + && message.as_deref().is_none_or(|message| !message.is_empty()) + }) + } + HostedMaintenanceAction::Retention { + snapshot_guard, + removed_versions, + expected_authority, + } => { + if !valid_sha256(snapshot_guard) + || removed_versions.is_empty() + || !removed_versions + .iter() + .all(|(version, digest)| is_valid_npm_version(version) && valid_sha256(digest)) + || expected_authority.is_empty() + { + return false; + } + expected_authority.iter().all(|(key, digest)| { + if !valid_sha256(digest) { + return false; } - })?; - let mut value = serde_json::from_slice::(&bytes).map_err(|_| { - crate::metrics::METADATA_CORRUPT_TOTAL - .with_label_values(&["npm"]) - .inc(); - ReadError::Corrupt - })?; - if let Some(deprecation_key) = deprecation_key { - let deprecated = state - .storage - .get(&deprecation_key) - .await - .map_err(storage_read_error)?; - let Some(message) = read_string(deprecated) else { - return Err(ReadError::Corrupt); - }; - let Some(object) = value.as_object_mut() else { - return Err(ReadError::Corrupt); + let Some(parsed) = crate::npm_layout::parse_npm_object_key(key) else { + return false; }; - object.insert("deprecated".to_string(), serde_json::Value::String(message)); - } - Ok::<_, ReadError>((version, value)) + parsed.repository == operation.repository + && parsed.package == operation.package + && matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedPackage + | crate::npm_layout::NpmObjectKind::HostedVersion(_) + | crate::npm_layout::NpmObjectKind::HostedPublishComplete(_) + | crate::npm_layout::NpmObjectKind::HostedPublishPendingIndex + | crate::npm_layout::NpmObjectKind::HostedDistTag(_) + | crate::npm_layout::NpmObjectKind::HostedDeprecation(_) + ) + }) } - }) - .buffered(HOSTED_PACKUMENT_READ_CONCURRENCY) - .collect::>() - .await; - - let mut versions = serde_json::Map::new(); - for result in version_reads { - let (version, value) = result?; - versions.insert(version, value); } +} - let mut tags = serde_json::Map::new(); - let tag_prefix = format!("{prefix}/dist-tags/"); - for key in state - .storage - .list(&tag_prefix) - .await - .map_err(|_| ReadError::Unavailable)? - { - let Some(tag) = key.rsplit('/').next() else { - continue; - }; - let value = - state - .storage - .get(&key) - .await - .map_err(|error| match storage_read_error(error) { - ReadError::NotFound => ReadError::Unavailable, - other => other, - })?; - let Some(version) = read_string(value) else { - return Err(ReadError::Corrupt); - }; - tags.insert(tag.to_string(), serde_json::Value::String(version)); +fn hosted_maintenance_marker_for_operation( + operation: &HostedMaintenanceOperation, +) -> Result { + if !valid_hosted_maintenance_operation(operation) { + return Err(StorageError::IntegrityViolation); } + let operation_id = crate::npm_layout::hosted_maintenance_operation_id(operation) + .map_err(|_| StorageError::IntegrityViolation)?; + Ok(HostedMaintenanceMarker { + schema: operation.schema, + repository: operation.repository.clone(), + package: operation.package.clone(), + operation_id, + base: operation.base.clone(), + target: operation.target.clone(), + action: operation.action.clone(), + }) +} - let package_fields = - match optional_storage_get(state, &hosted_package_key(repository, package)).await? { - Some(bytes) => Some(serde_json::from_slice::(&bytes).map_err( - |_| { - crate::metrics::METADATA_CORRUPT_TOTAL - .with_label_values(&["npm"]) - .inc(); - ReadError::Corrupt - }, - )?), - None => None, - }; +fn valid_hosted_maintenance_marker( + marker: &HostedMaintenanceMarker, + repository: &str, + package: &str, +) -> bool { + marker.repository == repository + && marker.package == package + && valid_sha256(&marker.operation_id) + && valid_hosted_maintenance_operation(&marker.operation()) + && crate::npm_layout::hosted_maintenance_operation_id(&marker.operation()) + .is_ok_and(|operation_id| operation_id == marker.operation_id) +} - if versions.is_empty() && tags.is_empty() && package_fields.is_none() { - return Err(ReadError::NotFound); +pub(crate) async fn read_hosted_maintenance_marker( + storage: &Storage, + repository: &str, + package: &str, +) -> Result, StorageError> { + let key = crate::npm_layout::hosted_maintenance_active_key(repository, package); + let bytes = match storage.get(&key).await { + Ok(bytes) => bytes, + Err(StorageError::NotFound) => return Ok(None), + Err(error) => return Err(error), + }; + let marker: HostedMaintenanceMarker = + serde_json::from_slice(&bytes).map_err(|_| StorageError::IntegrityViolation)?; + if !valid_hosted_maintenance_marker(&marker, repository, package) { + return Err(StorageError::IntegrityViolation); + } + Ok(Some(marker)) +} + +pub(crate) async fn create_hosted_maintenance_marker( + storage: &Storage, + operation: &HostedMaintenanceOperation, +) -> Result { + let marker = hosted_maintenance_marker_for_operation(operation)?; + let bytes = serde_json::to_vec(&marker).map_err(|_| StorageError::IntegrityViolation)?; + let key = + crate::npm_layout::hosted_maintenance_active_key(&operation.repository, &operation.package); + let create = storage.put_if_absent(&key, &bytes).await; + match storage.get(&key).await { + Ok(stored) if stored.as_ref() == bytes.as_slice() => Ok(marker), + Ok(_) => match create { + Err(StorageError::AlreadyExists) => Err(StorageError::AlreadyExists), + _ => Err(StorageError::IntegrityViolation), + }, + Err(StorageError::NotFound) => match create { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Err(error) => Err(error), } +} - let mut packument = package_fields.unwrap_or_else(|| serde_json::json!({})); - let Some(object) = packument.as_object_mut() else { - return Err(ReadError::Corrupt); - }; - object.insert( - "name".to_string(), - serde_json::Value::String(package.to_string()), - ); - object.insert("versions".to_string(), serde_json::Value::Object(versions)); - object.insert("dist-tags".to_string(), serde_json::Value::Object(tags)); - Ok(packument) +pub(crate) async fn clear_hosted_maintenance_marker( + storage: &Storage, + marker: &HostedMaintenanceMarker, +) -> Result<(), StorageError> { + if !valid_hosted_maintenance_marker(marker, &marker.repository, &marker.package) { + return Err(StorageError::IntegrityViolation); + } + let expected = serde_json::to_vec(marker).map_err(|_| StorageError::IntegrityViolation)?; + let key = crate::npm_layout::hosted_maintenance_active_key(&marker.repository, &marker.package); + match storage.get(&key).await { + Ok(current) if current.as_ref() == expected.as_slice() => {} + Ok(_) => return Err(StorageError::AlreadyExists), + Err(StorageError::NotFound) => return Ok(()), + Err(error) => return Err(error), + } + let deleted = storage.delete(&key).await; + match storage.get(&key).await { + Err(StorageError::NotFound) => Ok(()), + Ok(current) if current.as_ref() == expected.as_slice() => match deleted { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Ok(_) => Err(StorageError::AlreadyExists), + Err(error) => Err(error), + } } -async fn hosted_packument( - state: &AppState, +async fn hosted_packument_for_pointer( + storage: &Storage, repository: &str, package: &str, - response_base: &str, -) -> Result { - if let Some(packument) = read_hosted_packument_cache(state, repository, package).await? { - return Ok(hosted_packument_response(packument, package, response_base)); + pointer: &HostedPackumentPointer, +) -> Result { + validate_hosted_packument_pointer(storage, repository, package, pointer).await?; + let full = storage + .get(&crate::npm_layout::hosted_packument_full_key( + repository, + package, + &pointer.generation, + )) + .await?; + valid_hosted_packument(&full, package).ok_or(StorageError::IntegrityViolation) +} + +fn apply_hosted_maintenance_action( + mut packument: serde_json::Value, + action: &HostedMaintenanceAction, +) -> Result { + match action { + HostedMaintenanceAction::DistTag { tag, value } => { + let tags = packument + .get_mut("dist-tags") + .and_then(serde_json::Value::as_object_mut) + .ok_or(StorageError::IntegrityViolation)?; + match value { + Some(version) => { + tags.insert(tag.clone(), serde_json::Value::String(version.clone())); + } + None => { + tags.remove(tag); + } + } + } + HostedMaintenanceAction::Deprecations { values } => { + let versions = packument + .get_mut("versions") + .and_then(serde_json::Value::as_object_mut) + .ok_or(StorageError::IntegrityViolation)?; + for (version, message) in values { + let manifest = versions + .get_mut(version) + .and_then(serde_json::Value::as_object_mut) + .ok_or(StorageError::IntegrityViolation)?; + match message { + Some(message) => { + manifest.insert( + "deprecated".to_string(), + serde_json::Value::String(message.clone()), + ); + } + None => { + manifest.remove("deprecated"); + } + } + } + } + HostedMaintenanceAction::Retention { .. } => return Err(StorageError::IntegrityViolation), } + Ok(packument) +} - // A cold read shares the exact package lock used by hosted mutations. - // This prevents a cache miss from materializing stale state after a - // concurrent mutation invalidated the previous cache. - let lock = state.publish_lock(&format!("npm:{repository}:{package}")); - let _guard = lock.lock().await; - if let Some(packument) = read_hosted_packument_cache(state, repository, package).await? { - return Ok(hosted_packument_response(packument, package, response_base)); +fn optional_packument_string( + packument: &serde_json::Value, + action: &HostedMaintenanceAction, +) -> Result)>, StorageError> { + match action { + HostedMaintenanceAction::DistTag { tag, .. } => { + let value = packument + .get("dist-tags") + .and_then(serde_json::Value::as_object) + .ok_or(StorageError::IntegrityViolation)? + .get(tag) + .map(|value| { + value + .as_str() + .map(str::to_string) + .ok_or(StorageError::IntegrityViolation) + }) + .transpose()?; + Ok(vec![(tag.clone(), value)]) + } + HostedMaintenanceAction::Deprecations { values } => { + let versions = packument + .get("versions") + .and_then(serde_json::Value::as_object) + .ok_or(StorageError::IntegrityViolation)?; + values + .keys() + .map(|version| { + let manifest = versions + .get(version) + .and_then(serde_json::Value::as_object) + .ok_or(StorageError::IntegrityViolation)?; + let message = manifest + .get("deprecated") + .map(|value| { + value + .as_str() + .map(str::to_string) + .ok_or(StorageError::IntegrityViolation) + }) + .transpose()?; + Ok((version.clone(), message)) + }) + .collect() + } + HostedMaintenanceAction::Retention { .. } => Err(StorageError::IntegrityViolation), } +} - let packument = build_hosted_packument(state, repository, package).await?; - let key = crate::npm_layout::hosted_packument_cache_key(repository, package); - match serde_json::to_vec(&packument) { - Ok(bytes) => { - if let Err(error) = state.storage.put(&key, &bytes).await { - // The cache is derived. A failed cache write must not turn a - // complete authoritative read into an unavailable response. - tracing::warn!( - repository, - package, - key, - ?error, - "failed to persist rebuildable npm hosted packument cache" - ); - } +async fn read_optional_string( + storage: &Storage, + key: &str, +) -> Result, StorageError> { + match storage.get(key).await { + Ok(bytes) => read_string(bytes) + .map(Some) + .ok_or(StorageError::IntegrityViolation), + Err(StorageError::NotFound) => Ok(None), + Err(error) => Err(error), + } +} + +async fn commit_authority_value( + storage: &Storage, + key: &str, + base: Option<&str>, + target: Option<&str>, +) -> Result<(), StorageError> { + let current = read_optional_string(storage, key).await?; + if current.as_deref() == target { + return Ok(()); + } + if current.as_deref() != base { + return Err(StorageError::AlreadyExists); + } + let mutation = match target { + Some(value) => storage.put(key, value.as_bytes()).await, + None => storage.delete(key).await, + }; + let readback = read_optional_string(storage, key).await; + match readback { + Ok(value) if value.as_deref() == target => Ok(()), + Ok(value) if value.as_deref() != base => Err(StorageError::AlreadyExists), + Ok(_) => match mutation { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Err(error) => Err(error), + } +} + +async fn resume_hosted_metadata_maintenance( + storage: &Storage, + marker: &HostedMaintenanceMarker, +) -> Result<(), StorageError> { + let HostedMaintenanceTarget::Live { pointer: target } = &marker.target else { + return Err(StorageError::IntegrityViolation); + }; + let base_packument = + hosted_packument_for_pointer(storage, &marker.repository, &marker.package, &marker.base) + .await?; + let expected_packument = + apply_hosted_maintenance_action(base_packument.clone(), &marker.action)?; + let expected_full = + serde_json::to_vec(&expected_packument).map_err(|_| StorageError::IntegrityViolation)?; + if hex::encode(sha2::Sha256::digest(&expected_full)) != target.full_sha256 { + return Err(StorageError::IntegrityViolation); + } + let expected_install = install_v1_packument(&expected_packument) + .map_err(|_| StorageError::IntegrityViolation) + .and_then(|value| { + serde_json::to_vec(&value).map_err(|_| StorageError::IntegrityViolation) + })?; + if hex::encode(sha2::Sha256::digest(&expected_install)) != target.install_v1_sha256 { + return Err(StorageError::IntegrityViolation); + } + validate_hosted_packument_pointer(storage, &marker.repository, &marker.package, target).await?; + match read_hosted_packument_pointer(storage, &marker.repository, &marker.package).await? { + Some(current) if current == marker.base || current == *target => {} + _ => return Err(StorageError::AlreadyExists), + } + + let base_values = optional_packument_string(&base_packument, &marker.action)?; + match &marker.action { + HostedMaintenanceAction::DistTag { tag, value } => { + let (_, base) = base_values + .first() + .ok_or(StorageError::IntegrityViolation)?; + commit_authority_value( + storage, + &hosted_tag_key(&marker.repository, &marker.package, tag), + base.as_deref(), + value.as_deref(), + ) + .await?; } - Err(error) => { - tracing::error!( - repository, - package, - ?error, - "failed to serialize npm packument" - ); - return Err(ReadError::Corrupt); + HostedMaintenanceAction::Deprecations { values } => { + let base_values = base_values.into_iter().collect::>(); + for (version, target) in values { + let base = base_values + .get(version) + .ok_or(StorageError::IntegrityViolation)?; + commit_authority_value( + storage, + &hosted_deprecation_key(&marker.repository, &marker.package, version), + base.as_deref(), + target.as_deref(), + ) + .await?; + } } + HostedMaintenanceAction::Retention { .. } => return Err(StorageError::IntegrityViolation), } - Ok(hosted_packument_response(packument, package, response_base)) + commit_hosted_packument_pointer(storage, &marker.repository, &marker.package, target).await } -fn negative_fresh(modified: u64, ttl: i64) -> bool { - if ttl <= 0 { - return false; +pub(crate) async fn resume_hosted_maintenance_operation( + storage: &Storage, + repository: &str, + package: &str, +) -> Result { + let Some(marker) = read_hosted_maintenance_marker(storage, repository, package).await? else { + return Ok(false); + }; + match &marker.action { + HostedMaintenanceAction::DistTag { .. } | HostedMaintenanceAction::Deprecations { .. } => { + resume_hosted_metadata_maintenance(storage, &marker).await? + } + HostedMaintenanceAction::Retention { .. } => { + crate::retention::resume_npm_retention_operation(storage, &marker).await? + } } - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - now.saturating_sub(modified) < ttl as u64 + clear_hosted_maintenance_marker(storage, &marker).await?; + Ok(true) } -async fn proxy_packument_raw( - state: &AppState, - repository: &ProxyRepository, +async fn execute_hosted_metadata_maintenance( + storage: &Storage, + repository: &str, package: &str, -) -> Result { - if is_internal(state, package) { - return Err(ReadError::NotFound); + base: HostedPackumentPointer, + target_packument: &serde_json::Value, + action: HostedMaintenanceAction, +) -> Result<(), StorageError> { + let full = + serde_json::to_vec(target_packument).map_err(|_| StorageError::IntegrityViolation)?; + let target = write_hosted_packument_generation_documents( + storage, + repository, + package, + target_packument, + &full, + ) + .await?; + let operation = HostedMaintenanceOperation { + schema: crate::npm_layout::HOSTED_MAINTENANCE_SCHEMA_V1, + repository: repository.to_string(), + package: package.to_string(), + base: base.clone(), + target: HostedMaintenanceTarget::Live { pointer: target }, + action, + }; + let marker = match create_hosted_maintenance_marker(storage, &operation).await { + Ok(marker) => marker, + Err(StorageError::AlreadyExists) => { + resume_hosted_maintenance_operation(storage, repository, package).await?; + return Err(StorageError::AlreadyExists); + } + Err(error) => return Err(error), + }; + match read_hosted_packument_pointer(storage, repository, package).await? { + Some(current) if current == base => {} + Some(current) + if matches!( + &marker.target, + HostedMaintenanceTarget::Live { pointer } if *pointer == current + ) => {} + _ => { + clear_hosted_maintenance_marker(storage, &marker).await?; + return Err(StorageError::AlreadyExists); + } + } + if !resume_hosted_maintenance_operation(storage, repository, package).await? { + return Err(StorageError::IntegrityViolation); } + Ok(()) +} - let key = proxy_packument_key(&repository.name, package); - let negative_key = proxy_negative_key(&repository.name, package); - if state - .storage - .stat(&negative_key) - .await - .is_some_and(|meta| negative_fresh(meta.modified, repository.negative_ttl)) - { - return Err(ReadError::NotFound); +fn import_packument_sha256(headers: &HeaderMap) -> Result, NpmHttpError> { + let values = headers + .get_all(NPM_IMPORT_PACKUMENT_HEADER) + .iter() + .collect::>(); + if values.is_empty() { + return Ok(None); + } + if values.len() != 1 { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Exactly one npm import packument hash is required", + )); } + let value = values[0].to_str().ok().filter(|value| valid_sha256(value)); + value.map(|value| Some(value.to_string())).ok_or_else(|| { + NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Invalid npm import packument SHA-256", + ) + }) +} - let cached = state.storage.get(&key).await.ok(); - let fresh = cached.is_some() - && state - .storage - .stat(&key) - .await - .is_some_and(|meta| negative_fresh(meta.modified, repository.metadata_ttl)); - if fresh { - return serde_json::from_slice(cached.as_ref().expect("cached when fresh")) - .map(PackumentRead::fresh) - .map_err(|_| ReadError::Corrupt); +fn valid_hosted_packument(bytes: &[u8], package: &str) -> Option { + let value = serde_json::from_slice::(bytes).ok()?; + let object = value.as_object()?; + (object.get("name").and_then(|value| value.as_str()) == Some(package) + && object + .get("versions") + .is_some_and(|value| value.is_object()) + && object + .get("dist-tags") + .is_some_and(|value| value.is_object())) + .then_some(value) +} + +fn install_v1_packument(packument: &serde_json::Value) -> Result { + let object = packument.as_object().ok_or(ReadError::Corrupt)?; + let mut abbreviated = serde_json::Map::new(); + for field in ["name", "modified", "dist-tags"] { + if let Some(value) = object.get(field) { + abbreviated.insert(field.to_string(), value.clone()); + } } + let versions = object + .get("versions") + .and_then(serde_json::Value::as_object) + .ok_or(ReadError::Corrupt)?; + let allowed = [ + "name", + "version", + "dist", + "dependencies", + "optionalDependencies", + "peerDependencies", + "peerDependenciesMeta", + "devDependencies", + "bundleDependencies", + "engines", + "funding", + "os", + "cpu", + "deprecated", + "bin", + "directories", + "acceptDependencies", + "_hasShrinkwrap", + "hasInstallScript", + ]; + let mut abbreviated_versions = serde_json::Map::new(); + for (version, manifest) in versions { + let manifest = manifest.as_object().ok_or(ReadError::Corrupt)?; + let mut projected = serde_json::Map::new(); + for field in allowed { + if let Some(value) = manifest.get(field) { + projected.insert(field.to_string(), value.clone()); + } + } + abbreviated_versions.insert(version.clone(), serde_json::Value::Object(projected)); + } + abbreviated.insert( + "versions".to_string(), + serde_json::Value::Object(abbreviated_versions), + ); + Ok(serde_json::Value::Object(abbreviated)) +} - // A singleton Nora still receives concurrent cold misses. Serialize the - // refresh for this package so a slower, older upstream response cannot - // overwrite a newer one. - let refresh_lock = state.publish_lock(&format!("npm-proxy:{}:{package}", repository.name)); - let _refresh_guard = refresh_lock.lock().await; - let cached = state.storage.get(&key).await.ok(); - let fresh = cached.is_some() - && state - .storage - .stat(&key) - .await - .is_some_and(|meta| negative_fresh(meta.modified, repository.metadata_ttl)); - if fresh { - return serde_json::from_slice(cached.as_ref().expect("cached when fresh")) - .map(PackumentRead::fresh) - .map_err(|_| ReadError::Corrupt); +fn project_packument( + packument: serde_json::Value, + flavor: PackumentFlavor, +) -> Result { + match flavor { + PackumentFlavor::Full => Ok(packument), + PackumentFlavor::InstallV1 => install_v1_packument(&packument), + } +} + +async fn put_immutable_storage( + storage: &Storage, + key: &str, + data: &[u8], +) -> Result<(), StorageError> { + let created = storage.put_if_absent(key, data).await; + match storage.get(key).await { + Ok(existing) if existing.as_ref() == data => Ok(()), + Ok(_) => Err(StorageError::IntegrityViolation), + Err(StorageError::NotFound) => match created { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Err(error) => Err(error), + } +} + +async fn write_hosted_packument_generation_documents( + storage: &Storage, + repository: &str, + package: &str, + packument: &serde_json::Value, + full: &[u8], +) -> Result { + let parsed = valid_hosted_packument(full, package).ok_or(StorageError::IntegrityViolation)?; + if &parsed != packument { + return Err(StorageError::IntegrityViolation); + } + let full_sha256 = hex::encode(sha2::Sha256::digest(full)); + let install_v1 = install_v1_packument(packument) + .map_err(|_| StorageError::IntegrityViolation) + .and_then(|value| { + serde_json::to_vec(&value).map_err(|_| StorageError::IntegrityViolation) + })?; + let install_v1_sha256 = hex::encode(sha2::Sha256::digest(&install_v1)); + let generation = full_sha256.clone(); + let full_key = crate::npm_layout::hosted_packument_full_key(repository, package, &generation); + let install_key = + crate::npm_layout::hosted_packument_install_v1_key(repository, package, &generation); + put_immutable_storage(storage, &full_key, full).await?; + put_immutable_storage(storage, &install_key, &install_v1).await?; + + Ok(WrittenPackumentGeneration { + generation, + full_sha256, + install_v1_sha256, + }) +} + +pub(crate) async fn commit_hosted_packument_pointer( + storage: &Storage, + repository: &str, + package: &str, + generation: &HostedPackumentPointer, +) -> Result<(), StorageError> { + // The mutable pointer is the sole read-model commit point and is written + // only after both immutable documents (and, for imports, the receipt) are + // durable. + if !valid_hosted_packument_pointer(generation) { + return Err(StorageError::IntegrityViolation); + } + validate_hosted_packument_pointer(storage, repository, package, generation).await?; + let retired_key = crate::npm_layout::hosted_packument_retired_key(repository, package); + match storage.get(&retired_key).await { + Ok(marker) if marker.as_ref() == crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 => { + let deleted = storage.delete(&retired_key).await; + match storage.get(&retired_key).await { + Err(StorageError::NotFound) => {} + Ok(current) + if current.as_ref() == crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 => + { + return match deleted { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + } + } + Ok(_) => return Err(StorageError::IntegrityViolation), + Err(error) => return Err(error), + } + } + Ok(_) => return Err(StorageError::IntegrityViolation), + Err(StorageError::NotFound) => {} + Err(error) => return Err(error), + } + let pointer = serde_json::to_vec(generation).map_err(|_| StorageError::IntegrityViolation)?; + let key = crate::npm_layout::hosted_packument_current_key(repository, package); + let committed = storage.put(&key, &pointer).await; + match storage.get(&key).await { + Ok(stored) if stored.as_ref() == pointer.as_slice() => Ok(()), + Ok(_) => Err(StorageError::IntegrityViolation), + Err(StorageError::NotFound) => match committed { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Err(error) => Err(error), + } +} + +pub(crate) async fn validate_hosted_packument_pointer( + storage: &Storage, + repository: &str, + package: &str, + generation: &HostedPackumentPointer, +) -> Result<(), StorageError> { + if !valid_hosted_packument_pointer(generation) { + return Err(StorageError::IntegrityViolation); + } + let full = storage + .get(&crate::npm_layout::hosted_packument_full_key( + repository, + package, + &generation.generation, + )) + .await?; + let install_v1 = storage + .get(&crate::npm_layout::hosted_packument_install_v1_key( + repository, + package, + &generation.generation, + )) + .await?; + if hex::encode(sha2::Sha256::digest(&full)) != generation.full_sha256 + || hex::encode(sha2::Sha256::digest(&install_v1)) != generation.install_v1_sha256 + || valid_hosted_packument(&full, package).is_none() + { + return Err(StorageError::IntegrityViolation); + } + Ok(()) +} + +pub(crate) async fn prepare_hosted_packument_after_retention( + storage: &Storage, + repository: &str, + package: &str, + removed_versions: &HashSet, +) -> Result { + // Retention derives its target from the exact developer-visible snapshot. + // LIST is neither an authority source nor an availability dependency. + let base = read_hosted_packument_pointer(storage, repository, package) + .await? + .ok_or(StorageError::NotFound)?; + validate_hosted_packument_pointer(storage, repository, package, &base).await?; + let mut packument = hosted_packument_for_pointer(storage, repository, package, &base).await?; + let versions = packument + .get_mut("versions") + .and_then(serde_json::Value::as_object_mut) + .ok_or(StorageError::IntegrityViolation)?; + versions.retain(|version, _| !removed_versions.contains(version)); + if versions.is_empty() { + return Ok(HostedMaintenanceTarget::Retired); + } + let tags = packument + .get_mut("dist-tags") + .and_then(serde_json::Value::as_object_mut) + .ok_or(StorageError::IntegrityViolation)?; + tags.retain(|_, target| { + !target + .as_str() + .is_some_and(|version| removed_versions.contains(version)) + }); + let full = serde_json::to_vec(&packument).map_err(|_| StorageError::IntegrityViolation)?; + let generation = write_hosted_packument_generation_documents( + storage, repository, package, &packument, &full, + ) + .await?; + // The package-wide maintenance marker owns the eventual pointer swap. + // Preparing immutable documents cannot change developer visibility. + Ok(HostedMaintenanceTarget::Live { + pointer: generation, + }) +} + +fn hosted_packument_response( + mut packument: serde_json::Value, + package: &str, + response_base: &str, +) -> serde_json::Value { + if let Some(versions) = packument + .get_mut("versions") + .and_then(|value| value.as_object_mut()) + { + for (version, manifest) in versions { + set_tarball_url(manifest, response_base, package, version); + } + } + packument +} + +async fn read_hosted_packument_generation( + state: &AppState, + repository: &str, + package: &str, + flavor: PackumentFlavor, +) -> Result { + let pointer_key = crate::npm_layout::hosted_packument_current_key(repository, package); + let pointer = match state.storage.get(&pointer_key).await { + Ok(pointer) => pointer, + Err(StorageError::NotFound) => { + // Developer GET remains strictly read-only. Bounded intent probes + // distinguish a quiescent absence/retirement from a live package + // or crash-recoverable import/publish/maintenance whose pointer is + // temporarily missing. No LIST or repair is allowed here. + let package_key = hosted_package_key(repository, package); + let import_key = crate::npm_layout::hosted_import_pending_key(repository, package); + let pending_index_key = hosted_publish_pending_index_key(repository, package); + let retired_key = crate::npm_layout::hosted_packument_retired_key(repository, package); + let (maintenance, package_state, import, pending_index, retired) = tokio::join!( + read_hosted_maintenance_marker(&state.storage, repository, package), + optional_hosted_materialization_get(state, &package_key), + optional_hosted_materialization_get(state, &import_key), + optional_hosted_materialization_get(state, &pending_index_key), + optional_hosted_materialization_get(state, &retired_key), + ); + let maintenance = maintenance.map_err(|_| ReadError::MaterializationUnavailable)?; + let package_state = package_state?; + let import = import?; + let pending_index = pending_index?; + let retired = retired?; + if import.as_ref().is_some_and(|marker| { + parse_hosted_import_session(marker, repository, package).is_err() + }) || pending_index.as_ref().is_some_and(|pending| { + parse_hosted_publish_pending(pending, repository, package).is_err() + }) { + return Err(ReadError::MaterializationUnavailable); + } + if maintenance.is_some() + || package_state.is_some() + || import.is_some() + || pending_index.is_some() + { + return Err(ReadError::MaterializationUnavailable); + } + return match retired { + Some(marker) + if marker.as_ref() == crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 => + { + Err(ReadError::NotFound) + } + Some(_) => Err(ReadError::MaterializationUnavailable), + None => Err(ReadError::NotFound), + }; + } + Err(_) => return Err(ReadError::MaterializationUnavailable), + }; + let pointer: HostedPackumentPointer = + serde_json::from_slice(&pointer).map_err(|_| ReadError::MaterializationUnavailable)?; + if !valid_sha256(&pointer.generation) + || !valid_sha256(&pointer.full_sha256) + || !valid_sha256(&pointer.install_v1_sha256) + || pointer.generation != pointer.full_sha256 + { + return Err(ReadError::MaterializationUnavailable); + } + let (key, expected) = match flavor { + PackumentFlavor::Full => ( + crate::npm_layout::hosted_packument_full_key(repository, package, &pointer.generation), + pointer.full_sha256, + ), + PackumentFlavor::InstallV1 => ( + crate::npm_layout::hosted_packument_install_v1_key( + repository, + package, + &pointer.generation, + ), + pointer.install_v1_sha256, + ), + }; + let bytes = state + .storage + .get(&key) + .await + .map_err(|_| ReadError::MaterializationUnavailable)?; + if hex::encode(sha2::Sha256::digest(&bytes)) != expected { + return Err(ReadError::MaterializationUnavailable); + } + valid_hosted_packument(&bytes, package).ok_or(ReadError::MaterializationUnavailable) +} + +async fn hosted_packument( + state: &AppState, + repository: &str, + package: &str, + response_base: &str, + flavor: PackumentFlavor, +) -> Result { + let packument = read_hosted_packument_generation(state, repository, package, flavor).await?; + Ok(hosted_packument_response(packument, package, response_base)) +} + +fn negative_fresh(modified: u64, ttl: i64) -> bool { + if ttl <= 0 { + return false; + } + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + now.saturating_sub(modified) < ttl as u64 +} + +async fn proxy_packument_raw( + state: &AppState, + repository: &ProxyRepository, + package: &str, +) -> Result { + if is_internal(state, package) { + return Err(ReadError::NotFound); + } + + let key = proxy_packument_key(&repository.name, package); + let negative_key = proxy_negative_key(&repository.name, package); + if state + .storage + .stat(&negative_key) + .await + .is_some_and(|meta| negative_fresh(meta.modified, repository.negative_ttl)) + { + return Err(ReadError::NotFound); + } + + let cached = state.storage.get(&key).await.ok(); + let fresh = cached.is_some() + && state + .storage + .stat(&key) + .await + .is_some_and(|meta| negative_fresh(meta.modified, repository.metadata_ttl)); + if fresh { + return serde_json::from_slice(cached.as_ref().expect("cached when fresh")) + .map(PackumentRead::fresh) + .map_err(|_| ReadError::Corrupt); + } + + // A singleton Nora still receives concurrent cold misses. Serialize the + // refresh for this package so a slower, older upstream response cannot + // overwrite a newer one. + let refresh_lock = state.publish_lock(&format!("npm-proxy:{}:{package}", repository.name)); + let _refresh_guard = refresh_lock.lock().await; + let cached = state.storage.get(&key).await.ok(); + let fresh = cached.is_some() + && state + .storage + .stat(&key) + .await + .is_some_and(|meta| negative_fresh(meta.modified, repository.metadata_ttl)); + if fresh { + return serde_json::from_slice(cached.as_ref().expect("cached when fresh")) + .map(PackumentRead::fresh) + .map_err(|_| ReadError::Corrupt); } let url = format!( @@ -1264,6 +2270,7 @@ async fn group_packument( members: &[String], package: &str, response_base: &str, + flavor: PackumentFlavor, ) -> Result { let mut packuments = Vec::new(); let mut stale = false; @@ -1273,24 +2280,24 @@ async fn group_packument( }; let result = match repository { NpmRepository::Hosted { name, .. } => { - hosted_packument(state, &name, package, response_base) + hosted_packument(state, &name, package, response_base, flavor) .await .map(PackumentRead::fresh) } NpmRepository::Proxy { .. } if is_internal(state, package) => Err(ReadError::NotFound), NpmRepository::Proxy { .. } => { let proxy = configured_proxy(state, &repository).expect("proxy config"); - proxy_packument_raw(state, &proxy, package) - .await - .map(|read| PackumentRead { - value: rewrite_packument_urls( - read.value, - response_base, - package, - &proxy.url, - ), + match proxy_packument_raw(state, &proxy, package).await { + Ok(read) => project_packument( + rewrite_packument_urls(read.value, response_base, package, &proxy.url), + flavor, + ) + .map(|value| PackumentRead { + value, stale: read.stale, - }) + }), + Err(error) => Err(error), + } } NpmRepository::Group { .. } => continue, }; @@ -1318,29 +2325,32 @@ async fn target_packument( target: &RepositoryTarget, package: &str, response_base: &str, + flavor: PackumentFlavor, ) -> Result { match target { RepositoryTarget::Named(NpmRepository::Hosted { name, .. }) => { - hosted_packument(state, name, package, response_base) + hosted_packument(state, name, package, response_base, flavor) .await .map(PackumentRead::fresh) } RepositoryTarget::Named(repository @ NpmRepository::Proxy { .. }) => { let proxy = configured_proxy(state, repository).expect("proxy config"); - proxy_packument_raw(state, &proxy, package) - .await - .map(|read| PackumentRead { - value: rewrite_packument_urls(read.value, response_base, package, &proxy.url), - stale: read.stale, - }) + let read = proxy_packument_raw(state, &proxy, package).await?; + Ok(PackumentRead { + value: project_packument( + rewrite_packument_urls(read.value, response_base, package, &proxy.url), + flavor, + )?, + stale: read.stale, + }) } RepositoryTarget::Named(NpmRepository::Group { members, .. }) => { - group_packument(state, members, package, response_base).await + group_packument(state, members, package, response_base, flavor).await } RepositoryTarget::Legacy => { let mut packuments = Vec::new(); let mut stale = false; - match hosted_packument(state, LEGACY_HOSTED, package, response_base).await { + match hosted_packument(state, LEGACY_HOSTED, package, response_base, flavor).await { Ok(hosted) => packuments.push(hosted), Err(ReadError::NotFound) => {} Err(error) => return Err(error), @@ -1350,12 +2360,15 @@ async fn target_packument( match proxy_packument_raw(state, &proxy, package).await { Ok(read) => { stale |= read.stale; - packuments.push(rewrite_packument_urls( - read.value, - response_base, - package, - &proxy.url, - )) + packuments.push(project_packument( + rewrite_packument_urls( + read.value, + response_base, + package, + &proxy.url, + ), + flavor, + )?) } Err(ReadError::NotFound) => {} Err(error) if packuments.is_empty() => return Err(error), @@ -1377,6 +2390,12 @@ fn read_error_response(error: ReadError) -> Response { ReadError::NotFound => StatusCode::NOT_FOUND.into_response(), ReadError::CircuitOpen(name) => circuit_open_response(&name), ReadError::Unavailable => StatusCode::BAD_GATEWAY.into_response(), + ReadError::MaterializationUnavailable => ( + StatusCode::SERVICE_UNAVAILABLE, + [(header::RETRY_AFTER, "1")], + "Hosted npm packument is not ready; retry after package repair/finalize", + ) + .into_response(), ReadError::Corrupt => StatusCode::INTERNAL_SERVER_ERROR.into_response(), ReadError::SearchScanLimit => ( StatusCode::BAD_GATEWAY, @@ -1386,15 +2405,69 @@ fn read_error_response(error: ReadError) -> Response { } } -async fn hosted_has_version( +enum HostedVersionResolution { + Absent, + AuthoritativelyAbsent, + Visible(serde_json::Value), +} + +async fn hosted_version_from_current( state: &AppState, repository: &str, package: &str, version: &str, -) -> Result { - optional_storage_get(state, &hosted_version_key(repository, package, version)) - .await - .map(|manifest| manifest.is_some()) +) -> Result { + match read_hosted_packument_generation(state, repository, package, PackumentFlavor::Full).await + { + Ok(packument) => { + if let Some(manifest) = packument + .get("versions") + .and_then(serde_json::Value::as_object) + .and_then(|versions| versions.get(version)) + .cloned() + { + return Ok(HostedVersionResolution::Visible(manifest)); + } + // A leftover exact split manifest is a bounded tombstone probe + // for a version removed from the committed generation. It must + // not be served or bypassed through a later group member. + return match state + .storage + .get(&hosted_version_key(repository, package, version)) + .await + { + Ok(_) => Ok(HostedVersionResolution::AuthoritativelyAbsent), + Err(StorageError::NotFound) => Ok(HostedVersionResolution::Absent), + Err(error) => Err(storage_read_error(error)), + }; + } + Err(ReadError::NotFound) => { + // A valid retired marker is an explicit absence authority even + // while best-effort split cleanup is still pending. + let retired_key = crate::npm_layout::hosted_packument_retired_key(repository, package); + match state.storage.get(&retired_key).await { + Ok(marker) if marker.as_ref() == crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 => { + return Ok(HostedVersionResolution::AuthoritativelyAbsent) + } + Ok(_) => return Err(ReadError::MaterializationUnavailable), + Err(StorageError::NotFound) => {} + Err(error) => return Err(storage_read_error(error)), + } + // The requested exact split key is only an intent/corruption + // probe when current is absent; it can never make a version + // visible or cause group/legacy fallthrough. + match state + .storage + .get(&hosted_version_key(repository, package, version)) + .await + { + Ok(_) => Err(ReadError::MaterializationUnavailable), + Err(StorageError::NotFound) => Ok(HostedVersionResolution::Absent), + Err(error) => Err(storage_read_error(error)), + } + } + Err(error) => Err(error), + } } fn dist_digest_matches(data: &[u8], version_data: &serde_json::Value) -> bool { @@ -1439,22 +2512,39 @@ async fn serve_hosted_tarball( let Some(version) = crate::curation::parse_npm_tarball_version(package, filename) else { return StatusCode::NOT_FOUND.into_response(); }; - let manifest = match state - .storage - .get(&hosted_version_key(repository, package, &version)) - .await + let version_data = match hosted_version_from_current(state, repository, package, &version).await { - Ok(manifest) => manifest, - Err(error) => return read_error_response(storage_read_error(error)), + Ok(HostedVersionResolution::Visible(version_data)) => version_data, + Ok(HostedVersionResolution::Absent | HostedVersionResolution::AuthoritativelyAbsent) => { + return StatusCode::NOT_FOUND.into_response() + } + Err(error) => return read_error_response(error), }; - let Ok(version_data) = serde_json::from_slice::(&manifest) else { - crate::metrics::METADATA_CORRUPT_TOTAL - .with_label_values(&["npm"]) - .inc(); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + serve_hosted_tarball_version( + state, + repository, + package, + &version, + version_data, + publish_date, + ) + .await +} + +async fn serve_hosted_tarball_version( + state: &AppState, + repository: &str, + package: &str, + version: &str, + version_data: serde_json::Value, + publish_date: Option, +) -> Response { + let manifest = match serde_json::to_vec(&version_data) { + Ok(manifest) => manifest, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), }; let Some(key) = - crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) + crate::npm_layout::hosted_blob_key_from_manifest(repository, package, manifest.as_slice()) else { crate::metrics::METADATA_CORRUPT_TOTAL .with_label_values(&["npm"]) @@ -1471,7 +2561,7 @@ async fn serve_hosted_tarball( if !dist_digest_matches(&data, &version_data) { return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - curated_tarball_response(state, package, &version, data, "hosted", publish_date) + curated_tarball_response(state, package, version, data, "hosted", publish_date) } async fn serve_proxy_tarball( @@ -1655,15 +2745,25 @@ async fn group_tarball( }; match repository { NpmRepository::Hosted { name, .. } => { - match hosted_has_version(state, &name, package, &version).await { - Ok(true) => { + match hosted_version_from_current(state, &name, package, &version).await { + Ok(HostedVersionResolution::Visible(version_data)) => { // Once a member claims the version, its tarball is the // only legal origin. Do not fall through after an // incomplete or corrupt member. - return serve_hosted_tarball(state, &name, package, filename, publish_date) - .await; + return serve_hosted_tarball_version( + state, + &name, + package, + &version, + version_data, + publish_date, + ) + .await; } - Ok(false) => {} + Ok(HostedVersionResolution::AuthoritativelyAbsent) => { + return StatusCode::NOT_FOUND.into_response() + } + Ok(HostedVersionResolution::Absent) => {} Err(error) => return read_error_response(error), } } @@ -1713,18 +2813,22 @@ async fn target_tarball( else { return StatusCode::NOT_FOUND.into_response(); }; - match hosted_has_version(state, LEGACY_HOSTED, package, &version).await { - Ok(true) => { - return serve_hosted_tarball( + match hosted_version_from_current(state, LEGACY_HOSTED, package, &version).await { + Ok(HostedVersionResolution::Visible(version_data)) => { + return serve_hosted_tarball_version( state, LEGACY_HOSTED, package, - filename, + &version, + version_data, publish_date, ) .await } - Ok(false) => {} + Ok(HostedVersionResolution::AuthoritativelyAbsent) => { + return StatusCode::NOT_FOUND.into_response() + } + Ok(HostedVersionResolution::Absent) => {} Err(error) => return read_error_response(error), } if let Some(proxy) = legacy_proxy(state) { @@ -2464,6 +3568,7 @@ async fn handle_get( query: Option, user: AuthenticatedUser, ) -> Response { + let packument_flavor = PackumentFlavor::from_headers(&headers); if path == "-/ping" { return axum::Json(serde_json::json!({})).into_response(); } @@ -2504,8 +3609,13 @@ async fn handle_get( } target_tarball(&state, &target, &package, &filename, publish_date).await } else { - match target_packument(&state, &target, &package, &response_base).await { - Ok(packument) => json_response_with_stale(&headers, &packument.value, packument.stale), + match target_packument(&state, &target, &package, &response_base, packument_flavor).await { + Ok(packument) => packument_response( + &headers, + &packument.value, + packument.stale, + packument_flavor, + ), Err(error) => read_error_response(error), } }; @@ -2909,171 +4019,440 @@ fn validate_publish( }) } -async fn publish( +#[derive(Debug)] +struct ImportVersionPreflight { + completed_receipt: bool, + session: Option, + pending_present: bool, + blob_present: bool, + manifest_present: bool, + completion_present: bool, + evidence_present: bool, +} + +async fn import_version_preflight( state: &AppState, repository: &str, - write_policy: NpmWritePolicy, package: &str, - payload: &serde_json::Value, -) -> Response { - let validated = match validate_publish(package, payload) { - Ok(validated) => validated, - Err(error) => return error.into_response(), + expected_packument_sha256: &str, + validated: &ValidatedPublish, +) -> Result { + let manifest_digest = crate::npm_layout::hosted_manifest_digest(&validated.manifest); + let receipt_key = crate::npm_layout::hosted_import_receipt_key( + repository, + package, + expected_packument_sha256, + ); + let receipt = match state.storage.get(&receipt_key).await { + Ok(bytes) => { + let receipt: HostedImportReceipt = + serde_json::from_slice(&bytes).map_err(|_| StorageError::IntegrityViolation)?; + if receipt.package != package + || receipt.generation != expected_packument_sha256 + || receipt.full_sha256 != expected_packument_sha256 + || !valid_sha256(&receipt.install_v1_sha256) + { + return Err(StorageError::IntegrityViolation); + } + Some(receipt) + } + Err(StorageError::NotFound) => None, + Err(error) => return Err(error), }; - let lock_key = format!("npm:{repository}:{package}"); - let lock = state.publish_lock(&lock_key); - let _guard = lock.lock().await; - match incomplete_publish_versions(state, repository, package).await { - Ok(incomplete) - if incomplete - .iter() - .any(|version| version != &validated.version) => - { - return incomplete_publish_response() - } - Ok(_) => {} - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), - } - if invalidate_hosted_packument_cache(state, repository, package) - .await - .is_err() + + let session = read_optional_import_session(&state.storage, repository, package).await?; + if session + .as_ref() + .is_some_and(|session| session.packument_sha256 != expected_packument_sha256) + || session + .as_ref() + .and_then(|session| session.versions.get(&validated.version)) + .is_some_and(|digest| digest != &manifest_digest) { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } + return Err(StorageError::AlreadyExists); + } + + let pending_present = + match read_optional_publish_pending(&state.storage, repository, package).await? { + Some(pending) + if pending.version == validated.version + && pending.manifest_sha256 == manifest_digest + && pending.blob_sha512 == validated.blob_digest + && pending.target + == (HostedPublishPendingTarget::Import { + packument_sha256: expected_packument_sha256.to_string(), + }) => + { + true + } + Some(_) => return Err(StorageError::AlreadyExists), + None => false, + }; - let pending_key = hosted_publish_pending_key(repository, package, &validated.version); - let completion_digest = crate::npm_layout::hosted_manifest_digest(&validated.manifest); - let previous_pending = match state.storage.get(&pending_key).await { - Ok(previous) if previous.as_ref() == completion_digest.as_bytes() => Some(previous), - Ok(_) => return incomplete_publish_response(), - Err(StorageError::NotFound) => None, - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + let manifest_key = hosted_version_key(repository, package, &validated.version); + let manifest_present = match state.storage.get(&manifest_key).await { + Ok(manifest) if manifest.as_ref() == validated.manifest.as_slice() => true, + Ok(_) => return Err(StorageError::AlreadyExists), + Err(StorageError::NotFound) => false, + Err(error) => return Err(error), + }; + let completion_key = hosted_publish_complete_key(repository, package, &validated.version); + let completion_present = match state.storage.get(&completion_key).await { + Ok(completion) if completion.as_ref() == manifest_digest.as_bytes() => true, + Ok(_) => return Err(StorageError::AlreadyExists), + Err(StorageError::NotFound) => false, + Err(error) => return Err(error), }; - if state - .storage - .put(&pending_key, completion_digest.as_bytes()) - .await - .is_err() - { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } let blob_key = crate::npm_layout::hosted_blob_key_for_digest(repository, package, &validated.blob_digest); - match put_immutable(state, &blob_key, &validated.tarball).await { - Ok(ImmutableWrite::Created | ImmutableWrite::ExistingSame) => {} - Ok(ImmutableWrite::Conflict) => { - if restore_publish_pending(state, &pending_key, previous_pending.as_deref()) - .await - .is_err() - { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + let blob_present = match state.storage.get(&blob_key).await { + Ok(blob) if blob.as_ref() == validated.tarball.as_slice() => true, + Ok(_) => return Err(StorageError::AlreadyExists), + Err(StorageError::NotFound) => false, + Err(error) => return Err(error), + }; + + let evidence_key = crate::npm_layout::hosted_import_evidence_key( + repository, + package, + expected_packument_sha256, + &validated.version, + &manifest_digest, + ); + let evidence_present = match state.storage.get(&evidence_key).await { + Ok(value) if value.as_ref() == b"1" => true, + Ok(_) => return Err(StorageError::AlreadyExists), + Err(StorageError::NotFound) => false, + Err(error) => return Err(error), + }; + + if let Some(receipt) = &receipt { + if pending_present + || !(manifest_present && completion_present && blob_present && evidence_present) + { + return Err(StorageError::AlreadyExists); + } + let generation = HostedPackumentPointer { + generation: receipt.generation.clone(), + full_sha256: receipt.full_sha256.clone(), + install_v1_sha256: receipt.install_v1_sha256.clone(), + }; + match validate_hosted_packument_pointer(&state.storage, repository, package, &generation) + .await + { + Ok(()) => {} + Err(StorageError::NotFound | StorageError::IntegrityViolation) => { + return Err(StorageError::AlreadyExists) } - tracing::error!(key = %blob_key, "npm content digest collision"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + Err(error) => return Err(error), } - Err(error) => { - if restore_publish_pending(state, &pending_key, previous_pending.as_deref()) + let full = match state + .storage + .get(&crate::npm_layout::hosted_packument_full_key( + repository, + package, + &receipt.generation, + )) + .await + { + Ok(full) => full, + Err(StorageError::NotFound | StorageError::IntegrityViolation) => { + return Err(StorageError::AlreadyExists) + } + Err(error) => return Err(error), + }; + let full = valid_hosted_packument(&full, package).ok_or(StorageError::AlreadyExists)?; + let versions = full + .get("versions") + .and_then(serde_json::Value::as_object) + .ok_or(StorageError::AlreadyExists)?; + if versions.len() != receipt.version_count { + return Err(StorageError::AlreadyExists); + } + let mut receipt_manifest = versions + .get(&validated.version) + .cloned() + .ok_or(StorageError::AlreadyExists)?; + receipt_manifest + .as_object_mut() + .ok_or(StorageError::AlreadyExists)? + .remove("deprecated"); + let expected_manifest: serde_json::Value = serde_json::from_slice(&validated.manifest) + .map_err(|_| StorageError::IntegrityViolation)?; + if receipt_manifest != expected_manifest { + return Err(StorageError::AlreadyExists); + } + } else if manifest_present { + // A missing completion is resumable only behind this exact pending + // digest. The tarball blob was ordered before the manifest, so its + // absence is corruption rather than permission to recreate history. + if !blob_present + || (!completion_present && !pending_present) + || (!completion_present && evidence_present) + || (evidence_present + && session + .as_ref() + .and_then(|session| session.versions.get(&validated.version)) + != Some(&manifest_digest)) + { + return Err(StorageError::AlreadyExists); + } + } else if completion_present || evidence_present { + return Err(StorageError::AlreadyExists); + } + + Ok(ImportVersionPreflight { + completed_receipt: receipt.is_some(), + session, + pending_present, + blob_present, + manifest_present, + completion_present, + evidence_present, + }) +} + +async fn put_exact_with_readback( + storage: &Storage, + key: &str, + value: &[u8], +) -> Result<(), StorageError> { + let written = storage.put(key, value).await; + match storage.get(key).await { + Ok(current) if current.as_ref() == value => Ok(()), + Ok(_) => Err(StorageError::AlreadyExists), + Err(StorageError::NotFound) => match written { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Err(error) => Err(error), + } +} + +async fn ensure_import_session_version( + state: &AppState, + repository: &str, + package: &str, + packument_sha256: &str, + version: &str, + manifest_sha256: &str, + existing: Option, +) -> Result { + let marker_key = crate::npm_layout::hosted_import_pending_key(repository, package); + let expected_previous = existing + .as_ref() + .map(serde_json::to_vec) + .transpose() + .map_err(|_| StorageError::IntegrityViolation)?; + let mut session = if let Some(session) = existing { + session + } else { + let base = read_hosted_packument_pointer(&state.storage, repository, package).await?; + if let Some(base) = &base { + validate_hosted_packument_pointer(&state.storage, repository, package, base).await?; + } else { + match state + .storage + .get(&hosted_package_key(repository, package)) .await - .is_err() { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + Err(StorageError::NotFound) => {} + Ok(_) => return Err(StorageError::IntegrityViolation), + Err(error) => return Err(error), } - tracing::error!(key = %blob_key, error = ?error, "npm tarball blob create failed"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } + HostedImportSession { + schema: crate::npm_layout::HOSTED_IMPORT_SESSION_SCHEMA_V1, + repository: repository.to_string(), + package: package.to_string(), + packument_sha256: packument_sha256.to_string(), + base, + versions: BTreeMap::new(), + } + }; + if session.packument_sha256 != packument_sha256 { + return Err(StorageError::AlreadyExists); + } + match session.versions.get(version) { + Some(existing) if existing == manifest_sha256 => return Ok(session), + Some(_) => return Err(StorageError::AlreadyExists), + None => { + session + .versions + .insert(version.to_string(), manifest_sha256.to_string()); + } + } + let bytes = serde_json::to_vec(&session).map_err(|_| StorageError::IntegrityViolation)?; + match (state.storage.get(&marker_key).await, expected_previous) { + (Ok(current), Some(previous)) if current.as_ref() == previous.as_slice() => { + put_exact_with_readback(&state.storage, &marker_key, &bytes).await? + } + (Ok(_), _) => return Err(StorageError::AlreadyExists), + (Err(StorageError::NotFound), None) => { + put_immutable_storage(&state.storage, &marker_key, &bytes).await? + } + (Err(StorageError::NotFound), Some(_)) => return Err(StorageError::AlreadyExists), + (Err(error), _) => return Err(error), } + let stored = state.storage.get(&marker_key).await?; + let stored = parse_hosted_import_session(&stored, repository, package)?; + if stored != session { + return Err(StorageError::AlreadyExists); + } + Ok(session) +} - // The version manifest is the sole visibility/commit point. The referenced - // tarball blob is content-addressed and is always durable before this write. - let manifest_key = hosted_version_key(repository, package, &validated.version); - let manifest_outcome = - match commit_hosted_manifest(state, &manifest_key, &validated.manifest, write_policy).await - { - Ok(outcome) => outcome, - Err(ManifestCommitError::Conflict) => { - if restore_publish_pending(state, &pending_key, previous_pending.as_deref()) - .await - .is_err() - { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - return ( - StatusCode::CONFLICT, - "Version already exists with other metadata or tarball bytes", - ) - .into_response(); - } - Err(ManifestCommitError::Storage(error)) => { - if restore_publish_pending(state, &pending_key, previous_pending.as_deref()) - .await - .is_err() - { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - tracing::error!(key = %manifest_key, error = ?error, "npm version commit failed"); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - }; +async fn delete_exact_with_readback( + storage: &Storage, + key: &str, + expected: &[u8], +) -> Result<(), StorageError> { + match storage.get(key).await { + Ok(current) if current.as_ref() == expected => {} + Ok(_) => return Err(StorageError::AlreadyExists), + Err(StorageError::NotFound) => return Ok(()), + Err(error) => return Err(error), + } + let deleted = storage.delete(key).await; + match storage.get(key).await { + Err(StorageError::NotFound) => Ok(()), + Ok(current) if current.as_ref() == expected => match deleted { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Ok(_) => Err(StorageError::AlreadyExists), + Err(error) => Err(error), + } +} - let completion_key = hosted_publish_complete_key(repository, package, &validated.version); - let needs_completion = if manifest_outcome == ManifestCommit::ExistingSame - && write_policy != NpmWritePolicy::Allow +async fn publish_import_version_locked( + state: &AppState, + repository: &str, + package: &str, + expected_packument_sha256: &str, + validated: &ValidatedPublish, +) -> Response { + fn immutable_write_error(error: StorageError) -> Response { + match error { + StorageError::AlreadyExists | StorageError::IntegrityViolation => ( + StatusCode::CONFLICT, + "Imported npm version state changed during exact preflight", + ) + .into_response(), + _ => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + let preflight = match import_version_preflight( + state, + repository, + package, + expected_packument_sha256, + validated, + ) + .await { - match state.storage.get(&completion_key).await { - Ok(existing) if existing.as_ref() == completion_digest.as_bytes() => false, - Ok(_) => { - tracing::error!( - key = %completion_key, - "npm publish completion marker does not match the committed manifest" - ); - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - Err(StorageError::NotFound) => { - // An allow-once retry must repair an interrupted first - // publish without overwriting a newer explicit - // tag/deprecation value. - if fill_missing_retry_state(state, repository, package, &validated) - .await - .is_err() - { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - true - } - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + Ok(preflight) => preflight, + Err(StorageError::AlreadyExists) => { + return ( + StatusCode::CONFLICT, + "Imported npm version state differs from the exact import payload", + ) + .into_response() } - } else { - // `allow` is a real redeploy even when the immutable version - // manifest happens to be byte-identical: npm publish's selected - // dist-tag and package fields are mutable payload state. Remove - // the completion marker before touching that state so a failed - // post-commit phase cannot be acknowledged as complete on retry. - match state.storage.delete(&completion_key).await { - Ok(()) | Err(StorageError::NotFound) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if preflight.completed_receipt { + // A delayed version PUT from an already completed import is an exact, + // read-only acknowledgement. Never rewind package metadata, tags, + // deprecations or a later packument pointer. + return StatusCode::CREATED.into_response(); + } + let active_pending = + match read_optional_publish_pending(&state.storage, repository, package).await { + Ok(pending) => pending, Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if active_pending + .as_ref() + .is_some_and(|pending| pending.version != validated.version) + { + return incomplete_publish_response(); + } + + let manifest_digest = crate::npm_layout::hosted_manifest_digest(&validated.manifest); + if let Err(error) = ensure_import_session_version( + state, + repository, + package, + expected_packument_sha256, + &validated.version, + &manifest_digest, + preflight.session.clone(), + ) + .await + { + return immutable_write_error(error); + } + let pending = HostedPublishPending { + schema: crate::npm_layout::HOSTED_PUBLISH_PENDING_SCHEMA_V1, + repository: repository.to_string(), + package: package.to_string(), + version: validated.version.clone(), + manifest_sha256: manifest_digest.clone(), + blob_sha512: validated.blob_digest.clone(), + target: HostedPublishPendingTarget::Import { + packument_sha256: expected_packument_sha256.to_string(), + }, + }; + if !preflight.pending_present { + if let Err(error) = create_hosted_publish_pending(&state.storage, &pending).await { + return immutable_write_error(error); } - if replace_publish_state(state, repository, package, &validated) - .await - .is_err() + } + let blob_key = + crate::npm_layout::hosted_blob_key_for_digest(repository, package, &validated.blob_digest); + if !preflight.blob_present { + if let Err(error) = + put_immutable_storage(&state.storage, &blob_key, &validated.tarball).await { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + return immutable_write_error(error); } - true - }; - if needs_completion - && state - .storage - .put(&completion_key, completion_digest.as_bytes()) - .await - .is_err() + } + let manifest_key = hosted_version_key(repository, package, &validated.version); + if !preflight.manifest_present { + if let Err(error) = + put_immutable_storage(&state.storage, &manifest_key, &validated.manifest).await + { + return immutable_write_error(error); + } + } + let completion_key = hosted_publish_complete_key(repository, package, &validated.version); + if !preflight.completion_present { + if let Err(error) = + put_immutable_storage(&state.storage, &completion_key, manifest_digest.as_bytes()).await + { + return immutable_write_error(error); + } + } + let evidence_key = crate::npm_layout::hosted_import_evidence_key( + repository, + package, + expected_packument_sha256, + &validated.version, + &manifest_digest, + ); + if !preflight.evidence_present { + if let Err(error) = put_immutable_storage(&state.storage, &evidence_key, b"1").await { + return immutable_write_error(error); + } + } + if clear_hosted_publish_pending(&state.storage, &pending) + .await + .is_err() { return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - match state.storage.delete(&pending_key).await { - Ok(()) | Err(StorageError::NotFound) => {} - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), - } state.metrics.record_upload("npm"); state @@ -3089,1093 +4468,3631 @@ async fn publish( StatusCode::CREATED.into_response() } -async fn replace_publish_state( +async fn publish_with_import( state: &AppState, repository: &str, + write_policy: NpmWritePolicy, package: &str, - validated: &ValidatedPublish, -) -> Result<(), ()> { - state - .storage - .put( - &hosted_package_key(repository, package), - &validated.package_fields, - ) - .await - .map_err(|_| ())?; - for (tag, version) in &validated.tags { - state - .storage - .put( - &hosted_tag_key(repository, package, tag), - version.as_bytes(), - ) - .await - .map_err(|_| ())?; + payload: &serde_json::Value, + import_packument_sha256: Option<&str>, +) -> Response { + let validated = match validate_publish(package, payload) { + Ok(validated) => validated, + Err(error) => return error.into_response(), + }; + let lock_key = format!("npm:{repository}:{package}"); + let lock = state.publish_lock(&lock_key); + let _guard = lock.lock().await; + match resume_hosted_maintenance_operation(&state.storage, repository, package).await { + Ok(true) => state.repo_index.invalidate("npm"), + Ok(false) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } - if let Some(message) = &validated.deprecation { - let key = hosted_deprecation_key(repository, package, &validated.version); - if message.is_empty() { - match state.storage.delete(&key).await { - Ok(()) | Err(StorageError::NotFound) => {} - Err(_) => return Err(()), - } - } else { - state - .storage - .put(&key, message.as_bytes()) - .await - .map_err(|_| ())?; + if let Some(expected) = import_packument_sha256 { + return publish_import_version_locked(state, repository, package, expected, &validated) + .await; + } + match hosted_import_active(&state.storage, repository, package).await { + Ok(true) => { + return ( + StatusCode::CONFLICT, + "Package has an active bulk import; finalize it before normal mutation", + ) + .into_response() } + Ok(false) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + let active_pending = + match read_optional_publish_pending(&state.storage, repository, package).await { + Ok(pending) => pending, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if active_pending + .as_ref() + .is_some_and(|pending| pending.version != validated.version) + { + return incomplete_publish_response(); } - Ok(()) -} - -#[derive(Debug, PartialEq, Eq)] -enum ManifestCommit { - Created, - ExistingSame, - Replaced, -} -#[derive(Debug)] -enum ManifestCommitError { - Conflict, - Storage(StorageError), -} + let manifest_key = hosted_version_key(repository, package, &validated.version); + let completion_key = hosted_publish_complete_key(repository, package, &validated.version); + let completion_digest = crate::npm_layout::hosted_manifest_digest(&validated.manifest); + let (completed_exact_allow_once, missing_completion_allow_once) = + if write_policy == NpmWritePolicy::AllowOnce { + match state.storage.get(&manifest_key).await { + Ok(existing) if existing.as_ref() == validated.manifest.as_slice() => { + match state.storage.get(&completion_key).await { + Ok(completion) if completion.as_ref() == completion_digest.as_bytes() => { + (true, false) + } + Ok(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + Err(StorageError::NotFound) => (false, true), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + Ok(_) => { + return ( + StatusCode::CONFLICT, + "Version already exists with other metadata or tarball bytes", + ) + .into_response() + } + Err(StorageError::NotFound) => (false, false), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } else { + (false, false) + }; -async fn commit_hosted_manifest( - state: &AppState, - key: &str, - manifest: &[u8], - write_policy: NpmWritePolicy, -) -> Result { - match write_policy { - NpmWritePolicy::Deny => Err(ManifestCommitError::Conflict), - NpmWritePolicy::AllowOnce => match put_immutable(state, key, manifest) + if completed_exact_allow_once && active_pending.is_none() { + // An accepted exact retry is a no-op. In particular, do not rewind a + // dist-tag/deprecation changed after the original publish. The + // committed current generation must already contain this version. + if ensure_completed_publish_materialized(state, repository, package, &validated.version) .await - .map_err(ManifestCommitError::Storage)? + .is_err() { - ImmutableWrite::Created => Ok(ManifestCommit::Created), - ImmutableWrite::ExistingSame => Ok(ManifestCommit::ExistingSame), - ImmutableWrite::Conflict => Err(ManifestCommitError::Conflict), - }, - NpmWritePolicy::Allow => match state.storage.get(key).await { - Ok(existing) if existing.as_ref() == manifest => Ok(ManifestCommit::ExistingSame), - Ok(_) => { - state - .storage - .put(key, manifest) - .await - .map_err(ManifestCommitError::Storage)?; - Ok(ManifestCommit::Replaced) - } - Err(StorageError::NotFound) => { - state - .storage - .put(key, manifest) - .await - .map_err(ManifestCommitError::Storage)?; - Ok(ManifestCommit::Created) - } - Err(error) => Err(ManifestCommitError::Storage(error)), - }, + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + return StatusCode::CREATED.into_response(); } -} -async fn fill_missing_retry_state( - state: &AppState, - repository: &str, - package: &str, - validated: &ValidatedPublish, -) -> Result<(), ()> { - let package_key = hosted_package_key(repository, package); - match state.storage.get(&package_key).await { - Ok(existing) => { - let mut current = - serde_json::from_slice::(&existing).map_err(|_| ())?; - let candidate = serde_json::from_slice::(&validated.package_fields) - .map_err(|_| ())?; - let (Some(current), Some(candidate)) = (current.as_object_mut(), candidate.as_object()) - else { - return Err(()); - }; - let mut changed = false; - for (field, value) in candidate { - if !current.contains_key(field) { - current.insert(field.clone(), value.clone()); - changed = true; + if missing_completion_allow_once && active_pending.is_none() { + // Completion is committed before the current pointer and pending is + // cleared only after both verify exactly. No legitimate crash can + // therefore leave an exact manifest with missing completion and no + // pending intent, regardless of whether current exists. Treat it as + // corruption and never manufacture recovery state from visibility. + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + + let (base, previous_packument, recorded_target) = match active_pending.as_ref() { + Some(HostedPublishPending { + target: HostedPublishPendingTarget::Publish { base, target }, + .. + }) => { + let previous = match base { + Some(base) => { + match hosted_packument_for_pointer(&state.storage, repository, package, base) + .await + { + Ok(packument) => Some(packument), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } } + None => None, + }; + let current = + match read_hosted_packument_pointer(&state.storage, repository, package).await { + Ok(current) => current, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if current.as_ref() != base.as_ref() && current.as_ref() != Some(target) { + return incomplete_publish_response(); } - if changed { - let merged = serde_json::to_vec(&serde_json::Value::Object(current.clone())) - .map_err(|_| ())?; - state - .storage - .put(&package_key, &merged) - .await - .map_err(|_| ())?; - } + (base.clone(), previous, Some(target.clone())) } - Err(StorageError::NotFound) => state - .storage - .put(&package_key, &validated.package_fields) + Some(_) => return incomplete_publish_response(), + None => { + let base = + match read_hosted_packument_pointer(&state.storage, repository, package).await { + Ok(base) => base, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let previous = match &base { + Some(base) => { + match hosted_packument_for_pointer(&state.storage, repository, package, base) + .await + { + Ok(packument) => Some(packument), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + None => { + match state + .storage + .get(&hosted_package_key(repository, package)) + .await + { + Err(StorageError::NotFound) => None, + Ok(_) => return StatusCode::SERVICE_UNAVAILABLE.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + }; + (base, previous, None) + } + }; + let target_packument = + match packument_after_publish(package, previous_packument.clone(), &validated) { + Ok(packument) => packument, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let (target_full, target_pointer) = match hosted_packument_pointer_for_value(&target_packument) + { + Ok(target) => target, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if recorded_target + .as_ref() + .is_some_and(|recorded| recorded != &target_pointer) + { + return incomplete_publish_response(); + } + let base_mutable = match hosted_mutable_state_from_packument(previous_packument.as_ref()) { + Ok(state) => state, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let target_mutable = match hosted_mutable_state_from_packument(Some(&target_packument)) { + Ok(state) => state, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let mutable_authorities = + hosted_mutable_authorities(repository, package, &base_mutable, &target_mutable); + let pending = HostedPublishPending { + schema: crate::npm_layout::HOSTED_PUBLISH_PENDING_SCHEMA_V1, + repository: repository.to_string(), + package: package.to_string(), + version: validated.version.clone(), + manifest_sha256: completion_digest.clone(), + blob_sha512: validated.blob_digest.clone(), + target: HostedPublishPendingTarget::Publish { + base, + target: target_pointer.clone(), + }, + }; + if active_pending + .as_ref() + .is_some_and(|active| active != &pending) + { + return incomplete_publish_response(); + } + if active_pending.is_none() + && create_hosted_publish_pending(&state.storage, &pending) .await - .map_err(|_| ())?, - Err(_) => return Err(()), + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - for (tag, version) in &validated.tags { - let key = hosted_tag_key(repository, package, tag); - match state.storage.get(&key).await { - Ok(_) => {} - Err(StorageError::NotFound) => state - .storage - .put(&key, version.as_bytes()) - .await - .map_err(|_| ())?, - Err(_) => return Err(()), + let blob_key = + crate::npm_layout::hosted_blob_key_for_digest(repository, package, &validated.blob_digest); + match put_immutable(state, &blob_key, &validated.tarball).await { + Ok(ImmutableWrite::Created | ImmutableWrite::ExistingSame) => {} + Ok(ImmutableWrite::Conflict) => { + tracing::error!(key = %blob_key, "npm content digest collision"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + Err(error) => { + tracing::error!(key = %blob_key, error = ?error, "npm tarball blob create failed"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } } - if let Some(message) = validated - .deprecation - .as_deref() - .filter(|message| !message.is_empty()) + + // Split objects are prepared behind the immutable current-generation + // pointer. The referenced tarball blob is content-addressed and durable + // before the version manifest changes. + let manifest_outcome = + match commit_hosted_manifest(state, &manifest_key, &validated.manifest, write_policy).await + { + Ok(outcome) => outcome, + Err(ManifestCommitError::Conflict) => { + return ( + StatusCode::CONFLICT, + "Version already exists with other metadata or tarball bytes", + ) + .into_response(); + } + Err(ManifestCommitError::Storage(error)) => { + tracing::error!(key = %manifest_key, error = ?error, "npm version commit failed"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + + let needs_completion = if manifest_outcome == ManifestCommit::ExistingSame + && write_policy != NpmWritePolicy::Allow { - let key = hosted_deprecation_key(repository, package, &validated.version); - match state.storage.get(&key).await { - Ok(_) => {} - Err(StorageError::NotFound) => state - .storage - .put(&key, message.as_bytes()) - .await - .map_err(|_| ())?, - Err(_) => return Err(()), + match state.storage.get(&completion_key).await { + Ok(existing) if existing.as_ref() == completion_digest.as_bytes() => false, + Ok(_) => { + tracing::error!( + key = %completion_key, + "npm publish completion marker does not match the committed manifest" + ); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + Err(StorageError::NotFound) => true, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } + } else { + // `allow` is a real redeploy even when the immutable version + // manifest happens to be byte-identical: npm publish's selected + // dist-tag and package fields are mutable payload state. Remove + // the completion marker before touching that state so a failed + // post-commit phase cannot be acknowledged as complete on retry. + match read_optional_exact(&state.storage, &completion_key).await { + Ok(Some(current)) => { + if delete_exact_with_readback(&state.storage, &completion_key, ¤t) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + } + Ok(None) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + true + }; + if converge_hosted_mutable_target(&state.storage, &mutable_authorities) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - Ok(()) -} - -async fn deprecate( - state: &AppState, - repository: &str, - package: &str, - payload: &serde_json::Value, -) -> Response { - if payload.get("name").and_then(|value| value.as_str()) != Some(package) { - return (StatusCode::BAD_REQUEST, "Package name mismatch").into_response(); + if needs_completion + && put_exact_with_readback( + &state.storage, + &completion_key, + completion_digest.as_bytes(), + ) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - let Some(versions) = payload.get("versions").and_then(|value| value.as_object()) else { - return (StatusCode::BAD_REQUEST, "Missing versions").into_response(); - }; - let lock = state.publish_lock(&format!("npm:{repository}:{package}")); - let _guard = lock.lock().await; - match incomplete_publish_versions(state, repository, package).await { - Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), - Ok(_) => {} + let written_target = match write_hosted_packument_generation_documents( + &state.storage, + repository, + package, + &target_packument, + &target_full, + ) + .await + { + Ok(target) if target == target_pointer => target, + Ok(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let verification = HostedPublishVerification { + manifest_key: &manifest_key, + manifest: &validated.manifest, + blob_key: &blob_key, + blob: &validated.tarball, + completion_key: &completion_key, + completion_digest: &completion_digest, + mutable_authorities: &mutable_authorities, + }; + if verify_publish_target_state(&state.storage, &verification) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - if invalidate_hosted_packument_cache(state, repository, package) + if commit_hosted_packument_pointer(&state.storage, repository, package, &written_target) .await .is_err() { return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - let mut changed = 0usize; - for (version, data) in versions { - match hosted_has_version(state, repository, package, version).await { - Ok(true) => {} - Ok(false) => continue, - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), - } - let Some(message) = data.get("deprecated").and_then(|value| value.as_str()) else { - continue; - }; - let key = hosted_deprecation_key(repository, package, version); - let result = if message.is_empty() { - match state.storage.delete(&key).await { - Ok(()) | Err(StorageError::NotFound) => Ok(()), - Err(error) => Err(error), - } - } else { - state.storage.put(&key, message.as_bytes()).await - }; - if result.is_err() { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); - } - changed += 1; + // The pending marker is the crash-recovery boundary. Remove it only after + // the developer-visible generation pointer and every exact target have + // committed successfully. + let pointer_matches = matches!( + read_hosted_packument_pointer(&state.storage, repository, package).await, + Ok(Some(current)) if current == written_target + ); + if !pointer_matches + || verify_publish_target_state(&state.storage, &verification) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - if changed == 0 { - return StatusCode::NOT_FOUND.into_response(); + if clear_hosted_publish_pending(&state.storage, &pending) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } + + state.metrics.record_upload("npm"); + state + .audit + .log(AuditEntry::new("push", "api", package, "npm", repository)); + state.activity.push(ActivityEntry::new( + ActionType::Push, + package.to_string(), + RegistryType::Npm, + "LOCAL", + )); state.repo_index.invalidate("npm"); StatusCode::CREATED.into_response() } -async fn handle_put( - state: AppState, - target: RepositoryTarget, - path: String, - authority: NamespaceAuthority, - body: Bytes, -) -> Response { - let Some(package) = decode_package_name(&path) else { - return StatusCode::NOT_FOUND.into_response(); - }; - if enforce_namespace_scope(&authority, &package).is_err() { - return StatusCode::FORBIDDEN.into_response(); +#[cfg(test)] +async fn publish( + state: &AppState, + repository: &str, + write_policy: NpmWritePolicy, + package: &str, + payload: &serde_json::Value, +) -> Response { + publish_with_import(state, repository, write_policy, package, payload, None).await +} + +struct DesiredImportVersion { + manifest: serde_json::Value, + manifest_sha256: String, + deprecation: Option, +} + +struct ValidatedHostedImport { + versions: std::collections::BTreeMap, + tags: std::collections::BTreeMap, + package_fields: serde_json::Value, +} + +fn validate_hosted_import( + package: &str, + packument: &serde_json::Value, +) -> Result { + let object = packument.as_object().ok_or_else(|| { + NpmHttpError::new(StatusCode::BAD_REQUEST, "Invalid npm import packument") + })?; + if object.get("name").and_then(serde_json::Value::as_str) != Some(package) { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm package name does not match the route", + )); } - let hosted = match writable_hosted(&state, &target, true) { - Ok(hosted) => hosted, - Err(error) => return error.into_response(), - }; - let payload = match serde_json::from_slice::(&body) { - Ok(value) => value, - Err(_) => return (StatusCode::BAD_REQUEST, "Invalid JSON").into_response(), - }; - if payload.get("_attachments").is_some() { - publish( - &state, - &hosted.name, - hosted.write_policy, - &package, - &payload, + let allowed_top = [ + "name", + "_id", + "description", + "readme", + "license", + "versions", + "dist-tags", + ]; + if object + .keys() + .any(|field| !allowed_top.contains(&field.as_str())) + { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm packument has unsupported package fields", + )); + } + let desired_versions = object + .get("versions") + .and_then(serde_json::Value::as_object) + .filter(|versions| !versions.is_empty()) + .ok_or_else(|| { + NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm packument has no versions", + ) + })?; + let mut versions = std::collections::BTreeMap::new(); + for (version, desired) in desired_versions { + if !is_valid_npm_version(version) + || desired.get("name").and_then(serde_json::Value::as_str) != Some(package) + || desired.get("version").and_then(serde_json::Value::as_str) != Some(version.as_str()) + { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm version metadata has an invalid coordinate", + )); + } + let mut manifest = desired.clone(); + let manifest_object = manifest.as_object_mut().ok_or_else(|| { + NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm version metadata is not an object", + ) + })?; + let deprecation = match manifest_object.remove("deprecated") { + Some(serde_json::Value::String(message)) => Some(message), + Some(_) => { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm deprecation is invalid", + )) + } + None => None, + }; + let dist = manifest_object + .get("dist") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| { + NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm version has invalid dist metadata", + ) + })?; + if dist.contains_key("tarball") + || !dist + .get("shasum") + .and_then(serde_json::Value::as_str) + .is_some_and(|digest| { + digest.len() == 40 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + || crate::npm_layout::hosted_blob_digest_from_manifest( + &serde_json::to_vec(&manifest).unwrap_or_default(), + ) + .is_none() + { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm dist checksums are invalid or include a route URL", + )); + } + let manifest_bytes = serde_json::to_vec(&manifest).map_err(|_| { + NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm version metadata is not serializable", + ) + })?; + versions.insert( + version.clone(), + DesiredImportVersion { + manifest, + manifest_sha256: crate::npm_layout::hosted_manifest_digest(&manifest_bytes), + deprecation, + }, + ); + } + + let desired_tags = object + .get("dist-tags") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| { + NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm dist-tags are invalid", + ) + })?; + let mut tags = std::collections::BTreeMap::new(); + for (tag, target) in desired_tags { + let target = target.as_str().ok_or_else(|| { + NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm dist-tag target is invalid", + ) + })?; + if !is_valid_dist_tag(tag) || !versions.contains_key(target) { + return Err(NpmHttpError::new( + StatusCode::BAD_REQUEST, + "Imported npm dist-tag references a missing version", + )); + } + tags.insert(tag.clone(), target.to_string()); + } + + let mut package_fields = serde_json::Map::new(); + for field in ["name", "_id", "description", "readme", "license"] { + if let Some(value) = object.get(field) { + package_fields.insert(field.to_string(), value.clone()); + } + } + Ok(ValidatedHostedImport { + versions, + tags, + package_fields: serde_json::Value::Object(package_fields), + }) +} + +async fn validate_import_version_state( + state: &AppState, + repository: &str, + package: &str, + full_sha256: &str, + desired: &ValidatedHostedImport, + session: &HostedImportSession, +) -> Result<(), StorageError> { + let desired_roster = desired + .versions + .iter() + .map(|(version, desired)| (version.clone(), desired.manifest_sha256.clone())) + .collect::>(); + if session.repository != repository + || session.package != package + || session.packument_sha256 != full_sha256 + || session.versions != desired_roster + { + return Err(StorageError::AlreadyExists); + } + + if let Some(base) = &session.base { + let base_packument = + hosted_packument_for_pointer(&state.storage, repository, package, base).await?; + let base_versions = base_packument + .get("versions") + .and_then(serde_json::Value::as_object) + .ok_or(StorageError::IntegrityViolation)?; + if !base_versions + .keys() + .all(|version| desired.versions.contains_key(version)) + { + return Err(StorageError::AlreadyExists); + } + } + + let desired_checks = desired + .versions + .iter() + .map(|(version, desired)| { + ( + version.clone(), + desired.manifest.clone(), + desired.manifest_sha256.clone(), + ) + }) + .collect::>(); + let checks = stream::iter(desired_checks) + .map( + |(version, desired_manifest, desired_manifest_sha256)| async move { + let manifest = state + .storage + .get(&hosted_version_key(repository, package, &version)) + .await?; + let desired_manifest_bytes = serde_json::to_vec(&desired_manifest) + .map_err(|_| StorageError::IntegrityViolation)?; + if manifest.as_ref() != desired_manifest_bytes.as_slice() { + return Err(StorageError::AlreadyExists); + } + let completion = state + .storage + .get(&hosted_publish_complete_key(repository, package, &version)) + .await?; + if completion.as_ref() != desired_manifest_sha256.as_bytes() { + return Err(StorageError::IntegrityViolation); + } + let blob_key = crate::npm_layout::hosted_blob_key_from_manifest( + repository, + package, + &desired_manifest_bytes, + ) + .ok_or(StorageError::IntegrityViolation)?; + let blob = state.storage.get(&blob_key).await?; + let actual_blob_digest = hex::encode(sha2::Sha512::digest(&blob)); + if !blob_key.ends_with(&format!("/{actual_blob_digest}.tgz")) { + return Err(StorageError::IntegrityViolation); + } + let evidence_key = crate::npm_layout::hosted_import_evidence_key( + repository, + package, + full_sha256, + &version, + &desired_manifest_sha256, + ); + let evidence = state.storage.get(&evidence_key).await?; + if evidence.as_ref() != b"1" { + return Err(StorageError::IntegrityViolation); + } + Ok::<(), StorageError>(()) + }, ) - .await - } else { - deprecate(&state, &hosted.name, &package, &payload).await + .buffer_unordered(32) + .collect::>() + .await; + for check in checks { + check?; } + Ok(()) } -pub(crate) async fn named_put_request( - state: AppState, - repository: String, - path: String, - authority: NamespaceAuthority, - body: Bytes, -) -> Response { - let Some(target) = named_target(&state, &repository) else { - return StatusCode::NOT_FOUND.into_response(); - }; - handle_put(state, target, path, authority, body).await +#[derive(Debug, Clone, Default)] +struct HostedMutableState { + package: Option>, + tags: BTreeMap>, + deprecations: BTreeMap>, } -async fn alias_put( - State(state): State, - Path(path): Path, - Extension(authority): Extension, - body: Bytes, -) -> Response { - let Some(target) = alias_target(&state) else { - return StatusCode::NOT_FOUND.into_response(); - }; - handle_put(state, target, path, authority, body).await +#[derive(Debug, Clone)] +struct HostedMutableAuthority { + key: String, + base: Option>, + target: Option>, } -async fn handle_dist_tags_get( - state: AppState, - target: RepositoryTarget, - response_base: String, - headers: HeaderMap, - package: String, -) -> Response { - let Some(package) = decode_package_name(&package) else { - return StatusCode::NOT_FOUND.into_response(); - }; - match target_packument(&state, &target, &package, &response_base).await { - Ok(packument) => { - let tags = packument - .value - .get("dist-tags") - .cloned() - .unwrap_or_else(|| serde_json::json!({})); - json_response_with_stale(&headers, &tags, packument.stale) +fn package_fields_from_packument(packument: &serde_json::Value) -> serde_json::Value { + let mut fields = serde_json::Map::new(); + for field in ["name", "_id", "description", "readme", "license"] { + if let Some(value) = packument.get(field) { + fields.insert(field.to_string(), value.clone()); } - Err(error) => read_error_response(error), } + serde_json::Value::Object(fields) } -async fn named_dist_tags_get( - State(state): State, - headers: HeaderMap, - Path((repository, package)): Path<(String, String)>, -) -> Response { - let Some(target) = named_target(&state, &repository) else { - return StatusCode::NOT_FOUND.into_response(); +fn hosted_mutable_state_from_packument( + packument: Option<&serde_json::Value>, +) -> Result { + let Some(packument) = packument else { + return Ok(HostedMutableState::default()); }; - handle_dist_tags_get( - state.clone(), - target, - public_base(&state, Some(&repository)), - headers, + let package = Some( + serde_json::to_vec(&package_fields_from_packument(packument)) + .map_err(|_| StorageError::IntegrityViolation)?, + ); + let tags = packument + .get("dist-tags") + .and_then(serde_json::Value::as_object) + .ok_or(StorageError::IntegrityViolation)? + .iter() + .map(|(tag, target)| { + target + .as_str() + .map(|target| (tag.clone(), target.as_bytes().to_vec())) + .ok_or(StorageError::IntegrityViolation) + }) + .collect::, _>>()?; + let deprecations = packument + .get("versions") + .and_then(serde_json::Value::as_object) + .ok_or(StorageError::IntegrityViolation)? + .iter() + .filter_map(|(version, manifest)| { + manifest + .get("deprecated") + .and_then(serde_json::Value::as_str) + .filter(|message| !message.is_empty()) + .map(|message| (version.clone(), message.as_bytes().to_vec())) + }) + .collect(); + Ok(HostedMutableState { package, - ) - .await + tags, + deprecations, + }) } -async fn alias_dist_tags_get( - State(state): State, - headers: HeaderMap, - Path(package): Path, -) -> Response { - let Some(target) = alias_target(&state) else { - return StatusCode::NOT_FOUND.into_response(); - }; - handle_dist_tags_get( - state.clone(), - target, - public_base(&state, None), - headers, +fn hosted_mutable_state_from_import( + desired: &ValidatedHostedImport, +) -> Result { + let package = Some( + serde_json::to_vec(&desired.package_fields) + .map_err(|_| StorageError::IntegrityViolation)?, + ); + let tags = desired + .tags + .iter() + .map(|(tag, target)| (tag.clone(), target.as_bytes().to_vec())) + .collect(); + let deprecations = desired + .versions + .iter() + .filter_map(|(version, desired)| { + desired + .deprecation + .as_deref() + .filter(|message| !message.is_empty()) + .map(|message| (version.clone(), message.as_bytes().to_vec())) + }) + .collect(); + Ok(HostedMutableState { package, - ) - .await + tags, + deprecations, + }) } -async fn handle_dist_tag_put( - state: AppState, - target: RepositoryTarget, - package: String, - tag: String, - authority: NamespaceAuthority, - body: Bytes, -) -> Response { - let Some(package) = decode_package_name(&package) else { - return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); - }; - if !is_valid_dist_tag(&tag) { - return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); +fn hosted_mutable_authorities( + repository: &str, + package: &str, + base: &HostedMutableState, + target: &HostedMutableState, +) -> Vec { + let mut authorities = BTreeMap::>, Option>)>::new(); + authorities.insert( + hosted_package_key(repository, package), + (base.package.clone(), target.package.clone()), + ); + for tag in base.tags.keys().chain(target.tags.keys()) { + authorities + .entry(hosted_tag_key(repository, package, tag)) + .or_insert_with(|| (base.tags.get(tag).cloned(), target.tags.get(tag).cloned())); + } + for version in base.deprecations.keys().chain(target.deprecations.keys()) { + authorities + .entry(hosted_deprecation_key(repository, package, version)) + .or_insert_with(|| { + ( + base.deprecations.get(version).cloned(), + target.deprecations.get(version).cloned(), + ) + }); } - if enforce_namespace_scope(&authority, &package).is_err() { - return StatusCode::FORBIDDEN.into_response(); + authorities + .into_iter() + .map(|(key, (base, target))| HostedMutableAuthority { key, base, target }) + .collect() +} + +async fn read_optional_exact( + storage: &Storage, + key: &str, +) -> Result>, StorageError> { + match storage.get(key).await { + Ok(value) => Ok(Some(value.to_vec())), + Err(StorageError::NotFound) => Ok(None), + Err(error) => Err(error), } - let repository = match writable_hosted(&state, &target, false) { - Ok(hosted) => hosted.name, - Err(error) => return error.into_response(), - }; - let version = match serde_json::from_slice::(&body) { - Ok(version) if is_valid_npm_version(&version) => version, - _ => return (StatusCode::BAD_REQUEST, "Invalid version").into_response(), - }; - match hosted_has_version(&state, &repository, &package, &version).await { - Ok(true) => {} - Ok(false) => return StatusCode::NOT_FOUND.into_response(), - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), - } - let lock = state.publish_lock(&format!("npm:{repository}:{package}")); - let _guard = lock.lock().await; - match incomplete_publish_versions(&state, &repository, &package).await { - Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), - Ok(_) => {} - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), +} + +async fn verify_hosted_mutable_target( + storage: &Storage, + authorities: &[HostedMutableAuthority], +) -> Result<(), StorageError> { + let reads = stream::iter(authorities.iter().cloned()) + .map(|authority| async move { + let current = read_optional_exact(storage, &authority.key).await?; + if current.as_deref() == authority.target.as_deref() { + Ok(()) + } else { + Err(StorageError::AlreadyExists) + } + }) + .buffer_unordered(NPM_IMPORT_MUTATION_CONCURRENCY) + .collect::>() + .await; + for read in reads { + read?; } - match hosted_has_version(&state, &repository, &package, &version).await { - Ok(true) => {} - Ok(false) => return StatusCode::NOT_FOUND.into_response(), - Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + Ok(()) +} + +async fn converge_hosted_mutable_target( + storage: &Storage, + authorities: &[HostedMutableAuthority], +) -> Result<(), StorageError> { + #[derive(Debug)] + enum Mutation { + Put { key: String, value: Vec }, + Delete { key: String, expected: Vec }, + } + + // Exact preflight is the recovery fence: every authority must still be at + // the recorded base or target before the first retry write. + let reads = stream::iter(authorities.iter().cloned()) + .map(|authority| async move { + let current = read_optional_exact(storage, &authority.key).await?; + if current.as_deref() == authority.target.as_deref() { + return Ok(None); + } + if current.as_deref() != authority.base.as_deref() { + return Err(StorageError::AlreadyExists); + } + Ok(match (¤t, &authority.target) { + (_, Some(value)) => Some(Mutation::Put { + key: authority.key.clone(), + value: value.clone(), + }), + (Some(expected), None) => Some(Mutation::Delete { + key: authority.key.clone(), + expected: expected.clone(), + }), + (None, None) => None, + }) + }) + .buffer_unordered(NPM_IMPORT_MUTATION_CONCURRENCY) + .collect::>() + .await; + let mut mutations = Vec::new(); + for read in reads { + if let Some(mutation) = read? { + mutations.push(mutation); + } } - if invalidate_hosted_packument_cache(&state, &repository, &package) - .await - .is_err() - { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + let writes = stream::iter(mutations) + .map(|mutation| async move { + match mutation { + Mutation::Put { key, value } => { + put_exact_with_readback(storage, &key, &value).await + } + Mutation::Delete { key, expected } => { + delete_exact_with_readback(storage, &key, &expected).await + } + } + }) + .buffer_unordered(NPM_IMPORT_MUTATION_CONCURRENCY) + .collect::>() + .await; + for write in writes { + write?; } - if state - .storage - .put( - &hosted_tag_key(&repository, &package, &tag), - version.as_bytes(), - ) - .await - .is_err() + verify_hosted_mutable_target(storage, authorities).await +} + +struct HostedPublishVerification<'a> { + manifest_key: &'a str, + manifest: &'a [u8], + blob_key: &'a str, + blob: &'a [u8], + completion_key: &'a str, + completion_digest: &'a str, + mutable_authorities: &'a [HostedMutableAuthority], +} + +async fn verify_publish_target_state( + storage: &Storage, + verification: &HostedPublishVerification<'_>, +) -> Result<(), StorageError> { + let (stored_manifest, stored_blob, stored_completion, mutable) = tokio::join!( + storage.get(verification.manifest_key), + storage.get(verification.blob_key), + storage.get(verification.completion_key), + verify_hosted_mutable_target(storage, verification.mutable_authorities), + ); + if stored_manifest?.as_ref() != verification.manifest + || stored_blob?.as_ref() != verification.blob + || stored_completion?.as_ref() != verification.completion_digest.as_bytes() { - return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + return Err(StorageError::AlreadyExists); } - state.repo_index.invalidate("npm"); - StatusCode::CREATED.into_response() + mutable } -async fn named_dist_tag_put( - State(state): State, - Path((repository, package, tag)): Path<(String, String, String)>, - Extension(authority): Extension, - body: Bytes, -) -> Response { - let Some(target) = named_target(&state, &repository) else { - return StatusCode::NOT_FOUND.into_response(); - }; - handle_dist_tag_put(state, target, package, tag, authority, body).await +async fn replace_import_mutable_state( + state: &AppState, + repository: &str, + package: &str, + desired: &ValidatedHostedImport, + base_packument: Option<&serde_json::Value>, +) -> Result<(), StorageError> { + let base = hosted_mutable_state_from_packument(base_packument)?; + let target = hosted_mutable_state_from_import(desired)?; + let authorities = hosted_mutable_authorities(repository, package, &base, &target); + converge_hosted_mutable_target(&state.storage, &authorities).await } -async fn alias_dist_tag_put( +fn import_receipt_response(receipt: HostedImportReceipt, status: StatusCode) -> Response { + let mut response = axum::Json(receipt).into_response(); + *response.status_mut() = status; + response +} + +async fn named_import_finalize( State(state): State, - Path((package, tag)): Path<(String, String)>, + Path((repository, encoded_package)): Path<(String, String)>, + headers: HeaderMap, Extension(authority): Extension, body: Bytes, ) -> Response { - let Some(target) = alias_target(&state) else { - return StatusCode::NOT_FOUND.into_response(); - }; - handle_dist_tag_put(state, target, package, tag, authority, body).await -} - -async fn handle_dist_tag_delete( - state: AppState, - target: RepositoryTarget, - package: String, - tag: String, - authority: NamespaceAuthority, -) -> Response { - let Some(package) = decode_package_name(&package) else { - return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + let Some(package) = decode_package_name(&encoded_package) else { + return StatusCode::BAD_REQUEST.into_response(); }; - if !is_valid_dist_tag(&tag) { - return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + if enforce_namespace_scope(&authority, &package).is_err() { + return StatusCode::FORBIDDEN.into_response(); } - if tag == "latest" { + let Some(NpmRepository::Hosted { write_policy, .. }) = state.config.npm.repository(&repository) + else { return ( StatusCode::BAD_REQUEST, - "The latest dist-tag cannot be deleted", + "npm import finalize must target a named hosted repository", ) .into_response(); + }; + if *write_policy == NpmWritePolicy::Deny { + return StatusCode::METHOD_NOT_ALLOWED.into_response(); } - if enforce_namespace_scope(&authority, &package).is_err() { - return StatusCode::FORBIDDEN.into_response(); + let expected = match import_packument_sha256(&headers) { + Ok(Some(expected)) => expected, + Ok(None) => { + return ( + StatusCode::BAD_REQUEST, + "npm import finalize requires a packument SHA-256", + ) + .into_response() + } + Err(error) => return error.into_response(), + }; + if hex::encode(sha2::Sha256::digest(&body)) != expected { + return ( + StatusCode::CONFLICT, + "npm import packument body does not match its SHA-256 header", + ) + .into_response(); } - let repository = match writable_hosted(&state, &target, false) { - Ok(hosted) => hosted.name, + let packument = match serde_json::from_slice::(&body) { + Ok(packument) => packument, + Err(_) => return (StatusCode::BAD_REQUEST, "Invalid JSON").into_response(), + }; + let desired = match validate_hosted_import(&package, &packument) { + Ok(desired) => desired, Err(error) => return error.into_response(), }; + let lock = state.publish_lock(&format!("npm:{repository}:{package}")); let _guard = lock.lock().await; + match resume_hosted_maintenance_operation(&state.storage, &repository, &package).await { + Ok(true) => state.repo_index.invalidate("npm"), + Ok(false) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + let receipt_key = + crate::npm_layout::hosted_import_receipt_key(&repository, &package, &expected); + let marker_key = crate::npm_layout::hosted_import_pending_key(&repository, &package); + let session = match read_optional_import_session(&state.storage, &repository, &package).await { + Ok(Some(session)) if session.packument_sha256 == expected => Some(session), + Ok(Some(_)) => { + return ( + StatusCode::CONFLICT, + "Package bulk import is bound to a different packument", + ) + .into_response() + } + Ok(None) => None, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let receipt = match state.storage.get(&receipt_key).await { + Ok(receipt) => Some(receipt), + Err(StorageError::NotFound) => None, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if let Some(receipt) = receipt { + let receipt: HostedImportReceipt = match serde_json::from_slice(&receipt) { + Ok(receipt) => receipt, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if receipt.package != package + || receipt.full_sha256 != expected + || receipt.generation != expected + || receipt.version_count != desired.versions.len() + || !valid_sha256(&receipt.install_v1_sha256) + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + let generation = WrittenPackumentGeneration { + generation: receipt.generation.clone(), + full_sha256: receipt.full_sha256.clone(), + install_v1_sha256: receipt.install_v1_sha256.clone(), + }; + if validate_hosted_packument_pointer(&state.storage, &repository, &package, &generation) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + let Some(session) = session.as_ref() else { + // A fully completed receipt is immutable history, not a request to + // restore its generation. Later versions, tags, deprecations and + // pointer generations are deliberately ignored on replay. + return import_receipt_response(receipt, StatusCode::OK); + }; + if let Err(error) = validate_import_version_state( + &state, + &repository, + &package, + &expected, + &desired, + session, + ) + .await + { + return match error { + StorageError::AlreadyExists => ( + StatusCode::CONFLICT, + "Imported npm version state no longer matches the completed receipt", + ) + .into_response(), + _ => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + } + let base_packument = match &session.base { + Some(base) => { + match hosted_packument_for_pointer(&state.storage, &repository, &package, base) + .await + { + Ok(packument) => Some(packument), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + None => None, + }; + if replace_import_mutable_state( + &state, + &repository, + &package, + &desired, + base_packument.as_ref(), + ) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + match read_hosted_packument_pointer(&state.storage, &repository, &package).await { + Ok(current) if current.as_ref() == session.base.as_ref() => { + if commit_hosted_packument_pointer( + &state.storage, + &repository, + &package, + &generation, + ) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + } + Ok(Some(current)) if current == generation => {} + Ok(_) => { + return ( + StatusCode::CONFLICT, + "Completed npm import cannot replace a later package generation", + ) + .into_response() + } + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + let session_bytes = match serde_json::to_vec(session) { + Ok(bytes) => bytes, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if delete_exact_with_readback(&state.storage, &marker_key, &session_bytes) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + return import_receipt_response(receipt, StatusCode::OK); + } + + // Fresh-install imports must have an exact journal written by version PUTs; + // adopting unjournaled pre-existing objects would make omitted LIST keys + // indistinguishable from absence. + let Some(session) = session.as_ref() else { + return ( + StatusCode::CONFLICT, + "Package bulk import has no exact version journal", + ) + .into_response(); + }; match incomplete_publish_versions(&state, &repository, &package).await { Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), Ok(_) => {} Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } - if invalidate_hosted_packument_cache(&state, &repository, &package) + if let Err(error) = + validate_import_version_state(&state, &repository, &package, &expected, &desired, session) + .await + { + return match error { + StorageError::AlreadyExists => ( + StatusCode::CONFLICT, + "Imported npm version set or metadata does not match committed hosted state", + ) + .into_response(), + _ => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + } + match read_hosted_packument_pointer(&state.storage, &repository, &package).await { + Ok(current) if current.as_ref() == session.base.as_ref() => {} + Ok(_) => { + return ( + StatusCode::CONFLICT, + "Package generation changed after bulk import started", + ) + .into_response() + } + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + let base_packument = match &session.base { + Some(base) => { + match hosted_packument_for_pointer(&state.storage, &repository, &package, base).await { + Ok(packument) => Some(packument), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + None => None, + }; + if replace_import_mutable_state( + &state, + &repository, + &package, + &desired, + base_packument.as_ref(), + ) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + let generation = match write_hosted_packument_generation_documents( + &state.storage, + &repository, + &package, + &packument, + &body, + ) + .await + { + Ok(generation) if generation.full_sha256 == expected => generation, + Ok(_) => return StatusCode::CONFLICT.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let receipt = HostedImportReceipt { + package: package.clone(), + version_count: desired.versions.len(), + full_sha256: generation.full_sha256.clone(), + install_v1_sha256: generation.install_v1_sha256.clone(), + generation: generation.generation.clone(), + }; + let receipt_bytes = match serde_json::to_vec(&receipt) { + Ok(bytes) => bytes, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if put_immutable_storage(&state.storage, &receipt_key, &receipt_bytes) .await .is_err() + || commit_hosted_packument_pointer(&state.storage, &repository, &package, &generation) + .await + .is_err() { return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } - let key = hosted_tag_key(&repository, &package, &tag); - match state.storage.delete(&key).await { - Ok(()) | Err(StorageError::NotFound) => {} + let session_bytes = match serde_json::to_vec(session) { + Ok(bytes) => bytes, Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if delete_exact_with_readback(&state.storage, &marker_key, &session_bytes) + .await + .is_err() + { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); } state.repo_index.invalidate("npm"); - StatusCode::NO_CONTENT.into_response() + import_receipt_response(receipt, StatusCode::CREATED) } -async fn named_dist_tag_delete( - State(state): State, - Path((repository, package, tag)): Path<(String, String, String)>, - Extension(authority): Extension, -) -> Response { - let Some(target) = named_target(&state, &repository) else { - return StatusCode::NOT_FOUND.into_response(); - }; - handle_dist_tag_delete(state, target, package, tag, authority).await +#[derive(Debug, PartialEq, Eq)] +enum ManifestCommit { + Created, + ExistingSame, + Replaced, } -async fn alias_dist_tag_delete( - State(state): State, - Path((package, tag)): Path<(String, String)>, - Extension(authority): Extension, -) -> Response { - let Some(target) = alias_target(&state) else { - return StatusCode::NOT_FOUND.into_response(); - }; - handle_dist_tag_delete(state, target, package, tag, authority).await +#[derive(Debug)] +enum ManifestCommitError { + Conflict, + Storage(StorageError), } -fn proxy_for_audit(state: &AppState, target: &RepositoryTarget) -> Option { - match target { - RepositoryTarget::Legacy => legacy_proxy(state), - RepositoryTarget::Named(repository @ NpmRepository::Proxy { .. }) => { - configured_proxy(state, repository) - } - RepositoryTarget::Named(NpmRepository::Hosted { .. }) => None, - RepositoryTarget::Named(NpmRepository::Group { members, .. }) => members - .iter() - .filter_map(|name| state.config.npm.repository(name)) - .find_map(|repository| configured_proxy(state, repository)), +async fn commit_hosted_manifest( + state: &AppState, + key: &str, + manifest: &[u8], + write_policy: NpmWritePolicy, +) -> Result { + match write_policy { + NpmWritePolicy::Deny => Err(ManifestCommitError::Conflict), + NpmWritePolicy::AllowOnce => match put_immutable(state, key, manifest) + .await + .map_err(ManifestCommitError::Storage)? + { + ImmutableWrite::Created => Ok(ManifestCommit::Created), + ImmutableWrite::ExistingSame => Ok(ManifestCommit::ExistingSame), + ImmutableWrite::Conflict => Err(ManifestCommitError::Conflict), + }, + NpmWritePolicy::Allow => match state.storage.get(key).await { + Ok(existing) if existing.as_ref() == manifest => Ok(ManifestCommit::ExistingSame), + Ok(_) => { + state + .storage + .put(key, manifest) + .await + .map_err(ManifestCommitError::Storage)?; + Ok(ManifestCommit::Replaced) + } + Err(StorageError::NotFound) => { + state + .storage + .put(key, manifest) + .await + .map_err(ManifestCommitError::Storage)?; + Ok(ManifestCommit::Created) + } + Err(error) => Err(ManifestCommitError::Storage(error)), + }, } } -fn npm_audit_error(status: StatusCode, message: &'static str) -> Response { - ( - status, - [( - header::CONTENT_TYPE, - HeaderValue::from_static("application/json"), - )], - serde_json::to_vec(&serde_json::json!({"error": message})) - .expect("static audit error JSON"), - ) - .into_response() -} - -#[derive(Debug, PartialEq, Eq)] -enum AuditBodyError { - Invalid, - TooLarge, -} - -fn decode_audit_body(headers: &HeaderMap, body: &[u8]) -> Result<(Vec, bool), AuditBodyError> { - let encoding = headers - .get(header::CONTENT_ENCODING) - .and_then(|value| value.to_str().ok()) - .unwrap_or("identity") - .trim(); - if encoding.eq_ignore_ascii_case("identity") || encoding.is_empty() { - if body.is_empty() { - return Err(AuditBodyError::Invalid); - } - return Ok((body.to_vec(), false)); +async fn deprecate( + state: &AppState, + repository: &str, + package: &str, + payload: &serde_json::Value, +) -> Response { + if payload.get("name").and_then(|value| value.as_str()) != Some(package) { + return (StatusCode::BAD_REQUEST, "Package name mismatch").into_response(); } - if !encoding.eq_ignore_ascii_case("gzip") { - return Err(AuditBodyError::Invalid); + let Some(versions) = payload.get("versions").and_then(|value| value.as_object()) else { + return (StatusCode::BAD_REQUEST, "Missing versions").into_response(); + }; + let lock = state.publish_lock(&format!("npm:{repository}:{package}")); + let _guard = lock.lock().await; + match resume_hosted_maintenance_operation(&state.storage, repository, package).await { + Ok(true) => state.repo_index.invalidate("npm"), + Ok(false) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } - let mut decoded = Vec::new(); - flate2::read::GzDecoder::new(body) - .take((NPM_AUDIT_BODY_CAP + 1) as u64) - .read_to_end(&mut decoded) - .map_err(|_| AuditBodyError::Invalid)?; - if decoded.len() > NPM_AUDIT_BODY_CAP { - return Err(AuditBodyError::TooLarge); + match state + .storage + .get(&crate::npm_layout::hosted_import_pending_key( + repository, package, + )) + .await + { + Ok(_) => { + return ( + StatusCode::CONFLICT, + "Package has an active bulk import; finalize it before normal mutation", + ) + .into_response() + } + Err(StorageError::NotFound) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } - if decoded.is_empty() { - return Err(AuditBodyError::Invalid); + match incomplete_publish_versions(state, repository, package).await { + Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), + Ok(_) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } - Ok((decoded, true)) -} - -fn gzip_audit_body(body: &[u8]) -> Result, AuditBodyError> { - let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); - encoder - .write_all(body) - .map_err(|_| AuditBodyError::Invalid)?; - encoder.finish().map_err(|_| AuditBodyError::Invalid) -} - -fn retain_public_dependencies( - value: &mut serde_json::Value, - engine: &crate::curation::CurationEngine, -) { - let Some(object) = value.as_object_mut() else { - return; - }; - for field in ["dependencies", "requires"] { - if let Some(dependencies) = object - .get_mut(field) - .and_then(serde_json::Value::as_object_mut) + for attempt in 0..2 { + let base = match read_hosted_packument_pointer(&state.storage, repository, package).await { + Ok(Some(pointer)) => pointer, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let base_packument = + match hosted_packument_for_pointer(&state.storage, repository, package, &base).await { + Ok(packument) => packument, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let Some(committed_versions) = base_packument + .get("versions") + .and_then(serde_json::Value::as_object) + else { + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + }; + let mut matched = 0usize; + let mut changes = std::collections::BTreeMap::new(); + for (version, data) in versions { + let Some(manifest) = committed_versions + .get(version) + .and_then(serde_json::Value::as_object) + else { + continue; + }; + let Some(message) = data.get("deprecated").and_then(serde_json::Value::as_str) else { + continue; + }; + matched += 1; + let current = match manifest.get("deprecated") { + Some(value) => match value.as_str() { + Some(value) => Some(value.to_string()), + None => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }, + None => None, + }; + let desired = (!message.is_empty()).then(|| message.to_string()); + let key = hosted_deprecation_key(repository, package, version); + match read_optional_string(&state.storage, &key).await { + Ok(authoritative) if authoritative == current => {} + _ => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + if current != desired { + changes.insert(version.clone(), desired); + } + } + if matched == 0 { + return StatusCode::NOT_FOUND.into_response(); + } + if changes.is_empty() { + return StatusCode::CREATED.into_response(); + } + let action = HostedMaintenanceAction::Deprecations { values: changes }; + let target_packument = match apply_hosted_maintenance_action(base_packument, &action) { + Ok(packument) => packument, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + match execute_hosted_metadata_maintenance( + &state.storage, + repository, + package, + base, + &target_packument, + action, + ) + .await { - dependencies.retain(|package, _| { - !crate::curation::is_internal_namespace( - engine, - crate::curation::RegistryType::Npm, - package, - ) - }); - for dependency in dependencies.values_mut() { - retain_public_dependencies(dependency, engine); + Ok(()) => { + state.repo_index.invalidate("npm"); + return StatusCode::CREATED.into_response(); } + Err(StorageError::AlreadyExists) if attempt == 0 => continue, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), } } - if let Some(packages) = object - .get_mut("packages") - .and_then(serde_json::Value::as_object_mut) - { - packages.retain(|path, _| { - let package = path - .rsplit_once("node_modules/") - .map(|(_, package)| package) - .unwrap_or(path); - package.is_empty() - || !crate::curation::is_internal_namespace( - engine, - crate::curation::RegistryType::Npm, - package, - ) - }); - for package in packages.values_mut() { - retain_public_dependencies(package, engine); - } + StatusCode::INTERNAL_SERVER_ERROR.into_response() +} + +async fn handle_put( + state: AppState, + target: RepositoryTarget, + path: String, + headers: HeaderMap, + body: Bytes, + authorize: F, +) -> Response +where + F: FnOnce(&str) -> bool, +{ + let Some(package) = decode_package_name(&path) else { + return StatusCode::NOT_FOUND.into_response(); + }; + if !authorize(&package) { + return StatusCode::FORBIDDEN.into_response(); } - if object - .get("name") - .and_then(|value| value.as_str()) - .is_some_and(|package| { - crate::curation::is_internal_namespace( - engine, - crate::curation::RegistryType::Npm, - package, - ) - }) + let import_hash = match import_packument_sha256(&headers) { + Ok(value) => value, + Err(error) => return error.into_response(), + }; + if import_hash.is_some() + && !matches!( + target, + RepositoryTarget::Named(NpmRepository::Hosted { .. }) + ) { - object.remove("name"); + return ( + StatusCode::BAD_REQUEST, + "Bulk import version PUT must target a named hosted npm repository", + ) + .into_response(); } - for (field, child) in object { - if field != "dependencies" && field != "requires" && field != "packages" { - match child { - serde_json::Value::Array(values) => { - for value in values { - retain_public_dependencies(value, engine); - } + let hosted = if import_hash.is_some() { + match &target { + RepositoryTarget::Named(NpmRepository::Hosted { name, write_policy }) + if *write_policy != NpmWritePolicy::Deny => + { + // Bulk import is exact-only. Repository `allow` must not turn + // a delayed or conflicting import PUT into a redeploy. + WritableHosted { + name: name.clone(), + write_policy: NpmWritePolicy::AllowOnce, } - serde_json::Value::Object(_) => retain_public_dependencies(child, engine), - _ => {} } + RepositoryTarget::Named(NpmRepository::Hosted { .. }) => { + return StatusCode::METHOD_NOT_ALLOWED.into_response() + } + _ => unreachable!("import target shape was validated above"), + } + } else { + match writable_hosted(&state, &target, true) { + Ok(hosted) => hosted, + Err(error) => return error.into_response(), } + }; + let payload = match serde_json::from_slice::(&body) { + Ok(value) => value, + Err(_) => return (StatusCode::BAD_REQUEST, "Invalid JSON").into_response(), + }; + if payload.get("_attachments").is_some() { + publish_with_import( + &state, + &hosted.name, + hosted.write_policy, + &package, + &payload, + import_hash.as_deref(), + ) + .await + } else if import_hash.is_some() { + ( + StatusCode::BAD_REQUEST, + "npm import hash is valid only for a version publish PUT", + ) + .into_response() + } else { + deprecate(&state, &hosted.name, &package, &payload).await } } -fn text_contains_internal_package(text: &str, engine: &crate::curation::CurationEngine) -> bool { - let decoded = percent_encoding::percent_decode_str(text) - .decode_utf8_lossy() - .into_owned(); - if crate::curation::is_internal_namespace(engine, crate::curation::RegistryType::Npm, &decoded) - { - return true; - } - let is_internal_candidate = |candidate: &str| { - !candidate.is_empty() - && crate::curation::is_internal_namespace( - engine, - crate::curation::RegistryType::Npm, - candidate, - ) +pub(crate) async fn named_put_request( + state: AppState, + repository: String, + path: String, + headers: HeaderMap, + body: Bytes, + authorize: F, +) -> Response +where + F: FnOnce(&str) -> bool, +{ + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); }; - for token in decoded.split(|character: char| { - !(character.is_ascii_alphanumeric() - || matches!(character, '@' | '/' | '.' | '_' | '-' | '~')) - }) { - let token = token.trim_matches('/'); - if is_internal_candidate(token) { - return true; - } - let segments: Vec<&str> = token - .split('/') - .filter(|segment| !segment.is_empty()) - .collect(); - for (index, segment) in segments.iter().enumerate() { - if is_internal_candidate(segment) { - return true; - } - if segment.starts_with('@') - && index + 1 < segments.len() - && is_internal_candidate(&format!("{segment}/{}", segments[index + 1])) - { - return true; - } - } - } - let bytes = decoded.as_bytes(); - for start in 0..bytes.len() { - if bytes[start] != b'@' { - continue; - } - let mut end = start + 1; - while end < bytes.len() - && (bytes[end].is_ascii_alphanumeric() - || matches!(bytes[end], b'@' | b'/' | b'.' | b'_' | b'-')) - { - end += 1; - } - if let Some(candidate) = decoded.get(start..end) { - if crate::curation::is_internal_namespace( - engine, - crate::curation::RegistryType::Npm, - candidate, - ) { - return true; - } - } - } - false -} - -fn audit_json_contains_internal( - value: &serde_json::Value, - engine: &crate::curation::CurationEngine, -) -> bool { - match value { - serde_json::Value::String(text) => text_contains_internal_package(text, engine), - serde_json::Value::Array(values) => values - .iter() - .any(|value| audit_json_contains_internal(value, engine)), - serde_json::Value::Object(object) => object.iter().any(|(key, value)| { - text_contains_internal_package(key, engine) - || audit_json_contains_internal(value, engine) - }), - _ => false, - } + handle_put(state, target, path, headers, body, authorize).await } -fn filter_audit_json( - path: &str, - body: &[u8], - engine: &crate::curation::CurationEngine, - filter_active: bool, -) -> Option> { - if path == "-/npm/v1/security/advisories/bulk" { - let mut map = - serde_json::from_slice::>(body).ok()?; - if filter_active { - map.retain(|package, _| { - !crate::curation::is_internal_namespace( - engine, - crate::curation::RegistryType::Npm, - package, - ) - }); - } - if map.is_empty() { - return None; - } - let value = serde_json::Value::Object(map); - if filter_active && audit_json_contains_internal(&value, engine) { - return None; - } - return serde_json::to_vec(&value).ok(); - } - - let mut value = serde_json::from_slice::(body).ok()?; - if !value.is_object() { - return None; - } - if filter_active { - retain_public_dependencies(&mut value, engine); - if audit_json_contains_internal(&value, engine) { - return None; - } - } - serde_json::to_vec(&value).ok() +async fn alias_put( + State(state): State, + Path(path): Path, + headers: HeaderMap, + Extension(authority): Extension, + body: Bytes, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let authorize = move |package: &str| enforce_namespace_scope(&authority, package).is_ok(); + handle_put(state, target, path, headers, body, authorize).await } -async fn handle_post( +async fn handle_dist_tags_get( state: AppState, target: RepositoryTarget, - path: String, + response_base: String, headers: HeaderMap, - body: Body, + package: String, ) -> Response { - let is_bulk = path == "-/npm/v1/security/advisories/bulk"; - let is_quick = path == "-/npm/v1/security/audits/quick"; - let is_full = path == "-/npm/v1/security/audits"; - if !is_bulk && !is_quick && !is_full { - return method_not_allowed("GET, PUT"); - } - let body = match axum::body::to_bytes(body, NPM_AUDIT_BODY_CAP).await { - Ok(body) => body, - Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), - }; - let Some(proxy) = proxy_for_audit(&state, &target) else { - return npm_audit_error( - StatusCode::BAD_REQUEST, - "Audit requires a configured proxy repository", - ); - }; - let engine = &state.curation().curation_engine; - let filter_active = crate::curation::namespace_filter_active(engine); - let (decoded, was_gzip) = match decode_audit_body(&headers, &body) { - Ok(decoded) => decoded, - Err(AuditBodyError::TooLarge) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), - Err(AuditBodyError::Invalid) => { - return npm_audit_error(StatusCode::BAD_REQUEST, "Invalid audit request body") - } - }; - let Some(filtered) = filter_audit_json(&path, &decoded, engine, filter_active) else { - return npm_audit_error(StatusCode::BAD_REQUEST, "Empty or invalid audit request"); - }; - let forward = if was_gzip { - match gzip_audit_body(&filtered) { - Ok(body) => body, - Err(_) => { - return npm_audit_error(StatusCode::BAD_REQUEST, "Invalid audit request body") - } - } - } else { - filtered + let Some(package) = decode_package_name(&package) else { + return StatusCode::NOT_FOUND.into_response(); }; - let mut forwarded_headers = Vec::new(); - if let Some(value) = headers - .get(header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - { - forwarded_headers.push(("content-type", value)); - } - if let Some(value) = headers - .get(header::CONTENT_ENCODING) - .and_then(|value| value.to_str().ok()) - { - forwarded_headers.push(("content-encoding", value)); - } - if let Some(value) = headers - .get(header::ACCEPT) - .and_then(|value| value.to_str().ok()) - { - forwarded_headers.push(("accept", value)); - } - let url = format!("{}/{}", proxy.url.trim_end_matches('/'), path); - match proxy_forward_post( - &state.no_redirect_http_client, - &url, - Duration::from_secs(state.config.npm.proxy_timeout), - expose_opt(&proxy.auth), - &forwarded_headers, - &forward, - &state.circuit_breaker, - RegistryType::Npm, - MAX_NPM_PROXY_REDIRECTS, - |next_url| validated_proxy_url(&proxy, next_url.as_str()).is_some(), + match target_packument( + &state, + &target, + &package, + &response_base, + PackumentFlavor::Full, ) .await { - Ok((status, response_body, content_type)) => { - state - .audit - .log(AuditEntry::new("proxy_fetch", "api", "", "npm", "audit")); - let status = StatusCode::from_u16(status).unwrap_or(StatusCode::BAD_GATEWAY); - let content_type = content_type - .as_deref() - .and_then(|value| HeaderValue::from_str(value).ok()) - .unwrap_or_else(|| HeaderValue::from_static("application/json")); - ( - status, - [(header::CONTENT_TYPE, content_type)], - response_body, - ) - .into_response() - } - Err(ProxyError::CircuitOpen(name)) => circuit_open_response(&name), - Err(error) => { - tracing::warn!(error = ?error, "npm audit upstream forward failed"); - StatusCode::BAD_GATEWAY.into_response() + Ok(packument) => { + let tags = packument + .value + .get("dist-tags") + .cloned() + .unwrap_or_else(|| serde_json::json!({})); + json_response_with_stale(&headers, &tags, packument.stale) } + Err(error) => read_error_response(error), } } -pub(crate) async fn named_post_request( - state: AppState, - repository: String, - path: String, +async fn named_dist_tags_get( + State(state): State, headers: HeaderMap, - body: Body, + Path((repository, package)): Path<(String, String)>, ) -> Response { let Some(target) = named_target(&state, &repository) else { return StatusCode::NOT_FOUND.into_response(); }; - handle_post(state, target, path, headers, body).await + handle_dist_tags_get( + state.clone(), + target, + public_base(&state, Some(&repository)), + headers, + package, + ) + .await } -async fn alias_post( +async fn alias_dist_tags_get( State(state): State, - Path(path): Path, headers: HeaderMap, - body: Body, + Path(package): Path, ) -> Response { let Some(target) = alias_target(&state) else { return StatusCode::NOT_FOUND.into_response(); }; - handle_post(state, target, path, headers, body).await -} - -#[cfg(test)] -mod tests { - use super::*; + handle_dist_tags_get( + state.clone(), + target, + public_base(&state, None), + headers, + package, + ) + .await +} - #[test] - fn group_merge_is_member_ordered_and_latest_is_not_derived() { - let first = serde_json::json!({ - "versions": { - "1.0.0": {"name": "p", "version": "1.0.0"}, - "2.0.0": {"name": "hosted", "version": "2.0.0"} +async fn handle_dist_tag_put( + state: AppState, + target: RepositoryTarget, + package: String, + tag: String, + authority: NamespaceAuthority, + body: Bytes, +) -> Response { + let Some(package) = decode_package_name(&package) else { + return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + }; + if !is_valid_dist_tag(&tag) { + return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + } + if enforce_namespace_scope(&authority, &package).is_err() { + return StatusCode::FORBIDDEN.into_response(); + } + let repository = match writable_hosted(&state, &target, false) { + Ok(hosted) => hosted.name, + Err(error) => return error.into_response(), + }; + let version = match serde_json::from_slice::(&body) { + Ok(version) if is_valid_npm_version(&version) => version, + _ => return (StatusCode::BAD_REQUEST, "Invalid version").into_response(), + }; + let lock = state.publish_lock(&format!("npm:{repository}:{package}")); + let _guard = lock.lock().await; + match resume_hosted_maintenance_operation(&state.storage, &repository, &package).await { + Ok(true) => state.repo_index.invalidate("npm"), + Ok(false) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + match hosted_import_active(&state.storage, &repository, &package).await { + Ok(true) => { + return ( + StatusCode::CONFLICT, + "Package has an active bulk import; finalize it before normal mutation", + ) + .into_response() + } + Ok(false) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + match incomplete_publish_versions(&state, &repository, &package).await { + Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), + Ok(_) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + for attempt in 0..2 { + let base = match read_hosted_packument_pointer(&state.storage, &repository, &package).await + { + Ok(Some(pointer)) => pointer, + Ok(None) => return StatusCode::NOT_FOUND.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let base_packument = match hosted_packument_for_pointer( + &state.storage, + &repository, + &package, + &base, + ) + .await + { + Ok(packument) => packument, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + if !base_packument + .get("versions") + .and_then(serde_json::Value::as_object) + .is_some_and(|versions| versions.contains_key(&version)) + { + return StatusCode::NOT_FOUND.into_response(); + } + let current = match base_packument + .get("dist-tags") + .and_then(serde_json::Value::as_object) + .and_then(|tags| tags.get(&tag)) + { + Some(value) => match value.as_str() { + Some(value) => Some(value.to_string()), + None => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), }, - "dist-tags": {"latest": "1.0.0"} - }); - let second = serde_json::json!({ - "versions": { - "2.0.0": {"name": "proxy", "version": "2.0.0"}, - "9.0.0": {"name": "p", "version": "9.0.0"} + None => None, + }; + let tag_key = hosted_tag_key(&repository, &package, &tag); + match read_optional_string(&state.storage, &tag_key).await { + Ok(authoritative) if authoritative == current => {} + _ => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + if current.as_deref() == Some(version.as_str()) { + return StatusCode::CREATED.into_response(); + } + let action = HostedMaintenanceAction::DistTag { + tag: tag.clone(), + value: Some(version.clone()), + }; + let target_packument = match apply_hosted_maintenance_action(base_packument, &action) { + Ok(packument) => packument, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + match execute_hosted_metadata_maintenance( + &state.storage, + &repository, + &package, + base, + &target_packument, + action, + ) + .await + { + Ok(()) => { + state.repo_index.invalidate("npm"); + return StatusCode::CREATED.into_response(); + } + Err(StorageError::AlreadyExists) if attempt == 0 => continue, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + StatusCode::INTERNAL_SERVER_ERROR.into_response() +} + +async fn named_dist_tag_put( + State(state): State, + Path((repository, package, tag)): Path<(String, String, String)>, + Extension(authority): Extension, + body: Bytes, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tag_put(state, target, package, tag, authority, body).await +} + +async fn alias_dist_tag_put( + State(state): State, + Path((package, tag)): Path<(String, String)>, + Extension(authority): Extension, + body: Bytes, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tag_put(state, target, package, tag, authority, body).await +} + +async fn handle_dist_tag_delete( + state: AppState, + target: RepositoryTarget, + package: String, + tag: String, + authority: NamespaceAuthority, +) -> Response { + let Some(package) = decode_package_name(&package) else { + return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + }; + if !is_valid_dist_tag(&tag) { + return (StatusCode::BAD_REQUEST, "Invalid package name or dist-tag").into_response(); + } + if tag == "latest" { + return ( + StatusCode::BAD_REQUEST, + "The latest dist-tag cannot be deleted", + ) + .into_response(); + } + if enforce_namespace_scope(&authority, &package).is_err() { + return StatusCode::FORBIDDEN.into_response(); + } + let repository = match writable_hosted(&state, &target, false) { + Ok(hosted) => hosted.name, + Err(error) => return error.into_response(), + }; + let lock = state.publish_lock(&format!("npm:{repository}:{package}")); + let _guard = lock.lock().await; + match resume_hosted_maintenance_operation(&state.storage, &repository, &package).await { + Ok(true) => state.repo_index.invalidate("npm"), + Ok(false) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + match hosted_import_active(&state.storage, &repository, &package).await { + Ok(true) => { + return ( + StatusCode::CONFLICT, + "Package has an active bulk import; finalize it before normal mutation", + ) + .into_response() + } + Ok(false) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + match incomplete_publish_versions(&state, &repository, &package).await { + Ok(incomplete) if !incomplete.is_empty() => return incomplete_publish_response(), + Ok(_) => {} + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + for attempt in 0..2 { + let base = match read_hosted_packument_pointer(&state.storage, &repository, &package).await + { + Ok(Some(pointer)) => pointer, + Ok(None) => return StatusCode::NO_CONTENT.into_response(), + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let base_packument = match hosted_packument_for_pointer( + &state.storage, + &repository, + &package, + &base, + ) + .await + { + Ok(packument) => packument, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + let current = match base_packument + .get("dist-tags") + .and_then(serde_json::Value::as_object) + .and_then(|tags| tags.get(&tag)) + { + Some(value) => match value.as_str() { + Some(value) => Some(value.to_string()), + None => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), }, - "dist-tags": {"latest": "9.0.0", "next": "9.0.0"} - }); - let merged = merge_packuments("p", "https://nora/repository/group", vec![first, second]) - .expect("merge"); - assert_eq!(merged["dist-tags"]["latest"], "1.0.0"); - assert_eq!(merged["dist-tags"]["next"], "9.0.0"); - assert_eq!(merged["versions"]["2.0.0"]["name"], "hosted"); + None => None, + }; + let key = hosted_tag_key(&repository, &package, &tag); + match read_optional_string(&state.storage, &key).await { + Ok(authoritative) if authoritative == current => {} + _ => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + if current.is_none() { + return StatusCode::NO_CONTENT.into_response(); + } + let action = HostedMaintenanceAction::DistTag { + tag: tag.clone(), + value: None, + }; + let target_packument = match apply_hosted_maintenance_action(base_packument, &action) { + Ok(packument) => packument, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + }; + match execute_hosted_metadata_maintenance( + &state.storage, + &repository, + &package, + base, + &target_packument, + action, + ) + .await + { + Ok(()) => { + state.repo_index.invalidate("npm"); + return StatusCode::NO_CONTENT.into_response(); + } + Err(StorageError::AlreadyExists) if attempt == 0 => continue, + Err(_) => return StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } + StatusCode::INTERNAL_SERVER_ERROR.into_response() +} + +async fn named_dist_tag_delete( + State(state): State, + Path((repository, package, tag)): Path<(String, String, String)>, + Extension(authority): Extension, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tag_delete(state, target, package, tag, authority).await +} + +async fn alias_dist_tag_delete( + State(state): State, + Path((package, tag)): Path<(String, String)>, + Extension(authority): Extension, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_dist_tag_delete(state, target, package, tag, authority).await +} + +fn proxy_for_audit(state: &AppState, target: &RepositoryTarget) -> Option { + match target { + RepositoryTarget::Legacy => legacy_proxy(state), + RepositoryTarget::Named(repository @ NpmRepository::Proxy { .. }) => { + configured_proxy(state, repository) + } + RepositoryTarget::Named(NpmRepository::Hosted { .. }) => None, + RepositoryTarget::Named(NpmRepository::Group { members, .. }) => members + .iter() + .filter_map(|name| state.config.npm.repository(name)) + .find_map(|repository| configured_proxy(state, repository)), } +} + +fn npm_audit_error(status: StatusCode, message: &'static str) -> Response { + ( + status, + [( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + )], + serde_json::to_vec(&serde_json::json!({"error": message})) + .expect("static audit error JSON"), + ) + .into_response() +} + +#[derive(Debug, PartialEq, Eq)] +enum AuditBodyError { + Invalid, + TooLarge, +} + +fn decode_audit_body(headers: &HeaderMap, body: &[u8]) -> Result<(Vec, bool), AuditBodyError> { + let encoding = headers + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .unwrap_or("identity") + .trim(); + if encoding.eq_ignore_ascii_case("identity") || encoding.is_empty() { + if body.is_empty() { + return Err(AuditBodyError::Invalid); + } + return Ok((body.to_vec(), false)); + } + if !encoding.eq_ignore_ascii_case("gzip") { + return Err(AuditBodyError::Invalid); + } + let mut decoded = Vec::new(); + flate2::read::GzDecoder::new(body) + .take((NPM_AUDIT_BODY_CAP + 1) as u64) + .read_to_end(&mut decoded) + .map_err(|_| AuditBodyError::Invalid)?; + if decoded.len() > NPM_AUDIT_BODY_CAP { + return Err(AuditBodyError::TooLarge); + } + if decoded.is_empty() { + return Err(AuditBodyError::Invalid); + } + Ok((decoded, true)) +} + +fn gzip_audit_body(body: &[u8]) -> Result, AuditBodyError> { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(body) + .map_err(|_| AuditBodyError::Invalid)?; + encoder.finish().map_err(|_| AuditBodyError::Invalid) +} + +fn retain_public_dependencies( + value: &mut serde_json::Value, + engine: &crate::curation::CurationEngine, +) { + let Some(object) = value.as_object_mut() else { + return; + }; + for field in ["dependencies", "requires"] { + if let Some(dependencies) = object + .get_mut(field) + .and_then(serde_json::Value::as_object_mut) + { + dependencies.retain(|package, _| { + !crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + package, + ) + }); + for dependency in dependencies.values_mut() { + retain_public_dependencies(dependency, engine); + } + } + } + if let Some(packages) = object + .get_mut("packages") + .and_then(serde_json::Value::as_object_mut) + { + packages.retain(|path, _| { + let package = path + .rsplit_once("node_modules/") + .map(|(_, package)| package) + .unwrap_or(path); + package.is_empty() + || !crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + package, + ) + }); + for package in packages.values_mut() { + retain_public_dependencies(package, engine); + } + } + if object + .get("name") + .and_then(|value| value.as_str()) + .is_some_and(|package| { + crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + package, + ) + }) + { + object.remove("name"); + } + for (field, child) in object { + if field != "dependencies" && field != "requires" && field != "packages" { + match child { + serde_json::Value::Array(values) => { + for value in values { + retain_public_dependencies(value, engine); + } + } + serde_json::Value::Object(_) => retain_public_dependencies(child, engine), + _ => {} + } + } + } +} + +fn text_contains_internal_package(text: &str, engine: &crate::curation::CurationEngine) -> bool { + let decoded = percent_encoding::percent_decode_str(text) + .decode_utf8_lossy() + .into_owned(); + if crate::curation::is_internal_namespace(engine, crate::curation::RegistryType::Npm, &decoded) + { + return true; + } + let is_internal_candidate = |candidate: &str| { + !candidate.is_empty() + && crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + candidate, + ) + }; + for token in decoded.split(|character: char| { + !(character.is_ascii_alphanumeric() + || matches!(character, '@' | '/' | '.' | '_' | '-' | '~')) + }) { + let token = token.trim_matches('/'); + if is_internal_candidate(token) { + return true; + } + let segments: Vec<&str> = token + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + for (index, segment) in segments.iter().enumerate() { + if is_internal_candidate(segment) { + return true; + } + if segment.starts_with('@') + && index + 1 < segments.len() + && is_internal_candidate(&format!("{segment}/{}", segments[index + 1])) + { + return true; + } + } + } + let bytes = decoded.as_bytes(); + for start in 0..bytes.len() { + if bytes[start] != b'@' { + continue; + } + let mut end = start + 1; + while end < bytes.len() + && (bytes[end].is_ascii_alphanumeric() + || matches!(bytes[end], b'@' | b'/' | b'.' | b'_' | b'-')) + { + end += 1; + } + if let Some(candidate) = decoded.get(start..end) { + if crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + candidate, + ) { + return true; + } + } + } + false +} + +fn audit_json_contains_internal( + value: &serde_json::Value, + engine: &crate::curation::CurationEngine, +) -> bool { + match value { + serde_json::Value::String(text) => text_contains_internal_package(text, engine), + serde_json::Value::Array(values) => values + .iter() + .any(|value| audit_json_contains_internal(value, engine)), + serde_json::Value::Object(object) => object.iter().any(|(key, value)| { + text_contains_internal_package(key, engine) + || audit_json_contains_internal(value, engine) + }), + _ => false, + } +} + +fn filter_audit_json( + path: &str, + body: &[u8], + engine: &crate::curation::CurationEngine, + filter_active: bool, +) -> Option> { + if path == "-/npm/v1/security/advisories/bulk" { + let mut map = + serde_json::from_slice::>(body).ok()?; + if filter_active { + map.retain(|package, _| { + !crate::curation::is_internal_namespace( + engine, + crate::curation::RegistryType::Npm, + package, + ) + }); + } + if map.is_empty() { + return None; + } + let value = serde_json::Value::Object(map); + if filter_active && audit_json_contains_internal(&value, engine) { + return None; + } + return serde_json::to_vec(&value).ok(); + } + + let mut value = serde_json::from_slice::(body).ok()?; + if !value.is_object() { + return None; + } + if filter_active { + retain_public_dependencies(&mut value, engine); + if audit_json_contains_internal(&value, engine) { + return None; + } + } + serde_json::to_vec(&value).ok() +} + +async fn handle_post( + state: AppState, + target: RepositoryTarget, + path: String, + headers: HeaderMap, + body: Body, +) -> Response { + let is_bulk = path == "-/npm/v1/security/advisories/bulk"; + let is_quick = path == "-/npm/v1/security/audits/quick"; + let is_full = path == "-/npm/v1/security/audits"; + if !is_bulk && !is_quick && !is_full { + return method_not_allowed("GET, PUT"); + } + let body = match axum::body::to_bytes(body, NPM_AUDIT_BODY_CAP).await { + Ok(body) => body, + Err(_) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), + }; + let Some(proxy) = proxy_for_audit(&state, &target) else { + return npm_audit_error( + StatusCode::BAD_REQUEST, + "Audit requires a configured proxy repository", + ); + }; + let engine = &state.curation().curation_engine; + let filter_active = crate::curation::namespace_filter_active(engine); + let (decoded, was_gzip) = match decode_audit_body(&headers, &body) { + Ok(decoded) => decoded, + Err(AuditBodyError::TooLarge) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), + Err(AuditBodyError::Invalid) => { + return npm_audit_error(StatusCode::BAD_REQUEST, "Invalid audit request body") + } + }; + let Some(filtered) = filter_audit_json(&path, &decoded, engine, filter_active) else { + return npm_audit_error(StatusCode::BAD_REQUEST, "Empty or invalid audit request"); + }; + let forward = if was_gzip { + match gzip_audit_body(&filtered) { + Ok(body) => body, + Err(_) => { + return npm_audit_error(StatusCode::BAD_REQUEST, "Invalid audit request body") + } + } + } else { + filtered + }; + let mut forwarded_headers = Vec::new(); + if let Some(value) = headers + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + { + forwarded_headers.push(("content-type", value)); + } + if let Some(value) = headers + .get(header::CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + { + forwarded_headers.push(("content-encoding", value)); + } + if let Some(value) = headers + .get(header::ACCEPT) + .and_then(|value| value.to_str().ok()) + { + forwarded_headers.push(("accept", value)); + } + let url = format!("{}/{}", proxy.url.trim_end_matches('/'), path); + match proxy_forward_post( + &state.no_redirect_http_client, + &url, + Duration::from_secs(state.config.npm.proxy_timeout), + expose_opt(&proxy.auth), + &forwarded_headers, + &forward, + &state.circuit_breaker, + RegistryType::Npm, + MAX_NPM_PROXY_REDIRECTS, + |next_url| validated_proxy_url(&proxy, next_url.as_str()).is_some(), + ) + .await + { + Ok((status, response_body, content_type)) => { + state + .audit + .log(AuditEntry::new("proxy_fetch", "api", "", "npm", "audit")); + let status = StatusCode::from_u16(status).unwrap_or(StatusCode::BAD_GATEWAY); + let content_type = content_type + .as_deref() + .and_then(|value| HeaderValue::from_str(value).ok()) + .unwrap_or_else(|| HeaderValue::from_static("application/json")); + ( + status, + [(header::CONTENT_TYPE, content_type)], + response_body, + ) + .into_response() + } + Err(ProxyError::CircuitOpen(name)) => circuit_open_response(&name), + Err(error) => { + tracing::warn!(error = ?error, "npm audit upstream forward failed"); + StatusCode::BAD_GATEWAY.into_response() + } + } +} + +pub(crate) async fn named_post_request( + state: AppState, + repository: String, + path: String, + headers: HeaderMap, + body: Body, +) -> Response { + let Some(target) = named_target(&state, &repository) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_post(state, target, path, headers, body).await +} + +async fn alias_post( + State(state): State, + Path(path): Path, + headers: HeaderMap, + body: Body, +) -> Response { + let Some(target) = alias_target(&state) else { + return StatusCode::NOT_FOUND.into_response(); + }; + handle_post(state, target, path, headers, body).await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn group_merge_is_member_ordered_and_latest_is_not_derived() { + let first = serde_json::json!({ + "versions": { + "1.0.0": {"name": "p", "version": "1.0.0"}, + "2.0.0": {"name": "hosted", "version": "2.0.0"} + }, + "dist-tags": {"latest": "1.0.0"} + }); + let second = serde_json::json!({ + "versions": { + "2.0.0": {"name": "proxy", "version": "2.0.0"}, + "9.0.0": {"name": "p", "version": "9.0.0"} + }, + "dist-tags": {"latest": "9.0.0", "next": "9.0.0"} + }); + let merged = merge_packuments("p", "https://nora/repository/group", vec![first, second]) + .expect("merge"); + assert_eq!(merged["dist-tags"]["latest"], "1.0.0"); + assert_eq!(merged["dist-tags"]["next"], "9.0.0"); + assert_eq!(merged["versions"]["2.0.0"]["name"], "hosted"); + } + + #[test] + fn attachment_validation_rejects_paths() { + assert!(is_valid_attachment_name("pkg-1.0.0.tgz")); + assert!(!is_valid_attachment_name("../pkg.tgz")); + assert!(!is_valid_attachment_name("scope/pkg.tgz")); + } + + #[test] + fn dist_tags_reject_semver_like_names() { + assert!(is_valid_dist_tag("latest")); + assert!(!is_valid_dist_tag("1.2.3")); + assert!(!is_valid_dist_tag("^1")); + } + + #[tokio::test] + async fn maintenance_marker_and_pointer_ambiguous_results_use_exact_readback() { + use crate::test_helpers::{create_test_context_with_config, send, FaultInjectBackend}; + use axum::http::Method; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + let base = read_hosted_packument_pointer(&ctx.state.storage, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + let base_packument = + hosted_packument_for_pointer(&ctx.state.storage, "npm-private", "pkg", &base) + .await + .unwrap(); + let action = HostedMaintenanceAction::DistTag { + tag: "next".to_string(), + value: Some("1.0.0".to_string()), + }; + let target_packument = apply_hosted_maintenance_action(base_packument, &action).unwrap(); + let full = serde_json::to_vec(&target_packument).unwrap(); + let target = write_hosted_packument_generation_documents( + &ctx.state.storage, + "npm-private", + "pkg", + &target_packument, + &full, + ) + .await + .unwrap(); + let operation = HostedMaintenanceOperation { + schema: crate::npm_layout::HOSTED_MAINTENANCE_SCHEMA_V1, + repository: "npm-private".to_string(), + package: "pkg".to_string(), + base, + target: HostedMaintenanceTarget::Live { + pointer: target.clone(), + }, + action, + }; + let marker_key = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + + let create_backend = + FaultInjectBackend::new(ctx.state.storage.clone()).fail_create_after(&marker_key); + let create_storage = Storage::from_backend(Arc::new(create_backend)); + let marker = create_hosted_maintenance_marker(&create_storage, &operation) + .await + .expect("post-commit create error is resolved by exact readback"); + + let delete_backend = + FaultInjectBackend::new(ctx.state.storage.clone()).fail_delete_after(&marker_key); + let delete_storage = Storage::from_backend(Arc::new(delete_backend)); + clear_hosted_maintenance_marker(&delete_storage, &marker) + .await + .expect("post-commit delete error is resolved by NotFound readback"); + assert!(ctx.state.storage.stat(&marker_key).await.is_none()); + + let create_backend = + FaultInjectBackend::new(ctx.state.storage.clone()).fail_create(&marker_key); + let create_storage = Storage::from_backend(Arc::new(create_backend)); + assert!( + create_hosted_maintenance_marker(&create_storage, &operation) + .await + .is_err() + ); + assert!(ctx.state.storage.stat(&marker_key).await.is_none()); + + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let pointer_backend = + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put_after(&pointer_key); + let pointer_storage = Storage::from_backend(Arc::new(pointer_backend)); + commit_hosted_packument_pointer(&pointer_storage, "npm-private", "pkg", &target) + .await + .expect("post-commit pointer error is resolved by exact readback"); + assert_eq!( + read_hosted_packument_pointer(&ctx.state.storage, "npm-private", "pkg") + .await + .unwrap(), + Some(target) + ); + } + + #[cfg(test)] + fn npm_tarball(package: &str, version: &str) -> Vec { + npm_tarball_with_marker(package, version, "") + } + + #[cfg(test)] + fn npm_tarball_with_marker(package: &str, version: &str, marker: &str) -> Vec { + use flate2::write::GzEncoder; + use flate2::Compression; + let encoder = GzEncoder::new(Vec::new(), Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let package_json = serde_json::to_vec(&serde_json::json!({ + "name": package, + "version": version + })) + .unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_size(package_json.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "package/package.json", package_json.as_slice()) + .unwrap(); + if !marker.is_empty() { + let mut marker_header = tar::Header::new_gnu(); + marker_header.set_size(marker.len() as u64); + marker_header.set_mode(0o644); + marker_header.set_cksum(); + archive + .append_data( + &mut marker_header, + "package/republish-marker.txt", + marker.as_bytes(), + ) + .unwrap(); + } + archive.into_inner().unwrap().finish().unwrap() + } + + fn named_config(config: &mut crate::config::Config) { + config.npm.proxy = None; + config.npm.repositories = vec![ + NpmRepository::Hosted { + name: "npm-private".into(), + write_policy: NpmWritePolicy::AllowOnce, + }, + NpmRepository::Proxy { + name: "npm-registry".into(), + url: "http://127.0.0.1:1".into(), + auth: None, + metadata_ttl: Some(300), + negative_ttl: 0, + }, + NpmRepository::Group { + name: "npm-group".into(), + members: vec!["npm-private".into(), "npm-registry".into()], + writable_member: Some("npm-private".into()), + }, + ]; + config.npm.default_repository = Some("npm-group".into()); + } + + #[test] + fn invalid_named_config_without_default_never_falls_into_legacy_layout() { + let ctx = crate::test_helpers::create_test_context_with_config(|config| { + config.npm.repositories = vec![NpmRepository::Hosted { + name: "packages".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }]; + config.npm.default_repository = None; + }); + assert!(alias_target(&ctx.state).is_none()); + } + + fn publish_payload(package: &str, version: &str, tag: &str) -> Vec { + let tgz = npm_tarball(package, version); + serde_json::to_vec(&serde_json::json!({ + "name": package, + "versions": { + (version): { + "name": package, + "version": version, + "dist": {} + } + }, + "_attachments": { + (canonical_tarball_filename(package, version)): { + "data": base64::engine::general_purpose::STANDARD.encode(&tgz), + "length": tgz.len() + } + }, + "dist-tags": {(tag): version} + })) + .unwrap() + } + + fn import_fixture( + package: &str, + version: &str, + tag: &str, + marker: &str, + ) -> (serde_json::Value, Vec, String) { + let payload: serde_json::Value = + serde_json::from_slice(&publish_payload_with_tarball(package, version, tag, marker)) + .unwrap(); + let validated = validate_publish(package, &payload).unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&validated.manifest).unwrap(); + let body = serde_json::to_vec(&serde_json::json!({ + "name": package, + "versions": {(version): manifest}, + "dist-tags": {(tag): version}, + })) + .unwrap(); + let sha256 = hex::encode(sha2::Sha256::digest(&body)); + (payload, body, sha256) + } + + fn publish_payload_with_tarball( + package: &str, + version: &str, + tag: &str, + marker: &str, + ) -> Vec { + let mut payload: serde_json::Value = + serde_json::from_slice(&publish_payload(package, version, tag)).unwrap(); + let tarball = npm_tarball_with_marker(package, version, marker); + let filename = canonical_tarball_filename(package, version); + payload["_attachments"][&filename]["data"] = + serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(&tarball)); + payload["_attachments"][&filename]["length"] = + serde_json::Value::Number(tarball.len().into()); + serde_json::to_vec(&payload).unwrap() + } + + #[test] + fn install_v1_accept_requires_a_positive_quality() { + let mut headers = HeaderMap::new(); + headers.insert( + header::ACCEPT, + HeaderValue::from_static( + "application/vnd.npm.install-v1+json;q=0, application/json;q=1", + ), + ); + assert_eq!( + PackumentFlavor::from_headers(&headers), + PackumentFlavor::Full + ); + headers.insert( + header::ACCEPT, + HeaderValue::from_static( + "application/vnd.npm.install-v1+json;q=1, application/json;q=0.8, */*", + ), + ); + assert_eq!( + PackumentFlavor::from_headers(&headers), + PackumentFlavor::InstallV1 + ); + headers.insert( + header::ACCEPT, + HeaderValue::from_static( + "application/vnd.npm.install-v1+json;q=0.9, application/json;q=0.8, */*", + ), + ); + assert_eq!( + PackumentFlavor::from_headers(&headers), + PackumentFlavor::InstallV1 + ); + headers.insert( + header::ACCEPT, + HeaderValue::from_static("application/vnd.npm.install-v1+json;q=0, */*;q=1"), + ); + assert_eq!( + PackumentFlavor::from_headers(&headers), + PackumentFlavor::Full + ); + headers.insert( + header::ACCEPT, + HeaderValue::from_static( + "application/vnd.npm.install-v1+json;q=1, application/json;q=1", + ), + ); + assert_eq!( + PackumentFlavor::from_headers(&headers), + PackumentFlavor::InstallV1 + ); + headers.insert( + header::ACCEPT, + HeaderValue::from_static("application/vnd.npm.install-v1+json; q=0.5"), + ); + assert_eq!( + PackumentFlavor::from_headers(&headers), + PackumentFlavor::InstallV1 + ); + headers.insert( + header::ACCEPT, + HeaderValue::from_static( + "application/json;q=1, application/vnd.npm.install-v1+json;q=0.9", + ), + ); + assert_eq!( + PackumentFlavor::from_headers(&headers), + PackumentFlavor::Full + ); + } + + #[tokio::test] + async fn bulk_import_withholds_until_finalize_and_serves_both_packuments() { + use crate::test_helpers::{ + body_bytes, create_test_context_with_config, send, send_with_headers, + }; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + let (payload, full, sha256) = import_fixture("pkg", "1.0.0", "latest", ""); + let response = send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + serde_json::to_vec(&payload).unwrap(), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + assert!(ctx + .state + .storage + .stat(&crate::npm_layout::hosted_packument_current_key( + "npm-private", + "pkg" + )) + .await + .is_none()); + let hidden = send(&ctx.app, Method::GET, "/repository/npm-private/pkg", "").await; + assert_eq!(hidden.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(hidden.headers().get(header::RETRY_AFTER).unwrap(), "1"); + + let finalized = send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/nora/import/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + full.clone(), + ) + .await; + assert_eq!(finalized.status(), StatusCode::CREATED); + let receipt: serde_json::Value = + serde_json::from_slice(&body_bytes(finalized).await).unwrap(); + assert_eq!(receipt.as_object().unwrap().len(), 5); + assert_eq!(receipt["generation"], sha256); + + let full_response = send(&ctx.app, Method::GET, "/repository/npm-private/pkg", "").await; + assert_eq!(full_response.status(), StatusCode::OK); + let install = send_with_headers( + &ctx.app, + Method::GET, + "/repository/npm-private/pkg", + vec![( + "accept", + "application/vnd.npm.install-v1+json, application/json;q=0.5", + )], + "", + ) + .await; + assert_eq!(install.status(), StatusCode::OK); + assert_eq!( + install.headers().get(header::CONTENT_TYPE).unwrap(), + "application/vnd.npm.install-v1+json" + ); + assert_eq!(install.headers().get(header::VARY).unwrap(), "Accept"); + let etag = install + .headers() + .get(header::ETAG) + .unwrap() + .to_str() + .unwrap(); + let not_modified = send_with_headers( + &ctx.app, + Method::GET, + "/repository/npm-private/pkg", + vec![ + ( + "accept", + "application/vnd.npm.install-v1+json, application/json;q=0.5", + ), + ("if-none-match", etag), + ], + "", + ) + .await; + assert_eq!(not_modified.status(), StatusCode::NOT_MODIFIED); + assert_eq!(not_modified.headers().get(header::VARY).unwrap(), "Accept"); + + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/nora/import/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + full.clone(), + ) + .await + .status(), + StatusCode::OK + ); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/package/pkg/dist-tags/next", + serde_json::to_vec("1.0.0").unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/nora/import/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + full, + ) + .await + .status(), + StatusCode::OK + ); + let after_replay = send(&ctx.app, Method::GET, "/repository/npm-private/pkg", "").await; + assert_eq!(after_replay.status(), StatusCode::OK); + let after_replay: serde_json::Value = + serde_json::from_slice(&body_bytes(after_replay).await).unwrap(); + assert_eq!(after_replay["dist-tags"]["next"], "1.0.0"); + } + + #[tokio::test] + async fn deferred_version_evidence_is_last_and_get_is_two_reads_without_list() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use axum::http::HeaderName; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let (payload, full, sha256) = import_fixture("pkg", "1.0.0", "latest", ""); + let backend = FaultInjectBackend::new(ctx.state.storage.clone()); + let writes = backend.write_attempts(); + let mut state = ctx.state.clone(); + state.storage = crate::storage::Storage::from_backend(Arc::new(backend)); + assert_eq!( + publish_with_import( + &state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + Some(&sha256), + ) + .await + .status(), + StatusCode::CREATED + ); + let writes = writes.lock().clone(); + let evidence = writes + .iter() + .position(|entry| entry.contains("/import/generations/")) + .unwrap(); + for needle in [ + "/blobs/sha512/", + "/versions/1.0.0.json", + "/publish-complete/1.0.0", + ] { + assert!( + writes + .iter() + .position(|entry| entry.contains(needle)) + .unwrap() + < evidence + ); + } + assert!( + writes + .iter() + .all(|entry| !entry.contains("/hosted-packuments/")), + "deferred version PUT must not build any package generation" + ); + assert!(state + .storage + .stat(&crate::npm_layout::hosted_packument_current_key( + "npm-private", + "pkg" + )) + .await + .is_none()); + + let response = named_import_finalize( + State(state.clone()), + Path(("npm-private".to_string(), "pkg".to_string())), + HeaderMap::from_iter([( + HeaderName::from_static(NPM_IMPORT_PACKUMENT_HEADER), + HeaderValue::from_str(&sha256).unwrap(), + )]), + Extension(NamespaceAuthority::Unrestricted), + Bytes::from(full), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + + let backend = FaultInjectBackend::new(state.storage.clone()); + let gets = backend.get_attempts(); + let lists = backend.list_attempts(); + let mut read_state = state; + read_state.storage = crate::storage::Storage::from_backend(Arc::new(backend)); + hosted_packument( + &read_state, + "npm-private", + "pkg", + "https://nora.example/repository/npm-private", + PackumentFlavor::Full, + ) + .await + .unwrap(); + assert_eq!(gets.lock().len(), 2); + assert!(lists.lock().is_empty()); + } + + #[tokio::test] + async fn import_finalize_rejects_omitted_roster_entries_without_list() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use axum::http::HeaderName; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let (first, full, sha256) = import_fixture("pkg", "1.0.0", "latest", ""); + let (second, _, _) = import_fixture("pkg", "2.0.0", "latest", ""); + for payload in [&first, &second] { + assert_eq!( + publish_with_import( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + payload, + Some(&sha256), + ) + .await + .status(), + StatusCode::CREATED + ); + } + + let second_manifest = validate_publish("pkg", &second).unwrap(); + let second_digest = crate::npm_layout::hosted_manifest_digest(&second_manifest.manifest); + let omitted_manifest = hosted_version_key("npm-private", "pkg", "2.0.0"); + let omitted_evidence = crate::npm_layout::hosted_import_evidence_key( + "npm-private", + "pkg", + &sha256, + "2.0.0", + &second_digest, + ); + let backend = FaultInjectBackend::new(ctx.state.storage.clone()) + .omit_from_list(omitted_manifest) + .omit_from_list(omitted_evidence); + let lists = backend.list_attempts(); + let mut state = ctx.state.clone(); + state.storage = Storage::from_backend(Arc::new(backend)); + let response = named_import_finalize( + State(state), + Path(("npm-private".to_string(), "pkg".to_string())), + HeaderMap::from_iter([( + HeaderName::from_static(NPM_IMPORT_PACKUMENT_HEADER), + HeaderValue::from_str(&sha256).unwrap(), + )]), + Extension(NamespaceAuthority::Unrestricted), + Bytes::from(full), + ) + .await; + assert_eq!(response.status(), StatusCode::CONFLICT); + assert!(lists.lock().is_empty()); + assert!(ctx + .state + .storage + .stat(&crate::npm_layout::hosted_import_pending_key( + "npm-private", + "pkg" + )) + .await + .is_some()); + assert!(ctx + .state + .storage + .stat(&crate::npm_layout::hosted_packument_current_key( + "npm-private", + "pkg" + )) + .await + .is_none()); + } + + #[tokio::test] + async fn import_finalize_exactly_removes_base_authority_when_list_omits_it() { + use crate::test_helpers::{ + create_test_context_with_config, send, send_with_headers, FaultInjectBackend, + }; + use axum::http::{HeaderName, Method}; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let (payload, full, sha256) = import_fixture("pkg", "1.0.0", "latest", ""); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + serde_json::to_vec(&payload).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/package/pkg/dist-tags/next", + serde_json::to_vec("1.0.0").unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + serde_json::to_vec(&serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": "superseded"}} + })) + .unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + serde_json::to_vec(&payload).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + + let stale_tag = hosted_tag_key("npm-private", "pkg", "next"); + let stale_deprecation = hosted_deprecation_key("npm-private", "pkg", "1.0.0"); + let backend = FaultInjectBackend::new(ctx.state.storage.clone()) + .omit_from_list(stale_tag.clone()) + .omit_from_list(stale_deprecation.clone()); + let lists = backend.list_attempts(); + let mut state = ctx.state.clone(); + state.storage = Storage::from_backend(Arc::new(backend)); + let response = named_import_finalize( + State(state), + Path(("npm-private".to_string(), "pkg".to_string())), + HeaderMap::from_iter([( + HeaderName::from_static(NPM_IMPORT_PACKUMENT_HEADER), + HeaderValue::from_str(&sha256).unwrap(), + )]), + Extension(NamespaceAuthority::Unrestricted), + Bytes::from(full), + ) + .await; + assert_eq!(response.status(), StatusCode::CREATED); + assert!(lists.lock().is_empty()); + assert!(matches!( + ctx.state.storage.get(&stale_tag).await, + Err(StorageError::NotFound) + )); + assert!(matches!( + ctx.state.storage.get(&stale_deprecation).await, + Err(StorageError::NotFound) + )); + } + + #[tokio::test] + async fn omitted_publish_intent_blocks_other_publish_and_exact_retry_recovers_without_list() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let base: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &base, + ) + .await + .status(), + StatusCode::CREATED + ); + let second: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "2.0.0", "latest")).unwrap(); + let base_version = hosted_version_key("npm-private", "pkg", "1.0.0"); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let pending_key = hosted_publish_pending_index_key("npm-private", "pkg"); + let failing_backend = FaultInjectBackend::new(ctx.state.storage.clone()) + .omit_from_list(base_version.clone()) + .fail_put(&pointer_key); + let failing_lists = failing_backend.list_attempts(); + let mut failing = ctx.state.clone(); + failing.storage = Storage::from_backend(Arc::new(failing_backend)); + assert_eq!( + publish( + &failing, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx.state.storage.stat(&pending_key).await.is_some()); + + let backend = FaultInjectBackend::new(ctx.state.storage.clone()) + .omit_from_list(base_version) + .omit_from_list(pending_key.clone()); + let lists = backend.list_attempts(); + let mut retry = ctx.state.clone(); + retry.storage = Storage::from_backend(Arc::new(backend)); + let other: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "3.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &retry, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &other, + ) + .await + .status(), + StatusCode::CONFLICT + ); + assert_eq!( + publish( + &retry, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(failing_lists.lock().is_empty()); + assert!(lists.lock().is_empty()); + assert!(ctx.state.storage.stat(&pending_key).await.is_none()); + let current = current_full_for_mutation(&ctx.state, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + assert!(current["versions"].get("1.0.0").is_some()); + assert!(current["versions"].get("2.0.0").is_some()); + } + + #[tokio::test] + async fn retention_target_uses_exact_pointer_when_list_omits_authority() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let payload: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + ) + .await + .status(), + StatusCode::CREATED + ); + let omitted_version = hosted_version_key("npm-private", "pkg", "1.0.0"); + let backend = + FaultInjectBackend::new(ctx.state.storage.clone()).omit_from_list(omitted_version); + let lists = backend.list_attempts(); + let storage = Storage::from_backend(Arc::new(backend)); + let target = prepare_hosted_packument_after_retention( + &storage, + "npm-private", + "pkg", + &HashSet::new(), + ) + .await + .unwrap(); + assert!(matches!(target, HostedMaintenanceTarget::Live { .. })); + assert!(lists.lock().is_empty()); + } + + #[tokio::test] + async fn completed_receipt_delayed_put_is_read_only_and_exact_only() { + use crate::test_helpers::{ + create_test_context_with_config, send, send_with_headers, FaultInjectBackend, + }; + use axum::http::Method; + use std::sync::Arc; + + let ctx = create_test_context_with_config(|config| { + named_config(config); + let NpmRepository::Hosted { write_policy, .. } = &mut config.npm.repositories[0] else { + unreachable!() + }; + *write_policy = NpmWritePolicy::Allow; + }); + let (first, full, sha256) = import_fixture("pkg", "1.0.0", "latest", "first"); + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + serde_json::to_vec(&first).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/nora/import/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + full.clone(), + ) + .await + .status(), + StatusCode::CREATED + ); + + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/package/pkg/dist-tags/next", + serde_json::to_vec("1.0.0").unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + + let backend = FaultInjectBackend::new(ctx.state.storage.clone()); + let writes = backend.write_attempts(); + let deletes = backend.delete_attempts(); + let mut delayed_state = ctx.state.clone(); + delayed_state.storage = Storage::from_backend(Arc::new(backend)); + assert_eq!( + publish_with_import( + &delayed_state, + "npm-private", + NpmWritePolicy::Allow, + "pkg", + &first, + Some(&sha256), + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(writes.lock().is_empty()); + assert!(deletes.lock().is_empty()); + + let (changed, _, _) = import_fixture("pkg", "1.0.0", "latest", "changed"); + assert_eq!( + publish_with_import( + &delayed_state, + "npm-private", + NpmWritePolicy::Allow, + "pkg", + &changed, + Some(&sha256), + ) + .await + .status(), + StatusCode::CONFLICT + ); + assert!(writes.lock().is_empty()); + assert!(deletes.lock().is_empty()); + + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "2.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + serde_json::to_vec(&serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": "superseded"}} + })) + .unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + + let replay = named_import_finalize( + State(delayed_state), + Path(("npm-private".to_string(), "pkg".to_string())), + HeaderMap::from_iter([( + header::HeaderName::from_static(NPM_IMPORT_PACKUMENT_HEADER), + HeaderValue::from_str(&sha256).unwrap(), + )]), + Extension(NamespaceAuthority::Unrestricted), + Bytes::from(full), + ) + .await; + assert_eq!(replay.status(), StatusCode::OK); + assert!(writes.lock().is_empty()); + assert!(deletes.lock().is_empty()); + } + + #[tokio::test] + async fn interrupted_finalize_receipt_resumes_only_the_missing_target_pointer() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use axum::http::HeaderName; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let (payload, full, sha256) = import_fixture("pkg", "1.0.0", "latest", ""); + assert_eq!( + publish_with_import( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + Some(&sha256), + ) + .await + .status(), + StatusCode::CREATED + ); + let headers = || { + HeaderMap::from_iter([( + HeaderName::from_static(NPM_IMPORT_PACKUMENT_HEADER), + HeaderValue::from_str(&sha256).unwrap(), + )]) + }; + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let marker_key = crate::npm_layout::hosted_import_pending_key("npm-private", "pkg"); + let receipt_key = + crate::npm_layout::hosted_import_receipt_key("npm-private", "pkg", &sha256); + let mut failing_state = ctx.state.clone(); + failing_state.storage = Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); + let failed = named_import_finalize( + State(failing_state), + Path(("npm-private".to_string(), "pkg".to_string())), + headers(), + Extension(NamespaceAuthority::Unrestricted), + Bytes::from(full.clone()), + ) + .await; + assert_eq!(failed.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(ctx.state.storage.stat(&pointer_key).await.is_none()); + assert!(ctx.state.storage.stat(&marker_key).await.is_some()); + assert!(ctx.state.storage.stat(&receipt_key).await.is_some()); + + let resumed = named_import_finalize( + State(ctx.state.clone()), + Path(("npm-private".to_string(), "pkg".to_string())), + headers(), + Extension(NamespaceAuthority::Unrestricted), + Bytes::from(full), + ) + .await; + assert_eq!(resumed.status(), StatusCode::OK); + assert!(ctx.state.storage.stat(&pointer_key).await.is_some()); + assert!(ctx.state.storage.stat(&marker_key).await.is_none()); + } + + #[tokio::test] + async fn import_preflight_rejects_blob_collision_before_any_mutation() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use std::sync::Arc; + + let ctx = create_test_context_with_config(|config| { + named_config(config); + let NpmRepository::Hosted { write_policy, .. } = &mut config.npm.repositories[0] else { + unreachable!() + }; + *write_policy = NpmWritePolicy::Allow; + }); + let (payload, _, sha256) = import_fixture("pkg", "1.0.0", "latest", ""); + let validated = validate_publish("pkg", &payload).unwrap(); + let blob_key = crate::npm_layout::hosted_blob_key_for_digest( + "npm-private", + "pkg", + &validated.blob_digest, + ); + ctx.state + .storage + .put(&blob_key, b"digest-key collision") + .await + .unwrap(); - #[test] - fn attachment_validation_rejects_paths() { - assert!(is_valid_attachment_name("pkg-1.0.0.tgz")); - assert!(!is_valid_attachment_name("../pkg.tgz")); - assert!(!is_valid_attachment_name("scope/pkg.tgz")); + let backend = FaultInjectBackend::new(ctx.state.storage.clone()); + let writes = backend.write_attempts(); + let deletes = backend.delete_attempts(); + let mut state = ctx.state.clone(); + state.storage = Storage::from_backend(Arc::new(backend)); + assert_eq!( + publish_with_import( + &state, + "npm-private", + NpmWritePolicy::Allow, + "pkg", + &payload, + Some(&sha256), + ) + .await + .status(), + StatusCode::CONFLICT + ); + assert!(writes.lock().is_empty()); + assert!(deletes.lock().is_empty()); + assert!(ctx + .state + .storage + .stat(&crate::npm_layout::hosted_import_pending_key( + "npm-private", + "pkg" + )) + .await + .is_none()); } - #[test] - fn dist_tags_reject_semver_like_names() { - assert!(is_valid_dist_tag("latest")); - assert!(!is_valid_dist_tag("1.2.3")); - assert!(!is_valid_dist_tag("^1")); + #[tokio::test] + async fn finalize_rejects_an_unjournaled_prepopulated_package() { + use crate::test_helpers::{create_test_context_with_config, send, send_with_headers}; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + let pointer: HostedPackumentPointer = serde_json::from_slice( + &ctx.state + .storage + .get(&crate::npm_layout::hosted_packument_current_key( + "npm-private", + "pkg", + )) + .await + .unwrap(), + ) + .unwrap(); + let full = ctx + .state + .storage + .get(&crate::npm_layout::hosted_packument_full_key( + "npm-private", + "pkg", + &pointer.generation, + )) + .await + .unwrap(); + let sha256 = hex::encode(sha2::Sha256::digest(&full)); + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/nora/import/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + full, + ) + .await + .status(), + StatusCode::CONFLICT + ); } - #[cfg(test)] - fn npm_tarball(package: &str, version: &str) -> Vec { - npm_tarball_with_marker(package, version, "") + #[tokio::test] + async fn allow_publish_retry_repairs_a_failed_first_pointer_commit() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let mut failing = ctx.state.clone(); + failing.storage = crate::storage::Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); + let payload: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &failing, + "npm-private", + NpmWritePolicy::Allow, + "pkg", + &payload, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx.state.storage.stat(&pointer_key).await.is_none()); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::Allow, + "pkg", + &payload, + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(ctx.state.storage.stat(&pointer_key).await.is_some()); } - #[cfg(test)] - fn npm_tarball_with_marker(package: &str, version: &str, marker: &str) -> Vec { - use flate2::write::GzEncoder; - use flate2::Compression; - let encoder = GzEncoder::new(Vec::new(), Compression::fast()); - let mut archive = tar::Builder::new(encoder); - let package_json = serde_json::to_vec(&serde_json::json!({ - "name": package, - "version": version - })) - .unwrap(); - let mut header = tar::Header::new_gnu(); - header.set_size(package_json.len() as u64); - header.set_mode(0o644); - header.set_cksum(); - archive - .append_data(&mut header, "package/package.json", package_json.as_slice()) + #[tokio::test] + async fn additive_publish_pointer_failure_blocks_later_publish_until_exact_retry() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let first: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &first, + ) + .await + .status(), + StatusCode::CREATED + ); + + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let second: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "2.0.0", "latest")).unwrap(); + let mut failing = ctx.state.clone(); + failing.storage = crate::storage::Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); + assert_eq!( + publish( + &failing, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx + .state + .storage + .stat(&hosted_publish_pending_index_key("npm-private", "pkg")) + .await + .is_some()); + let visible = current_full_for_mutation(&ctx.state, "npm-private", "pkg") + .await + .unwrap() .unwrap(); - if !marker.is_empty() { - let mut marker_header = tar::Header::new_gnu(); - marker_header.set_size(marker.len() as u64); - marker_header.set_mode(0o644); - marker_header.set_cksum(); - archive - .append_data( - &mut marker_header, - "package/republish-marker.txt", - marker.as_bytes(), - ) - .unwrap(); + assert!(visible["versions"].get("1.0.0").is_some()); + assert!(visible["versions"].get("2.0.0").is_none()); + + let third: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "3.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &third, + ) + .await + .status(), + StatusCode::CONFLICT + ); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(ctx + .state + .storage + .stat(&hosted_publish_pending_index_key("npm-private", "pkg")) + .await + .is_none()); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &third, + ) + .await + .status(), + StatusCode::CREATED + ); + let visible = current_full_for_mutation(&ctx.state, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + for version in ["1.0.0", "2.0.0", "3.0.0"] { + assert!(visible["versions"].get(version).is_some(), "{version}"); } - archive.into_inner().unwrap().finish().unwrap() } - fn named_config(config: &mut crate::config::Config) { - config.npm.proxy = None; - config.npm.repositories = vec![ - NpmRepository::Hosted { - name: "npm-private".into(), - write_policy: NpmWritePolicy::AllowOnce, - }, - NpmRepository::Proxy { - name: "npm-registry".into(), - url: "http://127.0.0.1:1".into(), - auth: None, - metadata_ttl: Some(300), - negative_ttl: 0, - }, - NpmRepository::Group { - name: "npm-group".into(), - members: vec!["npm-private".into(), "npm-registry".into()], - writable_member: Some("npm-private".into()), - }, - ]; - config.npm.default_repository = Some("npm-group".into()); - } + #[tokio::test] + async fn import_header_is_rejected_outside_named_hosted_version_publish() { + use crate::test_helpers::{create_test_context_with_config, send_with_headers}; + use axum::http::Method; - #[test] - fn invalid_named_config_without_default_never_falls_into_legacy_layout() { - let ctx = crate::test_helpers::create_test_context_with_config(|config| { - config.npm.repositories = vec![NpmRepository::Hosted { - name: "packages".to_string(), - write_policy: NpmWritePolicy::AllowOnce, - }]; - config.npm.default_repository = None; - }); - assert!(alias_target(&ctx.state).is_none()); + let ctx = create_test_context_with_config(named_config); + let (payload, _, sha256) = import_fixture("pkg", "1.0.0", "latest", ""); + let payload = serde_json::to_vec(&payload).unwrap(); + for uri in ["/repository/npm-group/pkg", "/repository/npm-registry/pkg"] { + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + uri, + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + payload.clone(), + ) + .await + .status(), + StatusCode::BAD_REQUEST, + "{uri}" + ); + } + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + serde_json::to_vec(&serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": "old"}}, + })) + .unwrap(), + ) + .await + .status(), + StatusCode::BAD_REQUEST + ); } - fn publish_payload(package: &str, version: &str, tag: &str) -> Vec { - let tgz = npm_tarball(package, version); - serde_json::to_vec(&serde_json::json!({ - "name": package, - "versions": { - (version): { - "name": package, - "version": version, - "dist": {} - } - }, - "_attachments": { - (canonical_tarball_filename(package, version)): { - "data": base64::engine::general_purpose::STANDARD.encode(&tgz), - "length": tgz.len() - } - }, - "dist-tags": {(tag): version} - })) - .unwrap() - } + #[tokio::test] + async fn bulk_import_finalize_supports_scoped_packages() { + use crate::test_helpers::{create_test_context_with_config, send_with_headers}; + use axum::http::Method; - fn publish_payload_with_tarball( - package: &str, - version: &str, - tag: &str, - marker: &str, - ) -> Vec { - let mut payload: serde_json::Value = - serde_json::from_slice(&publish_payload(package, version, tag)).unwrap(); - let tarball = npm_tarball_with_marker(package, version, marker); - let filename = canonical_tarball_filename(package, version); - payload["_attachments"][&filename]["data"] = - serde_json::Value::String(base64::engine::general_purpose::STANDARD.encode(&tarball)); - payload["_attachments"][&filename]["length"] = - serde_json::Value::Number(tarball.len().into()); - serde_json::to_vec(&payload).unwrap() + let ctx = create_test_context_with_config(named_config); + let (payload, full, sha256) = import_fixture("@scope/pkg", "1.0.0", "latest", ""); + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/%40scope%2Fpkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + serde_json::to_vec(&payload).unwrap(), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + send_with_headers( + &ctx.app, + Method::PUT, + "/repository/npm-private/-/nora/import/%40scope%2Fpkg", + vec![(NPM_IMPORT_PACKUMENT_HEADER, sha256.as_str())], + full, + ) + .await + .status(), + StatusCode::CREATED + ); } #[tokio::test] async fn publish_through_group_commits_only_hosted_and_exact_retry_repairs() { - use crate::test_helpers::{create_test_context_with_config, send}; + use crate::test_helpers::{create_test_context_with_config, send, FaultInjectBackend}; use axum::http::Method; + use std::sync::Arc; let ctx = create_test_context_with_config(named_config); let body = publish_payload("pkg", "1.0.0", "latest"); - let first = send( - &ctx.app, - Method::PUT, - "/repository/npm-group/pkg", - body.clone(), - ) - .await; - assert_eq!(first.status(), StatusCode::CREATED); - // Model a crash after the version-manifest commit and before the - // durable post-commit marker. Only this incomplete state is repairable. - ctx.state - .storage - .delete(&hosted_tag_key("npm-private", "pkg", "latest")) - .await - .unwrap(); - ctx.state - .storage - .delete(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0")) - .await - .unwrap(); - let manifest = ctx - .state - .storage - .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) - .await - .unwrap(); - ctx.state - .storage - .put( - &hosted_publish_pending_key("npm-private", "pkg", "1.0.0"), - crate::npm_layout::hosted_manifest_digest(&manifest).as_bytes(), + let payload: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let mut failing = ctx.state.clone(); + failing.storage = Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); + assert_eq!( + publish( + &failing, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, ) .await - .unwrap(); + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); let retry = send(&ctx.app, Method::PUT, "/repository/npm-group/pkg", body).await; assert_eq!(retry.status(), StatusCode::CREATED); assert!(ctx @@ -4202,7 +8119,7 @@ mod tests { } #[tokio::test] - async fn fresh_publish_scans_only_pending_markers() { + async fn fresh_publish_uses_only_the_exact_pending_intent() { let ctx = crate::test_helpers::create_test_context_with_config(named_config); let backend = crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()); let list_attempts = backend.list_attempts(); @@ -4226,41 +8143,32 @@ mod tests { ); } - assert_eq!( - list_attempts.lock().as_slice(), - [ - "npm/repositories/npm-private/pkg/publish-pending/", - "npm/repositories/npm-private/pkg/publish-pending/", - ] - ); - assert_eq!( + assert!(list_attempts.lock().is_empty()); + assert!(matches!( state .storage .get(&hosted_publish_pending_index_key("npm-private", "pkg")) - .await - .unwrap() - .as_ref(), - b"1" - ); - for version in ["1.0.0", "2.0.0"] { - assert!(matches!( - state - .storage - .get(&hosted_publish_pending_key("npm-private", "pkg", version)) - .await, - Err(StorageError::NotFound) - )); - } + .await, + Err(StorageError::NotFound) + )); } #[tokio::test] - async fn completed_pending_marker_is_cleaned_before_the_next_publish() { + async fn completed_pending_marker_blocks_until_the_exact_cleanup_retry() { + use crate::test_helpers::FaultInjectBackend; + use std::sync::Arc; + let ctx = crate::test_helpers::create_test_context_with_config(named_config); let first: serde_json::Value = serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + let pending_key = hosted_publish_pending_index_key("npm-private", "pkg"); + let mut failing = ctx.state.clone(); + failing.storage = Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_delete(&pending_key), + )); assert_eq!( publish( - &ctx.state, + &failing, "npm-private", NpmWritePolicy::AllowOnce, "pkg", @@ -4268,24 +8176,9 @@ mod tests { ) .await .status(), - StatusCode::CREATED + StatusCode::INTERNAL_SERVER_ERROR ); - - let manifest = ctx - .state - .storage - .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) - .await - .unwrap(); - let pending_key = hosted_publish_pending_key("npm-private", "pkg", "1.0.0"); - ctx.state - .storage - .put( - &pending_key, - crate::npm_layout::hosted_manifest_digest(&manifest).as_bytes(), - ) - .await - .unwrap(); + assert!(ctx.state.storage.stat(&pending_key).await.is_some()); let second: serde_json::Value = serde_json::from_slice(&publish_payload("pkg", "2.0.0", "latest")).unwrap(); @@ -4299,37 +8192,67 @@ mod tests { ) .await .status(), + StatusCode::CONFLICT + ); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &first, + ) + .await + .status(), StatusCode::CREATED ); assert!(matches!( ctx.state.storage.get(&pending_key).await, Err(StorageError::NotFound) )); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::CREATED + ); } #[tokio::test] async fn pending_publish_accepts_only_the_exact_manifest_retry() { + use crate::test_helpers::FaultInjectBackend; + use std::sync::Arc; + let ctx = crate::test_helpers::create_test_context_with_config(named_config); let exact: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( "pkg", "1.0.0", "latest", "exact", )) .unwrap(); - let exact_publish = validate_publish("pkg", &exact).unwrap(); - let exact_digest = crate::npm_layout::hosted_manifest_digest(&exact_publish.manifest); - let pending_key = hosted_publish_pending_key("npm-private", "pkg", "1.0.0"); - ctx.state - .storage - .put( - &hosted_publish_pending_index_key("npm-private", "pkg"), - b"1", + let pending_key = hosted_publish_pending_index_key("npm-private", "pkg"); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let mut failing = ctx.state.clone(); + failing.storage = Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); + assert_eq!( + publish( + &failing, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &exact, ) .await - .unwrap(); - ctx.state - .storage - .put(&pending_key, exact_digest.as_bytes()) - .await - .unwrap(); + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + let exact_pending = ctx.state.storage.get(&pending_key).await.unwrap(); let different: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( "pkg", @@ -4351,8 +8274,8 @@ mod tests { StatusCode::CONFLICT ); assert_eq!( - ctx.state.storage.get(&pending_key).await.unwrap().as_ref(), - exact_digest.as_bytes() + ctx.state.storage.get(&pending_key).await.unwrap(), + exact_pending ); assert_eq!( publish( @@ -4375,7 +8298,7 @@ mod tests { #[tokio::test] async fn failed_pending_marker_cleanup_is_recovered_by_exact_retry() { let ctx = crate::test_helpers::create_test_context_with_config(named_config); - let pending_key = hosted_publish_pending_key("npm-private", "pkg", "1.0.0"); + let pending_key = hosted_publish_pending_index_key("npm-private", "pkg"); let backend = crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) .fail_delete(&pending_key); let mut failing_state = ctx.state.clone(); @@ -4415,6 +8338,46 @@ mod tests { )); } + #[tokio::test] + async fn import_evidence_commits_before_pending_cleanup_and_exact_retry_cleans_it() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let (payload, _, sha256) = import_fixture("pkg", "1.0.0", "latest", ""); + let pending_key = hosted_publish_pending_index_key("npm-private", "pkg"); + let backend = crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) + .fail_delete(&pending_key); + let mut failing_state = ctx.state.clone(); + failing_state.storage = crate::storage::Storage::from_backend(std::sync::Arc::new(backend)); + + assert_eq!( + publish_with_import( + &failing_state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + Some(&sha256), + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx.state.storage.stat(&pending_key).await.is_some()); + assert_eq!( + publish_with_import( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + Some(&sha256), + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(ctx.state.storage.stat(&pending_key).await.is_none()); + } + #[tokio::test] async fn completed_publish_retry_does_not_resurrect_deleted_tag() { use crate::test_helpers::{create_test_context_with_config, send}; @@ -4460,9 +8423,12 @@ mod tests { } #[tokio::test] - async fn publish_deprecation_uses_overlay_and_completed_retry_does_not_resurrect_it() { - use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + async fn publish_deprecation_markerless_corruption_fails_closed_without_repair() { + use crate::test_helpers::{ + body_bytes, create_test_context_with_config, send, FaultInjectBackend, + }; use axum::http::Method; + use std::sync::Arc; let ctx = create_test_context_with_config(named_config); let mut payload: serde_json::Value = @@ -4499,87 +8465,59 @@ mod tests { b"do not use" ); - // An exact retry repairs a publish that reached the manifest commit - // but not the mutable overlay/completion phase. + // Completion precedes pointer and pending cleanup, so deleting both a + // completed overlay and completion marker is corruption rather than a + // reachable crash state. An exact body must not manufacture repairs. ctx.state.storage.delete(&deprecation_key).await.unwrap(); ctx.state.storage.delete(&completion_key).await.unwrap(); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let pointer_before = ctx.state.storage.get(&pointer_key).await.unwrap(); + let pending_key = hosted_publish_pending_index_key("npm-private", "pkg"); + let backend = FaultInjectBackend::new(ctx.state.storage.clone()); + let write_attempts = backend.write_attempts(); + let mut corrupt = ctx.state.clone(); + corrupt.storage = Storage::from_backend(Arc::new(backend)); assert_eq!( - send( - &ctx.app, - Method::PUT, - "/repository/npm-group/pkg", - body.clone(), - ) - .await - .status(), - StatusCode::CREATED - ); - assert_eq!( - ctx.state - .storage - .get(&deprecation_key) - .await - .unwrap() - .as_ref(), - b"do not use" - ); - let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; - assert_eq!(response.status(), StatusCode::OK); - let packument: serde_json::Value = - serde_json::from_slice(&body_bytes(response).await).unwrap(); - assert_eq!(packument["versions"]["1.0.0"]["deprecated"], "do not use"); - - let clear = serde_json::json!({ - "name": "pkg", - "versions": { - "1.0.0": {"deprecated": ""} - } - }); - assert_eq!( - send( - &ctx.app, - Method::PUT, - "/repository/npm-group/pkg", - serde_json::to_vec(&clear).unwrap(), + publish( + &corrupt, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, ) .await .status(), - StatusCode::CREATED + StatusCode::INTERNAL_SERVER_ERROR ); + assert!(write_attempts.lock().is_empty()); assert!(matches!( ctx.state.storage.get(&deprecation_key).await, Err(StorageError::NotFound) )); - - let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; - assert_eq!(response.status(), StatusCode::OK); - let packument: serde_json::Value = - serde_json::from_slice(&body_bytes(response).await).unwrap(); - assert!(packument["versions"]["1.0.0"].get("deprecated").is_none()); - - // Once the original publish completed, retrying that exact body must - // not rewind the later mutable undeprecation. - assert_eq!( - send(&ctx.app, Method::PUT, "/repository/npm-group/pkg", body) - .await - .status(), - StatusCode::CREATED - ); assert!(matches!( - ctx.state.storage.get(&deprecation_key).await, + ctx.state.storage.get(&completion_key).await, + Err(StorageError::NotFound) + )); + assert!(matches!( + ctx.state.storage.get(&pending_key).await, Err(StorageError::NotFound) )); + assert_eq!( + ctx.state.storage.get(&pointer_key).await.unwrap(), + pointer_before + ); let response = send(&ctx.app, Method::GET, "/repository/npm-group/pkg", "").await; assert_eq!(response.status(), StatusCode::OK); let packument: serde_json::Value = serde_json::from_slice(&body_bytes(response).await).unwrap(); - assert!(packument["versions"]["1.0.0"].get("deprecated").is_none()); + assert_eq!(packument["versions"]["1.0.0"]["deprecated"], "do not use"); } #[tokio::test] async fn incomplete_publish_must_be_retried_before_later_mutable_operations() { - use crate::test_helpers::{create_test_context_with_config, send}; + use crate::test_helpers::{create_test_context_with_config, send, FaultInjectBackend}; use axum::http::Method; + use std::sync::Arc; let ctx = create_test_context_with_config(named_config); let mut first: serde_json::Value = @@ -4587,41 +8525,24 @@ mod tests { first["description"] = serde_json::Value::String("old description".to_string()); first["versions"]["1.0.0"]["deprecated"] = serde_json::Value::String("old deprecation".to_string()); - let first = serde_json::to_vec(&first).unwrap(); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let mut failing = ctx.state.clone(); + failing.storage = Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); assert_eq!( - send( - &ctx.app, - Method::PUT, - "/repository/npm-private/pkg", - first.clone(), + publish( + &failing, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &first, ) .await .status(), - StatusCode::CREATED + StatusCode::INTERNAL_SERVER_ERROR ); - - // Model a process loss after the manifest became visible but before - // the durable completion marker. No later mutable operation may race - // the exact retry that repairs this state. - ctx.state - .storage - .delete(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0")) - .await - .unwrap(); - let manifest = ctx - .state - .storage - .get(&hosted_version_key("npm-private", "pkg", "1.0.0")) - .await - .unwrap(); - ctx.state - .storage - .put( - &hosted_publish_pending_key("npm-private", "pkg", "1.0.0"), - crate::npm_layout::hosted_manifest_digest(&manifest).as_bytes(), - ) - .await - .unwrap(); + let first = serde_json::to_vec(&first).unwrap(); let clear = serde_json::json!({ "name": "pkg", "versions": {"1.0.0": {"deprecated": ""}} @@ -4836,6 +8757,153 @@ mod tests { ); } + #[tokio::test] + async fn allow_once_retry_converges_the_exact_recorded_mutable_target() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let mut first: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( + "pkg", "1.0.0", "latest", "first", + )) + .unwrap(); + first["description"] = serde_json::json!("old package"); + first["versions"]["1.0.0"]["deprecated"] = serde_json::json!("old version"); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &first, + ) + .await + .status(), + StatusCode::CREATED + ); + + let mut second: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( + "pkg", "2.0.0", "latest", "second", + )) + .unwrap(); + second["description"] = serde_json::json!("new package"); + second["versions"]["2.0.0"]["deprecated"] = serde_json::json!("new version"); + let package_key = hosted_package_key("npm-private", "pkg"); + let pending_key = hosted_publish_pending_index_key("npm-private", "pkg"); + let backend = FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&package_key); + let interrupted_lists = backend.list_attempts(); + let mut interrupted = ctx.state.clone(); + interrupted.storage = Storage::from_backend(Arc::new(backend)); + assert_eq!( + publish( + &interrupted, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx.state.storage.stat(&pending_key).await.is_some()); + assert!(ctx + .state + .storage + .stat(&hosted_version_key("npm-private", "pkg", "2.0.0")) + .await + .is_some()); + + let retry_backend = FaultInjectBackend::new(ctx.state.storage.clone()) + .omit_from_list(package_key.clone()) + .omit_from_list(hosted_tag_key("npm-private", "pkg", "latest")) + .omit_from_list(hosted_deprecation_key("npm-private", "pkg", "2.0.0")); + let retry_lists = retry_backend.list_attempts(); + let mut retry = ctx.state.clone(); + retry.storage = Storage::from_backend(Arc::new(retry_backend)); + assert_eq!( + publish( + &retry, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &second, + ) + .await + .status(), + StatusCode::CREATED + ); + assert!(interrupted_lists.lock().is_empty()); + assert!(retry_lists.lock().is_empty()); + assert!(ctx.state.storage.stat(&pending_key).await.is_none()); + let package: serde_json::Value = + serde_json::from_slice(&ctx.state.storage.get(&package_key).await.unwrap()).unwrap(); + assert_eq!(package["description"], "new package"); + assert_eq!( + ctx.state + .storage + .get(&hosted_tag_key("npm-private", "pkg", "latest")) + .await + .unwrap() + .as_ref(), + b"2.0.0" + ); + assert_eq!( + ctx.state + .storage + .get(&hosted_deprecation_key("npm-private", "pkg", "2.0.0")) + .await + .unwrap() + .as_ref(), + b"new version" + ); + let visible = current_full_for_mutation(&ctx.state, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + assert_eq!(visible["description"], "new package"); + assert_eq!(visible["dist-tags"]["latest"], "2.0.0"); + assert_eq!(visible["versions"]["1.0.0"]["deprecated"], "old version"); + assert_eq!(visible["versions"]["2.0.0"]["deprecated"], "new version"); + } + + #[tokio::test] + async fn markerless_fresh_allow_once_manifest_fails_closed() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let payload: serde_json::Value = serde_json::from_slice(&publish_payload_with_tarball( + "pkg", "1.0.0", "latest", "orphan", + )) + .unwrap(); + let validated = validate_publish("pkg", &payload).unwrap(); + let manifest_key = hosted_version_key("npm-private", "pkg", "1.0.0"); + ctx.state + .storage + .put(&manifest_key, &validated.manifest) + .await + .unwrap(); + + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &payload, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + for key in [ + hosted_publish_pending_index_key("npm-private", "pkg"), + hosted_publish_complete_key("npm-private", "pkg", "1.0.0"), + hosted_package_key("npm-private", "pkg"), + crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"), + ] { + assert!(ctx.state.storage.stat(&key).await.is_none(), "{key}"); + } + } + #[tokio::test] async fn group_tarball_does_not_fall_through_on_hosted_manifest_read_error() { let ctx = crate::test_helpers::create_test_context_with_config(named_config); @@ -4890,33 +8958,32 @@ mod tests { &["npm-private".to_string(), "npm-registry".to_string()], "pkg", "http://localhost/repository/npm-group", + PackumentFlavor::Full, ) .await, - Err(ReadError::Unavailable) + Err(ReadError::MaterializationUnavailable) )); } #[tokio::test] async fn deprecation_and_dist_tag_delete_are_idempotent_but_not_error_blind() { + use crate::test_helpers::send; + use axum::http::Method; + let ctx = crate::test_helpers::create_test_context_with_config(named_config); - let manifest = hosted_version_key("npm-private", "pkg", "1.0.0"); - let manifest_bytes = br#"{"name":"pkg","version":"1.0.0"}"#; - let completion = hosted_publish_complete_key("npm-private", "pkg", "1.0.0"); - let deprecation = hosted_deprecation_key("npm-private", "pkg", "1.0.0"); - let tag = hosted_tag_key("npm-private", "pkg", "next"); - ctx.state - .storage - .put(&manifest, manifest_bytes) - .await - .unwrap(); - ctx.state - .storage - .put( - &completion, - crate::npm_layout::hosted_manifest_digest(manifest_bytes).as_bytes(), + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), ) .await - .unwrap(); + .status(), + StatusCode::CREATED + ); + let deprecation = hosted_deprecation_key("npm-private", "pkg", "1.0.0"); + let tag = hosted_tag_key("npm-private", "pkg", "next"); ctx.state.storage.put(&deprecation, b"old").await.unwrap(); ctx.state.storage.put(&tag, b"1.0.0").await.unwrap(); let backend = crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) @@ -4925,69 +8992,208 @@ mod tests { let mut state = ctx.state.clone(); state.storage = crate::storage::Storage::from_backend(std::sync::Arc::new(backend)); - let payload = serde_json::json!({ + let payload = serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": ""}} + }); + assert_eq!( + deprecate(&state, "npm-private", "pkg", &payload) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert_eq!( + handle_dist_tag_delete( + state, + RepositoryTarget::Named(NpmRepository::Hosted { + name: "npm-private".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }), + "pkg".to_string(), + "next".to_string(), + NamespaceAuthority::Unrestricted, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + + let clean = crate::test_helpers::create_test_context_with_config(named_config); + assert_eq!( + send( + &clean.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload("pkg", "1.0.0", "latest"), + ) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + deprecate(&clean.state, "npm-private", "pkg", &payload) + .await + .status(), + StatusCode::CREATED + ); + assert_eq!( + handle_dist_tag_delete( + clean.state, + RepositoryTarget::Named(NpmRepository::Hosted { + name: "npm-private".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }), + "pkg".to_string(), + "next".to_string(), + NamespaceAuthority::Unrestricted, + ) + .await + .status(), + StatusCode::NO_CONTENT + ); + } + + #[tokio::test] + async fn overlay_pointer_failures_resume_from_immutable_maintenance_marker() { + use crate::test_helpers::{create_test_context_with_config, FaultInjectBackend}; + use std::sync::Arc; + + let ctx = create_test_context_with_config(named_config); + let initial: serde_json::Value = + serde_json::from_slice(&publish_payload("pkg", "1.0.0", "latest")).unwrap(); + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::AllowOnce, + "pkg", + &initial, + ) + .await + .status(), + StatusCode::CREATED + ); + let target = || { + RepositoryTarget::Named(NpmRepository::Hosted { + name: "npm-private".to_string(), + write_policy: NpmWritePolicy::AllowOnce, + }) + }; + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let marker_key = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + + let mut failing_deprecation = ctx.state.clone(); + failing_deprecation.storage = crate::storage::Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); + let deprecated = serde_json::json!({ "name": "pkg", - "versions": {"1.0.0": {"deprecated": ""}} + "versions": {"1.0.0": {"deprecated": "old"}} }); assert_eq!( - deprecate(&state, "npm-private", "pkg", &payload) + deprecate(&failing_deprecation, "npm-private", "pkg", &deprecated,) .await .status(), StatusCode::INTERNAL_SERVER_ERROR ); + assert!(ctx.state.storage.stat(&pointer_key).await.is_some()); + assert!(ctx.state.storage.stat(&marker_key).await.is_some()); + let still_visible = current_full_for_mutation(&ctx.state, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + assert!(still_visible["versions"]["1.0.0"] + .get("deprecated") + .is_none()); assert_eq!( - handle_dist_tag_delete( - state, - RepositoryTarget::Named(NpmRepository::Hosted { - name: "npm-private".to_string(), - write_policy: NpmWritePolicy::AllowOnce, - }), + handle_dist_tag_put( + ctx.state.clone(), + target(), "pkg".to_string(), "next".to_string(), NamespaceAuthority::Unrestricted, + Bytes::from(serde_json::to_vec("1.0.0").unwrap()), ) .await .status(), - StatusCode::INTERNAL_SERVER_ERROR + StatusCode::CREATED ); - - let clean = crate::test_helpers::create_test_context_with_config(named_config); - clean - .state - .storage - .put(&manifest, manifest_bytes) + let visible = current_full_for_mutation(&ctx.state, "npm-private", "pkg") .await + .unwrap() .unwrap(); - clean - .state - .storage - .put( - &completion, - crate::npm_layout::hosted_manifest_digest(manifest_bytes).as_bytes(), + assert_eq!(visible["versions"]["1.0.0"]["deprecated"], "old"); + assert_eq!(visible["dist-tags"]["next"], "1.0.0"); + assert!(ctx.state.storage.stat(&marker_key).await.is_none()); + + let mut failing_tag_put = ctx.state.clone(); + failing_tag_put.storage = crate::storage::Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); + assert_eq!( + handle_dist_tag_put( + failing_tag_put, + target(), + "pkg".to_string(), + "beta".to_string(), + NamespaceAuthority::Unrestricted, + Bytes::from(serde_json::to_vec("1.0.0").unwrap()), ) .await - .unwrap(); + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx.state.storage.stat(&pointer_key).await.is_some()); + assert!(ctx.state.storage.stat(&marker_key).await.is_some()); + let newer = serde_json::json!({ + "name": "pkg", + "versions": {"1.0.0": {"deprecated": "newer"}} + }); assert_eq!( - deprecate(&clean.state, "npm-private", "pkg", &payload) + deprecate(&ctx.state, "npm-private", "pkg", &newer) .await .status(), StatusCode::CREATED ); + let visible = current_full_for_mutation(&ctx.state, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + assert_eq!(visible["versions"]["1.0.0"]["deprecated"], "newer"); + assert_eq!(visible["dist-tags"]["beta"], "1.0.0"); + + let mut failing_tag_delete = ctx.state.clone(); + failing_tag_delete.storage = crate::storage::Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); assert_eq!( handle_dist_tag_delete( - clean.state, - RepositoryTarget::Named(NpmRepository::Hosted { - name: "npm-private".to_string(), - write_policy: NpmWritePolicy::AllowOnce, - }), + failing_tag_delete, + target(), "pkg".to_string(), "next".to_string(), NamespaceAuthority::Unrestricted, ) .await .status(), - StatusCode::NO_CONTENT + StatusCode::INTERNAL_SERVER_ERROR + ); + assert!(ctx.state.storage.stat(&pointer_key).await.is_some()); + assert!(ctx.state.storage.stat(&marker_key).await.is_some()); + assert_eq!( + deprecate(&ctx.state, "npm-private", "pkg", &newer) + .await + .status(), + StatusCode::CREATED ); + let visible = current_full_for_mutation(&ctx.state, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + assert!(visible["dist-tags"].get("next").is_none()); + assert_eq!(visible["dist-tags"]["beta"], "1.0.0"); + assert_eq!(visible["versions"]["1.0.0"]["deprecated"], "newer"); + assert!(ctx.state.storage.stat(&marker_key).await.is_none()); } #[tokio::test] @@ -5047,7 +9253,7 @@ mod tests { } #[tokio::test] - async fn hosted_packument_cache_removes_storage_fanout_and_rewrites_per_route() { + async fn hosted_packument_generation_removes_storage_fanout_and_rewrites_per_route() { use crate::test_helpers::{create_test_context_with_config, send, FaultInjectBackend}; use axum::http::Method; use std::sync::Arc; @@ -5076,35 +9282,42 @@ mod tests { "npm-private", "pkg", "https://nora.example/repository/npm-private", + PackumentFlavor::Full, ) .await - .expect("cold hosted packument"); + .expect("hosted packument"); assert_eq!( hosted["versions"]["1.0.0"]["dist"]["tarball"], "https://nora.example/repository/npm-private/pkg/-/pkg-1.0.0.tgz" ); - assert_eq!(list_attempts.lock().len(), 3); - - let cache_key = crate::npm_layout::hosted_packument_cache_key("npm-private", "pkg"); - let cache: serde_json::Value = serde_json::from_slice( + assert!(list_attempts.lock().is_empty()); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let pointer_before = state.storage.get(&pointer_key).await.unwrap(); + let pointer: HostedPackumentPointer = serde_json::from_slice(&pointer_before).unwrap(); + let stored: serde_json::Value = serde_json::from_slice( &state .storage - .get(&cache_key) + .get(&crate::npm_layout::hosted_packument_full_key( + "npm-private", + "pkg", + &pointer.generation, + )) .await - .expect("persisted cache"), + .unwrap(), ) .unwrap(); - assert!(cache["versions"]["1.0.0"]["dist"].get("tarball").is_none()); + assert!(stored["versions"]["1.0.0"]["dist"].get("tarball").is_none()); let grouped = hosted_packument( &state, "npm-private", "pkg", "https://nora.example/repository/npm-group", + PackumentFlavor::Full, ) .await - .expect("warm hosted packument"); - assert_eq!(list_attempts.lock().len(), 3); + .expect("group-route hosted packument"); + assert!(list_attempts.lock().is_empty()); assert_eq!( grouped["versions"]["1.0.0"]["dist"]["tarball"], "https://nora.example/repository/npm-group/pkg/-/pkg-1.0.0.tgz" @@ -5124,21 +9337,25 @@ mod tests { .status(), StatusCode::CREATED ); - assert!(state.storage.stat(&cache_key).await.is_none()); + assert_ne!( + state.storage.get(&pointer_key).await.unwrap(), + pointer_before + ); let rebuilt = hosted_packument( &state, "npm-private", "pkg", "https://nora.example/repository/npm-private", + PackumentFlavor::Full, ) .await - .expect("rebuilt hosted packument"); + .expect("next hosted generation"); assert_eq!(rebuilt["versions"].as_object().unwrap().len(), 2); assert_eq!(rebuilt["dist-tags"]["next"], "2.0.0"); } #[tokio::test] - async fn hosted_mutation_fails_closed_when_packument_cache_cannot_be_invalidated() { + async fn hosted_mutation_fails_closed_when_retired_marker_cannot_be_cleared() { use crate::test_helpers::{create_test_context_with_config, send, FaultInjectBackend}; use axum::http::Method; use std::sync::Arc; @@ -5155,19 +9372,17 @@ mod tests { .status(), StatusCode::CREATED ); - hosted_packument( - &ctx.state, - "npm-private", - "pkg", - "https://nora.example/repository/npm-private", - ) - .await - .expect("materialize hosted cache"); - - let cache_key = crate::npm_layout::hosted_packument_cache_key("npm-private", "pkg"); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let pointer_before = ctx.state.storage.get(&pointer_key).await.unwrap(); + let retired_key = crate::npm_layout::hosted_packument_retired_key("npm-private", "pkg"); + ctx.state + .storage + .put(&retired_key, crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1) + .await + .unwrap(); let mut failing_state = ctx.state.clone(); failing_state.storage = crate::storage::Storage::from_backend(Arc::new( - FaultInjectBackend::new(ctx.state.storage.clone()).fail_delete(&cache_key), + FaultInjectBackend::new(ctx.state.storage.clone()).fail_delete(&retired_key), )); let payload: serde_json::Value = serde_json::from_slice(&publish_payload("pkg", "2.0.0", "latest")).unwrap(); @@ -5183,17 +9398,52 @@ mod tests { .status(), StatusCode::INTERNAL_SERVER_ERROR ); - assert!(ctx - .state + assert_eq!( + ctx.state.storage.get(&pointer_key).await.unwrap(), + pointer_before + ); + assert!(ctx.state.storage.stat(&retired_key).await.is_some()); + } + + #[tokio::test] + async fn retired_marker_returns_not_found_only_after_resumable_state_is_gone() { + let ctx = crate::test_helpers::create_test_context_with_config(named_config); + let retired_key = crate::npm_layout::hosted_packument_retired_key("npm-private", "pkg"); + let pending_index = + crate::npm_layout::hosted_publish_pending_index_key("npm-private", "pkg"); + ctx.state .storage - .stat(&hosted_version_key("npm-private", "pkg", "2.0.0")) + .put(&retired_key, crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1) .await - .is_none()); - assert!(ctx.state.storage.stat(&cache_key).await.is_some()); + .unwrap(); + ctx.state.storage.put(&pending_index, b"1").await.unwrap(); + assert!(matches!( + hosted_packument( + &ctx.state, + "npm-private", + "pkg", + "https://nora.example/repository/npm-private", + PackumentFlavor::Full, + ) + .await, + Err(ReadError::MaterializationUnavailable) + )); + ctx.state.storage.delete(&pending_index).await.unwrap(); + assert!(matches!( + hosted_packument( + &ctx.state, + "npm-private", + "pkg", + "https://nora.example/repository/npm-private", + PackumentFlavor::Full, + ) + .await, + Err(ReadError::NotFound) + )); } #[tokio::test] - async fn hosted_overlays_invalidate_and_rebuild_packument_cache() { + async fn hosted_overlays_commit_new_packument_generations() { use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; use axum::http::Method; @@ -5210,12 +9460,8 @@ mod tests { .status(), StatusCode::CREATED ); - let cache_key = crate::npm_layout::hosted_packument_cache_key("npm-private", "pkg"); - assert_eq!( - send(&ctx.app, Method::GET, package_uri, "").await.status(), - StatusCode::OK - ); - assert!(ctx.state.storage.stat(&cache_key).await.is_some()); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let initial_pointer = ctx.state.storage.get(&pointer_key).await.unwrap(); let deprecation = serde_json::json!({ "name": "pkg", @@ -5232,7 +9478,8 @@ mod tests { .status(), StatusCode::CREATED ); - assert!(ctx.state.storage.stat(&cache_key).await.is_none()); + let deprecated_pointer = ctx.state.storage.get(&pointer_key).await.unwrap(); + assert_ne!(deprecated_pointer, initial_pointer); let response = send(&ctx.app, Method::GET, package_uri, "").await; let packument: serde_json::Value = serde_json::from_slice(&body_bytes(response).await).unwrap(); @@ -5250,7 +9497,8 @@ mod tests { .status(), StatusCode::CREATED ); - assert!(ctx.state.storage.stat(&cache_key).await.is_none()); + let tagged_pointer = ctx.state.storage.get(&pointer_key).await.unwrap(); + assert_ne!(tagged_pointer, deprecated_pointer); let response = send(&ctx.app, Method::GET, package_uri, "").await; let packument: serde_json::Value = serde_json::from_slice(&body_bytes(response).await).unwrap(); @@ -5260,7 +9508,8 @@ mod tests { send(&ctx.app, Method::DELETE, tag_uri, "").await.status(), StatusCode::NO_CONTENT ); - assert!(ctx.state.storage.stat(&cache_key).await.is_none()); + let untagged_pointer = ctx.state.storage.get(&pointer_key).await.unwrap(); + assert_ne!(untagged_pointer, tagged_pointer); let response = send(&ctx.app, Method::GET, package_uri, "").await; let packument: serde_json::Value = serde_json::from_slice(&body_bytes(response).await).unwrap(); @@ -5379,9 +9628,10 @@ mod tests { } #[tokio::test] - async fn allow_redeploy_stale_completion_blocks_later_mutations_until_retry() { - use crate::test_helpers::{create_test_context_with_config, send}; + async fn allow_redeploy_pointer_failure_blocks_later_mutations_until_retry() { + use crate::test_helpers::{create_test_context_with_config, send, FaultInjectBackend}; use axum::http::Method; + use std::sync::Arc; let ctx = create_test_context_with_config(|config| { named_config(config); @@ -5404,42 +9654,22 @@ mod tests { let second = publish_payload_with_tarball("pkg", "1.0.0", "latest", "second"); let second_value: serde_json::Value = serde_json::from_slice(&second).unwrap(); - let validated = validate_publish("pkg", &second_value).unwrap(); - let blob_key = crate::npm_layout::hosted_blob_key_for_digest( - "npm-private", - "pkg", - &validated.blob_digest, - ); - ctx.state - .storage - .put(&blob_key, &validated.tarball) - .await - .unwrap(); - ctx.state - .storage - .put( - &hosted_version_key("npm-private", "pkg", "1.0.0"), - &validated.manifest, - ) - .await - .unwrap(); - ctx.state - .storage - .put( - &hosted_publish_pending_key("npm-private", "pkg", "1.0.0"), - crate::npm_layout::hosted_manifest_digest(&validated.manifest).as_bytes(), + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let mut failing = ctx.state.clone(); + failing.storage = Storage::from_backend(Arc::new( + FaultInjectBackend::new(ctx.state.storage.clone()).fail_put(&pointer_key), + )); + assert_eq!( + publish( + &failing, + "npm-private", + NpmWritePolicy::Allow, + "pkg", + &second_value, ) .await - .unwrap(); - let stale_marker = ctx - .state - .storage - .get(&hosted_publish_complete_key("npm-private", "pkg", "1.0.0")) - .await - .unwrap(); - assert_ne!( - stale_marker.as_ref(), - crate::npm_layout::hosted_manifest_digest(&validated.manifest).as_bytes() + .status(), + StatusCode::INTERNAL_SERVER_ERROR ); let deprecation = serde_json::json!({ @@ -5502,6 +9732,167 @@ mod tests { ); } + #[tokio::test] + async fn allow_redeploy_pointer_failure_keeps_tarball_on_the_visible_generation() { + use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; + use axum::http::Method; + use std::sync::Arc; + + let ctx = create_test_context_with_config(|config| { + named_config(config); + let NpmRepository::Hosted { write_policy, .. } = &mut config.npm.repositories[0] else { + unreachable!() + }; + *write_policy = NpmWritePolicy::Allow; + }); + let first = publish_payload_with_tarball("pkg", "1.0.0", "latest", "first"); + let first_value: serde_json::Value = serde_json::from_slice(&first).unwrap(); + let first_tarball = base64::engine::general_purpose::STANDARD + .decode( + first_value["_attachments"]["pkg-1.0.0.tgz"]["data"] + .as_str() + .unwrap(), + ) + .unwrap(); + assert_eq!( + send(&ctx.app, Method::PUT, "/repository/npm-private/pkg", first,) + .await + .status(), + StatusCode::CREATED + ); + + let second = publish_payload_with_tarball("pkg", "1.0.0", "latest", "second"); + let second_value: serde_json::Value = serde_json::from_slice(&second).unwrap(); + let second_tarball = base64::engine::general_purpose::STANDARD + .decode( + second_value["_attachments"]["pkg-1.0.0.tgz"]["data"] + .as_str() + .unwrap(), + ) + .unwrap(); + let pointer_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let mut interrupted = ctx.state.clone(); + interrupted.storage = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(ctx.state.storage.clone()) + .fail_put(&pointer_key), + )); + assert_eq!( + publish( + &interrupted, + "npm-private", + NpmWritePolicy::Allow, + "pkg", + &second_value, + ) + .await + .status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + + let visible = send(&ctx.app, Method::GET, "/repository/npm-private/pkg", "").await; + assert_eq!(visible.status(), StatusCode::OK); + let visible: serde_json::Value = + serde_json::from_slice(&body_bytes(visible).await).unwrap(); + assert_eq!( + visible["versions"]["1.0.0"]["dist"]["integrity"], + format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD + .encode(sha2::Sha512::digest(&first_tarball)) + ) + ); + let tarball = send( + &ctx.app, + Method::GET, + "/repository/npm-private/pkg/-/pkg-1.0.0.tgz", + "", + ) + .await; + assert_eq!(tarball.status(), StatusCode::OK); + assert_eq!(body_bytes(tarball).await.as_ref(), first_tarball.as_slice()); + + assert_eq!( + publish( + &ctx.state, + "npm-private", + NpmWritePolicy::Allow, + "pkg", + &second_value, + ) + .await + .status(), + StatusCode::CREATED + ); + let visible = send(&ctx.app, Method::GET, "/repository/npm-private/pkg", "").await; + let visible: serde_json::Value = + serde_json::from_slice(&body_bytes(visible).await).unwrap(); + assert_eq!( + visible["versions"]["1.0.0"]["dist"]["integrity"], + format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD + .encode(sha2::Sha512::digest(&second_tarball)) + ) + ); + let tarball = send( + &ctx.app, + Method::GET, + "/repository/npm-private/pkg/-/pkg-1.0.0.tgz", + "", + ) + .await; + assert_eq!(tarball.status(), StatusCode::OK); + assert_eq!( + body_bytes(tarball).await.as_ref(), + second_tarball.as_slice() + ); + } + + #[tokio::test] + async fn retention_removed_version_is_not_served_from_leftover_split_state() { + use crate::test_helpers::{create_test_context_with_config, send}; + use axum::http::Method; + + let ctx = create_test_context_with_config(named_config); + for version in ["1.0.0", "2.0.0"] { + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/repository/npm-private/pkg", + publish_payload_with_tarball("pkg", version, "latest", version), + ) + .await + .status(), + StatusCode::CREATED + ); + } + let removed_manifest = hosted_version_key("npm-private", "pkg", "1.0.0"); + let target = prepare_hosted_packument_after_retention( + &ctx.state.storage, + "npm-private", + "pkg", + &HashSet::from(["1.0.0".to_string()]), + ) + .await + .unwrap(); + let HostedMaintenanceTarget::Live { pointer } = target else { + panic!("one retained version must keep the package live") + }; + commit_hosted_packument_pointer(&ctx.state.storage, "npm-private", "pkg", &pointer) + .await + .unwrap(); + assert!(ctx.state.storage.stat(&removed_manifest).await.is_some()); + + for uri in [ + "/repository/npm-private/pkg/-/pkg-1.0.0.tgz", + "/repository/npm-group/pkg/-/pkg-1.0.0.tgz", + ] { + let response = send(&ctx.app, Method::GET, uri, "").await; + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{uri}"); + } + } + #[tokio::test] async fn group_package_put_routes_deprecation_to_hosted_overlay() { use crate::test_helpers::{body_bytes, create_test_context_with_config, send}; diff --git a/nora-registry/src/registry/ns_isolation_metadata_tests.rs b/nora-registry/src/registry/ns_isolation_metadata_tests.rs index 6331f9d7..4a95aa1c 100644 --- a/nora-registry/src/registry/ns_isolation_metadata_tests.rs +++ b/nora-registry/src/registry/ns_isolation_metadata_tests.rs @@ -296,16 +296,19 @@ async fn npm_internal_stale_not_refetched() { c.npm.metadata_ttl = 0; // every pull is "stale" → would trigger refetch_metadata c.npm.serve_stale = false; // without the guard, the failed refetch → 502 }); - // A locally-published internal package in the fresh named hosted layout - // must be served from local state, NOT re-fetched upstream. - ctx.state - .storage - .put( - "npm/repositories/npm-private/internalpkg/pkg.json", - br#"{"name":"internalpkg"}"#, + // Seed through the legacy public publish route so this test exercises the + // same committed-generation contract as a real npm client. + assert_eq!( + send( + &ctx.app, + Method::PUT, + "/npm/internalpkg", + crate::test_helpers::npm_publish_payload("internalpkg", "1.0.0", "latest"), ) .await - .unwrap(); + .status(), + StatusCode::CREATED + ); let resp = send(&ctx.app, Method::GET, "/npm/internalpkg", "").await; assert_eq!( resp.status(), diff --git a/nora-registry/src/retention.rs b/nora-registry/src/retention.rs index 3624afb3..bdf01c34 100644 --- a/nora-registry/src/retention.rs +++ b/nora-registry/src/retention.rs @@ -641,6 +641,10 @@ struct NpmVersionGroup { repository: String, package: String, versions: Vec, + /// A durable package maintenance marker is resumed before rules are + /// consulted. Marker-only packages therefore remain discoverable even + /// after a last-version retention run removed all split authority. + active_maintenance: bool, /// Digest of the complete authoritative hosted package state observed /// while planning. Every plan in this package is validated against the /// same digest under the exact npm publish lock. @@ -651,6 +655,13 @@ struct NpmVersionGroup { struct NpmPackageSnapshot { versions: Vec, guard: String, + pointer: crate::npm_layout::HostedPackumentPointer, + packument: serde_json::Value, + /// Exact hashes/sizes of mutable authoritative objects. These become the + /// recovery oracle persisted in a retention marker; content-addressed + /// blobs are deliberately excluded and left to GC once unreachable. + authority_sha256: std::collections::BTreeMap, + authority_sizes: std::collections::BTreeMap, } fn is_hosted_npm_object(kind: &crate::npm_layout::NpmObjectKind) -> bool { @@ -658,11 +669,19 @@ fn is_hosted_npm_object(kind: &crate::npm_layout::NpmObjectKind) -> bool { kind, crate::npm_layout::NpmObjectKind::HostedPackage | crate::npm_layout::NpmObjectKind::HostedVersion(_) + | crate::npm_layout::NpmObjectKind::HostedPublishPending(_) + | crate::npm_layout::NpmObjectKind::HostedPublishPendingIndex | crate::npm_layout::NpmObjectKind::HostedPublishComplete(_) | crate::npm_layout::NpmObjectKind::HostedTarball(_) | crate::npm_layout::NpmObjectKind::HostedBlob { .. } | crate::npm_layout::NpmObjectKind::HostedDistTag(_) | crate::npm_layout::NpmObjectKind::HostedDeprecation(_) + | crate::npm_layout::NpmObjectKind::HostedMaintenanceActive + | crate::npm_layout::NpmObjectKind::HostedPackumentCurrent + | crate::npm_layout::NpmObjectKind::HostedPackumentRetired + | crate::npm_layout::NpmObjectKind::HostedPackumentFull(_) + | crate::npm_layout::NpmObjectKind::HostedPackumentInstallV1(_) + | crate::npm_layout::NpmObjectKind::HostedImportPending ) } @@ -671,181 +690,340 @@ fn npm_guard_bytes(hasher: &mut sha2::Sha256, value: &[u8]) { hasher.update(value); } -/// Read one exact hosted npm package snapshot. -/// -/// `LIST` establishes the relevant key set, `HEAD` supplies metadata for every -/// listed key, and mutable/visibility objects are read and hashed. Any -/// uncertainty fails closed for the whole package. Blob bodies are not read: -/// their content-addressed key plus metadata is sufficient identity, while -/// avoiding a retention-time download of every tarball. -async fn npm_package_snapshot( +async fn npm_exact_object( + storage: &Storage, + key: &str, +) -> Result, String> { + let bytes = match storage.get(key).await { + Ok(bytes) => bytes, + Err(StorageError::NotFound) => return Ok(None), + Err(error) => return Err(format!("cannot read {key}: {error}")), + }; + let meta = storage + .stat(key) + .await + .ok_or_else(|| format!("metadata unavailable for exact key {key}"))?; + if meta.size != bytes.len() as u64 { + return Err(format!("size changed while reading exact key {key}")); + } + let after = storage + .get(key) + .await + .map_err(|error| format!("cannot re-read exact key {key}: {error}"))?; + if after != bytes { + return Err(format!("contents changed while reading exact key {key}")); + } + Ok(Some((bytes, meta))) +} + +async fn npm_active_transaction_present( storage: &Storage, repository: &str, package: &str, - listed_keys: Vec, -) -> Result { - use std::collections::BTreeMap; +) -> Result { + match crate::registry::read_hosted_maintenance_marker(storage, repository, package).await { + Ok(Some(_)) => return Ok(true), + Ok(None) => {} + Err(error) => return Err(format!("maintenance marker unreadable: {error}")), + } + let transactions = + crate::registry::read_hosted_active_transactions(storage, repository, package) + .await + .map_err(|error| format!("active transaction journal unreadable: {error}"))?; + Ok(transactions.import.is_some() || transactions.publish.is_some()) +} - let mut objects = BTreeMap::new(); - for key in listed_keys { - let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { - continue; - }; - if parsed.repository != repository - || parsed.package != package - || !is_hosted_npm_object(&parsed.kind) +fn npm_version_key(repository: &str, package: &str, version: &str) -> Result { + let key = format!("npm/repositories/{repository}/{package}/versions/{version}.json"); + match crate::npm_layout::parse_npm_object_key(&key) { + Some(parsed) + if parsed.repository == repository + && parsed.package == package + && matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedVersion(ref parsed_version) + if parsed_version == version + ) => { - continue; + Ok(key) } - let meta = storage - .stat(&key) - .await - .ok_or_else(|| format!("metadata unavailable for {key}"))?; - let contents = if matches!( - parsed.kind, - crate::npm_layout::NpmObjectKind::HostedPackage - | crate::npm_layout::NpmObjectKind::HostedVersion(_) - | crate::npm_layout::NpmObjectKind::HostedPublishComplete(_) - | crate::npm_layout::NpmObjectKind::HostedDistTag(_) - | crate::npm_layout::NpmObjectKind::HostedDeprecation(_) - ) { - let contents = storage - .get(&key) - .await - .map_err(|error| format!("cannot read {key}: {error}"))?; - if contents.len() as u64 != meta.size { - return Err(format!("size changed while reading {key}")); - } - let after = storage - .stat(&key) - .await - .ok_or_else(|| format!("metadata disappeared for {key}"))?; - if after.size != meta.size || after.modified != meta.modified { - return Err(format!("metadata changed while reading {key}")); - } - Some(contents) - } else { - None - }; - if objects - .insert(key.clone(), (parsed.kind, meta, contents)) - .is_some() + _ => Err(format!( + "invalid npm version in committed packument: {version}" + )), + } +} + +fn npm_dist_tag_key(repository: &str, package: &str, tag: &str) -> Result { + let key = format!("npm/repositories/{repository}/{package}/dist-tags/{tag}"); + match crate::npm_layout::parse_npm_object_key(&key) { + Some(parsed) + if parsed.repository == repository + && parsed.package == package + && matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedDistTag(ref parsed_tag) + if parsed_tag == tag + ) => { - return Err(format!("duplicate key in listing: {key}")); + Ok(key) } + _ => Err(format!( + "invalid npm dist-tag in committed packument: {tag}" + )), } +} - let mut hasher = sha2::Sha256::new(); - npm_guard_bytes(&mut hasher, b"nora/npm-retention-package/v1"); - npm_guard_bytes(&mut hasher, repository.as_bytes()); - npm_guard_bytes(&mut hasher, package.as_bytes()); - for (key, (_, meta, contents)) in &objects { - npm_guard_bytes(&mut hasher, key.as_bytes()); - hasher.update(meta.size.to_be_bytes()); - hasher.update(meta.modified.to_be_bytes()); - match contents { - Some(contents) => { - hasher.update([1]); - npm_guard_bytes(&mut hasher, contents); - } - None => hasher.update([0]), +fn npm_deprecation_key(repository: &str, package: &str, version: &str) -> String { + format!("npm/repositories/{repository}/{package}/deprecations/{version}") +} + +fn npm_publish_complete_key(repository: &str, package: &str, version: &str) -> String { + format!("npm/repositories/{repository}/{package}/publish-complete/{version}") +} + +/// Read one exact hosted npm package snapshot. +/// +/// LIST is deliberately absent here. The committed pointer and its immutable +/// full document define the complete visible version/tag set. Every mutable +/// object reachable from that set is then read through its exact key and +/// compared with the immutable document. A LIST omission can therefore only +/// hide a package from discovery; it can never shrink a destructive snapshot. +async fn npm_package_snapshot( + storage: &Storage, + repository: &str, + package: &str, +) -> Result { + use std::collections::BTreeMap; + if npm_active_transaction_present(storage, repository, package).await? { + return Err("npm package has an active transaction".to_string()); + } + let pointer = crate::registry::read_hosted_packument_pointer(storage, repository, package) + .await + .map_err(|error| format!("current pointer unreadable: {error}"))? + .ok_or_else(|| "current pointer is missing".to_string())?; + crate::registry::validate_hosted_packument_pointer(storage, repository, package, &pointer) + .await + .map_err(|error| format!("current generation invalid: {error}"))?; + let full_key = + crate::npm_layout::hosted_packument_full_key(repository, package, &pointer.generation); + let (full, _) = npm_exact_object(storage, &full_key) + .await? + .ok_or_else(|| format!("current full packument is missing: {full_key}"))?; + if hex::encode(sha2::Sha256::digest(&full)) != pointer.full_sha256 { + return Err("current full packument hash mismatch".to_string()); + } + let packument: serde_json::Value = serde_json::from_slice(&full) + .map_err(|_| "current full packument is invalid JSON".to_string())?; + if packument.get("name").and_then(serde_json::Value::as_str) != Some(package) { + return Err("current full packument package name mismatch".to_string()); + } + let committed_versions = packument + .get("versions") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| "current full packument has invalid versions".to_string())?; + let committed_tags = packument + .get("dist-tags") + .and_then(serde_json::Value::as_object) + .ok_or_else(|| "current full packument has invalid dist-tags".to_string())?; + + let mut authority_sha256 = BTreeMap::new(); + let mut authority_sizes = BTreeMap::new(); + let mut authority_meta = BTreeMap::new(); + let mut exact_versions = serde_json::Map::new(); + let mut exact_tags = serde_json::Map::new(); + let mut tag_keys_by_version = BTreeMap::>::new(); + + for (tag, target) in committed_tags { + let target = target + .as_str() + .ok_or_else(|| format!("dist-tag {tag} has a non-string target"))?; + let key = npm_dist_tag_key(repository, package, tag)?; + let (bytes, meta) = npm_exact_object(storage, &key) + .await? + .ok_or_else(|| format!("committed dist-tag authority is missing: {key}"))?; + if bytes.as_ref() != target.as_bytes() { + return Err(format!("committed dist-tag authority mismatch: {key}")); } + authority_sha256.insert(key.clone(), hex::encode(sha2::Sha256::digest(&bytes))); + authority_sizes.insert(key.clone(), meta.size); + authority_meta.insert(key.clone(), meta); + tag_keys_by_version + .entry(target.to_string()) + .or_default() + .push(key); + exact_tags.insert(tag.clone(), serde_json::Value::String(target.to_string())); } - let mut manifests = BTreeMap::new(); - let mut deprecations = BTreeMap::new(); - let mut completions = BTreeMap::new(); - let mut tags = Vec::new(); - for (key, (kind, _, contents)) in &objects { - match kind { - crate::npm_layout::NpmObjectKind::HostedVersion(version) => { - if manifests + let mut versions = Vec::with_capacity(committed_versions.len()); + for (version, committed_manifest) in committed_versions { + let manifest_key = npm_version_key(repository, package, version)?; + let (manifest, manifest_meta) = npm_exact_object(storage, &manifest_key) + .await? + .ok_or_else(|| format!("committed manifest authority is missing: {manifest_key}"))?; + let mut exact_manifest: serde_json::Value = serde_json::from_slice(&manifest) + .map_err(|_| format!("committed manifest is invalid JSON: {manifest_key}"))?; + let deprecation_key = npm_deprecation_key(repository, package, version); + let deprecation = npm_exact_object(storage, &deprecation_key).await?; + match deprecation { + Some((bytes, meta)) => { + let message = std::str::from_utf8(&bytes) + .map_err(|_| format!("deprecation is not UTF-8: {deprecation_key}"))?; + exact_manifest + .as_object_mut() + .ok_or_else(|| format!("manifest is not an object: {manifest_key}"))? .insert( - version.clone(), - ( - key.clone(), - contents - .as_ref() - .ok_or_else(|| format!("manifest unreadable: {key}"))? - .clone(), - ), - ) - .is_some() - { - return Err(format!("duplicate manifest for npm version {version}")); - } - } - crate::npm_layout::NpmObjectKind::HostedDeprecation(version) => { - deprecations.insert(version.clone(), key.clone()); - } - crate::npm_layout::NpmObjectKind::HostedPublishComplete(version) => { - completions.insert(version.clone(), key.clone()); + "deprecated".to_string(), + serde_json::Value::String(message.to_string()), + ); + authority_sha256.insert( + deprecation_key.clone(), + hex::encode(sha2::Sha256::digest(&bytes)), + ); + authority_sizes.insert(deprecation_key.clone(), meta.size); + authority_meta.insert(deprecation_key.clone(), meta); } - crate::npm_layout::NpmObjectKind::HostedDistTag(_) => { - tags.push(( - key.clone(), - contents - .as_ref() - .ok_or_else(|| format!("dist-tag unreadable: {key}"))? - .clone(), - )); + None => { + if committed_manifest.get("deprecated").is_some() { + return Err(format!( + "committed deprecation authority is missing: {deprecation_key}" + )); + } } - _ => {} } - } + if &exact_manifest != committed_manifest { + return Err(format!( + "manifest authority does not match current full packument: {manifest_key}" + )); + } - let mut versions = Vec::with_capacity(manifests.len()); - for (version, (manifest_key, manifest)) in manifests { let blob_key = crate::npm_layout::hosted_blob_key_from_manifest(repository, package, &manifest) .ok_or_else(|| { format!("npm version manifest has no valid blob reference: {manifest_key}") })?; - let Some((blob_kind, _, _)) = objects.get(&blob_key) else { - return Err(format!("npm version blob is missing: {blob_key}")); - }; - if !matches!( - blob_kind, - crate::npm_layout::NpmObjectKind::HostedBlob { .. } - ) { - return Err(format!("npm version blob has invalid layout: {blob_key}")); - } + let (_blob_size, blob_reader) = storage + .get_reader(&blob_key) + .await + .map_err(|error| format!("committed npm blob is unreadable at {blob_key}: {error}"))?; + drop(blob_reader); - let mut keys = vec![manifest_key, blob_key]; - if let Some(key) = completions.get(&version) { - keys.push(key.clone()); + authority_sha256.insert( + manifest_key.clone(), + hex::encode(sha2::Sha256::digest(&manifest)), + ); + authority_sizes.insert(manifest_key.clone(), manifest_meta.size); + authority_meta.insert(manifest_key.clone(), manifest_meta); + let mut keys = vec![manifest_key.clone()]; + + let completion_key = npm_publish_complete_key(repository, package, version); + if let Some((completion, meta)) = npm_exact_object(storage, &completion_key).await? { + let expected = crate::npm_layout::hosted_manifest_digest(&manifest); + if completion.as_ref() != expected.as_bytes() { + return Err(format!("publish completion mismatch: {completion_key}")); + } + authority_sha256.insert( + completion_key.clone(), + hex::encode(sha2::Sha256::digest(&completion)), + ); + authority_sizes.insert(completion_key.clone(), meta.size); + authority_meta.insert(completion_key.clone(), meta); + keys.push(completion_key); } - if let Some(key) = deprecations.get(&version) { - keys.push(key.clone()); + if authority_meta.contains_key(&deprecation_key) { + keys.push(deprecation_key); } - for (tag_key, target) in &tags { - if target.as_ref() == version.as_bytes() { - keys.push(tag_key.clone()); - } + if let Some(tag_keys) = tag_keys_by_version.get(version) { + keys.extend(tag_keys.iter().cloned()); } keys.sort(); keys.dedup(); - let mut modified = 0u64; let mut size = 0u64; for key in &keys { - let (_, meta, _) = objects + let meta = authority_meta .get(key) - .ok_or_else(|| format!("snapshot key disappeared: {key}"))?; + .ok_or_else(|| format!("snapshot metadata disappeared for exact key {key}"))?; modified = modified.max(meta.modified); size += meta.size; } versions.push(VersionEntry { - name: version, + name: version.clone(), keys, modified, size, }); + exact_versions.insert(version.clone(), exact_manifest); + } + + let package_key = crate::npm_layout::hosted_package_key(repository, package); + let mut rebuilt = match npm_exact_object(storage, &package_key).await? { + Some((bytes, meta)) => { + authority_sha256.insert( + package_key.clone(), + hex::encode(sha2::Sha256::digest(&bytes)), + ); + authority_sizes.insert(package_key.clone(), meta.size); + authority_meta.insert(package_key, meta); + serde_json::from_slice(&bytes) + .map_err(|_| "npm package authority is invalid JSON".to_string())? + } + None => serde_json::json!({}), + }; + let rebuilt_object = rebuilt + .as_object_mut() + .ok_or_else(|| "npm package authority is not an object".to_string())?; + rebuilt_object.insert( + "name".to_string(), + serde_json::Value::String(package.to_string()), + ); + rebuilt_object.insert( + "versions".to_string(), + serde_json::Value::Object(exact_versions), + ); + rebuilt_object.insert( + "dist-tags".to_string(), + serde_json::Value::Object(exact_tags), + ); + if rebuilt != packument { + return Err("split npm authority does not match current full packument".to_string()); + } + + let pointer_after = + crate::registry::read_hosted_packument_pointer(storage, repository, package) + .await + .map_err(|error| format!("current pointer re-read failed: {error}"))?; + if pointer_after.as_ref() != Some(&pointer) { + return Err("current pointer changed while taking snapshot".to_string()); + } + if npm_active_transaction_present(storage, repository, package).await? { + return Err("npm transaction appeared while taking snapshot".to_string()); + } + + let mut hasher = sha2::Sha256::new(); + npm_guard_bytes(&mut hasher, b"nora/npm-retention-package/v2"); + npm_guard_bytes(&mut hasher, repository.as_bytes()); + npm_guard_bytes(&mut hasher, package.as_bytes()); + npm_guard_bytes( + &mut hasher, + &serde_json::to_vec(&pointer).map_err(|_| "cannot encode pointer".to_string())?, + ); + npm_guard_bytes(&mut hasher, &full); + for (key, digest) in &authority_sha256 { + npm_guard_bytes(&mut hasher, key.as_bytes()); + npm_guard_bytes(&mut hasher, digest.as_bytes()); + if let Some(meta) = authority_meta.get(key) { + hasher.update(meta.size.to_be_bytes()); + hasher.update(meta.modified.to_be_bytes()); + } } Ok(NpmPackageSnapshot { versions, guard: hex::encode(hasher.finalize()), + pointer, + packument, + authority_sha256, + authority_sizes, }) } @@ -854,12 +1032,7 @@ async fn read_npm_package_snapshot( repository: &str, package: &str, ) -> Result { - let prefix = format!("npm/repositories/{repository}/{package}/"); - let keys = storage - .list(&prefix) - .await - .map_err(|error| format!("cannot list {prefix}: {error}"))?; - npm_package_snapshot(storage, repository, package, keys).await + npm_package_snapshot(storage, repository, package).await } /// Collect npm packages, including packages that only contain cleanup state. @@ -873,28 +1046,49 @@ async fn collect_npm_versions(storage: &Storage) -> Vec { return Vec::new(); } }; - let mut packages: std::collections::BTreeMap<(String, String), Vec> = - std::collections::BTreeMap::new(); + let mut packages = std::collections::BTreeSet::new(); for key in all_keys { let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { continue; }; if is_hosted_npm_object(&parsed.kind) { - packages - .entry((parsed.repository, parsed.package)) - .or_default() - .push(key); + packages.insert((parsed.repository, parsed.package)); } } let mut result = Vec::with_capacity(packages.len()); - for ((repository, package), keys) in packages { - match npm_package_snapshot(storage, &repository, &package, keys).await { + for (repository, package) in packages { + match crate::registry::read_hosted_maintenance_marker(storage, &repository, &package).await + { + Ok(Some(_)) => { + result.push(NpmVersionGroup { + group_name: format!("npm:{repository}:{package}"), + repository, + package, + versions: Vec::new(), + active_maintenance: true, + snapshot_guard: String::new(), + }); + continue; + } + Ok(None) => {} + Err(error) => { + tracing::warn!( + repository, + package, + %error, + "retention: npm maintenance marker is unreadable; package skipped" + ); + continue; + } + } + match npm_package_snapshot(storage, &repository, &package).await { Ok(snapshot) => result.push(NpmVersionGroup { group_name: format!("npm:{repository}:{package}"), repository, package, versions: snapshot.versions, + active_maintenance: false, snapshot_guard: snapshot.guard, }), Err(error) => { @@ -911,361 +1105,283 @@ async fn collect_npm_versions(storage: &Storage) -> Vec { } #[derive(Debug, Default)] -struct NpmCleanupOutcome { - complete: bool, +struct NpmBatchOutcome { + applied_versions: usize, deleted_keys: usize, bytes_freed: u64, + changed: bool, } -/// Remove package-level hosted state after retention deleted the last version -/// manifest. Empty packages are collected independently, so a partial failure -/// leaves at least one discoverable key and is retried on a later run. -/// -/// The caller holds npm's exact package publish lock. -async fn clean_empty_npm_package( +async fn delete_npm_authority_exact( storage: &Storage, - repository: &str, - package: &str, -) -> NpmCleanupOutcome { - let versions_prefix = format!("npm/repositories/{repository}/{package}/versions/"); - let versions = match storage.list(&versions_prefix).await { - Ok(keys) => keys, - Err(error) => { - tracing::warn!( - repository, - package, - error = %error, - "retention: cannot verify empty npm package; package metadata kept" - ); - return NpmCleanupOutcome::default(); - } + key: &str, + expected_sha256: &str, +) -> Result { + let current = match storage.get(key).await { + Ok(bytes) => bytes, + Err(StorageError::NotFound) => return Ok(false), + Err(error) => return Err(error), }; - if !versions.is_empty() { - return NpmCleanupOutcome { - complete: true, - ..NpmCleanupOutcome::default() - }; + if hex::encode(sha2::Sha256::digest(¤t)) != expected_sha256 { + return Err(StorageError::AlreadyExists); } - let package_prefix = format!("npm/repositories/{repository}/{package}/"); - let mut keys = match storage.list(&package_prefix).await { - Ok(keys) => keys, - Err(error) => { - tracing::warn!( - repository, - package, - error = %error, - "retention: cannot list empty npm package state; metadata kept" - ); - return NpmCleanupOutcome::default(); - } - }; - keys.sort(); - let mut outcome = NpmCleanupOutcome::default(); - for key in keys { - let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { - continue; - }; - if parsed.repository != repository || parsed.package != package { - continue; - } - if !matches!( - parsed.kind, - crate::npm_layout::NpmObjectKind::HostedPackage - | crate::npm_layout::NpmObjectKind::HostedDistTag(_) - | crate::npm_layout::NpmObjectKind::HostedDeprecation(_) - | crate::npm_layout::NpmObjectKind::HostedPublishComplete(_) - ) { - continue; - } - let Some(meta) = storage.stat(&key).await else { - tracing::warn!( - repository, - package, - key, - "retention: cannot stat empty npm package state; cleanup will retry" - ); - return outcome; - }; - match storage.delete(&key).await { - Ok(()) => { - outcome.deleted_keys += 1; - outcome.bytes_freed += meta.size; - } - Err(error) => { - tracing::warn!( - repository, - package, - key, - %error, - "retention: empty npm package cleanup failed; cleanup will retry" - ); - return outcome; - } + let deletion = storage.delete(key).await; + match storage.get(key).await { + Err(StorageError::NotFound) => Ok(true), + Ok(bytes) if hex::encode(sha2::Sha256::digest(&bytes)) != expected_sha256 => { + Err(StorageError::AlreadyExists) } + Ok(_) => match deletion { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Err(error) => Err(error), } - outcome.complete = true; - outcome } -async fn npm_blob_still_referenced( +async fn ensure_npm_retired_marker( storage: &Storage, - blob_key: &str, -) -> Result { - let Some(parsed) = crate::npm_layout::parse_npm_object_key(blob_key) else { - return Ok(false); - }; - if !matches!( - parsed.kind, - crate::npm_layout::NpmObjectKind::HostedBlob { .. } - ) { - return Ok(false); - } - let prefix = format!( - "npm/repositories/{}/{}/versions/", - parsed.repository, parsed.package - ); - for manifest_key in storage.list(&prefix).await? { - let manifest = storage.get(&manifest_key).await?; - if crate::npm_layout::hosted_blob_key_from_manifest( - &parsed.repository, - &parsed.package, - &manifest, - ) - .as_deref() - == Some(blob_key) - { - return Ok(true); + repository: &str, + package: &str, +) -> Result<(), StorageError> { + let key = crate::npm_layout::hosted_packument_retired_key(repository, package); + match storage.get(&key).await { + Ok(bytes) if bytes.as_ref() == crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 => { + return Ok(()) } + Ok(_) => return Err(StorageError::IntegrityViolation), + Err(StorageError::NotFound) => {} + Err(error) => return Err(error), + } + let write = storage + .put_if_absent(&key, crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1) + .await; + match storage.get(&key).await { + Ok(bytes) if bytes.as_ref() == crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 => Ok(()), + Ok(_) => Err(StorageError::IntegrityViolation), + Err(StorageError::NotFound) => match write { + Ok(()) => Err(StorageError::IntegrityViolation), + Err(error) => Err(error), + }, + Err(error) => Err(error), } - Ok(false) } -#[derive(Debug, Default)] -struct NpmDeleteOutcome { - applied: bool, - deleted_keys: usize, - bytes_freed: u64, +async fn ensure_npm_retired_authority_absent( + storage: &Storage, + repository: &str, + package: &str, + expected_authority: &std::collections::BTreeMap, +) -> Result<(), StorageError> { + let transactions = + crate::registry::read_hosted_active_transactions(storage, repository, package).await?; + if transactions.import.is_some() || transactions.publish.is_some() { + return Err(StorageError::AlreadyExists); + } + // Every destructive key came from the immutable base generation and was + // persisted in the maintenance marker before the pointer disappeared. + // Recheck that exact roster; LIST omission is never accepted as proof that + // retirement cleanup completed. + for key in expected_authority.keys() { + match storage.get(key).await { + Err(StorageError::NotFound) => {} + Ok(_) => return Err(StorageError::AlreadyExists), + Err(error) => return Err(error), + } + } + // The package root has a fixed exact key and may legitimately have been + // absent from the base generation. Probe it independently so a late or + // omitted live root cannot be hidden by a LIST snapshot. + let package_key = crate::npm_layout::hosted_package_key(repository, package); + match storage.get(&package_key).await { + Err(StorageError::NotFound) => {} + Ok(_) => return Err(StorageError::AlreadyExists), + Err(error) => return Err(error), + } + Ok(()) } -/// Delete one hosted npm version with its version manifest as the visibility -/// commit point. Mutable dependants are removed first, with -/// `publish-complete` first of all: if a later pre-commit operation fails, an -/// exact publish retry observes the missing completion marker and can repair -/// the original publish state. Arbitrary tags or deprecations added later are -/// not reconstructible, so this phase is fail-safe rather than atomic. The -/// content-addressed blob is considered only after the manifest commit and may -/// safely remain as a GC-healable orphan. -async fn delete_npm_plan( +/// Resume one durable npm retention operation. The package publish lock is +/// held by every caller. This function intentionally does not clear the active +/// marker; the shared npm maintenance dispatcher removes it only after this +/// complete state machine returns success. +pub(crate) async fn resume_npm_retention_operation( storage: &Storage, - group_name: &str, - plan: &DeletionPlan, -) -> NpmDeleteOutcome { - let mut manifest_key = None; - let mut completion_keys = Vec::new(); - let mut tag_keys = Vec::new(); - let mut deprecation_keys = Vec::new(); - let mut blob_keys = Vec::new(); - for key in &plan.keys { - let Some(parsed) = crate::npm_layout::parse_npm_object_key(key) else { - tracing::error!( - group = group_name, - version = %plan.version_name, - key, - "retention: npm plan contains an invalid key; deletion aborted" - ); - return NpmDeleteOutcome::default(); - }; - match parsed.kind { - crate::npm_layout::NpmObjectKind::HostedVersion(version) - if version == plan.version_name => + marker: &crate::npm_layout::HostedMaintenanceMarker, +) -> Result<(), StorageError> { + let crate::npm_layout::HostedMaintenanceAction::Retention { + removed_versions, + expected_authority, + .. + } = &marker.action + else { + return Err(StorageError::IntegrityViolation); + }; + if removed_versions.is_empty() || expected_authority.is_empty() { + return Err(StorageError::IntegrityViolation); + } + + match &marker.target { + crate::npm_layout::HostedMaintenanceTarget::Live { pointer: target } => { + crate::registry::validate_hosted_packument_pointer( + storage, + &marker.repository, + &marker.package, + target, + ) + .await?; + match crate::registry::read_hosted_packument_pointer( + storage, + &marker.repository, + &marker.package, + ) + .await? { - if manifest_key.replace(key.clone()).is_some() { - tracing::error!( - group = group_name, - version = %plan.version_name, - "retention: npm plan has multiple version manifests; deletion aborted" - ); - return NpmDeleteOutcome::default(); + Some(current) if current == marker.base => { + crate::registry::commit_hosted_packument_pointer( + storage, + &marker.repository, + &marker.package, + target, + ) + .await?; } - } - crate::npm_layout::NpmObjectKind::HostedPublishComplete(version) - if version == plan.version_name => - { - completion_keys.push(key.clone()); - } - crate::npm_layout::NpmObjectKind::HostedDeprecation(version) - if version == plan.version_name => - { - deprecation_keys.push(key.clone()); - } - crate::npm_layout::NpmObjectKind::HostedDistTag(_) => tag_keys.push(key.clone()), - crate::npm_layout::NpmObjectKind::HostedBlob { .. } => blob_keys.push(key.clone()), - _ => { - tracing::error!( - group = group_name, - version = %plan.version_name, - key, - "retention: npm plan contains an unrelated key; deletion aborted" - ); - return NpmDeleteOutcome::default(); + Some(current) if current == *target => {} + _ => return Err(StorageError::AlreadyExists), } } - } - let Some(manifest_key) = manifest_key else { - tracing::error!( - group = group_name, - version = %plan.version_name, - "retention: npm plan has no version manifest; deletion aborted" - ); - return NpmDeleteOutcome::default(); - }; - completion_keys.sort(); - tag_keys.sort(); - deprecation_keys.sort(); - blob_keys.sort(); - - let mut outcome = NpmDeleteOutcome::default(); - - // Pre-commit phase. A failure may leave already-deleted mutable state, but - // never removes the version manifest. Deleting the completion marker first - // makes the original publish state explicitly retryable by npm's - // exact-publish repair path. Later operator mutations are not - // reconstructible; the manifest nevertheless remains a safe visibility - // boundary and no tag can dangle from a deleted version. - for key in completion_keys - .iter() - .chain(tag_keys.iter()) - .chain(deprecation_keys.iter()) - { - if tag_keys.binary_search(key).is_ok() { - match storage.get(key).await { - Ok(target) if target.as_ref() == plan.version_name.as_bytes() => {} - Ok(_) => { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key, - "retention: npm dist-tag target changed; deletion aborted" - ); - return outcome; - } - Err(error) => { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key, - %error, - "retention: cannot verify npm dist-tag target; deletion aborted" - ); - return outcome; + crate::npm_layout::HostedMaintenanceTarget::Retired => { + match crate::registry::read_hosted_packument_pointer( + storage, + &marker.repository, + &marker.package, + ) + .await? + { + Some(current) if current == marker.base => { + let pointer = serde_json::to_vec(&marker.base) + .map_err(|_| StorageError::IntegrityViolation)?; + let expected = hex::encode(sha2::Sha256::digest(pointer)); + delete_npm_authority_exact( + storage, + &crate::npm_layout::hosted_packument_current_key( + &marker.repository, + &marker.package, + ), + &expected, + ) + .await?; } - } - } - let Some(meta) = storage.stat(key).await else { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key, - "retention: cannot stat dependent npm object; deletion aborted" - ); - return outcome; - }; - match storage.delete(key).await { - Ok(()) => { - outcome.deleted_keys += 1; - outcome.bytes_freed += meta.size; - } - Err(error) => { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key, - %error, - "retention: dependent npm object deletion failed; manifest kept" - ); - return outcome; + None => {} + Some(_) => return Err(StorageError::AlreadyExists), } } } - let Some(manifest_meta) = storage.stat(&manifest_key).await else { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key = %manifest_key, - "retention: cannot stat npm manifest; deletion aborted" - ); - return outcome; - }; - match storage.delete(&manifest_key).await { - Ok(()) => { - outcome.deleted_keys += 1; - outcome.bytes_freed += manifest_meta.size; - outcome.applied = true; - } - Err(error) => { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key = %manifest_key, - %error, - "retention: npm manifest deletion failed; blob kept" - ); - return outcome; - } + // The pointer is the visibility boundary. Only after it is at the durable + // target do we remove exact source authority. Missing is an already-done + // step; a replacement body or read error fails closed and leaves the + // marker blocking every package writer. + let mut expected = expected_authority.iter().collect::>(); + expected.sort_by_key(|(key, _)| { + crate::npm_layout::parse_npm_object_key(key).is_some_and(|parsed| { + matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedVersion(_) + ) + }) + }); + for (key, expected_sha256) in expected { + delete_npm_authority_exact(storage, key, expected_sha256).await?; } - // Post-commit phase: failures only leave content-addressed orphan blobs. - for key in blob_keys { - match npm_blob_still_referenced(storage, &key).await { - Ok(true) => continue, - Ok(false) => {} - Err(error) => { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key, - %error, - "retention: cannot prove npm blob is unreferenced; blob kept" - ); - continue; - } - } - let Some(meta) = storage.stat(&key).await else { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key, - "retention: cannot stat unreferenced npm blob; blob kept" - ); - continue; - }; - match storage.delete(&key).await { - Ok(()) => { - outcome.deleted_keys += 1; - outcome.bytes_freed += meta.size; + match &marker.target { + crate::npm_layout::HostedMaintenanceTarget::Live { pointer: target } => { + crate::registry::validate_hosted_packument_pointer( + storage, + &marker.repository, + &marker.package, + target, + ) + .await?; + match crate::registry::read_hosted_packument_pointer( + storage, + &marker.repository, + &marker.package, + ) + .await? + { + Some(current) if current == *target => Ok(()), + _ => Err(StorageError::AlreadyExists), } - Err(error) => { - tracing::warn!( - group = group_name, - version = %plan.version_name, - key, - %error, - "retention: unreferenced npm blob deletion failed; GC will retry" - ); + } + crate::npm_layout::HostedMaintenanceTarget::Retired => { + ensure_npm_retired_authority_absent( + storage, + &marker.repository, + &marker.package, + expected_authority, + ) + .await?; + ensure_npm_retired_marker(storage, &marker.repository, &marker.package).await?; + match crate::registry::read_hosted_packument_pointer( + storage, + &marker.repository, + &marker.package, + ) + .await? + { + None => Ok(()), + Some(_) => Err(StorageError::AlreadyExists), } } } - - outcome } -#[derive(Debug, Default)] -struct NpmBatchOutcome { - applied_versions: usize, - deleted_keys: usize, - bytes_freed: u64, +async fn npm_retention_target_matches_snapshot( + storage: &Storage, + repository: &str, + package: &str, + snapshot: &NpmPackageSnapshot, + removed: &std::collections::HashSet, + target: &crate::npm_layout::HostedMaintenanceTarget, +) -> Result { + let mut expected = snapshot.packument.clone(); + let versions = expected + .get_mut("versions") + .and_then(serde_json::Value::as_object_mut) + .ok_or_else(|| "snapshot packument has invalid versions".to_string())?; + versions.retain(|version, _| !removed.contains(version)); + if versions.is_empty() { + return Ok(matches!( + target, + crate::npm_layout::HostedMaintenanceTarget::Retired + )); + } + let tags = expected + .get_mut("dist-tags") + .and_then(serde_json::Value::as_object_mut) + .ok_or_else(|| "snapshot packument has invalid dist-tags".to_string())?; + tags.retain(|_, value| { + !value + .as_str() + .is_some_and(|version| removed.contains(version)) + }); + let crate::npm_layout::HostedMaintenanceTarget::Live { pointer } = target else { + return Ok(false); + }; + crate::registry::validate_hosted_packument_pointer(storage, repository, package, pointer) + .await + .map_err(|error| format!("prepared target generation is invalid: {error}"))?; + let key = + crate::npm_layout::hosted_packument_full_key(repository, package, &pointer.generation); + let (full, _) = npm_exact_object(storage, &key) + .await? + .ok_or_else(|| format!("prepared target full document is missing: {key}"))?; + let prepared: serde_json::Value = serde_json::from_slice(&full) + .map_err(|_| "prepared target full document is invalid JSON".to_string())?; + Ok(prepared == expected) } /// Validate and apply a complete npm package plan while holding the same lock @@ -1284,6 +1400,40 @@ async fn apply_npm_plans( let lock = crate::acquire_publish_lock(publish_locks, &group.group_name); let _guard = lock.lock().await; + match crate::registry::resume_hosted_maintenance_operation( + storage, + &group.repository, + &group.package, + ) + .await + { + Ok(true) => { + tracing::info!( + repository = group.repository, + package = group.package, + "retention: resumed prior npm maintenance operation; stale plan skipped" + ); + return NpmBatchOutcome { + changed: true, + ..NpmBatchOutcome::default() + }; + } + Ok(false) => {} + Err(error) => { + tracing::warn!( + repository = group.repository, + package = group.package, + %error, + "retention: active npm maintenance cannot be resumed; package kept" + ); + return NpmBatchOutcome::default(); + } + } + + if group.active_maintenance { + return NpmBatchOutcome::default(); + } + let current = match read_npm_package_snapshot(storage, &group.repository, &group.package).await { Ok(snapshot) => snapshot, @@ -1307,46 +1457,144 @@ async fn apply_npm_plans( } let mut outcome = NpmBatchOutcome::default(); - // The hosted packument cache is derived and deliberately excluded from - // the authoritative retention snapshot. Remove it before the first - // mutation while holding the exact publish lock; a later read will rebuild - // from whatever authoritative state the batch commits. - let cache_key = - crate::npm_layout::hosted_packument_cache_key(&group.repository, &group.package); - let cache_size = storage - .stat(&cache_key) - .await - .map(|meta| meta.size) - .unwrap_or(0); - match storage.delete(&cache_key).await { - Ok(()) => { - outcome.deleted_keys += 1; - outcome.bytes_freed += cache_size; + if plans.is_empty() { + return outcome; + } + let base = current.pointer.clone(); + + let removed = plans + .iter() + .map(|plan| plan.version_name.clone()) + .collect::>(); + let target = match crate::registry::prepare_hosted_packument_after_retention( + storage, + &group.repository, + &group.package, + &removed, + ) + .await + { + Ok(target) => target, + Err(error) => { + tracing::warn!( + repository = group.repository, + package = group.package, + %error, + "retention: cannot prepare next npm packument target; batch skipped" + ); + return outcome; + } + }; + match npm_retention_target_matches_snapshot( + storage, + &group.repository, + &group.package, + ¤t, + &removed, + &target, + ) + .await + { + Ok(true) => {} + Ok(false) => { + tracing::warn!( + repository = group.repository, + package = group.package, + "retention: LIST-built target differs from exact current generation; batch skipped" + ); + return outcome; } - Err(StorageError::NotFound) => {} Err(error) => { tracing::warn!( repository = group.repository, package = group.package, - key = cache_key, %error, - "retention: cannot invalidate npm hosted packument cache; batch skipped" + "retention: cannot validate prepared target; batch skipped" ); return outcome; } } + + let mut removed_versions = std::collections::BTreeMap::new(); + let mut expected_authority = std::collections::BTreeMap::new(); for plan in plans { - let deleted = delete_npm_plan(storage, &group.group_name, plan).await; - outcome.deleted_keys += deleted.deleted_keys; - outcome.bytes_freed += deleted.bytes_freed; - if !deleted.applied { - // A pre-commit failure may already have removed mutable state. - // Stop the package batch. The manifest remains visible and no tag - // can dangle from a deleted version. Removing publish-complete - // first also lets an exact retry repair original publish state. - return outcome; + let mut manifest_digest = None; + for key in &plan.keys { + let Some(digest) = current.authority_sha256.get(key) else { + tracing::warn!( + repository = group.repository, + package = group.package, + key, + "retention: planned npm authority lacks an exact snapshot hash" + ); + return outcome; + }; + let Some(parsed) = crate::npm_layout::parse_npm_object_key(key) else { + return outcome; + }; + if matches!( + parsed.kind, + crate::npm_layout::NpmObjectKind::HostedVersion(ref version) + if version == &plan.version_name + ) && manifest_digest.replace(digest.clone()).is_some() + { + return outcome; + } + expected_authority.insert(key.clone(), digest.clone()); } - outcome.applied_versions += 1; + let Some(manifest_digest) = manifest_digest else { + return outcome; + }; + removed_versions.insert(plan.version_name.clone(), manifest_digest); + } + if matches!(&target, crate::npm_layout::HostedMaintenanceTarget::Retired) { + expected_authority = current.authority_sha256.clone(); + } + + let bytes_freed = expected_authority + .keys() + .filter_map(|key| current.authority_sizes.get(key)) + .sum(); + let operation = crate::npm_layout::HostedMaintenanceOperation { + schema: crate::npm_layout::HOSTED_MAINTENANCE_SCHEMA_V1, + repository: group.repository.clone(), + package: group.package.clone(), + base, + target, + action: crate::npm_layout::HostedMaintenanceAction::Retention { + snapshot_guard: current.guard, + removed_versions, + expected_authority: expected_authority.clone(), + }, + }; + if let Err(error) = crate::registry::create_hosted_maintenance_marker(storage, &operation).await + { + tracing::warn!( + repository = group.repository, + package = group.package, + %error, + "retention: cannot create durable npm maintenance marker; authority untouched" + ); + return outcome; + } + outcome.changed = true; + if let Err(error) = crate::registry::resume_hosted_maintenance_operation( + storage, + &group.repository, + &group.package, + ) + .await + { + tracing::warn!( + repository = group.repository, + package = group.package, + %error, + "retention: npm maintenance remains active for recovery" + ); + return outcome; + } + + for plan in plans { info!( group = %group.group_name, version = %plan.version_name, @@ -1354,18 +1602,12 @@ async fn apply_npm_plans( "Retention: deleted" ); } - - let cleanup = clean_empty_npm_package(storage, &group.repository, &group.package).await; - outcome.deleted_keys += cleanup.deleted_keys; - outcome.bytes_freed += cleanup.bytes_freed; - if !cleanup.complete { - tracing::warn!( - repository = group.repository, - package = group.package, - "retention: empty npm package cleanup remains pending" - ); + NpmBatchOutcome { + applied_versions: plans.len(), + deleted_keys: expected_authority.len(), + bytes_freed, + changed: true, } - outcome } /// Collect PyPI package files. @@ -1826,6 +2068,31 @@ pub(crate) async fn run_retention_configured( std::collections::BTreeSet::new(); for (group_name, versions) in all_groups { + // Durable npm maintenance is a recovery obligation, not a fresh + // policy decision. Resume it before rule lookup so changing/removing + // rules cannot strand a package behind an active marker. A dry-run is + // strictly observational and deliberately leaves the marker intact. + if let Some(group) = npm_groups + .get(&group_name) + .filter(|group| group.active_maintenance) + { + if dry_run { + info!( + repository = group.repository, + package = group.package, + "[dry-run] Retention: active npm maintenance requires recovery" + ); + } else { + let outcome = apply_npm_plans(storage, publish_locks, group, &[]).await; + total_deleted_keys += outcome.deleted_keys; + total_bytes += outcome.bytes_freed; + if outcome.changed { + mutated_registries.insert("npm".to_string()); + } + } + continue; + } + // Find matching rule for this group let registry = group_name.split(':').next().unwrap_or(""); let rule = match find_matching_rule(rules, registry, &group_name) { @@ -1844,7 +2111,7 @@ pub(crate) async fn run_retention_configured( let outcome = apply_npm_plans(storage, publish_locks, group, &[]).await; total_deleted_keys += outcome.deleted_keys; total_bytes += outcome.bytes_freed; - if outcome.deleted_keys > 0 { + if outcome.changed { mutated_registries.insert("npm".to_string()); } } @@ -1868,7 +2135,7 @@ pub(crate) async fn run_retention_configured( total_applied += outcome.applied_versions; total_deleted_keys += outcome.deleted_keys; total_bytes += outcome.bytes_freed; - if outcome.deleted_keys > 0 { + if outcome.changed { mutated_registries.insert("npm".to_string()); } } @@ -2206,6 +2473,130 @@ mod tests { Arc::new(parking_lot::Mutex::new(std::collections::HashMap::new())) } + async fn seed_npm_current(storage: &Storage, repository: &str, package: &str) { + // Build the first immutable generation directly. The production + // retention preparer intentionally requires an existing exact base + // pointer, so it cannot bootstrap a test package. + let package_key = crate::npm_layout::hosted_package_key(repository, package); + let mut packument: serde_json::Value = match storage.get(&package_key).await { + Ok(bytes) => serde_json::from_slice(&bytes).unwrap(), + Err(StorageError::NotFound) => serde_json::json!({}), + Err(error) => panic!("cannot read seeded package root: {error}"), + }; + let mut versions = serde_json::Map::new(); + let mut tags = serde_json::Map::new(); + let prefix = format!("npm/repositories/{repository}/{package}/"); + for key in storage.list(&prefix).await.unwrap() { + let Some(parsed) = crate::npm_layout::parse_npm_object_key(&key) else { + continue; + }; + if parsed.repository != repository || parsed.package != package { + continue; + } + match parsed.kind { + crate::npm_layout::NpmObjectKind::HostedVersion(version) => { + let bytes = storage.get(&key).await.unwrap(); + let mut manifest: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let deprecation = npm_deprecation_key(repository, package, &version); + if let Ok(message) = storage.get(&deprecation).await { + manifest.as_object_mut().unwrap().insert( + "deprecated".to_string(), + serde_json::Value::String( + std::str::from_utf8(&message).unwrap().to_string(), + ), + ); + } + versions.insert(version, manifest); + } + crate::npm_layout::NpmObjectKind::HostedDistTag(tag) => { + let target = storage.get(&key).await.unwrap(); + tags.insert( + tag, + serde_json::Value::String( + std::str::from_utf8(&target).unwrap().to_string(), + ), + ); + } + _ => {} + } + } + let object = packument.as_object_mut().unwrap(); + object.insert( + "name".to_string(), + serde_json::Value::String(package.to_string()), + ); + object.insert("versions".to_string(), serde_json::Value::Object(versions)); + object.insert("dist-tags".to_string(), serde_json::Value::Object(tags)); + let full = serde_json::to_vec(&packument).unwrap(); + let generation = hex::encode(sha2::Sha256::digest(&full)); + let install_v1 = full.clone(); + let pointer = crate::npm_layout::HostedPackumentPointer { + generation: generation.clone(), + full_sha256: generation.clone(), + install_v1_sha256: hex::encode(sha2::Sha256::digest(&install_v1)), + }; + storage + .put( + &crate::npm_layout::hosted_packument_full_key(repository, package, &generation), + &full, + ) + .await + .unwrap(); + storage + .put( + &crate::npm_layout::hosted_packument_install_v1_key( + repository, + package, + &generation, + ), + &install_v1, + ) + .await + .unwrap(); + crate::registry::commit_hosted_packument_pointer(storage, repository, package, &pointer) + .await + .unwrap(); + } + + async fn seed_npm_active_import(storage: &Storage, repository: &str, package: &str) -> String { + let key = crate::npm_layout::hosted_import_pending_key(repository, package); + let session = crate::npm_layout::HostedImportSession { + schema: crate::npm_layout::HOSTED_IMPORT_SESSION_SCHEMA_V1, + repository: repository.to_string(), + package: package.to_string(), + packument_sha256: "a".repeat(64), + base: None, + versions: std::collections::BTreeMap::from([("1.0.0".to_string(), "b".repeat(64))]), + }; + storage + .put(&key, &serde_json::to_vec(&session).unwrap()) + .await + .unwrap(); + key + } + + async fn npm_keep_zero_group_and_plans( + storage: &Storage, + repository: &str, + package: &str, + ) -> (NpmVersionGroup, Vec) { + let group_name = format!("npm:{repository}:{package}"); + let group = collect_npm_versions(storage) + .await + .into_iter() + .find(|group| group.group_name == group_name) + .unwrap(); + let rule = RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(0), + older_than_days: None, + exclude_tags: vec![], + }; + let plans = plan_deletions(group.versions.clone(), &rule, NOW); + (group, plans) + } + fn make_rule( keep_last: Option, older_than_days: Option, @@ -2971,6 +3362,7 @@ mod tests { ) .await .unwrap(); + seed_npm_current(&storage, repository, "pkg").await; } let proxy_key = "npm/repositories/npm-registry/proxy/tarballs/pkg/pkg-1.0.0.tgz"; storage.put(proxy_key, b"cache").await.unwrap(); @@ -2998,7 +3390,7 @@ mod tests { hex::encode(sha2::Sha512::digest(b"1.0.0")) )) .await - .is_none()); + .is_some()); assert!(storage .stat(&format!("npm/repositories/{repository}/pkg/dist-tags/old")) .await @@ -3014,7 +3406,29 @@ mod tests { repository, "pkg" )) .await - .is_none()); + .is_some()); + let pointer: serde_json::Value = serde_json::from_slice( + &storage + .get(&crate::npm_layout::hosted_packument_current_key( + repository, "pkg", + )) + .await + .unwrap(), + ) + .unwrap(); + let full: serde_json::Value = serde_json::from_slice( + &storage + .get(&crate::npm_layout::hosted_packument_full_key( + repository, + "pkg", + pointer["generation"].as_str().unwrap(), + )) + .await + .unwrap(), + ) + .unwrap(); + assert!(full["versions"].get("1.0.0").is_none()); + assert!(full["versions"].get("2.0.0").is_some()); } assert!(storage.get(proxy_key).await.is_ok()); } @@ -3027,6 +3441,7 @@ mod tests { let (_, first_blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"shared").await; let (_, second_blob) = seed_npm_version(&storage, prefix, "pkg", "2.0.0", b"shared").await; assert_eq!(first_blob, second_blob); + seed_npm_current(&storage, "npm-private", "pkg").await; let rules = vec![RetentionRule { registry: "npm".to_string(), name_glob: None, @@ -3049,6 +3464,119 @@ mod tests { assert!(storage.get(&first_blob).await.is_ok()); } + #[tokio::test] + async fn test_retention_npm_list_omission_cannot_shrink_exact_snapshot() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (old_manifest, old_blob) = + seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"old").await; + let (live_manifest, live_blob) = + seed_npm_version(&inner, prefix, "pkg", "2.0.0", b"live").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let pointer = crate::registry::read_hosted_packument_pointer(&inner, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + let storage = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()) + .omit_from_list(&old_manifest) + .omit_from_list(&live_manifest) + .omit_from_list(crate::npm_layout::hosted_packument_current_key( + "npm-private", + "pkg", + )) + .omit_from_list(crate::npm_layout::hosted_packument_full_key( + "npm-private", + "pkg", + &pointer.generation, + )) + .omit_from_list(crate::npm_layout::hosted_packument_install_v1_key( + "npm-private", + "pkg", + &pointer.generation, + )), + )); + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 1); + assert!(inner.get(&old_manifest).await.is_err()); + assert!( + inner.get(&old_blob).await.is_ok(), + "blob cleanup belongs to GC" + ); + assert!(inner.get(&live_manifest).await.is_ok()); + assert!(inner.get(&live_blob).await.is_ok()); + let current = crate::registry::read_hosted_packument_pointer(&inner, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + let full: serde_json::Value = serde_json::from_slice( + &inner + .get(&crate::npm_layout::hosted_packument_full_key( + "npm-private", + "pkg", + ¤t.generation, + )) + .await + .unwrap(), + ) + .unwrap(); + assert!(full["versions"].get("1.0.0").is_none()); + assert!(full["versions"].get("2.0.0").is_some()); + } + + #[tokio::test] + async fn test_retention_npm_list_omission_cannot_hide_active_import() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (old_manifest, old_blob) = + seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"old").await; + let (live_manifest, live_blob) = + seed_npm_version(&inner, prefix, "pkg", "2.0.0", b"live").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let current_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let pointer_before = inner.get(¤t_key).await.unwrap(); + let import = seed_npm_active_import(&inner, "npm-private", "pkg").await; + let storage = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).omit_from_list(&import), + )); + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }]; + + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 0); + assert_eq!(result.deleted_keys, 0); + assert_eq!(inner.get(¤t_key).await.unwrap(), pointer_before); + for key in [ + &old_manifest, + &old_blob, + &live_manifest, + &live_blob, + &import, + ] { + assert!( + inner.get(key).await.is_ok(), + "active package object lost: {key}" + ); + } + } + #[tokio::test] async fn test_retention_handles_hosted_package_named_proxy() { let dir = tempfile::tempdir().unwrap(); @@ -3063,6 +3591,7 @@ mod tests { ) .await; } + seed_npm_current(&storage, "npm-private", "proxy").await; let rules = vec![RetentionRule { registry: "npm".to_string(), @@ -3089,7 +3618,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); let prefix = "npm/repositories/npm-private/pkg"; - seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"tarball").await; + let (manifest, _) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"tarball").await; storage .put(&format!("{prefix}/pkg.json"), br#"{"name":"pkg"}"#) .await @@ -3102,10 +3631,14 @@ mod tests { .put(&format!("{prefix}/deprecations/1.0.0"), b"old") .await .unwrap(); + let completion = format!("{prefix}/publish-complete/1.0.0"); + let manifest_digest = + crate::npm_layout::hosted_manifest_digest(&storage.get(&manifest).await.unwrap()); storage - .put(&format!("{prefix}/publish-complete/1.0.0"), b"complete") + .put(&completion, manifest_digest.as_bytes()) .await .unwrap(); + seed_npm_current(&storage, "npm-private", "pkg").await; let rules = vec![RetentionRule { registry: "npm".to_string(), @@ -3117,13 +3650,26 @@ mod tests { let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; assert_eq!(result.planned, 1); - assert!( + for key in [ + format!("{prefix}/versions/1.0.0.json"), + format!("{prefix}/pkg.json"), + format!("{prefix}/dist-tags/latest"), + format!("{prefix}/deprecations/1.0.0"), + completion, + crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"), + ] { + assert!(storage.get(&key).await.is_err(), "authority remains: {key}"); + } + assert_eq!( storage - .list(&format!("{prefix}/")) + .get(&crate::npm_layout::hosted_packument_retired_key( + "npm-private", + "pkg" + )) .await .unwrap() - .is_empty(), - "last-version retention must not leave a 200-but-empty package shadow" + .as_ref(), + crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 ); } @@ -3134,6 +3680,7 @@ mod tests { let prefix = "npm/repositories/npm-private/pkg"; let (manifest, tarball) = seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"tarball").await; + seed_npm_current(&inner, "npm-private", "pkg").await; let backend = crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(&manifest); @@ -3167,6 +3714,7 @@ mod tests { let prefix = "npm/repositories/npm-private/pkg"; seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"old-tarball").await; seed_npm_version(&storage, prefix, "pkg", "2.0.0", b"newer-version").await; + seed_npm_current(&storage, "npm-private", "pkg").await; let group = collect_npm_versions(&storage) .await .into_iter() @@ -3187,6 +3735,7 @@ mod tests { // before retention acquires the package publish lock. let (replacement_manifest, replacement_blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"replacement-tarball").await; + seed_npm_current(&storage, "npm-private", "pkg").await; let replacement_manifest_bytes = storage.get(&replacement_manifest).await.unwrap(); let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; @@ -3207,6 +3756,7 @@ mod tests { let prefix = "npm/repositories/npm-private/pkg"; let (manifest, blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"old").await; seed_npm_version(&storage, prefix, "pkg", "2.0.0", b"new").await; + seed_npm_current(&storage, "npm-private", "pkg").await; let group = collect_npm_versions(&storage) .await .into_iter() @@ -3222,6 +3772,7 @@ mod tests { let plans = plan_deletions(group.versions.clone(), &rule, NOW); let tag = format!("{prefix}/dist-tags/stable"); storage.put(&tag, b"1.0.0").await.unwrap(); + seed_npm_current(&storage, "npm-private", "pkg").await; let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; @@ -3240,6 +3791,7 @@ mod tests { let (old_manifest, old_blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"old").await; seed_npm_version(&storage, prefix, "pkg", "2.0.0", b"new").await; + seed_npm_current(&storage, "npm-private", "pkg").await; let group = collect_npm_versions(&storage) .await .into_iter() @@ -3254,6 +3806,7 @@ mod tests { }; let plans = plan_deletions(group.versions.clone(), &rule, NOW); seed_npm_version(&storage, prefix, "pkg", "3.0.0", b"newest").await; + seed_npm_current(&storage, "npm-private", "pkg").await; let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; @@ -3272,8 +3825,14 @@ mod tests { seed_npm_version(&inner, prefix, "pkg", "2.0.0", b"new").await; let completion = format!("{prefix}/publish-complete/1.0.0"); let tag = format!("{prefix}/dist-tags/stable"); - inner.put(&completion, b"completed").await.unwrap(); + let manifest_digest = + crate::npm_layout::hosted_manifest_digest(&inner.get(&manifest).await.unwrap()); + inner + .put(&completion, manifest_digest.as_bytes()) + .await + .unwrap(); inner.put(&tag, b"1.0.0").await.unwrap(); + seed_npm_current(&inner, "npm-private", "pkg").await; let backend = crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(&tag); let attempts = backend.delete_attempts(); let storage = Storage::from_backend(Arc::new(backend)); @@ -3294,38 +3853,186 @@ mod tests { let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; assert_eq!(outcome.applied_versions, 0); - assert_eq!(outcome.deleted_keys, 1); - assert!( - inner.get(&completion).await.is_err(), - "publish-complete must be removed first so an exact retry repairs partial state" - ); + assert_eq!(outcome.deleted_keys, 0); + assert!(inner.get(&completion).await.is_ok()); assert_eq!(inner.get(&tag).await.unwrap().as_ref(), b"1.0.0"); assert!(inner.get(&manifest).await.is_ok()); assert!(inner.get(&blob).await.is_ok()); - let attempts = attempts.lock(); - assert_eq!( - attempts.first(), - Some(&crate::npm_layout::hosted_packument_cache_key( + assert!(inner + .get(&crate::npm_layout::hosted_maintenance_active_key( "npm-private", "pkg" - )), - "derived cache must be invalidated before authoritative state changes" - ); - assert_eq!(attempts.get(1), Some(&completion)); - assert_eq!(attempts.get(2), Some(&tag)); + )) + .await + .is_ok()); + let attempts = attempts.lock(); + assert!(attempts.contains(&tag)); assert!(!attempts.contains(&manifest)); assert!(!attempts.contains(&blob)); } #[tokio::test] - async fn empty_npm_package_cleanup_is_discoverable_and_retryable() { + async fn markerless_retired_npm_state_is_not_reinterpreted_as_a_retention_operation() { let dir = tempfile::tempdir().unwrap(); let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); let package_key = "npm/repositories/npm-private/pkg/pkg.json"; inner.put(package_key, br#"{"name":"pkg"}"#).await.unwrap(); + inner + .put( + &crate::npm_layout::hosted_packument_retired_key("npm-private", "pkg"), + crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1, + ) + .await + .unwrap(); + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(0), + older_than_days: None, + exclude_tags: vec![], + }]; + + let result = run_retention(&inner, &test_publish_locks(), None, &rules, false).await; + assert_eq!(result.planned, 0); + assert_eq!(result.deleted_keys, 0); + assert!(inner.get(package_key).await.is_ok()); + assert_eq!( + inner + .get(&crate::npm_layout::hosted_packument_retired_key( + "npm-private", + "pkg" + )) + .await + .unwrap() + .as_ref(), + crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 + ); + } + + #[tokio::test] + async fn npm_last_version_retention_marker_failure_leaves_current_and_authority_untouched() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, _) = seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let current_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let current = inner.get(¤t_key).await.unwrap(); + let retired_key = crate::npm_layout::hosted_packument_retired_key("npm-private", "pkg"); + let active_key = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + let storage = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_create(&active_key), + )); + let (group, plans) = npm_keep_zero_group_and_plans(&storage, "npm-private", "pkg").await; + + let outcome = apply_npm_plans(&storage, &test_publish_locks(), &group, &plans).await; + + assert_eq!(outcome.applied_versions, 0); + assert_eq!(inner.get(¤t_key).await.unwrap(), current); + assert!(inner.get(&manifest).await.is_ok()); + assert!(inner.get(&retired_key).await.is_err()); + assert!(inner.get(&active_key).await.is_err()); + } + + #[tokio::test] + async fn npm_last_version_retention_pointer_failure_is_safe_and_retryable() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, _) = seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let current_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let current = inner.get(¤t_key).await.unwrap(); + let retired_key = crate::npm_layout::hosted_packument_retired_key("npm-private", "pkg"); + let active_key = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + let failing = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(¤t_key), + )); + let (group, plans) = npm_keep_zero_group_and_plans(&failing, "npm-private", "pkg").await; + let first = apply_npm_plans(&failing, &test_publish_locks(), &group, &plans).await; + assert_eq!(first.applied_versions, 0); + assert_eq!(inner.get(¤t_key).await.unwrap(), current); + assert!(inner.get(&manifest).await.is_ok()); + assert!(inner.get(&retired_key).await.is_err()); + assert!(inner.get(&active_key).await.is_ok()); + + let retry = run_retention(&inner, &test_publish_locks(), None, &[], false).await; + assert_eq!(retry.planned, 0); + assert!(inner.get(¤t_key).await.is_err()); + assert!(inner.get(&manifest).await.is_err()); + assert!(inner.get(&active_key).await.is_err()); + assert_eq!( + inner.get(&retired_key).await.unwrap().as_ref(), + crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 + ); + } + + #[tokio::test] + async fn npm_last_version_delete_failure_stays_resumable_until_retry() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, _) = seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let current_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let retired_key = crate::npm_layout::hosted_packument_retired_key("npm-private", "pkg"); + let active_key = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); let failing = Storage::from_backend(Arc::new( - crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(package_key), + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(&manifest), )); + let (group, plans) = npm_keep_zero_group_and_plans(&failing, "npm-private", "pkg").await; + let first = apply_npm_plans(&failing, &test_publish_locks(), &group, &plans).await; + assert_eq!(first.applied_versions, 0); + assert!(inner.get(¤t_key).await.is_err()); + assert!(inner.get(&manifest).await.is_ok()); + assert!(inner.get(&retired_key).await.is_err()); + assert!(inner.get(&active_key).await.is_ok()); + + let retry = run_retention(&inner, &test_publish_locks(), None, &[], false).await; + assert_eq!(retry.planned, 0); + assert!(inner.get(&manifest).await.is_err()); + assert!(inner.get(&active_key).await.is_err()); + assert_eq!( + inner.get(&retired_key).await.unwrap().as_ref(), + crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 + ); + } + + #[tokio::test] + async fn npm_last_version_active_publish_pending_blocks_retention() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&storage, "npm-private", "pkg").await; + let pointer = + crate::registry::read_hosted_packument_pointer(&storage, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + let pending_index = + crate::npm_layout::hosted_publish_pending_index_key("npm-private", "pkg"); + let pending = crate::npm_layout::HostedPublishPending { + schema: crate::npm_layout::HOSTED_PUBLISH_PENDING_SCHEMA_V1, + repository: "npm-private".to_string(), + package: "pkg".to_string(), + version: "1.0.0".to_string(), + manifest_sha256: crate::npm_layout::hosted_manifest_digest( + &storage.get(&manifest).await.unwrap(), + ), + blob_sha512: hex::encode(sha2::Sha512::digest(b"one")), + target: crate::npm_layout::HostedPublishPendingTarget::Publish { + base: Some(pointer.clone()), + target: pointer, + }, + }; + storage + .put(&pending_index, &serde_json::to_vec(&pending).unwrap()) + .await + .unwrap(); + let current_key = crate::npm_layout::hosted_packument_current_key("npm-private", "pkg"); + let retired_key = crate::npm_layout::hosted_packument_retired_key("npm-private", "pkg"); + let active_key = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); let rules = vec![RetentionRule { registry: "npm".to_string(), name_glob: None, @@ -3334,14 +4041,247 @@ mod tests { exclude_tags: vec![], }]; - let first = run_retention(&failing, &test_publish_locks(), None, &rules, false).await; - assert_eq!(first.planned, 0); - assert!(inner.get(package_key).await.is_ok()); + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 0); + assert_eq!(result.deleted_keys, 0); + for key in [¤t_key, &manifest, &blob, &pending_index] { + assert!( + storage.get(key).await.is_ok(), + "active package object lost: {key}" + ); + } + assert!(storage.get(&retired_key).await.is_err()); + assert!(storage.get(&active_key).await.is_err()); + } + + #[tokio::test] + async fn npm_retention_recovery_rejects_replaced_expected_authority() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, _) = seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let failing = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(&manifest), + )); + let (group, plans) = npm_keep_zero_group_and_plans(&failing, "npm-private", "pkg").await; + let first = apply_npm_plans(&failing, &test_publish_locks(), &group, &plans).await; + assert_eq!(first.applied_versions, 0); + + let replacement = br#"{"name":"pkg","version":"9.9.9"}"#; + inner.put(&manifest, replacement).await.unwrap(); + let active = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + let retry = run_retention(&inner, &test_publish_locks(), None, &[], false).await; + + assert_eq!(retry.planned, 0); + assert_eq!(inner.get(&manifest).await.unwrap().as_ref(), replacement); + assert!(inner.get(&active).await.is_ok()); + assert!(inner + .get(&crate::npm_layout::hosted_packument_retired_key( + "npm-private", + "pkg" + )) + .await + .is_err()); + } + + #[tokio::test] + async fn npm_retention_without_committed_pointer_fails_closed() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"one").await; + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(0), + older_than_days: None, + exclude_tags: vec![], + }]; + + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 0); + assert_eq!(result.deleted_keys, 0); + assert!(storage.get(&manifest).await.is_ok()); + assert!(storage.get(&blob).await.is_ok()); + assert!(storage + .get(&crate::npm_layout::hosted_maintenance_active_key( + "npm-private", + "pkg" + )) + .await + .is_err()); + } + + #[tokio::test] + async fn npm_retention_requires_split_authority_to_match_base_pointer() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, blob) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&storage, "npm-private", "pkg").await; + let mut replacement: serde_json::Value = + serde_json::from_slice(&storage.get(&manifest).await.unwrap()).unwrap(); + replacement["description"] = serde_json::json!("changed outside the pointer"); + let replacement = serde_json::to_vec(&replacement).unwrap(); + storage.put(&manifest, &replacement).await.unwrap(); + let rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(0), + older_than_days: None, + exclude_tags: vec![], + }]; + + let result = run_retention(&storage, &test_publish_locks(), None, &rules, false).await; + + assert_eq!(result.planned, 0); + assert_eq!(result.deleted_keys, 0); + assert_eq!(storage.get(&manifest).await.unwrap().as_ref(), replacement); + assert!(storage.get(&blob).await.is_ok()); + assert!(storage + .get(&crate::npm_layout::hosted_maintenance_active_key( + "npm-private", + "pkg" + )) + .await + .is_err()); + } + + #[tokio::test] + async fn npm_retention_dry_run_never_resumes_active_marker() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, _) = seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let failing = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(&manifest), + )); + let (group, plans) = npm_keep_zero_group_and_plans(&failing, "npm-private", "pkg").await; + apply_npm_plans(&failing, &test_publish_locks(), &group, &plans).await; + let active = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + let marker_before = inner.get(&active).await.unwrap(); + let manifest_before = inner.get(&manifest).await.unwrap(); + + let dry_run = run_retention(&inner, &test_publish_locks(), None, &[], true).await; + + assert_eq!(dry_run.planned, 0); + assert_eq!(inner.get(&active).await.unwrap(), marker_before); + assert_eq!(inner.get(&manifest).await.unwrap(), manifest_before); + assert!(inner + .get(&crate::npm_layout::hosted_packument_retired_key( + "npm-private", + "pkg" + )) + .await + .is_err()); + } + + #[tokio::test] + async fn npm_non_last_recovery_finishes_after_rules_change() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (old_manifest, old_blob) = + seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"old").await; + let (new_manifest, _) = seed_npm_version(&inner, prefix, "pkg", "2.0.0", b"new").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let failing = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()).fail_delete(&old_manifest), + )); + let group = collect_npm_versions(&failing) + .await + .into_iter() + .find(|group| group.group_name == "npm:npm-private:pkg") + .unwrap(); + let plans = plan_deletions( + group.versions.clone(), + &RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(1), + older_than_days: None, + exclude_tags: vec![], + }, + NOW, + ); + let first = apply_npm_plans(&failing, &test_publish_locks(), &group, &plans).await; + assert_eq!(first.applied_versions, 0); + let active = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + assert!(inner.get(&active).await.is_ok()); + let pointer = crate::registry::read_hosted_packument_pointer(&inner, "npm-private", "pkg") + .await + .unwrap() + .unwrap(); + let full: serde_json::Value = serde_json::from_slice( + &inner + .get(&crate::npm_layout::hosted_packument_full_key( + "npm-private", + "pkg", + &pointer.generation, + )) + .await + .unwrap(), + ) + .unwrap(); + assert!(full["versions"].get("1.0.0").is_none()); + + let changed_rules = vec![RetentionRule { + registry: "npm".to_string(), + name_glob: None, + keep_last: Some(99), + older_than_days: None, + exclude_tags: vec![], + }]; + let retry = run_retention(&inner, &test_publish_locks(), None, &changed_rules, false).await; + assert_eq!(retry.planned, 0); + assert!(inner.get(&active).await.is_err()); + assert!(inner.get(&old_manifest).await.is_err()); + assert!(inner.get(&new_manifest).await.is_ok()); + assert!( + inner.get(&old_blob).await.is_ok(), + "blob cleanup belongs to GC" + ); + } + + #[tokio::test] + async fn npm_exact_delete_post_commit_error_is_accepted_by_readback() { + let dir = tempfile::tempdir().unwrap(); + let inner = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, _) = seed_npm_version(&inner, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&inner, "npm-private", "pkg").await; + let ambiguous = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(inner.clone()) + .fail_delete_after(&manifest), + )); + let (group, plans) = npm_keep_zero_group_and_plans(&ambiguous, "npm-private", "pkg").await; - let second = run_retention(&inner, &test_publish_locks(), None, &rules, false).await; - assert_eq!(second.planned, 0); - assert!(inner.get(package_key).await.is_err()); - assert_eq!(second.deleted_keys, 1); + let outcome = apply_npm_plans(&ambiguous, &test_publish_locks(), &group, &plans).await; + + assert_eq!(outcome.applied_versions, 1); + assert!(inner.get(&manifest).await.is_err()); + assert!(inner + .get(&crate::npm_layout::hosted_maintenance_active_key( + "npm-private", + "pkg" + )) + .await + .is_err()); + assert_eq!( + inner + .get(&crate::npm_layout::hosted_packument_retired_key( + "npm-private", + "pkg" + )) + .await + .unwrap() + .as_ref(), + crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 + ); } #[tokio::test] @@ -3445,6 +4385,60 @@ mod tests { handle.await.unwrap(); } + #[tokio::test] + async fn test_scheduler_boot_recovers_npm_marker_with_empty_rules() { + let dir = tempfile::tempdir().unwrap(); + let storage = Storage::new_local(dir.path().join("data").to_str().unwrap()); + let prefix = "npm/repositories/npm-private/pkg"; + let (manifest, _) = seed_npm_version(&storage, prefix, "pkg", "1.0.0", b"one").await; + seed_npm_current(&storage, "npm-private", "pkg").await; + let failing = Storage::from_backend(Arc::new( + crate::test_helpers::FaultInjectBackend::new(storage.clone()).fail_delete(&manifest), + )); + let (group, plans) = npm_keep_zero_group_and_plans(&failing, "npm-private", "pkg").await; + apply_npm_plans(&failing, &test_publish_locks(), &group, &plans).await; + let active = crate::npm_layout::hosted_maintenance_active_key("npm-private", "pkg"); + assert!(storage.get(&active).await.is_ok()); + + let cancel = tokio_util::sync::CancellationToken::new(); + let handle = spawn_retention_scheduler( + storage.clone(), + test_publish_locks(), + None, + MavenConfig::default(), + Arc::new(crate::repo_index::RepoIndex::new()), + Vec::new(), + 86400, + false, + None, + Arc::new(tokio::sync::Mutex::new(())), + cancel.clone(), + ); + + let deadline = Instant::now() + std::time::Duration::from_secs(10); + while storage.get(&active).await.is_ok() { + assert!( + Instant::now() < deadline, + "boot recovery never cleared marker" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!(storage.get(&manifest).await.is_err()); + assert_eq!( + storage + .get(&crate::npm_layout::hosted_packument_retired_key( + "npm-private", + "pkg" + )) + .await + .unwrap() + .as_ref(), + crate::npm_layout::HOSTED_PACKUMENT_RETIRED_V1 + ); + cancel.cancel(); + handle.await.unwrap(); + } + /// The boot pass waits on the shared cleanup lock instead of the /// periodic skip-if-held — losing the boot race to GC must delay the /// first run, not forfeit it for a whole interval. diff --git a/nora-registry/src/test_helpers.rs b/nora-registry/src/test_helpers.rs index 67868fcd..e21053a6 100644 --- a/nora-registry/src/test_helpers.rs +++ b/nora-registry/src/test_helpers.rs @@ -45,10 +45,18 @@ use parking_lot::RwLock; pub struct FaultInjectBackend { inner: Storage, get_failures: HashSet, + put_failures: HashSet, + put_after_failures: HashSet, + create_failures: HashSet, + create_after_failures: HashSet, delete_failures: HashSet, + delete_after_failures: HashSet, stat_none: HashSet, + list_omissions: HashSet, delete_attempts: Arc>>, + get_attempts: Arc>>, list_attempts: Arc>>, + write_attempts: Arc>>, } impl FaultInjectBackend { @@ -56,10 +64,18 @@ impl FaultInjectBackend { Self { inner, get_failures: HashSet::new(), + put_failures: HashSet::new(), + put_after_failures: HashSet::new(), + create_failures: HashSet::new(), + create_after_failures: HashSet::new(), delete_failures: HashSet::new(), + delete_after_failures: HashSet::new(), stat_none: HashSet::new(), + list_omissions: HashSet::new(), delete_attempts: Arc::new(parking_lot::Mutex::new(Vec::new())), + get_attempts: Arc::new(parking_lot::Mutex::new(Vec::new())), list_attempts: Arc::new(parking_lot::Mutex::new(Vec::new())), + write_attempts: Arc::new(parking_lot::Mutex::new(Vec::new())), } } @@ -68,37 +84,98 @@ impl FaultInjectBackend { self } + pub fn fail_put(mut self, key: impl Into) -> Self { + self.put_failures.insert(key.into()); + self + } + + pub fn fail_put_after(mut self, key: impl Into) -> Self { + self.put_after_failures.insert(key.into()); + self + } + + pub fn fail_create(mut self, key: impl Into) -> Self { + self.create_failures.insert(key.into()); + self + } + + pub fn fail_create_after(mut self, key: impl Into) -> Self { + self.create_after_failures.insert(key.into()); + self + } + pub fn fail_delete(mut self, key: impl Into) -> Self { self.delete_failures.insert(key.into()); self } + pub fn fail_delete_after(mut self, key: impl Into) -> Self { + self.delete_after_failures.insert(key.into()); + self + } + #[allow(dead_code)] // consumed by binary-only cleanup tests, not lib test target pub fn stat_none(mut self, key: impl Into) -> Self { self.stat_none.insert(key.into()); self } + /// Simulate an eventually-consistent object-store LIST that omits an + /// existing exact key while GET continues to return it. + pub fn omit_from_list(mut self, key: impl Into) -> Self { + self.list_omissions.insert(key.into()); + self + } + pub fn delete_attempts(&self) -> Arc>> { Arc::clone(&self.delete_attempts) } + pub fn get_attempts(&self) -> Arc>> { + Arc::clone(&self.get_attempts) + } + pub fn list_attempts(&self) -> Arc>> { Arc::clone(&self.list_attempts) } + + pub fn write_attempts(&self) -> Arc>> { + Arc::clone(&self.write_attempts) + } } #[async_trait] impl StorageBackend for FaultInjectBackend { async fn put(&self, key: &str, data: &[u8]) -> crate::storage::Result<()> { - self.inner.put(key, data).await + self.write_attempts.lock().push(format!("put:{key}")); + if self.put_failures.contains(key) { + return Err(StorageError::Network("injected put failure".to_string())); + } + let result = self.inner.put(key, data).await; + if result.is_ok() && self.put_after_failures.contains(key) { + return Err(StorageError::Network( + "injected post-commit put failure".to_string(), + )); + } + result } async fn put_if_absent(&self, key: &str, data: &[u8]) -> crate::storage::Result<()> { - self.inner.put_if_absent(key, data).await + self.write_attempts.lock().push(format!("create:{key}")); + if self.create_failures.contains(key) { + return Err(StorageError::Network("injected create failure".to_string())); + } + let result = self.inner.put_if_absent(key, data).await; + if result.is_ok() && self.create_after_failures.contains(key) { + return Err(StorageError::Network( + "injected post-commit create failure".to_string(), + )); + } + result } async fn get(&self, key: &str) -> crate::storage::Result { + self.get_attempts.lock().push(key.to_string()); if self.get_failures.contains(key) { return Err(StorageError::Network("injected get failure".to_string())); } @@ -110,12 +187,20 @@ impl StorageBackend for FaultInjectBackend { if self.delete_failures.contains(key) { return Err(StorageError::Network("injected delete failure".to_string())); } - self.inner.delete(key).await + let result = self.inner.delete(key).await; + if result.is_ok() && self.delete_after_failures.contains(key) { + return Err(StorageError::Network( + "injected post-commit delete failure".to_string(), + )); + } + result } async fn list(&self, prefix: &str) -> crate::storage::Result> { self.list_attempts.lock().push(prefix.to_string()); - self.inner.list(prefix).await + let mut entries = self.inner.list(prefix).await?; + entries.retain(|key| !self.list_omissions.contains(key)); + Ok(entries) } async fn stat(&self, key: &str) -> Option { @@ -130,7 +215,8 @@ impl StorageBackend for FaultInjectBackend { prefix: &str, ) -> crate::storage::Result> { let mut entries = self.inner.list_with_meta(prefix).await?; - entries.retain(|(key, _)| !self.stat_none.contains(key)); + entries + .retain(|(key, _)| !self.stat_none.contains(key) && !self.list_omissions.contains(key)); Ok(entries) } @@ -586,6 +672,54 @@ pub async fn send_with_headers( app.clone().oneshot(request).await.unwrap() } +/// Build a minimal, protocol-valid npm publish body for integration tests. +/// +/// Keeping this in the shared test harness lets cross-router tests seed hosted +/// state through the public publish contract instead of reconstructing Nora's +/// private storage layout. +pub fn npm_publish_payload(package: &str, version: &str, tag: &str) -> Vec { + use base64::Engine as _; + use flate2::write::GzEncoder; + use flate2::Compression; + + let encoder = GzEncoder::new(Vec::new(), Compression::fast()); + let mut archive = tar::Builder::new(encoder); + let package_json = serde_json::to_vec(&serde_json::json!({ + "name": package, + "version": version, + })) + .unwrap(); + let mut header = tar::Header::new_gnu(); + header.set_size(package_json.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, "package/package.json", package_json.as_slice()) + .unwrap(); + let tarball = archive.into_inner().unwrap().finish().unwrap(); + let package_basename = package.split('/').next_back().unwrap_or(package); + let filename = format!("{package_basename}-{version}.tgz"); + + serde_json::to_vec(&serde_json::json!({ + "name": package, + "versions": { + (version): { + "name": package, + "version": version, + "dist": {}, + }, + }, + "_attachments": { + (filename): { + "data": base64::engine::general_purpose::STANDARD.encode(&tarball), + "length": tarball.len(), + }, + }, + "dist-tags": {(tag): version}, + })) + .unwrap() +} + /// Read the full response body into bytes. pub async fn body_bytes(response: axum::http::Response) -> axum::body::Bytes { response