From e7de019a480de23dcc158906d14cc9d6711ee038 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 19:35:23 +0000 Subject: [PATCH] =?UTF-8?q?D-163:=20Maven=20Central=20release=20history=20?= =?UTF-8?q?=E2=80=94=20the=20temporal=20axis=20reaches=20maven=20nodes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit D-162 gave maven-coordinate nodes advisory coverage while registry metadata stayed dark: no release timeline, so VC-004 and the temporal axis never evaluated for the dependency class that motivated the adapter. This adds a RegistrySource for ecosystem "maven" via the existing Spec framework: Maven Central's solrsearch gav core, one page of 200 versions newest-first, epoch-milli timestamps decoded to UTC. - Clojars is deliberately absent: its API serves no per-version publish timestamps, and a zero-time Release would read as a 55-year dormancy awakening on every package — a fabricated timeline is worse than a disclosed absence. Clojars-hosted artifacts 404 on Central and count as NotFound: disclosed coverage. - Central is immutable, so Yanked stays false as a registry fact and the VC-012 yank-lure shape is structurally impossible from this source (pinned); no per-version publishers exist, so VC-011 declines to evaluate rather than claim continuity (pinned). Validation: two-sided parser tests (zero/negative timestamps and duplicate docs drop; malformed JSON errors); the request URL pinned fragment-by- fragment through the shared client with a fake doer (search.maven.org is egress-blocked in this environment); 404 pinned as NotFound-not-failure; a registration pin covering every emitting ecosystem's release-history source (fails on unwiring, mutation-checked); a consumer-side pin that a maven history fires VC-004 identically to npm. Live-fired: the built binary issued the exact expected solrsearch query per jepsen dependency and disclosed "degraded data source(s): maven-central-registry ... NOT an all-clear" under denied egress. Full suite green (35 packages), -race clean. D-163 entry appended to docs/DECISIONS.md. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_018d8nb38Prtn8dzwep6jaH9 --- cmd/depsnort/d163_registry_wiring_test.go | 22 +++ cmd/depsnort/main.go | 1 + docs/DECISIONS.md | 49 +++++ .../check/builtin/d163_maven_temporal_test.go | 41 +++++ internal/datasource/registry/mavenreg_test.go | 168 ++++++++++++++++++ internal/datasource/registry/specs.go | 88 +++++++++ 6 files changed, 369 insertions(+) create mode 100644 cmd/depsnort/d163_registry_wiring_test.go create mode 100644 internal/check/builtin/d163_maven_temporal_test.go create mode 100644 internal/datasource/registry/mavenreg_test.go diff --git a/cmd/depsnort/d163_registry_wiring_test.go b/cmd/depsnort/d163_registry_wiring_test.go new file mode 100644 index 0000000..b337bcf --- /dev/null +++ b/cmd/depsnort/d163_registry_wiring_test.go @@ -0,0 +1,22 @@ +package main + +import "testing" + +// D-163 registration pin: every ecosystem adapters emit nodes for must have +// its release-history source wired, or its temporal checks (VC-004 and +// friends) silently never evaluate — the maven shape: D-162 gave maven nodes +// advisory coverage while registry metadata stayed dark until this source +// landed. A source disappearing from this list is exactly the kind of quiet +// coverage regression a test must catch, since a scan without it still exits +// green. +func TestRegistrySourcesCoverEmittedEcosystems(t *testing.T) { + have := map[string]bool{} + for _, s := range registrySources(t.TempDir(), true) { + have[s.Ecosystem()] = true + } + for _, eco := range []string{"npm", "pypi", "gem", "cargo", "composer", "nuget", "gomod", "maven"} { + if !have[eco] { + t.Errorf("no registry release-history source wired for ecosystem %q", eco) + } + } +} diff --git a/cmd/depsnort/main.go b/cmd/depsnort/main.go index e7246e2..55f8867 100644 --- a/cmd/depsnort/main.go +++ b/cmd/depsnort/main.go @@ -305,6 +305,7 @@ func registrySources(cacheRoot string, offline bool) []datasource.RegistrySource registry.NewComposer(datasource.NewCache(filepath.Join(cacheRoot, "composer"), ttl), offline), registry.NewNuGet(datasource.NewCache(filepath.Join(cacheRoot, "nuget"), ttl), offline), goproxy.New(datasource.NewCache(filepath.Join(cacheRoot, "goproxy-temporal"), ttl), offline), + registry.NewMaven(datasource.NewCache(filepath.Join(cacheRoot, "maven"), ttl), offline), } } diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 24504e8..2721537 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -6799,3 +6799,52 @@ question, each its own decision), as are `pom.xml` (still a disclosed gap; prope parent chains are real work, not a name-table entry) and Homebrew formulae (unchanged from D-161's reasoning). And Maven version-range resolution is not implemented anywhere in this tool — a range is disclosed as unresolved, never evaluated. + +## D-163 — Maven Central release history: the temporal axis reaches maven nodes + +**Trigger:** D-162's own residual. The clojure adapter gave maven-coordinate nodes advisory coverage +(VC-008) while registry metadata stayed dark — no release timeline, so VC-004 and the rest of the temporal +axis silently never evaluated for the exact dependency class (`org.postgresql:postgresql`) that motivated +the adapter. This wires a `RegistrySource` for ecosystem `maven` through the existing Spec framework: one +Spec (`NewMaven`), the shared client's cache/concurrency/stats machinery unchanged. + +**The source is Maven Central's solrsearch gav core** — `q=g:"group" AND a:"artifact"`, one page of 200 +versions requested newest-first, `timestamp` in epoch millis decoded to UTC publish instants. Names arrive +in the `group:artifact` form maven nodes already carry; a bare name maps group == artifact, mirroring the +Leiningen convention the adapter established. + +**Clojars is deliberately absent, and the reason is a data honesty constraint, not effort.** Clojars' +artifact API serves no per-version publish timestamps, and a `Release` with a zero time would poison every +temporal computation downstream — a "dormancy gap" measured from the Unix epoch reads as a 55-year +awakening on every package. A fabricated timeline is strictly worse than a disclosed absence, so a +Clojars-hosted artifact (the jepsen fixture's `com.taoensso/carmine`) 404s on Central and counts as +NotFound in the source's stats: disclosed coverage, exactly like any other registry miss. Reaching Clojars +honestly would take per-version pom probing (one request per version) or an upstream API that serves +dates — either is its own decision, named here rather than approximated. + +**What the registry's own semantics give and withhold.** Central is immutable — a published artifact +cannot be withdrawn — so `Yanked` stays false as a fact of the registry rather than an unread field, and +VC-012's yank-lure shape is structurally impossible from this source (pinned in test). No per-version +publisher identity is exposed, so `Publishers` stays empty and VC-011's honesty predicate +(`PriorPublishers.Evaluable()`) declines to evaluate rather than claim continuity — also pinned. What +maven nodes genuinely gain is the timeline: VC-004 dormancy, median cadence, and the republish-burst +window, all ecosystem-neutral consumers of `ReleaseHistory`. + +**Validation:** parser tests two-sided (real-shaped response sorts oldest-first with correct UTC instants; +empty versions, zero/negative timestamps, and duplicate docs all drop — a fabricated epoch date corrupting +the dormancy math is the failure the skip exists to prevent; malformed JSON errors rather than returning +an empty history); the request URL pinned fragment-by-fragment through the shared client with a fake doer +(`search.maven.org` was egress-blocked in the landing environment, so the wire is the contract, as with +D-162's OSV mapping); the 404 path pinned as NotFound-not-failure; a registration pin +(`TestRegistrySourcesCoverEmittedEcosystems`) that fails if any emitting ecosystem loses its release-history +source — the quiet-regression shape this entry exists to close; and a consumer-side pin that a maven +history fires VC-004 identically to npm (`TestDormancyFiresOnMavenHistory`). Mutation-checked: unwiring +`NewMaven` fails the registration pin. Live-fired through the built binary on the jepsen fixture: the +source issued the exact expected solrsearch query per dependency and, with egress denied, the scan +disclosed `degraded data source(s): maven-central-registry … NOT an all-clear` — the D-24 machinery +working unmodified for the new source. Full suite green (35 packages), `-race` clean, gofmt/vet silent. + +Residual limitations: one page of 200 versions bounds the OLD end of a very long history (the recent +history every temporal check reads stays intact); pagination is the follow-up, not silently assumed away. +Clojars as above. And the first scan from a network that reaches both `api.osv.dev` and +`search.maven.org` remains the outstanding live confirmation for the whole D-162/D-163 chain. diff --git a/internal/check/builtin/d163_maven_temporal_test.go b/internal/check/builtin/d163_maven_temporal_test.go new file mode 100644 index 0000000..c2b287a --- /dev/null +++ b/internal/check/builtin/d163_maven_temporal_test.go @@ -0,0 +1,41 @@ +package builtin + +import ( + "testing" + "time" + + "ihbv.io/depsnort/internal/check" + "ihbv.io/depsnort/internal/datasource" + "ihbv.io/depsnort/internal/graph" +) + +// D-163 consumer-side pin: a maven-coordinate node with a Maven Central +// release history reaches the temporal checks like any other ecosystem — the +// check layer is ecosystem-neutral (D-24's coverage keys and D-03's +// extraction/judgment split both depend on that), and this guards it staying +// so. The jepsen shape: a JDBC driver pinned to a version published after a +// multi-year quiet stretch must produce the same VC-004 dormancy advisory it +// would as an npm package. +func TestDormancyFiresOnMavenHistory(t *testing.T) { + id := "pkg:maven/org.postgresql/postgresql@42.7.4" + g := graph.New() + g.AddNode(&graph.Node{ + ID: id, Kind: graph.KindPackage, Ecosystem: "maven", + Name: "org.postgresql:postgresql", Version: "42.7.4", + }) + h := &datasource.ReleaseHistory{Package: "org.postgresql:postgresql", Ecosystem: "maven", + Releases: []datasource.Release{ + {Version: "42.7.3", Published: time.Date(2021, 2, 1, 0, 0, 0, 0, time.UTC)}, + {Version: "42.7.4", Published: time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC)}, + }} + h.Sort() + + fs := (Dormancy{}).Run(&check.Context{Graph: g, Now: nowRef, + Releases: map[string]*datasource.ReleaseHistory{id: h}}) + if len(fs) != 1 { + t.Fatalf("VC-004 findings on a maven history = %d, want 1", len(fs)) + } + if fs[0].NodeID != id { + t.Errorf("finding attached to %q, want %q", fs[0].NodeID, id) + } +} diff --git a/internal/datasource/registry/mavenreg_test.go b/internal/datasource/registry/mavenreg_test.go new file mode 100644 index 0000000..79c9db4 --- /dev/null +++ b/internal/datasource/registry/mavenreg_test.go @@ -0,0 +1,168 @@ +package registry + +// D-163: the Maven Central release-history source serving the maven-coordinate +// nodes the clojure adapter produces (D-162). search.maven.org was +// egress-blocked in the environment this landed from, so the URL and response +// shapes are pinned here against the documented solrsearch gav API. + +import ( + "context" + "io" + "net/http" + "strings" + "testing" + "time" + + "ihbv.io/depsnort/internal/datasource" +) + +type mavenDoer struct { + body string + status int + calls int + lastURL string +} + +func (d *mavenDoer) Do(req *http.Request) (*http.Response, error) { + d.calls++ + d.lastURL = req.URL.String() + st := d.status + if st == 0 { + st = 200 + } + return &http.Response{ + StatusCode: st, + Body: io.NopCloser(strings.NewReader(d.body)), + Header: make(http.Header), + }, nil +} + +const mavenPostgresResp = `{"response":{"numFound":3,"docs":[ + {"id":"org.postgresql:postgresql:42.7.4","g":"org.postgresql","a":"postgresql","v":"42.7.4","timestamp":1724371200000}, + {"id":"org.postgresql:postgresql:42.7.3","g":"org.postgresql","a":"postgresql","v":"42.7.3","timestamp":1713139200000}, + {"id":"org.postgresql:postgresql:42.7.2","g":"org.postgresql","a":"postgresql","v":"42.7.2","timestamp":1708300800000} +]}}` + +func TestParseMavenVersions(t *testing.T) { + h, err := parseMavenVersions("org.postgresql:postgresql", []byte(mavenPostgresResp)) + if err != nil { + t.Fatal(err) + } + if h.Ecosystem != "maven" { + t.Errorf("ecosystem = %q, want maven", h.Ecosystem) + } + if len(h.Releases) != 3 { + t.Fatalf("releases = %d, want 3", len(h.Releases)) + } + // Sorted oldest -> newest regardless of the newest-first response order. + if h.Releases[0].Version != "42.7.2" || h.Releases[2].Version != "42.7.4" { + t.Errorf("sort order wrong: first %q last %q", h.Releases[0].Version, h.Releases[2].Version) + } + // Epoch millis decode to the real publish instant, in UTC. + want := time.UnixMilli(1724371200000).UTC() + if !h.Releases[2].Published.Equal(want) { + t.Errorf("published = %v, want %v", h.Releases[2].Published, want) + } + // Central is immutable: no release may parse as yanked, and the yank-lure + // shape must be structurally impossible from this source. + for _, r := range h.Releases { + if r.Yanked { + t.Errorf("version %s parsed as yanked; Central cannot yank", r.Version) + } + } + if _, _, ok := h.YankLureShape(); ok { + t.Error("yank-lure shape reported from an immutable registry") + } + // No per-version publisher identity exists here, so VC-011's honesty + // predicate must decline to evaluate rather than claim continuity. + if h.PriorPublishers("42.7.4").Evaluable() { + t.Error("publisher history must be non-evaluable: Central exposes none") + } +} + +func TestParseMavenSkipsUnusableDocs(t *testing.T) { + raw := `{"response":{"numFound":4,"docs":[ + {"v":"1.0.0","timestamp":1600000000000}, + {"v":"","timestamp":1600000000001}, + {"v":"0.9.0","timestamp":0}, + {"v":"1.0.0","timestamp":1600000000002} + ]}}` + h, err := parseMavenVersions("a:b", []byte(raw)) + if err != nil { + t.Fatal(err) + } + // The empty version, the zero timestamp (a fabricated epoch date would + // corrupt the dormancy math), and the duplicate must all drop. + if len(h.Releases) != 1 || h.Releases[0].Version != "1.0.0" { + t.Fatalf("releases = %+v, want exactly one 1.0.0", h.Releases) + } +} + +func TestParseMavenMalformed(t *testing.T) { + if _, err := parseMavenVersions("a:b", []byte(`{"response":`)); err == nil { + t.Error("malformed JSON must error, not return an empty history") + } +} + +func TestSplitMavenCoordinate(t *testing.T) { + cases := map[string][2]string{ + "org.postgresql:postgresql": {"org.postgresql", "postgresql"}, + "postgresql": {"postgresql", "postgresql"}, // lein bare-symbol convention + "a:": {"a:", "a:"}, // degenerate: no artifact, used whole + } + for in, want := range cases { + g, a := splitMavenCoordinate(in) + if g != want[0] || a != want[1] { + t.Errorf("splitMavenCoordinate(%q) = (%q,%q), want (%q,%q)", in, g, a, want[0], want[1]) + } + } +} + +func TestMavenHistoriesWireFormat(t *testing.T) { + doer := &mavenDoer{body: mavenPostgresResp} + c := NewMaven(datasource.NewCache(t.TempDir(), time.Hour), false) + c.HTTP = doer + + got, err := c.Histories(context.Background(), []string{"org.postgresql:postgresql"}) + if err != nil { + t.Fatalf("Histories: %v", err) + } + // The URL is the contract with search.maven.org: gav core, quoted g/a + // terms, bounded rows, JSON, newest-first. + for _, frag := range []string{ + "https://search.maven.org/solrsearch/select?q=", + "g%3A%22org.postgresql%22+AND+a%3A%22postgresql%22", + "core=gav", "rows=200", "wt=json", "sort=timestamp+desc", + } { + if !strings.Contains(doer.lastURL, frag) { + t.Errorf("request URL missing %q: %s", frag, doer.lastURL) + } + } + h := got["org.postgresql:postgresql"] + if h == nil || len(h.Releases) != 3 { + t.Fatalf("history did not round-trip: %+v", h) + } + if c.GetStats().FromNet != 1 { + t.Errorf("stats.FromNet = %d, want 1", c.GetStats().FromNet) + } +} + +func TestMavenClojarsHostedArtifactIsAGapNotATimeline(t *testing.T) { + // A Clojure-native artifact (Clojars-only) 404s on Central. That must be a + // counted gap — never an invented history, and never a hard failure that + // takes the whole batch down. + doer := &mavenDoer{status: 404} + c := NewMaven(datasource.NewCache(t.TempDir(), time.Hour), false) + c.HTTP = doer + + got, err := c.Histories(context.Background(), []string{"com.taoensso:carmine"}) + if err != nil { + t.Fatalf("a 404 must not fail the batch: %v", err) + } + if h := got["com.taoensso:carmine"]; h != nil { + t.Errorf("no history may be fabricated for a missing artifact, got %+v", h) + } + if c.GetStats().NotFound != 1 { + t.Errorf("stats.NotFound = %d, want 1 (disclosed coverage)", c.GetStats().NotFound) + } +} diff --git a/internal/datasource/registry/specs.go b/internal/datasource/registry/specs.go index 45bfd39..c7277e7 100644 --- a/internal/datasource/registry/specs.go +++ b/internal/datasource/registry/specs.go @@ -317,3 +317,91 @@ func parseNuGetVersions(name string, raw []byte) (*datasource.ReleaseHistory, er h.Sort() return h, nil } + +// ---------- Maven Central ---------- +// GET /solrsearch/select?q=g:""+AND+a:""&core=gav&rows=200&wt=json +// → {response: {numFound, docs: [{v, timestamp}]}} — timestamp is epoch millis. +// +// This serves the maven-coordinate nodes the clojure adapter produces (D-162); +// a future pom.xml or Gradle adapter shares it unchanged. Two deliberate +// boundaries, both disclosed in D-163 rather than smuggled: +// +// - Central only, no Clojars. Clojars' artifact API serves no per-version +// publish timestamps, and a release with a zero time would poison every +// temporal computation downstream (a "dormancy gap" measured from the +// epoch). A Clojars-hosted artifact 404s here and counts as NotFound in +// Stats — disclosed coverage, exactly like any other registry miss, +// never a fabricated timeline. +// - One page of 200 versions, requested newest-first. solrsearch caps rows; +// for the rare artifact with a longer history the OLD end truncates, +// which leaves the recent history every temporal check reads intact. +// Pagination is a follow-up, not silently assumed away (D-142 spirit). +// +// Central is immutable — a published artifact cannot be withdrawn — so Yanked +// stays false as a fact of the registry, not an unread field, and VC-012's +// yank-lure shape cannot occur here. No per-version publisher identity is +// exposed, so Publishers stays empty and VC-011 correctly declines to +// evaluate (PriorPublishers.Evaluable() == false). + +// NewMaven returns the Maven Central release-history source. Names are Maven +// coordinates in the "group:artifact" form maven nodes carry. +func NewMaven(cache *datasource.Cache, offline bool) *Client { + return New(Spec{ + SourceName: "maven-central-registry", + Eco: "maven", + CacheTag: "maven", + Endpoint: "https://search.maven.org", + BuildURL: func(endpoint, name string) string { + group, artifact := splitMavenCoordinate(name) + q := url.QueryEscape(`g:"` + group + `" AND a:"` + artifact + `"`) + return endpoint + "/solrsearch/select?q=" + q + + "&core=gav&rows=200&wt=json&sort=" + url.QueryEscape("timestamp desc") + }, + Parse: parseMavenVersions, + }, cache, offline) +} + +// splitMavenCoordinate splits "group:artifact"; a bare name (no colon) is used +// as both, mirroring the Leiningen bare-symbol convention the adapter maps. +func splitMavenCoordinate(name string) (group, artifact string) { + if i := strings.IndexByte(name, ':'); i > 0 && i < len(name)-1 { + return name[:i], name[i+1:] + } + return name, name +} + +type mavenSearchResponse struct { + Response struct { + NumFound int `json:"numFound"` + Docs []mavenDoc `json:"docs"` + } `json:"response"` +} + +type mavenDoc struct { + Version string `json:"v"` + Timestamp int64 `json:"timestamp"` // epoch millis +} + +func parseMavenVersions(name string, raw []byte) (*datasource.ReleaseHistory, error) { + var resp mavenSearchResponse + if err := json.Unmarshal(raw, &resp); err != nil { + return nil, fmt.Errorf("mavenreg: parsing versions for %s: %w", name, err) + } + h := &datasource.ReleaseHistory{Package: name, Ecosystem: "maven"} + seen := map[string]bool{} + for _, d := range resp.Response.Docs { + // A doc without a positive timestamp gets no release: a zero or + // negative time is not a publish date, and inventing one would corrupt + // the dormancy/cadence math this history exists to feed. + if d.Version == "" || d.Timestamp <= 0 || seen[d.Version] { + continue + } + seen[d.Version] = true + h.Releases = append(h.Releases, datasource.Release{ + Version: d.Version, + Published: time.UnixMilli(d.Timestamp).UTC(), + }) + } + h.Sort() + return h, nil +}