[evals] expand package metadata long-tail PPL slices (#5061)

Cover the five surfaces called out in the issue DoD (registry JSON,
dependency-graph rows, advisory text, release metadata, lockfile-like
records) across PyPI / npm / crates.io / RubyGems / NuGet / Maven / Go
plus GHSA + OSV. deps.dev BigQuery ingest is deferred per reviewer
approval; entries stay metadata-only so no bulk HF mirroring or
cross-region transfers are triggered by this change.

Keeps these slices in the long-tail registry only; does not wire them
into default_raw_validation_sets.

Co-authored-by: David Hall <dlwh@users.noreply.github.com>

Author: github-actions[bot]

experiments/evals/long_tail_ppl.py

@@ -379,15 +379,23 @@ def _slice(
         raw_relative_path="formal/hwmcc/aiger_btor.jsonl.gz",
         notes="Use textual renderings only; preserve solver and model-checking syntax.",
     ),
-    # Package metadata
+    # Package metadata — surfaces from issue #5061 DoD:
+    # registry JSON, dependency-graph rows, version constraints, advisory text,
+    # release metadata, and lockfile-like records. All entries are deterministic
+    # file paths under raw_root; ingestion is deferred (no BigQuery, no bulk HF
+    # mirroring). deps.dev BigQuery ingest is tracked separately per reviewer
+    # approval to defer the BigQuery tooling.
     _slice(
         name="deps_dev",
         family=LongTailPplFamily.PACKAGE_METADATA,
         issue_number=PACKAGE_METADATA_ISSUE,
         source_url="https://docs.deps.dev/bigquery/v1/",
         surface_form="dependency_rows",
         raw_relative_path="packages/deps_dev/rows.jsonl.gz",
-        notes="Preserve package names, semver constraints, hashes, and dependency edges.",
+        notes=(
+            "Preserve package names, semver constraints, hashes, and dependency edges. "
+            "BigQuery ingest is deferred; sample into region-local GCS with documented queries."
+        ),
     ),
     _slice(
         name="ecosystem_ms_libraries_io",
@@ -398,14 +406,113 @@ def _slice(
         raw_relative_path="packages/ecosystems_ms/metadata.jsonl.gz",
         notes="Keep repository/package metadata, licenses, and release records literal.",
     ),
+    _slice(
+        name="libraries_io_dependencies",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://libraries.io/data",
+        surface_form="dependency_edges",
+        raw_relative_path="packages/libraries_io/dependencies.jsonl.gz",
+        notes="Preserve package names, semver constraints, scopes, and platform markers per edge.",
+    ),
+    _slice(
+        name="pypi_registry_json",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://warehouse.pypa.io/api-reference/json.html",
+        surface_form="registry_json",
+        raw_relative_path="packages/pypi/registry.jsonl.gz",
+        notes="Preserve PyPI JSON API envelopes, classifiers, requires_dist strings, and file digests.",
+    ),
     _slice(
         name="npm_registry_metadata",
         family=LongTailPplFamily.PACKAGE_METADATA,
         issue_number=PACKAGE_METADATA_ISSUE,
         source_url="https://docs.npmjs.com/policies/crawlers/",
         surface_form="registry_json",
         raw_relative_path="packages/npm/registry.jsonl.gz",
-        notes="Preserve CouchDB-style package JSON and nested version fields.",
+        notes="Preserve CouchDB-style package JSON, scoped names, and nested version fields.",
+    ),
+    _slice(
+        name="crates_io_registry_json",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://doc.rust-lang.org/cargo/reference/registry-web-api.html",
+        surface_form="registry_json",
+        raw_relative_path="packages/crates_io/registry.jsonl.gz",
+        notes="Preserve crate versions, yanked flags, feature maps, and checksum fields.",
+    ),
+    _slice(
+        name="rubygems_registry_json",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://guides.rubygems.org/rubygems-org-api/",
+        surface_form="registry_json",
+        raw_relative_path="packages/rubygems/registry.jsonl.gz",
+        notes="Keep gem dependencies, runtime/development scopes, and SHA-256 digests literal.",
+    ),
+    _slice(
+        name="nuget_registry_index",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://learn.microsoft.com/nuget/api/registration-base-url-resource",
+        surface_form="registry_json",
+        raw_relative_path="packages/nuget/registration_index.jsonl.gz",
+        notes="Preserve NuGet registration envelopes, framework target strings, and catalog URLs.",
+    ),
+    _slice(
+        name="maven_central_metadata",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://central.sonatype.org/search/rest-api-guide/",
+        surface_form="maven_pom_xml",
+        raw_relative_path="packages/maven/metadata.jsonl.gz",
+        notes="Preserve group/artifact/version coordinates, scopes, exclusions, and XML layout.",
+    ),
+    _slice(
+        name="go_modules_proxy",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://proxy.golang.org/",
+        surface_form="go_mod_sum",
+        raw_relative_path="packages/go_modules/proxy.jsonl.gz",
+        notes="Keep module paths, pseudo-versions, and h1: sum strings exactly as served.",
+    ),
+    _slice(
+        name="ghsa_advisories",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://github.com/github/advisory-database",
+        surface_form="advisory_osv_json",
+        raw_relative_path="packages/ghsa/advisories.jsonl.gz",
+        notes="Preserve GHSA IDs, CVE aliases, affected-ranges, and OSV JSON punctuation.",
+    ),
+    _slice(
+        name="osv_advisories",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://osv.dev/",
+        surface_form="advisory_osv_json",
+        raw_relative_path="packages/osv/advisories.jsonl.gz",
+        notes="Keep OSV schema fields, semver ranges, and ecosystem tags literal.",
+    ),
+    _slice(
+        name="npm_release_metadata",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://docs.npmjs.com/cli/v10/using-npm/registry",
+        surface_form="release_metadata",
+        raw_relative_path="packages/npm/release_metadata.jsonl.gz",
+        notes="Keep dist-tags, tarball URLs, integrity hashes, and publish timestamps per release.",
+    ),
+    _slice(
+        name="pypi_release_metadata",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://warehouse.pypa.io/api-reference/json.html",
+        surface_form="release_metadata",
+        raw_relative_path="packages/pypi/release_metadata.jsonl.gz",
+        notes="Keep file digests, upload_time, requires_python, and yanked reason strings.",
     ),
     _slice(
         name="package_lock_corpora",
@@ -416,6 +523,18 @@ def _slice(
         raw_relative_path="packages/package_lock/lockfiles.jsonl.gz",
         notes="Later pipeline work should keep lockfile structure, URLs, and checksums intact.",
     ),
+    _slice(
+        name="the_stack_v2_lockfiles",
+        family=LongTailPplFamily.PACKAGE_METADATA,
+        issue_number=PACKAGE_METADATA_ISSUE,
+        source_url="https://huggingface.co/datasets/bigcode/the-stack-v2",
+        surface_form="lockfile",
+        raw_relative_path="packages/the_stack_v2/lockfiles.jsonl.gz",
+        notes=(
+            "Filename-filtered stopgap (package-lock.json, yarn.lock, poetry.lock, Pipfile.lock, "
+            "Cargo.lock, Gemfile.lock, go.sum) until the corpus from #4961 lands."
+        ),
+    ),
     # Game / music
     _slice(
         name="lichess_pgn",

tests/evals/test_long_tail_ppl.py

@@ -5,7 +5,9 @@
 
 from experiments.evals.long_tail_ppl import (
     GAME_MUSIC_ISSUE,
+    PACKAGE_METADATA_ISSUE,
     LongTailPplFamily,
+    long_tail_ppl_slices,
     long_tail_raw_validation_sets,
     render_long_tail_ppl_registry_markdown,
 )
@@ -47,6 +49,30 @@ def test_hf_backed_raw_dataset_preserves_requested_split():
     assert component.source.splits == ["test"]
 
 
+def test_package_metadata_family_covers_dod_surfaces():
+    """Issue #5061 DoD requires slices for registry JSON, dependency graph rows,
+    advisory text, release metadata, and lockfile-like records. Verify each surface
+    is registered at least once under the PACKAGE_METADATA family."""
+
+    slices = long_tail_ppl_slices(family=LongTailPplFamily.PACKAGE_METADATA)
+    surfaces = {slice_.surface_form for slice_ in slices}
+
+    required_surfaces = {
+        "registry_json",
+        "dependency_rows",
+        "dependency_edges",
+        "advisory_osv_json",
+        "release_metadata",
+        "lockfile",
+    }
+    missing = required_surfaces - surfaces
+    assert not missing, f"PACKAGE_METADATA family is missing DoD surfaces: {missing}"
+
+    for slice_ in slices:
+        assert slice_.issue_number == PACKAGE_METADATA_ISSUE
+        assert slice_.raw_relative_path.startswith("packages/")
+
+
 def test_file_backed_raw_dataset_rejects_non_validation_split():
     with pytest.raises(ValueError, match="Hugging Face dataset sources"):
         raw_text_dataset("gs://example-bucket/eval.jsonl", split="test")