[pull] master from DataDog:master

.builders/pyproject.toml

@@ -0,0 +1,36 @@
+[tool.mypy]
+python_version = "3.13"
+follow_imports = "normal"
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+check_untyped_defs = true
+warn_return_any = false
+warn_unused_ignores = false
+warn_redundant_casts = true
+warn_unused_configs = true
+strict_optional = false
+show_column_numbers = true
+show_error_codes = true
+pretty = true
+namespace_packages = true
+explicit_package_bases = true
+allow_redefinition = true
+allow_untyped_globals = true
+
+[[tool.mypy.overrides]]
+module = [
+    "google.cloud.*",
+    "auditwheel.*",
+    "delocate.*",
+    "pathspec",
+    "dotenv",
+    "utils",
+]
+ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "tests.*"
+disallow_untyped_defs = false
+disallow_incomplete_defs = false
+allow_untyped_calls = true
+allow_redefinition = true

.builders/scripts/build_wheels.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import argparse
-import email
+import email.message
 import json
 import os
 import re
@@ -57,6 +57,10 @@ class WheelSizes(TypedDict):
     uncompressed: int
 
 
+class VersionedWheelSizes(WheelSizes):
+    version: str
+
+
 if sys.platform == 'win32':
     PY3_PATH = Path('C:\\py3\\Scripts\\python.exe')
     PY2_PATH = Path('C:\\py2\\Scripts\\python.exe')
@@ -66,7 +70,7 @@ class WheelSizes(TypedDict):
     def join_command_args(args: list[str]) -> str:
         return subprocess.list2cmdline(args)
 
-    def path_to_uri(path: str) -> str:
+    def path_to_uri(path: str | Path) -> str:
         return f'file:///{os.path.abspath(path).replace(" ", "%20").replace(os.sep, "/")}'
 
 else:
@@ -80,7 +84,7 @@ def path_to_uri(path: str) -> str:
     def join_command_args(args: list[str]) -> str:
         return shlex.join(args)
 
-    def path_to_uri(path: str) -> str:
+    def path_to_uri(path: str | Path) -> str:
         return f'file://{os.path.abspath(path).replace(" ", "%20")}'
 
 
@@ -98,7 +102,7 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess:
     return process
 
 
-def extract_metadata(wheel: Path) -> email.Message:
+def extract_metadata(wheel: Path) -> email.message.Message:
     with ZipFile(str(wheel)) as zip_archive:
         for path in zip_archive.namelist():
             root = path.split('/', 1)[0]
@@ -245,7 +249,7 @@ def is_excluded_from_wheel(path: str | Path) -> bool:
     return False
 
 
-def add_dependency(dependencies: dict[str, str], sizes: dict[str, WheelSizes], wheel: Path) -> None:
+def add_dependency(dependencies: dict[str, str], sizes: dict[str, VersionedWheelSizes], wheel: Path) -> None:
     project_metadata = extract_metadata(wheel)
     project_name = normalize_project_name(project_metadata['Name'])
     project_version = project_metadata['Version']
@@ -359,8 +363,8 @@ def main():
             ]
         )
 
-    dependencies: dict[str, tuple[str, str]] = {}
-    sizes: dict[str, WheelSizes] = {}
+    dependencies: dict[str, str] = {}
+    sizes: dict[str, VersionedWheelSizes] = {}
 
     # Handle wheels currently in the external directory and move them to the built directory if they were modified
     for wheel in iter_wheels(external_wheels_dir):

.builders/scripts/utils.py

@@ -2,7 +2,7 @@
 from typing import Iterator
 
 
-def iter_wheels(source_dir: str) -> Iterator[Path]:
+def iter_wheels(source_dir: str | Path) -> Iterator[Path]:
     for entry in sorted(Path(source_dir).iterdir(), key=lambda entry: entry.name.casefold()):
         if entry.suffix == '.whl' and entry.is_file():
             yield entry

.builders/test_dependencies.txt

@@ -1 +1,2 @@
 pytest==8.0.2
+mypy==1.13.0

.builders/tests/test_upload.py

@@ -1,3 +1,4 @@
+import email.message
 import fnmatch
 from pathlib import Path
 from unittest import mock
@@ -453,3 +454,114 @@ def test_generate_lockfiles_accepts_string_path(tmp_path):
          mock.patch.object(upload, "LOCK_FILE_DIR", fake_resolved_dir):
         # Should not raise TypeError: unsupported operand type(s) for /: 'str' and 'str'
         upload.generate_lockfiles(str(tmp_path / "targets"), lockfile)
+
+
+def test_collect_and_validate_wheels(tmp_path):
+    """Test that collect_and_validate_wheels correctly collects and validates wheel metadata."""
+    wheel_dir = tmp_path / "wheels"
+    wheel_dir.mkdir()
+
+    write_dummy_wheel(wheel_dir / "package1-1.0.0-py3-none-any.whl", "package1", "1.0.0", ">=3.6")
+    write_dummy_wheel(wheel_dir / "package2-2.0.0-py3-none-any.whl", "package2", "2.0.0", ">=3.7")
+
+    upload_data = upload.collect_and_validate_wheels(wheel_dir)
+
+    assert len(upload_data) == 2
+    assert upload_data[0][0] == "package1"
+    assert upload_data[0][1]["Name"] == "package1"
+    assert upload_data[0][1]["Version"] == "1.0.0"
+    assert upload_data[1][0] == "package2"
+    assert upload_data[1][1]["Name"] == "package2"
+    assert upload_data[1][1]["Version"] == "2.0.0"
+
+
+def test_collect_and_validate_wheels_invalid_name(tmp_path):
+    """Test that collect_and_validate_wheels raises error for invalid project names."""
+    wheel_dir = tmp_path / "wheels"
+    wheel_dir.mkdir()
+
+    write_dummy_wheel(wheel_dir / "-invalid-1.0.0-py3-none-any.whl", "-invalid", "1.0.0", ">=3.6")
+
+    with pytest.raises(RuntimeError) as exc_info:
+        upload.collect_and_validate_wheels(wheel_dir)
+
+    assert "Invalid project name" in str(exc_info.value)
+
+
+def test_process_wheel_for_upload_external_new(setup_fake_hash):
+    """Test processing a new external wheel that needs to be uploaded."""
+    wheel_path = Path("test.whl")
+    metadata = email.message.Message()
+    metadata["Name"] = "test-pkg"
+    metadata["Version"] = "1.0.0"
+
+    mock_bucket = mock.Mock()
+    mock_blob = mock.Mock()
+    mock_blob.exists.return_value = False
+    mock_bucket.blob.return_value = mock_blob
+
+    setup_fake_hash({"test.whl": "abc123"})
+
+    lockfile_entry, artifact_name = upload.process_wheel_for_upload(
+        wheel_path, "external", "test-pkg", metadata, mock_bucket, "(1/1)"
+    )
+
+    assert artifact_name == "test.whl"
+    assert "test-pkg @ https://agent-int-packages.datadoghq.com/external/test-pkg/test.whl#sha256=abc123" == lockfile_entry
+
+
+def test_process_wheel_for_upload_external_existing(setup_fake_hash):
+    """Test processing an existing external wheel that doesn't need upload."""
+    wheel_path = Path("test.whl")
+    metadata = email.message.Message()
+    metadata["Name"] = "test-pkg"
+    metadata["Version"] = "1.0.0"
+
+    mock_bucket = mock.Mock()
+    mock_blob = mock.Mock()
+    mock_blob.exists.return_value = True
+    mock_blob.metadata = {"sha256": "existing123"}
+    mock_bucket.blob.return_value = mock_blob
+
+    setup_fake_hash({"test.whl": "abc123"})
+
+    lockfile_entry, artifact_name = upload.process_wheel_for_upload(
+        wheel_path, "external", "test-pkg", metadata, mock_bucket, "(1/1)"
+    )
+
+    assert artifact_name is None
+    assert "test-pkg @ https://agent-int-packages.datadoghq.com/external/test-pkg/test.whl#sha256=existing123" == lockfile_entry
+
+
+def test_generate_artifact_listings():
+    """Test that generate_artifact_listings creates proper HTML index pages."""
+    mock_bucket = mock.Mock()
+
+    mock_blob1 = mock.Mock()
+    mock_blob1.name = "external/package1/package1-1.0.0.whl"
+    mock_blob1.metadata = {"requires-python": ">=3.6", "sha256": "hash1"}
+
+    mock_blob2 = mock.Mock()
+    mock_blob2.name = "external/package2/package2-2.0.0.whl"
+    mock_blob2.metadata = {"requires-python": ">=3.7", "sha256": "hash2"}
+
+    mock_bucket.list_blobs.return_value = [mock_blob1, mock_blob2]
+
+    created_blobs = {}
+    def track_blob(name):
+        blob = mock.Mock()
+        blob.upload_from_string = mock.Mock(side_effect=lambda content, **kwargs: created_blobs.update({name: content}))
+        return blob
+
+    mock_bucket.blob.side_effect = track_blob
+
+    upload.generate_artifact_listings({"external"}, mock_bucket)
+
+    assert "external/" in created_blobs
+    assert "external/package1/" in created_blobs
+    assert "external/package2/" in created_blobs
+
+    root_html = created_blobs["external/"]
+    assert "<h1>Agent integrations dependencies</h1>" in root_html
+    assert 'href="package1/"' in root_html
+    assert 'href="package2/"' in root_html