Make pip cache {purge, remove} delete additional unneeded files (#9058)

These commands now remove:
- wheel cache folders without `.whl` files.
- empty folders in the HTTP cache.
- `selfcheck.json`, which pip does not use anymore.

Co-authored-by: Richard Si <sichard26@gmail.com>
Co-authored-by: Damian Shaw <damian.peter.shaw@gmail.com>

Author: 3 people

news/9058.feature.rst

@@ -0,0 +1,2 @@
+``pip cache purge`` and ``pip cache remove`` now clean up empty directories
+and legacy files left by older pip versions.

src/pip/_internal/commands/cache.py

@@ -189,7 +189,31 @@ def remove_cache_items(self, options: Values, args: list[str]) -> None:
             bytes_removed += os.stat(filename).st_size
             os.unlink(filename)
             logger.verbose("Removed %s", filename)
+
+        http_dirs = filesystem.subdirs_without_files(self._cache_dir(options, "http"))
+        wheel_dirs = filesystem.subdirs_without_wheels(
+            self._cache_dir(options, "wheels")
+        )
+        dirs = [*http_dirs, *wheel_dirs]
+
+        for subdir in dirs:
+            try:
+                for file in subdir.iterdir():
+                    file.unlink(missing_ok=True)
+                subdir.rmdir()
+            except FileNotFoundError:
+                # If the directory is already gone, that's fine.
+                pass
+            logger.verbose("Removed %s", subdir)
+
+        # selfcheck.json is no longer used by pip.
+        selfcheck_json = self._cache_dir(options, "selfcheck.json")
+        if os.path.isfile(selfcheck_json):
+            os.remove(selfcheck_json)
+            logger.verbose("Removed legacy selfcheck.json file")
+
         logger.info("Files removed: %s (%s)", len(files), format_size(bytes_removed))
+        logger.info("Directories removed: %s", len(dirs))
 
     def purge_cache(self, options: Values, args: list[str]) -> None:
         if args:

src/pip/_internal/utils/filesystem.py

@@ -7,8 +7,9 @@
 import sys
 from collections.abc import Generator
 from contextlib import contextmanager
+from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Any, BinaryIO, cast
+from typing import Any, BinaryIO, Callable, cast
 
 from pip._internal.utils.compat import get_path_uid
 from pip._internal.utils.misc import format_size
@@ -162,3 +163,41 @@ def copy_directory_permissions(directory: str, target_file: BinaryIO) -> None:
         os.chmod(target_file.fileno(), mode)
     elif os.chmod in os.supports_follow_symlinks:
         os.chmod(target_file.name, mode, follow_symlinks=False)
+
+
+def _subdirs_without_generic(
+    path: str, predicate: Callable[[str, list[str]], bool]
+) -> Generator[Path]:
+    """Yields every subdirectory of +path+ that has no files matching the
+    predicate under it."""
+
+    directories = []
+    excluded = set()
+
+    for root_str, _, filenames in os.walk(Path(path).resolve()):
+        root = Path(root_str)
+        if predicate(root_str, filenames):
+            # This directory should be excluded, so exclude it and all of its
+            # parent directories.
+            # The last item in root.parents is ".", so we ignore it.
+            #
+            # Wrapping this in `list()` is only needed for Python 3.9.
+            excluded.update(list(root.parents)[:-1])
+            excluded.add(root)
+        directories.append(root)
+
+    for d in sorted(directories, reverse=True):
+        if d not in excluded:
+            yield d
+
+
+def subdirs_without_files(path: str) -> Generator[Path]:
+    """Yields every subdirectory of +path+ that has no files under it."""
+    return _subdirs_without_generic(path, lambda root, filenames: len(filenames) > 0)
+
+
+def subdirs_without_wheels(path: str) -> Generator[Path]:
+    """Yields every subdirectory of +path+ that has no .whl files under it."""
+    return _subdirs_without_generic(
+        path, lambda root, filenames: any(x.endswith(".whl") for x in filenames)
+    )

tests/functional/test_cache.py

@@ -1,4 +1,6 @@
+import json
 import os
+import re
 import shutil
 from glob import glob
 from typing import Callable
@@ -256,7 +258,7 @@ def test_cache_purge_with_empty_cache(script: PipTestEnvironment) -> None:
     and exit without an error code."""
     result = script.pip("cache", "purge", allow_stderr_warning=True)
     assert result.stderr == "WARNING: No matching packages\n"
-    assert result.stdout == "Files removed: 0 (0 bytes)\n"
+    assert result.stdout == "Files removed: 0 (0 bytes)\nDirectories removed: 0\n"
 
 
 @pytest.mark.usefixtures("populate_wheel_cache")
@@ -265,7 +267,7 @@ def test_cache_remove_with_bad_pattern(script: PipTestEnvironment) -> None:
     and exit without an error code."""
     result = script.pip("cache", "remove", "aaa", allow_stderr_warning=True)
     assert result.stderr == 'WARNING: No matching packages for pattern "aaa"\n'
-    assert result.stdout == "Files removed: 0 (0 bytes)\n"
+    assert result.stdout == "Files removed: 0 (0 bytes)\nDirectories removed: 0\n"
 
 
 def test_cache_list_too_many_args(script: PipTestEnvironment) -> None:
@@ -413,3 +415,109 @@ def test_cache_abort_when_no_cache_dir(
         "ERROR: pip cache commands can not function"
         " since cache is disabled." in result.stderr.splitlines()
     )
+
+
+@pytest.fixture
+def populate_wheel_cache_with_empty_dirs(wheel_cache_dir: str) -> None:
+    metadata_dir = os.path.join(wheel_cache_dir, "metadata_only")
+    os.makedirs(metadata_dir)
+    with open(os.path.join(metadata_dir, "metadata.json"), "w"):
+        pass
+
+    empty_dir = os.path.join(wheel_cache_dir, "completely_empty")
+    os.makedirs(empty_dir)
+
+    nested_empty = os.path.join(wheel_cache_dir, "nested", "empty", "dirs")
+    os.makedirs(nested_empty)
+
+
+@pytest.fixture
+def populate_http_cache_with_empty_dirs(cache_dir: str) -> None:
+    http_cache_dir = os.path.join(cache_dir, "http")
+    empty1 = os.path.join(http_cache_dir, "empty1")
+    empty2 = os.path.join(http_cache_dir, "empty2", "nested")
+
+    os.makedirs(empty1)
+    os.makedirs(empty2)
+
+
+@pytest.fixture
+def create_selfcheck_json(cache_dir: str) -> None:
+    selfcheck_path = os.path.join(cache_dir, "selfcheck.json")
+    with open(selfcheck_path, "w") as statefile:
+        json.dump(
+            {
+                "/some/prefix": {
+                    "last_check": "2020-01-01T00:00:00",
+                    "pypi_version": "20.0.1",
+                }
+            },
+            statefile,
+        )
+
+
+@pytest.mark.usefixtures(
+    "populate_wheel_cache_with_empty_dirs",
+    "populate_http_cache_with_empty_dirs",
+    "create_selfcheck_json",
+)
+def test_cache_purge_removes_empty_dirs_and_legacy_files(
+    script: PipTestEnvironment,
+    cache_dir: str,
+    wheel_cache_dir: str,
+) -> None:
+    """Test pip cache purge/remove with empty dirs and legacy files.
+
+    Verifies purge removes:
+    - Wheel cache directories without .whl files
+    - HTTP cache empty directories
+    - Legacy selfcheck.json file
+    - Reports correct directory counts
+    Also tests that 'cache remove' works similarly.
+    """
+    selfcheck_path = os.path.join(cache_dir, "selfcheck.json")
+    http_cache_dir = os.path.join(cache_dir, "http")
+    metadata_dir = os.path.join(wheel_cache_dir, "metadata_only")
+
+    # Verify setup
+    assert os.path.exists(selfcheck_path)
+    assert os.path.exists(metadata_dir)
+    assert os.path.exists(os.path.join(http_cache_dir, "empty1"))
+
+    result = script.pip("cache", "purge", "--verbose", allow_stderr_warning=True)
+
+    # Verify all cleanup happened
+    assert not os.path.exists(selfcheck_path)
+    assert "Removed legacy selfcheck.json file" in result.stdout
+    assert not os.path.exists(metadata_dir)
+    assert not os.path.exists(os.path.join(wheel_cache_dir, "completely_empty"))
+    assert not os.path.exists(os.path.join(http_cache_dir, "empty1"))
+    assert "Directories removed:" in result.stdout
+
+    # Verify directory count is positive
+    dir_count = int(re.findall(r"Directories removed: (\d+)", result.stdout)[0])
+    assert dir_count > 0
+
+
+def test_cache_purge_with_mixed_content(
+    script: PipTestEnvironment,
+    populate_wheel_cache: list[tuple[str, str]],
+    wheel_cache_dir: str,
+) -> None:
+    """Test purge removes both wheel files and empty directories."""
+    # Add an empty directory alongside the wheels
+    empty_dir = os.path.join(wheel_cache_dir, "empty_subdir")
+    os.makedirs(empty_dir)
+
+    result = script.pip("cache", "purge", "--verbose")
+
+    # Verify wheels and empty directory were removed
+    for _name, filepath in populate_wheel_cache:
+        assert not os.path.exists(filepath)
+    assert not os.path.exists(empty_dir)
+
+    # Verify counts in output
+    assert "Files removed:" in result.stdout
+    assert "Directories removed:" in result.stdout
+    files_removed = int(re.findall(r"Files removed: (\d+)", result.stdout)[0])
+    assert files_removed == 4