Add workflow to clear persistent cache dirs on self-hosted runners (#…

…20755)

We have a few long-lived runners that have cache directories that grow
and grow until the machine runs out of space. This workflow is a
stop-gap measure that makes it easier for anyone to give the machine
more space, by making workflow to do it, rather than requiring someone
with credentials to SSH in and do it manually.

For instance, on March 18, the macOS 11 runner had:

| directory              | size (GB) |
|------------------------|-----------|
| `~/Library/Caches/nce` | 113       |
| `~/.cache`             | 34        |
| `~/.rustup`            | 12        |
| `~/.pex`               | 8.8       |
| `~/.nce`               | 1.2       |

This PR set-ups automatic deletion of these directories with some
logging.

.github/workflows/clear_self_hosted_persistent_caches.yaml

@@ -0,0 +1,67 @@
+# GENERATED, DO NOT EDIT!
+# To change, edit `src/python/pants_release/generate_github_workflows.py` and run:
+#   ./pants run src/python/pants_release/generate_github_workflows.py
+
+
+jobs:
+- name: clear_linux_arm64
+  runs-on:
+  - self-hosted
+  - Linux
+  - ARM64
+  steps:
+  - name: df before
+    run: df -h
+  - name: Deleting ~/Library/Caches
+    run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
+  - name: Deleting ~/.cache
+    run: du -sh ~/.cache || true; rm -rf ~/.cache
+  - name: Deleting ~/.nce
+    run: du -sh ~/.nce || true; rm -rf ~/.nce
+  - name: Deleting ~/.rustup
+    run: du -sh ~/.rustup || true; rm -rf ~/.rustup
+  - name: Deleting ~/.pex
+    run: du -sh ~/.pex || true; rm -rf ~/.pex
+  - name: df after
+    run: df -h
+- name: clear_macos10_15_x86_64
+  runs-on:
+  - self-hosted
+  - macOS-10.15-X64
+  steps:
+  - name: df before
+    run: df -h
+  - name: Deleting ~/Library/Caches
+    run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
+  - name: Deleting ~/.cache
+    run: du -sh ~/.cache || true; rm -rf ~/.cache
+  - name: Deleting ~/.nce
+    run: du -sh ~/.nce || true; rm -rf ~/.nce
+  - name: Deleting ~/.rustup
+    run: du -sh ~/.rustup || true; rm -rf ~/.rustup
+  - name: Deleting ~/.pex
+    run: du -sh ~/.pex || true; rm -rf ~/.pex
+  - name: df after
+    run: df -h
+- name: clear_macos11_arm64
+  runs-on:
+  - self-hosted
+  - macOS-11-ARM64
+  steps:
+  - name: df before
+    run: df -h
+  - name: Deleting ~/Library/Caches
+    run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
+  - name: Deleting ~/.cache
+    run: du -sh ~/.cache || true; rm -rf ~/.cache
+  - name: Deleting ~/.nce
+    run: du -sh ~/.nce || true; rm -rf ~/.nce
+  - name: Deleting ~/.rustup
+    run: du -sh ~/.rustup || true; rm -rf ~/.rustup
+  - name: Deleting ~/.pex
+    run: du -sh ~/.pex || true; rm -rf ~/.pex
+  - name: df after
+    run: df -h
+name: Clear persistent caches on long-lived self-hosted runners
+'on':
+  workflow_dispatch: {}

src/python/pants_release/generate_github_workflows.py

@@ -1570,6 +1570,38 @@ def gen_goals(use_default_version: bool) -> Sequence[object]:
     return PublicReposOutput(jobs=jobs, inputs=inputs, run_name=run_name)
 
 
+def clear_self_hosted_persistent_caches_jobs() -> list[dict[str, Any]]:
+    def make_job(platform: Platform) -> dict[str, Any]:
+        helper = Helper(platform)
+
+        clear_steps = [
+            {
+                "name": f"Deleting {directory}",
+                "run": f"du -sh {directory} || true; rm -rf {directory}",
+            }
+            for directory in [
+                # not all of these will necessarily exist (e.g. ~/Library/Caches is macOS-specific),
+                # but the script is resilient to this
+                "~/Library/Caches",
+                "~/.cache",
+                "~/.nce",
+                "~/.rustup",
+                "~/.pex",
+            ]
+        ]
+        return {
+            "name": helper.job_name("clear"),
+            "runs-on": helper.runs_on(),
+            "steps": [
+                {"name": "df before", "run": "df -h"},
+                *clear_steps,
+                {"name": "df after", "run": "df -h"},
+            ],
+        }
+
+    return [make_job(platform) for platform in sorted(SELF_HOSTED, key=lambda p: p.value)]
+
+
 # ----------------------------------------------------------------------
 # Main file
 # ----------------------------------------------------------------------
@@ -1740,12 +1772,24 @@ def generate() -> dict[Path, str]:
         Dumper=NoAliasDumper,
     )
 
+    clear_self_hosted_persistent_caches = clear_self_hosted_persistent_caches_jobs()
+    clear_self_hosted_persistent_caches_yaml = yaml.dump(
+        {
+            "name": "Clear persistent caches on long-lived self-hosted runners",
+            "on": {"workflow_dispatch": {}},
+            "jobs": clear_self_hosted_persistent_caches,
+        }
+    )
+
     return {
         Path(".github/workflows/audit.yaml"): f"{HEADER}\n\n{audit_yaml}",
         Path(".github/workflows/cache_comparison.yaml"): f"{HEADER}\n\n{cache_comparison_yaml}",
         Path(".github/workflows/test.yaml"): f"{HEADER}\n\n{test_yaml}",
         Path(".github/workflows/release.yaml"): f"{HEADER}\n\n{release_yaml}",
         Path(".github/workflows/public_repos.yaml"): f"{HEADER}\n\n{public_repos_yaml}",
+        Path(
+            ".github/workflows/clear_self_hosted_persistent_caches.yaml"
+        ): f"{HEADER}\n\n{clear_self_hosted_persistent_caches_yaml}",
     }