Skip to content

Commit 9897614

Browse files
huonwhuonw
authored
Add workflow to clear persistent cache dirs on self-hosted runners (#20755)
We have a few long-lived runners that have cache directories that grow and grow until the machine runs out of space. This workflow is a stop-gap measure that makes it easier for anyone to give the machine more space, by making workflow to do it, rather than requiring someone with credentials to SSH in and do it manually. For instance, on March 18, the macOS 11 runner had: | directory | size (GB) | |------------------------|-----------| | `~/Library/Caches/nce` | 113 | | `~/.cache` | 34 | | `~/.rustup` | 12 | | `~/.pex` | 8.8 | | `~/.nce` | 1.2 | This PR set-ups automatic deletion of these directories with some logging.
1 parent 7d850cf commit 9897614

File tree

2 files changed

+111
-0
lines changed

2 files changed

+111
-0
lines changed

‎.github/workflows/clear_self_hosted_persistent_caches.yaml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# GENERATED, DO NOT EDIT!
2+
# To change, edit `src/python/pants_release/generate_github_workflows.py` and run:
3+
# ./pants run src/python/pants_release/generate_github_workflows.py
4+
5+
6+
jobs:
7+
- name: clear_linux_arm64
8+
runs-on:
9+
- self-hosted
10+
- Linux
11+
- ARM64
12+
steps:
13+
- name: df before
14+
run: df -h
15+
- name: Deleting ~/Library/Caches
16+
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
17+
- name: Deleting ~/.cache
18+
run: du -sh ~/.cache || true; rm -rf ~/.cache
19+
- name: Deleting ~/.nce
20+
run: du -sh ~/.nce || true; rm -rf ~/.nce
21+
- name: Deleting ~/.rustup
22+
run: du -sh ~/.rustup || true; rm -rf ~/.rustup
23+
- name: Deleting ~/.pex
24+
run: du -sh ~/.pex || true; rm -rf ~/.pex
25+
- name: df after
26+
run: df -h
27+
- name: clear_macos10_15_x86_64
28+
runs-on:
29+
- self-hosted
30+
- macOS-10.15-X64
31+
steps:
32+
- name: df before
33+
run: df -h
34+
- name: Deleting ~/Library/Caches
35+
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
36+
- name: Deleting ~/.cache
37+
run: du -sh ~/.cache || true; rm -rf ~/.cache
38+
- name: Deleting ~/.nce
39+
run: du -sh ~/.nce || true; rm -rf ~/.nce
40+
- name: Deleting ~/.rustup
41+
run: du -sh ~/.rustup || true; rm -rf ~/.rustup
42+
- name: Deleting ~/.pex
43+
run: du -sh ~/.pex || true; rm -rf ~/.pex
44+
- name: df after
45+
run: df -h
46+
- name: clear_macos11_arm64
47+
runs-on:
48+
- self-hosted
49+
- macOS-11-ARM64
50+
steps:
51+
- name: df before
52+
run: df -h
53+
- name: Deleting ~/Library/Caches
54+
run: du -sh ~/Library/Caches || true; rm -rf ~/Library/Caches
55+
- name: Deleting ~/.cache
56+
run: du -sh ~/.cache || true; rm -rf ~/.cache
57+
- name: Deleting ~/.nce
58+
run: du -sh ~/.nce || true; rm -rf ~/.nce
59+
- name: Deleting ~/.rustup
60+
run: du -sh ~/.rustup || true; rm -rf ~/.rustup
61+
- name: Deleting ~/.pex
62+
run: du -sh ~/.pex || true; rm -rf ~/.pex
63+
- name: df after
64+
run: df -h
65+
name: Clear persistent caches on long-lived self-hosted runners
66+
'on':
67+
workflow_dispatch: {}

‎src/python/pants_release/generate_github_workflows.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1570,6 +1570,38 @@ def gen_goals(use_default_version: bool) -> Sequence[object]:
15701570
return PublicReposOutput(jobs=jobs, inputs=inputs, run_name=run_name)
15711571

15721572

1573+
def clear_self_hosted_persistent_caches_jobs() -> list[dict[str, Any]]:
1574+
def make_job(platform: Platform) -> dict[str, Any]:
1575+
helper = Helper(platform)
1576+
1577+
clear_steps = [
1578+
{
1579+
"name": f"Deleting {directory}",
1580+
"run": f"du -sh {directory} || true; rm -rf {directory}",
1581+
}
1582+
for directory in [
1583+
# not all of these will necessarily exist (e.g. ~/Library/Caches is macOS-specific),
1584+
# but the script is resilient to this
1585+
"~/Library/Caches",
1586+
"~/.cache",
1587+
"~/.nce",
1588+
"~/.rustup",
1589+
"~/.pex",
1590+
]
1591+
]
1592+
return {
1593+
"name": helper.job_name("clear"),
1594+
"runs-on": helper.runs_on(),
1595+
"steps": [
1596+
{"name": "df before", "run": "df -h"},
1597+
*clear_steps,
1598+
{"name": "df after", "run": "df -h"},
1599+
],
1600+
}
1601+
1602+
return [make_job(platform) for platform in sorted(SELF_HOSTED, key=lambda p: p.value)]
1603+
1604+
15731605
# ----------------------------------------------------------------------
15741606
# Main file
15751607
# ----------------------------------------------------------------------
@@ -1740,12 +1772,24 @@ def generate() -> dict[Path, str]:
17401772
Dumper=NoAliasDumper,
17411773
)
17421774

1775+
clear_self_hosted_persistent_caches = clear_self_hosted_persistent_caches_jobs()
1776+
clear_self_hosted_persistent_caches_yaml = yaml.dump(
1777+
{
1778+
"name": "Clear persistent caches on long-lived self-hosted runners",
1779+
"on": {"workflow_dispatch": {}},
1780+
"jobs": clear_self_hosted_persistent_caches,
1781+
}
1782+
)
1783+
17431784
return {
17441785
Path(".github/workflows/audit.yaml"): f"{HEADER}\n\n{audit_yaml}",
17451786
Path(".github/workflows/cache_comparison.yaml"): f"{HEADER}\n\n{cache_comparison_yaml}",
17461787
Path(".github/workflows/test.yaml"): f"{HEADER}\n\n{test_yaml}",
17471788
Path(".github/workflows/release.yaml"): f"{HEADER}\n\n{release_yaml}",
17481789
Path(".github/workflows/public_repos.yaml"): f"{HEADER}\n\n{public_repos_yaml}",
1790+
Path(
1791+
".github/workflows/clear_self_hosted_persistent_caches.yaml"
1792+
): f"{HEADER}\n\n{clear_self_hosted_persistent_caches_yaml}",
17491793
}
17501794

17511795

0 commit comments

Comments
 (0)