Skip to content

Commit d26fa82

Browse files
committed
Add golang.py for Go release downloads
Mirror Go release tarballs and installers from go.dev. The script uses the official go.dev/dl/?mode=json&include=all listing as the index and downloads files from dl.google.com/go via curl with atomic .tmp + rename. Compared to the previously common rsync upstream rsync.mirrors.ustc.edu.cn::golang/ this is a direct from-origin sync that does not depend on another mirror. Used by NJU mirror's golang job (~646G, 8118 files).
1 parent 93b8cff commit d26fa82

1 file changed

Lines changed: 125 additions & 0 deletions

File tree

golang.py

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
#!/usr/bin/env python3
2+
"""Golang mirror sync script.
3+
4+
Fetches version list from go.dev API and downloads all files from dl.google.com/go/.
5+
Cleans up stale files that are no longer in the version list.
6+
"""
7+
import json
8+
import os
9+
import sys
10+
import subprocess
11+
import hashlib
12+
13+
WORKDIR = os.environ.get("TUNASYNC_WORKING_DIR", "/data/mirrors/golang")
14+
BASE_URL = "https://dl.google.com/go/"
15+
API_URL = "https://go.dev/dl/?mode=json&include=all"
16+
17+
18+
def fetch_versions():
19+
"""Fetch version list from go.dev API."""
20+
print("Fetching version list from go.dev...")
21+
try:
22+
result = subprocess.run(
23+
["curl", "-s", "-m", "60", API_URL],
24+
capture_output=True, text=True, timeout=90
25+
)
26+
data = json.loads(result.stdout)
27+
except Exception as e:
28+
print(f"Error: failed to fetch version list: {e}")
29+
sys.exit(1)
30+
31+
files = {}
32+
for v in data:
33+
for f in v.get("files", []):
34+
filename = f["filename"]
35+
sha256 = f.get("sha256", "")
36+
files[filename] = {
37+
"url": BASE_URL + filename,
38+
"sha256": sha256,
39+
}
40+
return files
41+
42+
43+
def download_file(filename, url, filepath):
44+
"""Download a file if not present or size mismatch."""
45+
tmpfile = filepath + ".tmp"
46+
try:
47+
result = subprocess.run(
48+
["curl", "-s", "-L", "-m", "600", "-o", tmpfile, url],
49+
timeout=630
50+
)
51+
if result.returncode == 0:
52+
os.rename(tmpfile, filepath)
53+
return True
54+
else:
55+
if os.path.exists(tmpfile):
56+
os.remove(tmpfile)
57+
return False
58+
except Exception:
59+
if os.path.exists(tmpfile):
60+
os.remove(tmpfile)
61+
return False
62+
63+
64+
def get_remote_size(url):
65+
"""Get remote file size via HEAD request."""
66+
try:
67+
result = subprocess.run(
68+
["curl", "-sI", "-m", "10", url],
69+
capture_output=True, text=True, timeout=15
70+
)
71+
for line in result.stdout.splitlines():
72+
if line.lower().startswith("content-length:"):
73+
return int(line.split(":", 1)[1].strip())
74+
except Exception:
75+
pass
76+
return None
77+
78+
79+
def main():
80+
os.makedirs(WORKDIR, exist_ok=True)
81+
os.chdir(WORKDIR)
82+
83+
expected = fetch_versions()
84+
print(f"Total files to sync: {len(expected)}")
85+
86+
downloaded = 0
87+
skipped = 0
88+
errors = 0
89+
90+
for filename, info in expected.items():
91+
filepath = os.path.join(WORKDIR, filename)
92+
url = info["url"]
93+
94+
# Check if file exists and has correct size
95+
if os.path.isfile(filepath):
96+
remote_size = get_remote_size(url)
97+
if remote_size is not None:
98+
local_size = os.path.getsize(filepath)
99+
if local_size == remote_size:
100+
skipped += 1
101+
continue
102+
103+
print(f"Downloading {filename}...")
104+
if download_file(filename, url, filepath):
105+
downloaded += 1
106+
else:
107+
errors += 1
108+
print(f"ERROR: failed to download {filename}")
109+
110+
print(f"Done. Downloaded: {downloaded}, Skipped: {skipped}, Errors: {errors}")
111+
112+
# Clean up stale files
113+
print("Cleaning up stale files...")
114+
stale = 0
115+
for fname in os.listdir(WORKDIR):
116+
fpath = os.path.join(WORKDIR, fname)
117+
if os.path.isfile(fpath) and fname not in expected:
118+
print(f"Removing stale file: {fname}")
119+
os.remove(fpath)
120+
stale += 1
121+
print(f"Removed {stale} stale files.")
122+
123+
124+
if __name__ == "__main__":
125+
main()

0 commit comments

Comments
 (0)