@@ -8,6 +8,29 @@ import Cache.Hashing
88
99namespace Cache.Requests
1010
11+ /-- Attempts to determine the running project's GitHub repository from its `origin` Git remote. -/
12+ def getRemoteRepo : IO String := do
13+ let out ← IO.Process.output {cmd := "git" , args := #["remote" , "get-url" , "origin" ]}
14+ unless out.exitCode == 0 do
15+ throw <| IO.userError s! "\
16+ Failed to run Git to determine project repository (exit code: { out.exitCode} ).\n \
17+ Ensure Git is installed and the `origin` remote points to the project's GitHub repository.\n \
18+ Stdout:\n { out.stdout.trim} \n Stderr:{ out.stderr.trim} \n "
19+ -- No strong validation is done here because this is simply used as a smart default
20+ -- for `lake exe cache get`, which is freely modifiable by any user.
21+ let url := out.stdout.trim.stripSuffix ".git"
22+ let repo? : Option String := do
23+ let pos ← url.revFind (· == '/' )
24+ let pos ← url.revFindAux (fun c => c == '/' || c == ':' ) pos
25+ return url.extract (url.next pos) url.endPos
26+ if let some repo := repo? then
27+ return repo
28+ else
29+ throw <| IO.userError s! "\
30+ Failed to determine project repository from remote URL.\n \
31+ Ensure the `origin` Git remote points to the project's GitHub repository.\n \
32+ Detected URL: { url} "
33+
1134-- FRO cache is flaky so disable until we work out the kinks: https://leanprover.zulipchat.com/#narrow/channel/113488-general/topic/The.20cache.20doesn't.20work/near/411058849
1235def useFROCache : Bool := false
1336
@@ -27,18 +50,22 @@ def getToken : IO String := do
2750
2851open System (FilePath)
2952
53+ /-- The full name of the main Mathlib GitHub repository. -/
54+ def MATHLIBREPO := "leanprover-community/mathlib4"
55+
3056/--
3157Given a file name like `"1234.tar.gz"`, makes the URL to that file on the server.
3258
3359The `f/` prefix means that it's a common file for caching.
3460-/
35- def mkFileURL (URL fileName : String) : String :=
36- s! "{ URL} /f/{ fileName} "
61+ def mkFileURL (repo URL fileName : String) : String :=
62+ let pre := if repo == MATHLIBREPO then "" else s! "{ repo} /"
63+ s! "{ URL} /f/{ pre}{ fileName} "
3764
3865section Get
3966
4067/-- Formats the config file for `curl`, containing the list of files to be downloaded -/
41- def mkGetConfigContent (hashMap : IO.ModuleHashMap) : IO String := do
68+ def mkGetConfigContent (repo : String) ( hashMap : IO.ModuleHashMap) : IO String := do
4269 hashMap.toArray.foldlM (init := "" ) fun acc ⟨_, hash⟩ => do
4370 let fileName := hash.asLTar
4471 -- Below we use `String.quote`, which is intended for quoting for use in Lean code
@@ -54,13 +81,13 @@ def mkGetConfigContent (hashMap : IO.ModuleHashMap) : IO String := do
5481
5582 -- Note we append a '.part' to the filenames here,
5683 -- which `downloadFiles` then removes when the download is successful.
57- pure <| acc ++ s! "url = { mkFileURL URL fileName} \n \
84+ pure <| acc ++ s! "url = { mkFileURL repo URL fileName} \n \
5885 -o { (IO.CACHEDIR / (fileName ++ ".part" )).toString.quote} \n "
5986
6087/-- Calls `curl` to download a single file from the server to `CACHEDIR` (`.cache`) -/
61- def downloadFile (hash : UInt64) : IO Bool := do
88+ def downloadFile (repo : String) ( hash : UInt64) : IO Bool := do
6289 let fileName := hash.asLTar
63- let url := mkFileURL URL fileName
90+ let url := mkFileURL repo URL fileName
6491 let path := IO.CACHEDIR / fileName
6592 let partFileName := fileName ++ ".part"
6693 let partPath := IO.CACHEDIR / partFileName
@@ -75,15 +102,16 @@ def downloadFile (hash : UInt64) : IO Bool := do
75102
76103/-- Call `curl` to download files from the server to `CACHEDIR` (`.cache`).
77104Exit the process with exit code 1 if any files failed to download. -/
78- def downloadFiles (hashMap : IO.ModuleHashMap) (forceDownload : Bool) (parallel : Bool) :
79- IO Unit := do
105+ def downloadFiles
106+ (repo : String) (hashMap : IO.ModuleHashMap)
107+ (forceDownload : Bool) (parallel : Bool) (warnOnMissing : Bool): IO Unit := do
80108 let hashMap ← if forceDownload then pure hashMap else hashMap.filterExists false
81109 let size := hashMap.size
82110 if size > 0 then
83111 IO.FS.createDirAll IO.CACHEDIR
84- IO.println s! "Attempting to download { size} file(s)"
112+ IO.println s! "Attempting to download { size} file(s) from { repo } cache "
85113 let failed ← if parallel then
86- IO.FS.writeFile IO.CURLCFG (← mkGetConfigContent hashMap)
114+ IO.FS.writeFile IO.CURLCFG (← mkGetConfigContent repo hashMap)
87115 let args := #["--request" , "GET" , "--parallel" , "--fail" , "--silent" ,
88116 "--retry" , "5" , -- there seem to be some intermittent failures
89117 "--write-out" , "%{json}\n " , "--config" , IO.CURLCFG.toString]
@@ -125,15 +153,15 @@ def downloadFiles (hashMap : IO.ModuleHashMap) (forceDownload : Bool) (parallel
125153 msg := msg ++ s! ", { failed} failed"
126154 IO.eprintln msg
127155 IO.FS.removeFile IO.CURLCFG
128- if success + failed < done then
156+ if warnOnMissing && success + failed < done then
129157 IO.eprintln "Warning: some files were not found in the cache."
130158 IO.eprintln "This usually means that your local checkout of mathlib4 has diverged from upstream."
131159 IO.eprintln "If you push your commits to a branch of the mathlib4 repository, CI will build the oleans and they will be available later."
132160 IO.eprintln "Alternatively, if you already have pushed your commits to a branch, this may mean the CI build has failed part-way through building."
133161 pure failed
134162 else
135163 let r ← hashMap.foldM (init := []) fun acc _ hash => do
136- pure <| (← IO.asTask do downloadFile hash) :: acc
164+ pure <| (← IO.asTask do downloadFile repo hash) :: acc
137165 pure <| r.foldl (init := 0 ) fun f t => if let .ok true := t.get then f else f + 1
138166 if failed > 0 then
139167 IO.println s! "{ failed} download(s) failed"
@@ -186,12 +214,21 @@ def getProofWidgets (buildDir : FilePath) : IO Unit := do
186214 throw <| IO.userError s! "Failed to prune ProofWidgets cloud release: { e} "
187215
188216/-- Downloads missing files, and unpacks files. -/
189- def getFiles (hashMap : IO.ModuleHashMap) (forceDownload forceUnpack parallel decompress : Bool) :
190- IO.CacheM Unit := do
217+ def getFiles
218+ (repo? : Option String) (hashMap : IO.ModuleHashMap)
219+ (forceDownload forceUnpack parallel decompress : Bool)
220+ : IO.CacheM Unit := do
191221 let isMathlibRoot ← IO.isMathlibRoot
192222 unless isMathlibRoot do checkForToolchainMismatch
193223 getProofWidgets (← read).proofWidgetsBuildDir
194- downloadFiles hashMap forceDownload parallel
224+ if let some repo := repo? then
225+ downloadFiles repo hashMap forceDownload parallel (warnOnMissing := true )
226+ else
227+ let repo ← getRemoteRepo
228+ IO.println s! "Project repository: { repo} "
229+ downloadFiles repo hashMap forceDownload parallel (warnOnMissing := false )
230+ unless repo == MATHLIBREPO do
231+ downloadFiles MATHLIBREPO hashMap forceDownload parallel (warnOnMissing := true )
195232 if decompress then
196233 IO.unpackCache hashMap forceUnpack
197234 else
@@ -209,20 +246,22 @@ def UPLOAD_URL : String :=
209246 URL
210247
211248/-- Formats the config file for `curl`, containing the list of files to be uploaded -/
212- def mkPutConfigContent (fileNames : Array String) (token : String) : IO String := do
249+ def mkPutConfigContent (repo : String) ( fileNames : Array String) (token : String) : IO String := do
213250 let token := if useFROCache then "" else s! "?{ token} " -- the FRO cache doesn't pass the token here
214251 let l ← fileNames.toList.mapM fun fileName : String => do
215- pure s! "-T { (IO.CACHEDIR / fileName).toString} \n url = { mkFileURL UPLOAD_URL fileName}{ token} "
252+ pure s! "-T { (IO.CACHEDIR / fileName).toString} \n url = { mkFileURL repo UPLOAD_URL fileName}{ token} "
216253 return "\n " .intercalate l
217254
218255/-- Calls `curl` to send a set of cached files to the server -/
219- def putFiles (fileNames : Array String) (overwrite : Bool) (token : String) : IO Unit := do
256+ def putFiles
257+ (repo : String) (fileNames : Array String)
258+ (overwrite : Bool) (token : String) : IO Unit := do
220259 -- TODO: reimplement using HEAD requests?
221260 let _ := overwrite
222261 let size := fileNames.size
223262 if size > 0 then
224- IO.FS.writeFile IO.CURLCFG (← mkPutConfigContent fileNames token)
225- IO.println s! "Attempting to upload { size} file(s)"
263+ IO.FS.writeFile IO.CURLCFG (← mkPutConfigContent repo fileNames token)
264+ IO.println s! "Attempting to upload { size} file(s) to { repo } cache "
226265 let args := if useFROCache then
227266 -- TODO: reimplement using HEAD requests?
228267 let _ := overwrite
0 commit comments