chore: cleanup local repo handling and improve reliability

Author: mhx

.docker/build-linux.sh

@@ -11,6 +11,7 @@ if [[ -n "$GITHUB_RUN_ID" ]]; then
     LOCAL_REPO_PATH=/local/repos
     mkdir -p "$LOCAL_REPO_PATH"
     LAST_UPDATE_FILE="$LOCAL_REPO_PATH/last-update"
+    UPDATE_LOCK_FILE="$LOCAL_REPO_PATH/.update.lock"
 
     WORKFLOW_LOG_DIR="/artifacts/workflow-logs/${GITHUB_RUN_ID}"
     NINJA_LOG_FILE="${WORKFLOW_LOG_DIR}/ninja-${BUILD_ARCH},${CROSS_ARCH},${BUILD_DIST},${BUILD_TYPE}.log"
@@ -23,35 +24,55 @@ if [[ -n "$GITHUB_RUN_ID" ]]; then
         echo "$(python3 - <<<'from datetime import datetime; print(datetime.now().isoformat())')	$event" >> "$BUILD_LOG_FILE"
     }
 
-    if [[ "$BUILD_TYPE" != "*-static-*" ]]; then
-      if [ -f "$LAST_UPDATE_FILE" ] && [ $(find "$LAST_UPDATE_FILE" -mmin -180) ]; then
-          echo "Skipping git repo update because it already ran in the last three hours."
-      else
-          echo "Running git repo update."
-
-          log "begin:repo-update"
-
-          for repo in "fmtlib/fmt" \
-                      "google/googletest" \
-                      "ericniebler/range-v3" \
-                      "greg7mdp/parallel-hashmap"; do
-            reponame=$(basename "$repo")
-            cd "$LOCAL_REPO_PATH"
-            if [ -d "$reponame" ]; then
-              cd "$reponame"
-              time git fetch
-            else
-              time git clone "https://github.com/$repo.git"
-            fi
-          done
-
-          log "end:repo-update"
-
-          touch "$LAST_UPDATE_FILE"
+    retry() {
+      local attempt=1 max=5 delay=15
+      while true; do
+        if "$@"; then return 0; fi
+        if [ "$attempt" -ge "$max" ]; then
+          echo "'$*' still failing after $attempt attempts, giving up." >&2
+          return 1
+        fi
+        echo "'$*' failed (attempt $attempt/$max), retrying in ${delay}s..." >&2
+        sleep "$delay"
+        attempt=$((attempt + 1))
+        delay=$((delay * 2))
+      done
+    }
+
+    REPO_LIST=(
+      google/googletest
+      greg7mdp/parallel-hashmap
+      # fmtlib/fmt
+      # ericniebler/range-v3
+    )
+
+    (
+      flock -w 900 9 || { echo "Timed out waiting for repo lock." >&2; exit 1; }
+
+      if [ -f "$LAST_UPDATE_FILE" ] && [ -n "$(find "$LAST_UPDATE_FILE" -mmin -180)" ]; then
+        echo "Skipping git repo update because it already ran in the last three hours."
+        exit 0
       fi
 
-      export DWARFS_LOCAL_REPO_PATH="$LOCAL_REPO_PATH"
-    fi
+      echo "Running git repo update."
+      log "begin:repo-update"
+
+      for repo in "${REPO_LIST[@]}"; do
+        reponame=$(basename "$repo")
+        if [ -d "$LOCAL_REPO_PATH/$reponame/.git" ]; then
+          time retry git -C "$LOCAL_REPO_PATH/$reponame" fetch --prune
+        else
+          rm -rf "$LOCAL_REPO_PATH/$reponame.tmp" "$LOCAL_REPO_PATH/$reponame"
+          time retry git clone "https://github.com/$repo.git" "$LOCAL_REPO_PATH/$reponame.tmp"
+          mv "$LOCAL_REPO_PATH/$reponame.tmp" "$LOCAL_REPO_PATH/$reponame"
+        fi
+      done
+
+      log "end:repo-update"
+      touch "$LAST_UPDATE_FILE"
+    ) 9>"$UPDATE_LOCK_FILE"
+
+    export DWARFS_LOCAL_REPO_PATH="$LOCAL_REPO_PATH"
 
     if [[ "-$BUILD_TYPE-" == *-debug-* ]] && [[ "-$BUILD_TYPE-" != *-coverage-* ]] &&
        [[ "-$BUILD_TYPE-" != *-[at]san-* ]] && [[ "-$BUILD_TYPE-" != *-ubsan-* ]]; then