Retry dependency fetches and fail fast in entrypoint [CI 6/9]

docker/finn_entrypoint.sh

@@ -28,6 +28,9 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+# Fail fast so a partial deps/ tree is caught early
+set -e
+
 export HOME=/tmp/home_dir
 export SHELL=/bin/bash
 export LANG="en_US.UTF-8"
@@ -55,18 +58,23 @@ recho () {
 
 # qonnx (using workaround for https://github.com/pypa/pip/issues/7953)
 # to be fixed in future Ubuntu versions (https://bugs.launchpad.net/ubuntu/+source/setuptools/+bug/1994016)
-mv ${FINN_ROOT}/deps/qonnx/pyproject.toml ${FINN_ROOT}/deps/qonnx/pyproject.tmp
-pip install --user -e ${FINN_ROOT}/deps/qonnx
-mv ${FINN_ROOT}/deps/qonnx/pyproject.tmp ${FINN_ROOT}/deps/qonnx/pyproject.toml
+# set -e propagates pip failures, so the trap covers only the qonnx-only swap.
+_qonnx_pyproj_toml="${FINN_ROOT}/deps/qonnx/pyproject.toml"
+_qonnx_pyproj_tmp="${FINN_ROOT}/deps/qonnx/pyproject.tmp"
+mv "$_qonnx_pyproj_toml" "$_qonnx_pyproj_tmp"
+trap 'mv "$_qonnx_pyproj_tmp" "$_qonnx_pyproj_toml" 2>/dev/null || true' EXIT
+pip install --user -e "${FINN_ROOT}/deps/qonnx"
+mv "$_qonnx_pyproj_tmp" "$_qonnx_pyproj_toml"
+trap - EXIT
 
 # finn-experimental
-pip install --user -e ${FINN_ROOT}/deps/finn-experimental
+pip install --user -e "${FINN_ROOT}/deps/finn-experimental"
 # brevitas
-pip install --user -e ${FINN_ROOT}/deps/brevitas
+pip install --user -e "${FINN_ROOT}/deps/brevitas"
 
 if [ -f "${FINN_ROOT}/setup.py" ];then
   # run pip install for finn
-  pip install --user -e ${FINN_ROOT}
+  pip install --user -e "${FINN_ROOT}"
 else
   recho "Unable to find FINN source code in ${FINN_ROOT}"
   recho "Ensure you have passed -v <path-to-finn-repo>:<path-to-finn-repo> to the docker run command"
@@ -77,11 +85,12 @@ if [ -f "$VITIS_PATH/settings64.sh" ];then
   # source Vitis env.vars
   export XILINX_VITIS=$VITIS_PATH
   export XILINX_XRT=/opt/xilinx/xrt
-  source $VITIS_PATH/settings64.sh
+  # env scripts may return non-zero, so do not let that abort the container
+  source "$VITIS_PATH/settings64.sh" || true
   gecho "Found Vitis at $VITIS_PATH"
   if [ -f "$XILINX_XRT/setup.sh" ];then
     # source XRT
-    source $XILINX_XRT/setup.sh
+    source "$XILINX_XRT/setup.sh" || true
     gecho "Found XRT at $XILINX_XRT"
   else
     recho "XRT not found on $XILINX_XRT, did you skip the download or did the installation fail?"
@@ -94,7 +103,7 @@ else
   if [ -f "$VIVADO_PATH/settings64.sh" ];then
     # source Vivado env.vars
     export XILINX_VIVADO=$VIVADO_PATH
-    source $VIVADO_PATH/settings64.sh
+    source "$VIVADO_PATH/settings64.sh" || true
     gecho "Found Vivado at $VIVADO_PATH"
   else
     yecho "Unable to find $VIVADO_PATH/settings64.sh"
@@ -111,10 +120,10 @@ else
     gecho "Found existing finn_xsi at ${FINN_ROOT}/finn_xsi/xsi.so"
   else
     gecho "Building finn_xsi using finn.xsi.setup..."
-    python -m finn.xsi.setup --quiet
-    if [ $? -eq 0 ]; then
+    if python -m finn.xsi.setup --quiet; then
       gecho "finn_xsi built successfully"
     else
+      # finn_xsi is optional, so a failed build is non-fatal
       recho "Failed to build finn_xsi"
     fi
   fi
@@ -123,7 +132,7 @@ fi
 
 if [ -f "$HLS_PATH/settings64.sh" ];then
   # source Vitis HLS env.vars
-  source $HLS_PATH/settings64.sh
+  source "$HLS_PATH/settings64.sh" || true
   gecho "Found Vitis HLS at $HLS_PATH"
 else
   yecho "Unable to find $HLS_PATH/settings64.sh"
@@ -133,7 +142,7 @@ else
 fi
 
 if [ -d "$FINN_ROOT/.Xilinx" ]; then
-  mkdir "$HOME/.Xilinx"
+  mkdir -p "$HOME/.Xilinx"
   if [ -f "$FINN_ROOT/.Xilinx/HLS_init.tcl" ]; then
     cp "$FINN_ROOT/.Xilinx/HLS_init.tcl" "$HOME/.Xilinx/"
     gecho "Found HLS_init.tcl and copied to $HOME/.Xilinx/HLS_init.tcl"
@@ -142,7 +151,7 @@ if [ -d "$FINN_ROOT/.Xilinx" ]; then
   fi
 
   if [ -f "$FINN_ROOT/.Xilinx/Vivado/Vivado_init.tcl" ]; then
-    mkdir "$HOME/.Xilinx/Vivado/"
+    mkdir -p "$HOME/.Xilinx/Vivado/"
     cp "$FINN_ROOT/.Xilinx/Vivado/Vivado_init.tcl" "$HOME/.Xilinx/Vivado/"
     gecho "Found Vivado_init.tcl and copied to $HOME/.Xilinx/Vivado/Vivado_init.tcl"
 

fetch-repos.sh

@@ -28,6 +28,10 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+# Fail fast so a partial deps/ tree is caught early
+# `-u` is deliberately omitted: optional FINN_SKIP_BOARD_FILES flag may be unset.
+set -eo pipefail
+
 QONNX_COMMIT="f5c9819bd00f01f41e70639b8461c8e4b39432f7"
 FINN_EXP_COMMIT="0724be21111a21f0d81a072fccc1c446e053f851"
 BREVITAS_COMMIT="aad4d5a293db6f2ec622a92a5d3278e47072453e"
@@ -64,43 +68,69 @@ SCRIPT=$(readlink -f "$0")
 # absolute path this script is in, thus /home/user/bin
 SCRIPTPATH=$(dirname "$SCRIPT")
 
+# Retry a command with exponential back-off (github 5xx, DNS blips, rate-limit).
+retry() {
+    local n=0
+    local max=5
+    local delay=4
+    until "$@"; do
+        n=$((n+1))
+        if (( n >= max )); then
+            echo "fetch-repos: command failed after $n attempts: $*" >&2
+            return 1
+        fi
+        echo "fetch-repos: attempt $n/$max failed for: $* (retrying in ${delay}s)" >&2
+        sleep "$delay"
+        delay=$((delay*2))
+    done
+}
+
+# Clone fresh, dropping any leftover first so an interrupted clone never makes
+# the retry refuse a non-empty target or reuse a tree with no resolvable HEAD.
+clone_repo() {
+    rm -rf "$2"
+    git clone "$1" "$2"
+}
+
 fetch_repo() {
     # URL for git repo to be cloned
-    REPO_URL=$1
+    local REPO_URL=$1
     # commit hash for repo
-    REPO_COMMIT=$2
+    local REPO_COMMIT=$2
     # directory to clone to under deps/
-    REPO_DIR=$3
+    local REPO_DIR=$3
     # absolute path for the repo local copy
-    CLONE_TO=$SCRIPTPATH/deps/$REPO_DIR
+    local CLONE_TO=$SCRIPTPATH/deps/$REPO_DIR
 
-    # clone repo if dir not found
-    if [ ! -d "$CLONE_TO" ]; then
-        git clone $REPO_URL $CLONE_TO
+    # (re-)clone when the dir is missing or has no resolvable HEAD (what an
+    # interrupted clone leaves behind, and what the checkout below needs)
+    if [ ! -d "$CLONE_TO" ] || ! git -C "$CLONE_TO" rev-parse HEAD >/dev/null 2>&1; then
+        retry clone_repo "$REPO_URL" "$CLONE_TO"
     fi
-    # verify and try to pull repo if not at correct commit
-    CURRENT_COMMIT=$(git -C $CLONE_TO rev-parse HEAD)
-    if [ $CURRENT_COMMIT != $REPO_COMMIT ]; then
-        git -C $CLONE_TO pull
-        # checkout the expected commit
-        git -C $CLONE_TO checkout $REPO_COMMIT
+
+    local CURRENT_COMMIT
+    CURRENT_COMMIT=$(git -C "$CLONE_TO" rev-parse HEAD)
+    if [ "$CURRENT_COMMIT" != "$REPO_COMMIT" ]; then
+        # fetch+checkout instead of pull: working copy is a detached HEAD.
+        retry git -C "$CLONE_TO" fetch --tags --force
+        git -C "$CLONE_TO" checkout "$REPO_COMMIT"
     fi
-    # verify one last time
-    CURRENT_COMMIT=$(git -C $CLONE_TO rev-parse HEAD)
-    if [ $CURRENT_COMMIT == $REPO_COMMIT ]; then
-        echo "Successfully checked out $REPO_DIR at commit $CURRENT_COMMIT"
-    else
-        echo "Could not check out $REPO_DIR. Check your internet connection and try again."
+
+    CURRENT_COMMIT=$(git -C "$CLONE_TO" rev-parse HEAD)
+    if [ "$CURRENT_COMMIT" != "$REPO_COMMIT" ]; then
+        echo "fetch-repos: ERROR: $REPO_DIR is at $CURRENT_COMMIT, expected $REPO_COMMIT" >&2
+        return 1
     fi
+    echo "Successfully checked out $REPO_DIR at commit $CURRENT_COMMIT"
 }
 
 fetch_board_files() {
     echo "Downloading and extracting board files..."
     mkdir -p "$SCRIPTPATH/deps/board_files"
     OLD_PWD=$(pwd)
     cd "$SCRIPTPATH/deps/board_files"
-    wget -q https://github.com/cathalmccabe/pynq-z1_board_files/raw/master/pynq-z1.zip
-    wget -q https://dpoauwgwqsy2x.cloudfront.net/Download/pynq-z2.zip
+    retry wget -qO pynq-z1.zip https://github.com/cathalmccabe/pynq-z1_board_files/raw/master/pynq-z1.zip
+    retry wget -qO pynq-z2.zip https://dpoauwgwqsy2x.cloudfront.net/Download/pynq-z2.zip
     unzip -q pynq-z1.zip
     unzip -q pynq-z2.zip
     cp -r $SCRIPTPATH/deps/$AVNET_BDF_DIR/* $SCRIPTPATH/deps/board_files/

run-docker.sh

@@ -162,7 +162,7 @@ gecho "Port-forwarding for Netron $NETRON_PORT:$NETRON_PORT"
 
 # Ensure git-based deps are checked out at correct commit
 if [ "$FINN_SKIP_DEP_REPOS" = "0" ]; then
-  ./fetch-repos.sh
+  ./fetch-repos.sh || exit 1
 fi
 
 # If xrt path given, copy .deb file to this repo