Clean local-dir when loading it from a storage with no slots

This allows to blindly remove all slots in the storage in emergency and expect that the subsequent CI runs will start from scratch and not with the leftover of the runner's work directory from the previous run (more idempotent behavior).

Author: dimikot

ci-storage

@@ -17,7 +17,8 @@ STORAGE_MAX_AGE_SEC_DEFAULT = 3600
 STORAGE_MAX_AGE_SEC_BAK = 60
 STORAGE_DIR_DEFAULT = "~/ci-storage"
 META_FILE = ".ci-storage.meta"
-LOCAL_META_FILE_DIR = "/tmp" if os.access("/tmp", os.W_OK) else tempfile.gettempdir()
+EMPTY_DIR = ".ci-storage.empty-dir"
+TEMP_DIR = "/tmp" if os.access("/tmp", os.W_OK) else tempfile.gettempdir()
 MAX_FULL_SNAPSHOT_HISTORY = 10
 
 
@@ -214,7 +215,19 @@ def action_load(
         prefix = f'Checking slot-id="{id}"...'
         if id == "*":
             if not slot_infos:
-                print(f"{prefix} {in_storage} has no slots, so exiting with a no-op")
+                if layer:
+                    print(
+                        f"{prefix} {in_storage} has no slots, so exiting with a no-op"
+                    )
+                else:
+                    print(
+                        f"{prefix} {in_storage} has no slots, so cleaning f{local_dir}"
+                    )
+                    action_clean(
+                        local_dir=local_dir,
+                        exclude=exclude,
+                        verbose=verbose,
+                    )
                 return
             elif not layer:
                 slot_id = list(slot_infos.keys())[0]
@@ -283,6 +296,42 @@ def action_load(
         slot_info.meta.write_to(local_dir=local_dir)
 
 
+#
+# Removes everything in local_dir. We use rsync and not rm to keep the excludes
+# intact and compatible with the "load" action.
+#
+def action_clean(
+    *,
+    local_dir: str,
+    exclude: list[str],
+    verbose: bool,
+):
+    empty_dir = f"{TEMP_DIR}/{EMPTY_DIR}.{normalize_slot_id(local_dir)}"
+    os.makedirs(empty_dir, exist_ok=True)
+    try:
+        check_call(
+            cmd=[
+                "rsync",
+                *build_rsync_args(
+                    host=None,
+                    port=None,
+                    action="load",
+                    exclude=exclude,
+                    layer=[],
+                    verbose=verbose,
+                ),
+                f"{empty_dir}/",
+                f"{local_dir}/",
+            ],
+            print_elapsed=True,
+        )
+    finally:
+        try:
+            os.rmdir(empty_dir)
+        except Exception:
+            pass
+
+
 #
 # Stores the content of the local directory in the storage with the provided
 # slot id on a remote host.
@@ -684,7 +733,7 @@ class SlotMeta:
 
     @staticmethod
     def _path(local_dir: str) -> str:
-        return f"{LOCAL_META_FILE_DIR}/{META_FILE}.{normalize_slot_id(local_dir)}"
+        return f"{TEMP_DIR}/{META_FILE}.{normalize_slot_id(local_dir)}"
 
 
 #

tests/0030-load-star-does-clean-when-no-slots-in-storage.test.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+source ./common.sh
+
+test -e "$LOCAL_DIR/file-1"
+test -e "$LOCAL_DIR/dir-a"
+
+ci-storage \
+  --slot-id="*" \
+  load
+
+grep -qF 'Checking slot-id="*"... storage has no slots, so cleaning' "$OUT"
+
+test ! -e "$LOCAL_DIR/file-1"
+test ! -e "$LOCAL_DIR/dir-a"

tests/0030-load-star-is-okay-when-no-slots-in-storage.test.sh

@@ -4,10 +4,9 @@ source ./common.sh
 ci-storage \
   --slot-id=myslot \
   store
-  
+
 ci-storage \
   --slot-id="*" \
-  load || error=$?
+  load
 
-test "$error" == 0
 grep -qF 'Checking slot-id="*"... loading the most recent full (non-layer) slot-id="myslot"' "$OUT"

tests/0040-load-star-with-slots-in-storage.test.sh

@@ -5,7 +5,6 @@ ci-storage \
   --slot-id="absent1" \
   --slot-id="absent2" \
   --slot-id="*" \
-  load || error=$?
+  load
 
-test "$error" == 0
-grep -qF 'Checking slot-id="*"... storage has no slots, so exiting with a no-op' "$OUT"
+grep -qF 'Checking slot-id="*"... storage has no slots, so cleaning' "$OUT"

tests/0050-load-absent-slot-ids-and-star-is-okay-when-no-slots-in-storage.test.sh

@@ -4,14 +4,13 @@ source ./common.sh
 ci-storage \
   --slot-id=myslot \
   store
-  
+
 ci-storage \
   --slot-id=absent1 \
   --slot-id=absent2 \
   --slot-id="*" \
-  load || error=$?
+  load
 
-test "$error" == 0
 grep -qF 'Checking slot-id="absent1"... not found' "$OUT"
 grep -qF 'Checking slot-id="absent2"... not found' "$OUT"
 grep -qF 'Checking slot-id="*"... loading the most recent full (non-layer) slot-id="myslot"' "$OUT"

tests/0060-load-absent-slot-ids-and-star-with-slots-in-storage.test.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+source ./common.sh
+
+test -e "$LOCAL_DIR/file-1"
+test -e "$LOCAL_DIR/dir-a"
+
+ci-storage \
+  --slot-id="*" \
+  --layer="*" \
+  load
+
+test -e "$LOCAL_DIR/file-1"
+test -e "$LOCAL_DIR/dir-a"