Retry transient Windows atomic write failures

python/aibrix/aibrix/storage/local.py

@@ -17,6 +17,7 @@
 import os
 import shutil
 import tempfile
+import time
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import AsyncIterator, BinaryIO, Optional, TextIO, Union
@@ -173,7 +174,14 @@ def _write_file(self, path: Path, reader: Reader) -> None:
                     f.write(str(reader))
                     f.flush()
                     os.fsync(f.fileno())
-            os.replace(tmp_path, path)
+            for attempt in range(5):
+                try:
+                    os.replace(tmp_path, path)
+                    break
+                except PermissionError:
+                    if attempt == 4:
+                        raise
+                    time.sleep(0.01 * (attempt + 1))
         except Exception:
             try:
                 os.close(fd)

python/aibrix/tests/storage/test_local_storage.py

@@ -40,6 +40,25 @@ async def test_local_storage_initialization(self):
             assert storage.base_path == Path(tmp_dir)
             assert storage.base_path.exists()
 
+    def test_write_retries_when_destination_is_temporarily_locked(self, monkeypatch, tmp_path):
+        storage = LocalStorage(base_path=str(tmp_path))
+        target = tmp_path / "object"
+        attempts = 0
+        real_replace = os.replace
+
+        def replace_with_transient_lock(source, destination):
+            nonlocal attempts
+            attempts += 1
+            if attempts == 1:
+                raise PermissionError("destination is temporarily locked")
+            return real_replace(source, destination)
+
+        monkeypatch.setattr(os, "replace", replace_with_transient_lock)
+        storage._write_file(target, storage._wrap_data("content"))
+
+        assert target.read_text() == "content"
+        assert attempts == 2
+
     @pytest.mark.asyncio
     async def test_environment_variable_override(self):
         """Test that STORAGE_LOCAL_PATH environment variable is respected."""