Skip recreating ensemble directory in storage

When writing errors.json, we should not recreate the ensemble directory
in storage if it does not exist. Then the user has probably deleted it
manually while Ert is running, and whatever happens is undefined
behaviour.

The consequence of recreating the directory is essentially a garbled
storage directory that Ert later will fail to load.

Keep the possibility that the realization directory has not been written
yet, as it (at least in tests) set_failure is called immediately on a
realization after the ensemble is created in storage.

(cherry picked from commit 54c689a)

Author: berland

src/ert/storage/local_ensemble.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import logging
-import os
 import time
 from collections.abc import Iterable
 from datetime import datetime
@@ -325,8 +324,14 @@ def set_failure(
         """
 
         filename: Path = self._realization_dir(realization) / self._error_log_name
-        os.makedirs(os.path.dirname(filename), exist_ok=True)
         error = _Failure(type=failure_type, message=message or "", time=datetime.now())
+        if not filename.parent.parent.exists():
+            logger.warning(
+                f"Storage {filename.parent.parent} does not exist, "
+                f"skipping writing {error} to {filename}"
+            )
+            return
+        filename.parent.mkdir(exist_ok=True)
         self._storage._write_transaction(
             filename, error.model_dump_json(indent=2).encode("utf-8")
         )

tests/ert/unit_tests/storage/test_local_storage.py

@@ -1084,6 +1084,35 @@ def test_data_fetching_missing_key(snake_oil_case):
             assert dataframe.empty
 
 
+def test_set_failure_will_create_realization_directory(storage):
+    # Setup
+    dummy_ensemble = storage.create_experiment().create_ensemble(
+        name="dummy", ensemble_size=1
+    )
+    assert dummy_ensemble._path.exists(), "Assumptions for test has changed"
+    assert not (dummy_ensemble._path / "realization-0").exists()
+
+    # Function test:
+    dummy_ensemble.set_failure(0, RealizationStorageState.FAILURE_IN_CURRENT)
+
+    # Then
+    assert dummy_ensemble._path.glob(f"realization-0/{dummy_ensemble._error_log_name}")
+
+
+def test_set_failure_will_not_recreate_ensemble_directory(storage):
+    dummy_ensemble = storage.create_experiment().create_ensemble(
+        name="dummy", ensemble_size=1
+    )
+    # Emulate a user deleting storage:
+    shutil.rmtree(dummy_ensemble._path)
+
+    # Then when a realization fails internalizing,
+    dummy_ensemble.set_failure(0, RealizationStorageState.FAILURE_IN_CURRENT)
+
+    # the ensemble path will not be created
+    assert not dummy_ensemble._path.exists()
+
+
 @dataclass
 class Ensemble:
     uuid: UUID