Internalize run_tepmlates into storage

 All templates will be part of storage and thus for instance restart would
 rely only on templates from the storage. Additionally, this removes templates
 param from create_run_path, but it needs to be specified directly when calling
 create_experiment.
 Templates are stored in the experiment.mount_point / templates folder
 Example: [["templates/seed_template_0.txt", "seed.txt"]]

 It requires storage migration for run_templates.

Author: xjules

src/ert/enkf_main.py

@@ -214,7 +214,6 @@ def create_run_path(
     env_pr_fm_step: dict[str, dict[str, Any]],
     forward_model_steps: list[ForwardModelStep],
     substitutions: Substitutions,
-    templates: list[tuple[str, str]],
     parameters_file: str,
     runpaths: Runpaths,
     context_env: dict[str, str] | None = None,
@@ -226,19 +225,15 @@ def create_run_path(
         run_path = Path(run_arg.runpath)
         if run_arg.active:
             run_path.mkdir(parents=True, exist_ok=True)
-            for source_file, target_file in templates:
+            for (
+                source_file_content,
+                target_file,
+            ) in ensemble.experiment.templates_configuration:
                 target_file = substitutions.substitute_real_iter(
                     target_file, run_arg.iens, ensemble.iteration
                 )
-                try:
-                    file_content = Path(source_file).read_text("utf-8")
-                except UnicodeDecodeError as e:
-                    raise ValueError(
-                        f"Unsupported non UTF-8 character found in file: {source_file}"
-                    ) from e
-
                 result = substitutions.substitute_real_iter(
-                    file_content,
+                    source_file_content,
                     run_arg.iens,
                     ensemble.iteration,
                 )

src/ert/gui/tools/manage_experiments/storage_widget.py

@@ -176,6 +176,7 @@ def _addItem(self) -> None:
                 responses=self._ert_config.ensemble_config.response_configuration,
                 observations=self._ert_config.enkf_obs.datasets,
                 name=create_experiment_dialog.experiment_name,
+                templates=self._ert_config.ert_templates,
             ).create_ensemble(
                 name=create_experiment_dialog.ensemble_name,
                 ensemble_size=self._ensemble_size,

src/ert/run_models/base_run_model.py

@@ -183,7 +183,6 @@ def __init__(
         forward_model_steps: list[ForwardModelStep],
         status_queue: SimpleQueue[StatusEvents],
         substitutions: Substitutions,
-        templates: list[tuple[str, str]],
         hooked_workflows: defaultdict[HookRuntime, list[Workflow]],
         active_realizations: list[bool],
         log_path: Path,
@@ -213,7 +212,6 @@ def __init__(
         self._runpath_file: Path = runpath_file
         self._forward_model_steps: list[ForwardModelStep] = forward_model_steps
         self._user_config_file: Path = user_config_file
-        self._templates: list[tuple[str, str]] = templates
         self._hooked_workflows: defaultdict[HookRuntime, list[Workflow]] = (
             hooked_workflows
         )
@@ -792,7 +790,6 @@ def _evaluate_and_postprocess(
             env_pr_fm_step=self._env_pr_fm_step,
             forward_model_steps=self._forward_model_steps,
             substitutions=self._substitutions,
-            templates=self._templates,
             parameters_file=self._model_config.gen_kw_export_name,
             runpaths=self.run_paths,
             context_env=self._context_env,
@@ -865,7 +862,6 @@ def __init__(
         forward_model_steps: list[ForwardModelStep],
         status_queue: SimpleQueue[StatusEvents],
         substitutions: Substitutions,
-        templates: list[tuple[str, str]],
         hooked_workflows: defaultdict[HookRuntime, list[Workflow]],
         active_realizations: list[bool],
         total_iterations: int,
@@ -888,7 +884,6 @@ def __init__(
             forward_model_steps,
             status_queue,
             substitutions,
-            templates,
             hooked_workflows,
             active_realizations=active_realizations,
             total_iterations=total_iterations,

src/ert/run_models/ensemble_experiment.py

@@ -52,6 +52,7 @@ def __init__(
         self._observations = config.observations
         self._parameter_configuration = config.ensemble_config.parameter_configuration
         self._response_configuration = config.ensemble_config.response_configuration
+        self._templates = config.ert_templates
 
         super().__init__(
             storage,
@@ -64,7 +65,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             total_iterations=1,
             log_path=config.analysis_config.log_path,
@@ -107,6 +107,7 @@ def run_experiment(
                 ),
                 observations=self._observations,
                 responses=self._response_configuration,
+                templates=self._templates,
             )
             self.ensemble = self._storage.create_ensemble(
                 self.experiment,

src/ert/run_models/ensemble_smoother.py

@@ -53,7 +53,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             active_realizations=active_realizations,
             start_iteration=0,
@@ -71,6 +70,7 @@ def __init__(
         self._design_matrix = config.analysis_config.design_matrix
         self._observations = config.observations
         self._response_configuration = config.ensemble_config.response_configuration
+        self._templates = config.ert_templates
 
     @tracer.start_as_current_span(f"{__name__}.run_experiment")
     def run_experiment(
@@ -104,6 +104,7 @@ def run_experiment(
             observations=self._observations,
             responses=self._response_configuration,
             name=self.experiment_name,
+            templates=self._templates,
         )
 
         self.set_env_key("_ERT_EXPERIMENT_ID", str(experiment.id))

src/ert/run_models/evaluate_ensemble.py

@@ -60,7 +60,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             start_iteration=self.ensemble.iteration,
             total_iterations=1,

src/ert/run_models/everest_run_model.py

@@ -195,6 +195,7 @@ def __init__(
         self._parameter_configuration = ensemble_config.parameter_configuration
         self._parameter_configs = ensemble_config.parameter_configs
         self._response_configuration = ensemble_config.response_configuration
+        self._templates = ert_templates
 
         super().__init__(
             storage,
@@ -207,7 +208,6 @@ def __init__(
             forward_model_steps,
             status_queue,
             substitutions,
-            ert_templates,
             hooked_workflows,
             random_seed=123,  # No-op as far as Everest is concerned
             active_realizations=[],  # Set dynamically in run_forward_model()

src/ert/run_models/manual_update.py

@@ -56,7 +56,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             active_realizations=active_realizations,
             total_iterations=1,

src/ert/run_models/multiple_data_assimilation.py

@@ -86,7 +86,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             active_realizations=active_realizations,
             total_iterations=total_iterations,
@@ -99,6 +98,7 @@ def __init__(
         self._observations = config.observations
         self._parameter_configuration = config.ensemble_config.parameter_configuration
         self._response_configuration = config.ensemble_config.response_configuration
+        self._templates = config.ert_templates
 
     @tracer.start_as_current_span(f"{__name__}.run_experiment")
     def run_experiment(
@@ -154,6 +154,7 @@ def run_experiment(
                 responses=self._response_configuration,
                 simulation_arguments=sim_args,
                 name=self.experiment_name,
+                templates=self._templates,
             )
 
             prior = self._storage.create_ensemble(

src/ert/storage/local_experiment.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import shutil
 from collections.abc import Generator
 from datetime import datetime
 from functools import cached_property
@@ -47,6 +48,7 @@ class LocalExperiment(BaseMode):
     _parameter_file = Path("parameter.json")
     _responses_file = Path("responses.json")
     _metadata_file = Path("metadata.json")
+    _templates_file = Path("templates.json")
 
     def __init__(
         self,
@@ -86,6 +88,7 @@ def create(
         observations: dict[str, pl.DataFrame] | None = None,
         simulation_arguments: dict[Any, Any] | None = None,
         name: str | None = None,
+        templates: list[tuple[str, str]] | None = None,
     ) -> LocalExperiment:
         """
         Create a new LocalExperiment and store its configuration data.
@@ -108,6 +111,8 @@ def create(
             Simulation arguments for the experiment.
         name : str, optional
             Experiment name. Defaults to current date if None.
+        templates : list of tuple[str, str], optional
+            Run templates for the experiment. Defaults to None.
 
         Returns
         -------
@@ -130,6 +135,23 @@ def create(
             json.dumps(parameter_data, indent=2).encode("utf-8"),
         )
 
+        if templates:
+            templates_path = path / "templates"
+            templates_path.mkdir(parents=True, exist_ok=True)
+            templates_abs: list[tuple[str, str]] = []
+            for idx, (src, dst) in enumerate(templates):
+                incoming_template = Path(src)
+                template_file_path = (
+                    templates_path
+                    / f"{incoming_template.stem}_{idx}{incoming_template.suffix}"
+                )
+                shutil.copyfile(incoming_template, template_file_path)
+                templates_abs.append((str(template_file_path.relative_to(path)), dst))
+            storage._write_transaction(
+                path / cls._templates_file,
+                json.dumps(templates_abs).encode("utf-8"),
+            )
+
         response_data = {}
         for response in responses or []:
             response_data.update({response.response_type: response.to_dict()})
@@ -248,6 +270,27 @@ def parameter_info(self) -> dict[str, Any]:
             info = json.load(f)
         return info
 
+    @property
+    def templates_configuration(self) -> list[tuple[str, str]]:
+        try:
+            templates: list[tuple[str, str]] = []
+            with open(self.mount_point / self._templates_file, encoding="utf-8") as f:
+                templates = json.load(f)
+            templates_with_content: list[tuple[str, str]] = []
+            for source_file, target_file in templates:
+                try:
+                    file_content = (self.mount_point / source_file).read_text("utf-8")
+                    templates_with_content.append((file_content, target_file))
+                except UnicodeDecodeError as e:
+                    raise ValueError(
+                        f"Unsupported non UTF-8 character found in file: {source_file}"
+                    ) from e
+                return templates_with_content
+        except (FileNotFoundError, json.JSONDecodeError):
+            pass
+            # If the file is missing or broken, we return an empty list
+        return []
+
     @property
     def response_info(self) -> dict[str, Any]:
         info: dict[str, Any]

src/ert/storage/local_storage.py

@@ -307,6 +307,7 @@ def create_experiment(
         observations: dict[str, pl.DataFrame] | None = None,
         simulation_arguments: dict[Any, Any] | None = None,
         name: str | None = None,
+        templates: list[tuple[str, str]] | None = None,
     ) -> LocalExperiment:
         """
         Creates a new experiment in the storage.
@@ -323,6 +324,8 @@ def create_experiment(
             The simulation arguments for the experiment.
         name : str, optional
             The name of the experiment.
+        templates : list of tuple[str, str], optional
+            Run templates for the experiment. Defaults to None.
 
         Returns
         -------
@@ -343,6 +346,7 @@ def create_experiment(
             observations=observations,
             simulation_arguments=simulation_arguments,
             name=name,
+            templates=templates,
         )
 
         self._experiments[exp.id] = exp
@@ -459,6 +463,7 @@ def _migrate(self, version: int) -> None:
             to7,
             to8,
             to9,
+            to10,
         )
 
         try:
@@ -478,7 +483,8 @@ def _migrate(self, version: int) -> None:
 
                 logger.info("Blockfs storage backed up")
                 print(
-                    dedent(f"""
+                    dedent(
+                        f"""
                     Detected outdated storage (blockfs), which is no longer supported
                     by ERT. Its contents are copied to:
 
@@ -497,13 +503,14 @@ def _migrate(self, version: int) -> None:
                     This is not guaranteed to work. Other than setting the custom
                     ENSPATH, the ERT config should ideally be the same as it was
                     when the old blockfs storage was created.
-                """)
+                """
+                    )
                 )
                 return None
 
             elif version < _LOCAL_STORAGE_VERSION:
                 migrations = list(
-                    enumerate([to2, to3, to4, to5, to6, to7, to8, to9], start=1)
+                    enumerate([to2, to3, to4, to5, to6, to7, to8, to9, to10], start=1)
                 )
                 for from_version, migration in migrations[version - 1 :]:
                     print(f"* Updating storage to version: {from_version + 1}")

src/ert/storage/migration/to10.py

@@ -0,0 +1,25 @@
+import json
+import shutil
+from pathlib import Path
+
+from ert.storage.local_storage import local_storage_get_ert_config
+
+info = "Internalizes run templates in to the storage"
+
+
+def migrate(path: Path) -> None:
+    ert_config = local_storage_get_ert_config()
+    if ert_config.ert_templates:
+        for experiment in path.glob("experiments/*"):
+            templates_abs: list[tuple[str, str]] = []
+            templates_path = experiment / "templates"
+            templates_path.mkdir(parents=True, exist_ok=True)
+            for src, dst in ert_config.ert_templates:
+                incoming_template_file_path = Path(src)
+                template_file_path = Path(
+                    templates_path / incoming_template_file_path.name
+                )
+                shutil.copyfile(incoming_template_file_path, template_file_path)
+                templates_abs.append((str(template_file_path.resolve()), dst))
+            with open(experiment / "templates.json", "w", encoding="utf-8") as fout:
+                fout.write(json.dumps(templates_abs))

tests/ert/unit_tests/config/test_gen_kw_config.py

@@ -235,7 +235,6 @@ def test_gen_kw_is_log_or_not(
             env_vars=ert_config.env_vars,
             env_pr_fm_step=ert_config.env_pr_fm_step,
             substitutions=ert_config.substitutions,
-            templates=ert_config.ert_templates,
             parameters_file="parameters",
         )
         assert re.match(

tests/ert/unit_tests/conftest.py

@@ -33,6 +33,19 @@ def prior_ensemble(storage):
     return storage.create_ensemble(experiment_id, name="prior", ensemble_size=100)
 
 
+@pytest.fixture
+def prior_ensemble_args(storage):
+    def _create_prior_ensemble(
+        ensemble_name="prior", ensemble_size=100, **experiment_params
+    ):
+        experiment_id = storage.create_experiment(**experiment_params)
+        return storage.create_ensemble(
+            experiment_id, name=ensemble_name, ensemble_size=ensemble_size
+        )
+
+    return _create_prior_ensemble
+
+
 @pytest.fixture
 def run_paths():
     def func(ert_config: ErtConfig):

tests/ert/unit_tests/run_models/test_base_run_model.py

@@ -39,7 +39,6 @@ def create_base_run_model(**kwargs):
         "forward_model_steps": MagicMock(spec=dict),
         "status_queue": MagicMock(spec=SimpleQueue),
         "substitutions": MagicMock(spec=Substitutions),
-        "templates": MagicMock(spec=dict),
         "hooked_workflows": MagicMock(spec=dict),
         "active_realizations": MagicMock(spec=list),
         "random_seed": 123,

tests/ert/unit_tests/storage/create_runpath.py

@@ -48,7 +48,6 @@ def create_runpath(
         env_pr_fm_step=ert_config.env_pr_fm_step,
         forward_model_steps=ert_config.forward_model_steps,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=runpaths,
     )