Internalize run_tepmlates into storage

 All templates will be part of storage and thus restart could use only
 the storage templates. Additionally, this removes templates param from create_run_path,
 but it needs to be specified directly when calling create_experiment.

 It requires storage migration for run_templates.

Author: xjules

src/ert/enkf_main.py

@@ -214,7 +214,6 @@ def create_run_path(
     env_pr_fm_step: dict[str, dict[str, Any]],
     forward_model_steps: list[ForwardModelStep],
     substitutions: Substitutions,
-    templates: list[tuple[str, str]],
     parameters_file: str,
     runpaths: Runpaths,
     context_env: dict[str, str] | None = None,
@@ -226,7 +225,7 @@ def create_run_path(
         run_path = Path(run_arg.runpath)
         if run_arg.active:
             run_path.mkdir(parents=True, exist_ok=True)
-            for source_file, target_file in templates:
+            for source_file, target_file in ensemble.experiment.templates_configuration:
                 target_file = substitutions.substitute_real_iter(
                     target_file, run_arg.iens, ensemble.iteration
                 )

src/ert/gui/tools/manage_experiments/storage_widget.py

@@ -176,6 +176,7 @@ def _addItem(self) -> None:
                 responses=self._ert_config.ensemble_config.response_configuration,
                 observations=self._ert_config.enkf_obs.datasets,
                 name=create_experiment_dialog.experiment_name,
+                templates=self._ert_config.ert_templates,
             ).create_ensemble(
                 name=create_experiment_dialog.ensemble_name,
                 ensemble_size=self._ensemble_size,

src/ert/run_models/base_run_model.py

@@ -71,9 +71,7 @@
 from ert.utils import log_duration
 from ert.workflow_runner import WorkflowRunner
 
-from ..plugins.workflow_fixtures import (
-    create_workflow_fixtures_from_hooked,
-)
+from ..plugins.workflow_fixtures import create_workflow_fixtures_from_hooked
 from ..run_arg import RunArg
 from .event import (
     AnalysisStatusEvent,
@@ -777,7 +775,6 @@ def _evaluate_and_postprocess(
             env_pr_fm_step=self._env_pr_fm_step,
             forward_model_steps=self._forward_model_steps,
             substitutions=self._substitutions,
-            templates=self._templates,
             parameters_file=self._model_config.gen_kw_export_name,
             runpaths=self.run_paths,
             context_env=self._context_env,

src/ert/run_models/ensemble_experiment.py

@@ -107,6 +107,7 @@ def run_experiment(
                 ),
                 observations=self._observations,
                 responses=self._response_configuration,
+                templates=self._templates,
             )
             self.ensemble = self._storage.create_ensemble(
                 self.experiment,

src/ert/run_models/ensemble_smoother.py

@@ -104,6 +104,7 @@ def run_experiment(
             observations=self._observations,
             responses=self._response_configuration,
             name=self.experiment_name,
+            templates=self._templates,
         )
 
         self.set_env_key("_ERT_EXPERIMENT_ID", str(experiment.id))

src/ert/run_models/multiple_data_assimilation.py

@@ -154,6 +154,7 @@ def run_experiment(
                 responses=self._response_configuration,
                 simulation_arguments=sim_args,
                 name=self.experiment_name,
+                ert_templates=self._templates,
             )
 
             prior = self._storage.create_ensemble(

src/ert/storage/local_experiment.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import shutil
 from collections.abc import Generator
 from datetime import datetime
 from functools import cached_property
@@ -47,6 +48,7 @@ class LocalExperiment(BaseMode):
     _parameter_file = Path("parameter.json")
     _responses_file = Path("responses.json")
     _metadata_file = Path("metadata.json")
+    _templates_file = Path("templates.json")
 
     def __init__(
         self,
@@ -86,6 +88,7 @@ def create(
         observations: dict[str, pl.DataFrame] | None = None,
         simulation_arguments: dict[Any, Any] | None = None,
         name: str | None = None,
+        templates: list[tuple[str, str]] | None = None,
     ) -> LocalExperiment:
         """
         Create a new LocalExperiment and store its configuration data.
@@ -108,6 +111,8 @@ def create(
             Simulation arguments for the experiment.
         name : str, optional
             Experiment name. Defaults to current date if None.
+        templates : list of tuple[str, str], optional
+            Run templates for the experiment. Defaults to None.
 
         Returns
         -------
@@ -130,6 +135,18 @@ def create(
             json.dumps(parameter_data, indent=2).encode("utf-8"),
         )
 
+        if templates:
+            templates_abs: list[tuple[str, str]] = []
+            for src, dst in templates:
+                incoming_template_file_path = Path(src)
+                template_file_path = Path(path / incoming_template_file_path.name)
+                shutil.copyfile(incoming_template_file_path, template_file_path)
+                templates_abs.append((str(template_file_path.resolve()), dst))
+            storage._write_transaction(
+                path / cls._templates_file,
+                json.dumps(templates_abs).encode("utf-8"),
+            )
+
         response_data = {}
         for response in responses or []:
             response_data.update({response.response_type: response.to_dict()})
@@ -248,6 +265,16 @@ def parameter_info(self) -> dict[str, Any]:
             info = json.load(f)
         return info
 
+    @cached_property
+    def templates_configuration(self) -> list[tuple[str, str]]:
+        try:
+            with open(self.mount_point / self._templates_file, encoding="utf-8") as f:
+                return json.load(f)
+        except (FileNotFoundError, json.JSONDecodeError):
+            pass
+            # If the file is missing or broken, we return an empty list
+        return []
+
     @property
     def response_info(self) -> dict[str, Any]:
         info: dict[str, Any]

src/ert/storage/local_storage.py

@@ -307,6 +307,7 @@ def create_experiment(
         observations: dict[str, pl.DataFrame] | None = None,
         simulation_arguments: dict[Any, Any] | None = None,
         name: str | None = None,
+        templates: list[tuple[str, str]] | None = None,
     ) -> LocalExperiment:
         """
         Creates a new experiment in the storage.
@@ -323,6 +324,8 @@ def create_experiment(
             The simulation arguments for the experiment.
         name : str, optional
             The name of the experiment.
+        templates : list of tuple[str, str], optional
+            Run templates for the experiment. Defaults to None.
 
         Returns
         -------
@@ -343,6 +346,7 @@ def create_experiment(
             observations=observations,
             simulation_arguments=simulation_arguments,
             name=name,
+            templates=templates,
         )
 
         self._experiments[exp.id] = exp
@@ -459,6 +463,7 @@ def _migrate(self, version: int) -> None:
             to7,
             to8,
             to9,
+            to10,
         )
 
         try:
@@ -478,7 +483,8 @@ def _migrate(self, version: int) -> None:
 
                 logger.info("Blockfs storage backed up")
                 print(
-                    dedent(f"""
+                    dedent(
+                        f"""
                     Detected outdated storage (blockfs), which is no longer supported
                     by ERT. Its contents are copied to:
 
@@ -497,13 +503,14 @@ def _migrate(self, version: int) -> None:
                     This is not guaranteed to work. Other than setting the custom
                     ENSPATH, the ERT config should ideally be the same as it was
                     when the old blockfs storage was created.
-                """)
+                """
+                    )
                 )
                 return None
 
             elif version < _LOCAL_STORAGE_VERSION:
                 migrations = list(
-                    enumerate([to2, to3, to4, to5, to6, to7, to8, to9], start=1)
+                    enumerate([to2, to3, to4, to5, to6, to7, to8, to9, to10], start=1)
                 )
                 for from_version, migration in migrations[version - 1 :]:
                     print(f"* Updating storage to version: {from_version + 1}")

src/ert/storage/migration/to10.py

@@ -0,0 +1,21 @@
+import json
+import shutil
+from pathlib import Path
+
+from ert.storage.local_storage import local_storage_get_ert_config
+
+info = "Internalizes run templates in to the storage"
+
+
+def migrate(path: Path) -> None:
+    ert_config = local_storage_get_ert_config()
+    if ert_config.ert_templates:
+        for experiment in path.glob("experiments/*"):
+            templates_abs: list[tuple[str, str]] = []
+            for src, dst in ert_config.ert_templates:
+                incoming_template_file_path = Path(src)
+                template_file_path = Path(path / incoming_template_file_path.name)
+                shutil.copyfile(incoming_template_file_path, template_file_path)
+                templates_abs.append((str(template_file_path.resolve()), dst))
+            with open(experiment / "templates.json", "w", encoding="utf-8") as fout:
+                fout.write(json.dumps(templates_abs))

tests/ert/unit_tests/config/test_gen_kw_config.py

@@ -7,12 +7,7 @@
 import pytest
 from lark import Token
 
-from ert.config import (
-    ConfigValidationError,
-    ConfigWarning,
-    ErtConfig,
-    GenKwConfig,
-)
+from ert.config import ConfigValidationError, ConfigWarning, ErtConfig, GenKwConfig
 from ert.config.gen_kw_config import TransformFunctionDefinition
 from ert.config.parsing import ContextString
 from ert.config.parsing.file_context_token import FileContextToken
@@ -235,7 +230,6 @@ def test_gen_kw_is_log_or_not(
             env_vars=ert_config.env_vars,
             env_pr_fm_step=ert_config.env_pr_fm_step,
             substitutions=ert_config.substitutions,
-            templates=ert_config.ert_templates,
             parameters_file="parameters",
         )
         assert re.match(

tests/ert/unit_tests/conftest.py

@@ -33,6 +33,19 @@ def prior_ensemble(storage):
     return storage.create_ensemble(experiment_id, name="prior", ensemble_size=100)
 
 
+@pytest.fixture
+def prior_ensemble_args(storage):
+    def _create_prior_ensemble(
+        ensemble_name="prior", ensemble_size=100, **experiment_params
+    ):
+        experiment_id = storage.create_experiment(**experiment_params)
+        return storage.create_ensemble(
+            experiment_id, name=ensemble_name, ensemble_size=ensemble_size
+        )
+
+    return _create_prior_ensemble
+
+
 @pytest.fixture
 def run_paths():
     def func(ert_config: ErtConfig):

tests/ert/unit_tests/storage/create_runpath.py

@@ -48,7 +48,6 @@ def create_runpath(
         env_pr_fm_step=ert_config.env_pr_fm_step,
         forward_model_steps=ert_config.forward_model_steps,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=runpaths,
     )

tests/ert/unit_tests/test_enkf_main.py

@@ -90,7 +90,6 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage, run_paths):
         env_vars=ert_config.env_vars,
         env_pr_fm_step=ert_config.env_pr_fm_step,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=runpaths,
     )
@@ -112,7 +111,6 @@ def test_assert_symlink_deleted(snake_oil_field_example, storage, run_paths):
         env_vars=ert_config.env_vars,
         env_pr_fm_step=ert_config.env_pr_fm_step,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=runpaths,
     )

tests/ert/unit_tests/test_load_forward_model.py

@@ -35,7 +35,6 @@ def func(config_text):
             env_pr_fm_step=ert_config.env_pr_fm_step,
             forward_model_steps=ert_config.forward_model_steps,
             substitutions=ert_config.substitutions,
-            templates=ert_config.ert_templates,
             parameters_file="parameters",
             runpaths=run_paths(ert_config),
         )
@@ -148,7 +147,6 @@ def test_load_forward_model_summary(
         env_pr_fm_step=ert_config.env_pr_fm_step,
         forward_model_steps=ert_config.forward_model_steps,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=run_paths(ert_config),
     )
@@ -253,7 +251,6 @@ def test_loading_gen_data_without_restart(storage, run_paths, run_args):
         env_pr_fm_step=ert_config.env_pr_fm_step,
         forward_model_steps=ert_config.forward_model_steps,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=run_paths(ert_config),
     )
@@ -318,7 +315,6 @@ def test_loading_from_any_available_iter(storage, run_paths, run_args, itr):
         env_pr_fm_step=ert_config.env_pr_fm_step,
         forward_model_steps=ert_config.forward_model_steps,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=run_paths(ert_config),
     )

tests/ert/unit_tests/test_run_path_creation.py

@@ -36,7 +36,8 @@
 def make_run_path(run_paths, run_args, storage):
     def func(ert_config):
         experiment_id = storage.create_experiment(
-            parameters=ert_config.ensemble_config.parameter_configuration
+            parameters=ert_config.ensemble_config.parameter_configuration,
+            templates=ert_config.ert_templates,
         )
         prior_ensemble = storage.create_ensemble(
             experiment_id, name="prior", ensemble_size=1
@@ -52,7 +53,6 @@ def func(ert_config):
             env_vars=ert_config.env_vars,
             env_pr_fm_step=ert_config.env_pr_fm_step,
             substitutions=ert_config.substitutions,
-            templates=ert_config.ert_templates,
             parameters_file="parameters",
             runpaths=runpaths,
         )
@@ -101,7 +101,7 @@ def test_jobs_json_is_backed_up(make_run_path):
 
 @pytest.mark.usefixtures("use_tmpdir")
 def test_that_run_template_replace_symlink_does_not_write_to_source(
-    prior_ensemble, run_args, run_paths
+    prior_ensemble_args, run_args, run_paths
 ):
     """This test is meant to test that we can have a symlinked file in the
     run path before we do replacement on a target file with the same name,
@@ -121,6 +121,7 @@ def test_that_run_template_replace_symlink_does_not_write_to_source(
             """
         )
     )
+    prior_ensemble = prior_ensemble_args(templates=ert_config.ert_templates)
     run_arg = run_args(ert_config, prior_ensemble)
     run_path = Path(run_arg[0].runpath)
     os.makedirs(run_path)
@@ -138,7 +139,6 @@ def test_that_run_template_replace_symlink_does_not_write_to_source(
         env_pr_fm_step=ert_config.env_pr_fm_step,
         forward_model_steps=ert_config.forward_model_steps,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=run_paths(ert_config),
     )
@@ -468,7 +468,6 @@ def test_write_runpath_file(storage, itr, run_paths):
         env_pr_fm_step=ert_config.env_pr_fm_step,
         forward_model_steps=ert_config.forward_model_steps,
         substitutions=ert_config.substitutions,
-        templates=ert_config.ert_templates,
         parameters_file="parameters",
         runpaths=run_path,
     )
@@ -735,7 +734,6 @@ def test_when_manifest_files_are_written_forward_model_ok_succeeds(storage, itr)
         env_pr_fm_step=config.env_pr_fm_step,
         forward_model_steps=config.forward_model_steps,
         substitutions=config.substitutions,
-        templates=config.ert_templates,
         parameters_file="parameters",
         runpaths=run_paths,
     )

tests/ert/unit_tests/test_summary_response.py

@@ -47,7 +47,6 @@ def test_load_summary_response_restart_not_zero(
             env_vars=ert_config.env_vars,
             env_pr_fm_step=ert_config.env_pr_fm_step,
             substitutions=ert_config.substitutions,
-            templates=ert_config.ert_templates,
             parameters_file="parameters",
             runpaths=run_paths(ert_config),
         )