Internalize run_tepmlates into storage

 All templates will be part of storage and thus for instance restart would
 rely only on templates from the storage. Additionally, this removes templates
 param from create_run_path, but it needs to be specified directly when calling
 create_experiment.

 It requires storage migration for run_templates.

Author: xjules

src/ert/enkf_main.py

@@ -214,7 +214,6 @@ def create_run_path(
     env_pr_fm_step: dict[str, dict[str, Any]],
     forward_model_steps: list[ForwardModelStep],
     substitutions: Substitutions,
-    templates: list[tuple[str, str]],
     parameters_file: str,
     runpaths: Runpaths,
     context_env: dict[str, str] | None = None,
@@ -226,7 +225,7 @@ def create_run_path(
         run_path = Path(run_arg.runpath)
         if run_arg.active:
             run_path.mkdir(parents=True, exist_ok=True)
-            for source_file, target_file in templates:
+            for source_file, target_file in ensemble.experiment.templates_configuration:
                 target_file = substitutions.substitute_real_iter(
                     target_file, run_arg.iens, ensemble.iteration
                 )

src/ert/gui/tools/manage_experiments/storage_widget.py

@@ -176,6 +176,7 @@ def _addItem(self) -> None:
                 responses=self._ert_config.ensemble_config.response_configuration,
                 observations=self._ert_config.enkf_obs.datasets,
                 name=create_experiment_dialog.experiment_name,
+                templates=self._ert_config.ert_templates,
             ).create_ensemble(
                 name=create_experiment_dialog.ensemble_name,
                 ensemble_size=self._ensemble_size,

src/ert/run_models/base_run_model.py

@@ -72,9 +72,7 @@
 from ert.utils import log_duration
 from ert.workflow_runner import WorkflowRunner
 
-from ..plugins.workflow_fixtures import (
-    create_workflow_fixtures_from_hooked,
-)
+from ..plugins.workflow_fixtures import create_workflow_fixtures_from_hooked
 from ..run_arg import RunArg
 from .event import (
     AnalysisStatusEvent,
@@ -183,7 +181,6 @@ def __init__(
         forward_model_steps: list[ForwardModelStep],
         status_queue: SimpleQueue[StatusEvents],
         substitutions: Substitutions,
-        templates: list[tuple[str, str]],
         hooked_workflows: defaultdict[HookRuntime, list[Workflow]],
         active_realizations: list[bool],
         log_path: Path,
@@ -213,7 +210,6 @@ def __init__(
         self._runpath_file: Path = runpath_file
         self._forward_model_steps: list[ForwardModelStep] = forward_model_steps
         self._user_config_file: Path = user_config_file
-        self._templates: list[tuple[str, str]] = templates
         self._hooked_workflows: defaultdict[HookRuntime, list[Workflow]] = (
             hooked_workflows
         )
@@ -788,7 +784,6 @@ def _evaluate_and_postprocess(
             env_pr_fm_step=self._env_pr_fm_step,
             forward_model_steps=self._forward_model_steps,
             substitutions=self._substitutions,
-            templates=self._templates,
             parameters_file=self._model_config.gen_kw_export_name,
             runpaths=self.run_paths,
             context_env=self._context_env,
@@ -861,7 +856,6 @@ def __init__(
         forward_model_steps: list[ForwardModelStep],
         status_queue: SimpleQueue[StatusEvents],
         substitutions: Substitutions,
-        templates: list[tuple[str, str]],
         hooked_workflows: defaultdict[HookRuntime, list[Workflow]],
         active_realizations: list[bool],
         total_iterations: int,
@@ -884,7 +878,6 @@ def __init__(
             forward_model_steps,
             status_queue,
             substitutions,
-            templates,
             hooked_workflows,
             active_realizations=active_realizations,
             total_iterations=total_iterations,

src/ert/run_models/ensemble_experiment.py

@@ -52,6 +52,7 @@ def __init__(
         self._observations = config.observations
         self._parameter_configuration = config.ensemble_config.parameter_configuration
         self._response_configuration = config.ensemble_config.response_configuration
+        self._templates = config.ert_templates
 
         super().__init__(
             storage,
@@ -64,7 +65,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             total_iterations=1,
             log_path=config.analysis_config.log_path,
@@ -107,6 +107,7 @@ def run_experiment(
                 ),
                 observations=self._observations,
                 responses=self._response_configuration,
+                templates=self._templates,
             )
             self.ensemble = self._storage.create_ensemble(
                 self.experiment,

src/ert/run_models/ensemble_smoother.py

@@ -53,7 +53,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             active_realizations=active_realizations,
             start_iteration=0,
@@ -71,6 +70,7 @@ def __init__(
         self._design_matrix = config.analysis_config.design_matrix
         self._observations = config.observations
         self._response_configuration = config.ensemble_config.response_configuration
+        self._templates = config.ert_templates
 
     @tracer.start_as_current_span(f"{__name__}.run_experiment")
     def run_experiment(
@@ -104,6 +104,7 @@ def run_experiment(
             observations=self._observations,
             responses=self._response_configuration,
             name=self.experiment_name,
+            templates=self._templates,
         )
 
         self.set_env_key("_ERT_EXPERIMENT_ID", str(experiment.id))

src/ert/run_models/evaluate_ensemble.py

@@ -60,7 +60,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             start_iteration=self.ensemble.iteration,
             total_iterations=1,

src/ert/run_models/everest_run_model.py

@@ -37,10 +37,7 @@
 from ert.storage import open_storage
 from everest.config import ControlConfig, ControlVariableGuessListConfig, EverestConfig
 from everest.config.utils import FlattenedControls
-from everest.everest_storage import (
-    EverestStorage,
-    OptimalResult,
-)
+from everest.everest_storage import EverestStorage, OptimalResult
 from everest.optimizer.everest2ropt import everest2ropt
 from everest.optimizer.opt_model_transforms import (
     ConstraintScaler,
@@ -59,11 +56,7 @@
 from ..run_arg import RunArg, create_run_arguments
 from ..storage.local_ensemble import EverestRealizationInfo
 from .base_run_model import BaseRunModel, StatusEvents
-from .event import (
-    EverestBatchResultEvent,
-    EverestCacheHitEvent,
-    EverestStatusEvent,
-)
+from .event import EverestBatchResultEvent, EverestCacheHitEvent, EverestStatusEvent
 
 if TYPE_CHECKING:
     from ert.storage import Ensemble, Experiment
@@ -192,6 +185,7 @@ def __init__(
         self._parameter_configuration = ensemble_config.parameter_configuration
         self._parameter_configs = ensemble_config.parameter_configs
         self._response_configuration = ensemble_config.response_configuration
+        self._templates = ert_templates
 
         super().__init__(
             storage,
@@ -204,7 +198,6 @@ def __init__(
             forward_model_steps,
             status_queue,
             substitutions,
-            ert_templates,
             hooked_workflows,
             random_seed=123,  # No-op as far as Everest is concerned
             active_realizations=[],  # Set dynamically in run_forward_model()
@@ -298,9 +291,11 @@ def _handle_optimizer_results(self, results: tuple[Results, ...]) -> None:
                 EverestBatchResultEvent(
                     batch=r.batch_id,
                     everest_event="OPTIMIZATION_RESULT",
-                    result_type="FunctionResult"
-                    if isinstance(r, FunctionResults)
-                    else "GradientResult",
+                    result_type=(
+                        "FunctionResult"
+                        if isinstance(r, FunctionResults)
+                        else "GradientResult"
+                    ),
                     results=batch_data.to_dict() if batch_data else None,
                 )
             )
@@ -692,9 +687,11 @@ def _forward_model_evaluator(
         evaluation_infos, cache_hits_df = self._create_evaluation_infos(
             control_values=control_values,
             model_realizations=model_realizations,
-            perturbations=evaluator_context.perturbations.tolist()
-            if evaluator_context.perturbations is not None
-            else [-1] * len(model_realizations),
+            perturbations=(
+                evaluator_context.perturbations.tolist()
+                if evaluator_context.perturbations is not None
+                else [-1] * len(model_realizations)
+            ),
             active_controls=active_control_vectors,
             control_names=self._everest_config.formatted_control_names,
             objective_names=self._everest_config.objective_names,

src/ert/run_models/manual_update.py

@@ -56,7 +56,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             active_realizations=active_realizations,
             total_iterations=1,

src/ert/run_models/multiple_data_assimilation.py

@@ -8,12 +8,7 @@
 
 import numpy as np
 
-from ert.config import (
-    ConfigValidationError,
-    ErtConfig,
-    ESSettings,
-    UpdateSettings,
-)
+from ert.config import ConfigValidationError, ErtConfig, ESSettings, UpdateSettings
 from ert.enkf_main import sample_prior, save_design_matrix_to_ensemble
 from ert.ensemble_evaluator import EvaluatorServerConfig
 from ert.storage import Ensemble, Storage
@@ -86,7 +81,6 @@ def __init__(
             config.forward_model_steps,
             status_queue,
             config.substitutions,
-            config.ert_templates,
             config.hooked_workflows,
             active_realizations=active_realizations,
             total_iterations=total_iterations,
@@ -99,6 +93,7 @@ def __init__(
         self._observations = config.observations
         self._parameter_configuration = config.ensemble_config.parameter_configuration
         self._response_configuration = config.ensemble_config.response_configuration
+        self._templates = config.ert_templates
 
     @tracer.start_as_current_span(f"{__name__}.run_experiment")
     def run_experiment(
@@ -154,6 +149,7 @@ def run_experiment(
                 responses=self._response_configuration,
                 simulation_arguments=sim_args,
                 name=self.experiment_name,
+                templates=self._templates,
             )
 
             prior = self._storage.create_ensemble(

src/ert/storage/local_experiment.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import shutil
 from collections.abc import Generator
 from datetime import datetime
 from functools import cached_property
@@ -47,6 +48,7 @@ class LocalExperiment(BaseMode):
     _parameter_file = Path("parameter.json")
     _responses_file = Path("responses.json")
     _metadata_file = Path("metadata.json")
+    _templates_file = Path("templates.json")
 
     def __init__(
         self,
@@ -86,6 +88,7 @@ def create(
         observations: dict[str, pl.DataFrame] | None = None,
         simulation_arguments: dict[Any, Any] | None = None,
         name: str | None = None,
+        templates: list[tuple[str, str]] | None = None,
     ) -> LocalExperiment:
         """
         Create a new LocalExperiment and store its configuration data.
@@ -108,6 +111,8 @@ def create(
             Simulation arguments for the experiment.
         name : str, optional
             Experiment name. Defaults to current date if None.
+        templates : list of tuple[str, str], optional
+            Run templates for the experiment. Defaults to None.
 
         Returns
         -------
@@ -130,6 +135,22 @@ def create(
             json.dumps(parameter_data, indent=2).encode("utf-8"),
         )
 
+        if templates:
+            templates_path = path / "templates"
+            templates_path.mkdir(parents=True, exist_ok=True)
+            templates_abs: list[tuple[str, str]] = []
+            for src, dst in templates:
+                incoming_template_file_path = Path(src)
+                template_file_path = Path(
+                    templates_path / incoming_template_file_path.name
+                )
+                shutil.copyfile(incoming_template_file_path, template_file_path)
+                templates_abs.append((str(template_file_path.resolve()), dst))
+            storage._write_transaction(
+                path / cls._templates_file,
+                json.dumps(templates_abs).encode("utf-8"),
+            )
+
         response_data = {}
         for response in responses or []:
             response_data.update({response.response_type: response.to_dict()})
@@ -248,6 +269,16 @@ def parameter_info(self) -> dict[str, Any]:
             info = json.load(f)
         return info
 
+    @cached_property
+    def templates_configuration(self) -> list[tuple[str, str]]:
+        try:
+            with open(self.mount_point / self._templates_file, encoding="utf-8") as f:
+                return json.load(f)
+        except (FileNotFoundError, json.JSONDecodeError):
+            pass
+            # If the file is missing or broken, we return an empty list
+        return []
+
     @property
     def response_info(self) -> dict[str, Any]:
         info: dict[str, Any]