Move GenKw template feature into run_template functionality

 - Add parameter substition feature into substitute
 - Raise ConfigValidationError when absolute path is provided as output file
 - Storage migration
 - Remove forward_init from gen_kw
 - Update snapshot due to forward_init_file was removed

Author: xjules

src/ert/config/design_matrix.py

@@ -168,8 +168,6 @@ def merge_with_existing_parameters(
                     )
 
                 design_parameter_group.name = parameter_group.name
-                design_parameter_group.template_file = parameter_group.template_file
-                design_parameter_group.output_file = parameter_group.output_file
                 design_group_added = True
             elif set(design_keys) & set(existing_keys):
                 raise ConfigValidationError(
@@ -249,8 +247,6 @@ def read_and_validate_design_matrix(
         parameter_configuration = GenKwConfig(
             name=DESIGN_MATRIX_GROUP,
             forward_init=False,
-            template_file=None,
-            output_file=None,
             transform_function_definitions=transform_function_definitions,
             update=False,
         )

src/ert/config/ensemble_config.py

@@ -60,10 +60,6 @@ def __post_init__(self) -> None:
             [p for p in self.parameter_configs.values() if isinstance(p, GenKwConfig)]
         )
 
-        self._check_for_forward_init_in_gen_kw(
-            [p for p in self.parameter_configs.values() if isinstance(p, GenKwConfig)]
-        )
-
         self.grid_file = _get_abs_path(self.grid_file)
 
     @staticmethod
@@ -97,11 +93,15 @@ def _check_for_duplicate_gen_kw_param_names(gen_kw_list: list[GenKwConfig]) -> N
                 f"Found duplicate GEN_KW parameter names: {duplicates_formatted}"
             )
 
+    @no_type_check
     @staticmethod
-    def _check_for_forward_init_in_gen_kw(gen_kw_list: list[GenKwConfig]) -> None:
-        for gen_kw in gen_kw_list:
-            if gen_kw.forward_init_file is not None:
-                logger.info(f"GEN_KW uses FORWARD_INIT: {gen_kw}")
+    def get_gen_kw_templates(config_dict: ConfigDict) -> list[tuple[str, str]]:
+        gen_kw_list = config_dict.get(ConfigKeys.GEN_KW, [])
+        return [
+            template
+            for g in gen_kw_list
+            if (template := GenKwConfig.templates_from_config(g)) is not None
+        ]
 
     @no_type_check
     @classmethod

src/ert/config/ert_config.py

@@ -58,9 +58,7 @@
     ObservationConfigError,
     SummaryValues,
 )
-from .parsing.observations_parser import (
-    parse_content as parse_observations,
-)
+from .parsing.observations_parser import parse_content as parse_observations
 from .queue_config import QueueConfig
 from .workflow import Workflow
 from .workflow_job import (
@@ -322,6 +320,7 @@ def read_templates(config_dict) -> list[tuple[str, str]]:
                 "it is synced with your DATA file."
             )
         templates.append(template)
+    templates.extend(EnsembleConfig.get_gen_kw_templates(config_dict))
     return templates
 
 
@@ -1220,8 +1219,9 @@ def env_pr_fm_step(self) -> dict[str, dict[str, Any]]:
 
     @staticmethod
     def _create_observations(
-        obs_config_content: dict[str, HistoryValues | SummaryValues | GenObsValues]
-        | None,
+        obs_config_content: (
+            dict[str, HistoryValues | SummaryValues | GenObsValues] | None
+        ),
         ensemble_config: EnsembleConfig,
         time_map: list[datetime] | None,
         history: HistorySource,

src/ert/config/gen_kw_config.py

@@ -2,7 +2,6 @@
 
 import math
 import os
-import shutil
 import warnings
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -16,8 +15,6 @@
 from scipy.stats import norm
 from typing_extensions import TypedDict
 
-from ert.substitutions import substitute_runpath_name
-
 from ._str_to_bool import str_to_bool
 from .parameter_config import ParameterConfig
 from .parsing import ConfigValidationError, ConfigWarning, ErrorInfo
@@ -59,10 +56,7 @@ class TransformFunctionDefinition:
 
 @dataclass
 class GenKwConfig(ParameterConfig):
-    template_file: str | None
-    output_file: str | None
     transform_function_definitions: list[TransformFunctionDefinition]
-    forward_init_file: str | None = None
 
     def __post_init__(self) -> None:
         self.transform_functions: list[TransformFunction] = []
@@ -86,31 +80,19 @@ def __len__(self) -> int:
         return len(self.transform_functions)
 
     @classmethod
-    def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
+    def templates_from_config(
+        cls, gen_kw: list[str | dict[str, str]]
+    ) -> tuple[str, str] | None:
         gen_kw_key = cast(str, gen_kw[0])
-
-        options = cast(dict[str, str], gen_kw[-1])
         positional_args = cast(list[str], gen_kw[:-1])
-        forward_init = str_to_bool(options.get("FORWARD_INIT", "FALSE"))
-        init_file = _get_abs_path(options.get("INIT_FILES"))
-        update_parameter = str_to_bool(options.get("UPDATE", "TRUE"))
-        errors = []
 
-        if len(positional_args) == 2:
-            parameter_file_contents = positional_args[1][1]
-            parameter_file_context = positional_args[1][0]
-            template_file = None
-            output_file = None
-        elif len(positional_args) == 4:
+        if len(positional_args) == 4:
             output_file = positional_args[2]
-            parameter_file_contents = positional_args[3][1]
             parameter_file_context = positional_args[3][0]
             template_file = _get_abs_path(positional_args[1][0])
             if not os.path.isfile(template_file):
-                errors.append(
-                    ConfigValidationError.with_context(
-                        f"No such template file: {template_file}", positional_args[1]
-                    )
+                raise ConfigValidationError.with_context(
+                    f"No such template file: {template_file}", positional_args[1]
                 )
             elif Path(template_file).stat().st_size == 0:
                 token = getattr(parameter_file_context, "token", parameter_file_context)
@@ -121,31 +103,35 @@ def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
                     f"instead: GEN_KW {gen_kw_key} {token}",
                     positional_args[1],
                 )
+            if output_file.startswith("/"):
+                raise ConfigValidationError.with_context(
+                    f"Output file cannot have an absolute path {output_file}",
+                    positional_args[2],
+                )
+            return template_file, output_file
+        return None
+
+    @classmethod
+    def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
+        gen_kw_key = cast(str, gen_kw[0])
+
+        options = cast(dict[str, str], gen_kw[-1])
+        positional_args = cast(list[str], gen_kw[:-1])
+        update_parameter = str_to_bool(options.get("UPDATE", "TRUE"))
+        errors = []
+
+        if len(positional_args) == 2:
+            parameter_file_contents = positional_args[1][1]
+            parameter_file_context = positional_args[1][0]
+        elif len(positional_args) == 4:
+            parameter_file_contents = positional_args[3][1]
+            parameter_file_context = positional_args[3][0]
 
         else:
             raise ConfigValidationError(
                 f"Unexpected positional arguments: {positional_args}"
             )
 
-        if forward_init:
-            errors.append(
-                ConfigValidationError.with_context(
-                    "Loading GEN_KW from files created by the forward "
-                    "model is not supported.",
-                    gen_kw,
-                )
-            )
-
-        if init_file and "%" not in init_file:
-            errors.append(
-                ConfigValidationError.with_context(
-                    "Loading GEN_KW from files requires %d in file format", gen_kw
-                )
-            )
-
-        if errors:
-            raise ConfigValidationError.from_collected(errors)
-
         transform_function_definitions: list[TransformFunctionDefinition] = []
         for line_number, item in enumerate(parameter_file_contents.splitlines()):
             item = item.split("--")[0]  # remove comments
@@ -187,10 +173,7 @@ def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
             )
         return cls(
             name=gen_kw_key,
-            forward_init=forward_init,
-            template_file=template_file,
-            output_file=output_file,
-            forward_init_file=init_file,
+            forward_init=False,
             transform_function_definitions=transform_function_definitions,
             update=update_parameter,
         )
@@ -281,9 +264,6 @@ def _check_valid_derrf_parameters(prior: PriorDict) -> None:
     def sample_or_load(
         self, real_nr: int, random_seed: int, ensemble_size: int
     ) -> xr.Dataset:
-        if self.forward_init_file:
-            return self.read_from_runpath(Path(), real_nr, 0)
-
         keys = [e.name for e in self.transform_functions]
         parameter_value = self._sample_value(
             self.name,
@@ -306,22 +286,7 @@ def read_from_runpath(
         real_nr: int,
         iteration: int,
     ) -> xr.Dataset:
-        keys = [e.name for e in self.transform_functions]
-        if not self.forward_init_file:
-            raise ValueError("loading gen_kw values requires forward_init_file")
-
-        parameter_value = self._values_from_file(
-            substitute_runpath_name(self.forward_init_file, real_nr, iteration),
-            keys,
-        )
-
-        return xr.Dataset(
-            {
-                "values": ("names", parameter_value),
-                "transformed_values": ("names", self.transform(parameter_value)),
-                "names": keys,
-            }
-        )
+        return xr.Dataset()
 
     def write_to_runpath(
         self,
@@ -362,21 +327,6 @@ def parse_value(value: float | int | str) -> float | int | str:
             if tf.use_log
         }
 
-        if self.template_file is not None and self.output_file is not None:
-            target_file = substitute_runpath_name(
-                self.output_file, real_nr, ensemble.iteration
-            )
-            target_file = target_file.removeprefix("/")
-            (run_path / target_file).parent.mkdir(exist_ok=True, parents=True)
-            template_file_path = (
-                ensemble.experiment.mount_point / Path(self.template_file).name
-            )
-            with open(template_file_path, encoding="utf-8") as f:
-                template = f.read()
-            for key, value in data.items():
-                template = template.replace(f"<{key}>", f"{value:.6g}")
-            with open(run_path / target_file, "w", encoding="utf-8") as f:
-                f.write(template)
         if log10_data:
             return {self.name: data, f"LOG10_{self.name}": log10_data}
         else:
@@ -553,14 +503,6 @@ def _parse_transform_function_definition(
             calc_func=PRIOR_FUNCTIONS[t.param_name],
         )
 
-    def save_experiment_data(self, experiment_path: Path) -> None:
-        if self.template_file:
-            incoming_template_file_path = Path(self.template_file)
-            template_file_path = Path(
-                experiment_path / incoming_template_file_path.name
-            )
-            shutil.copyfile(incoming_template_file_path, template_file_path)
-
 
 @dataclass
 class TransformFunction:

src/ert/enkf_main.py

@@ -90,7 +90,7 @@ def _generate_parameter_files(
     iens: int,
     fs: Ensemble,
     iteration: int,
-) -> None:
+) -> dict[str, dict[str, float]]:
     """
     Generate parameter files that are placed in each runtime directory for
     forward-model jobs to consume.
@@ -119,6 +119,7 @@ def _generate_parameter_files(
 
     _value_export_txt(run_path, export_base_name, exports)
     _value_export_json(run_path, export_base_name, exports)
+    return exports
 
 
 def _manifest_to_json(ensemble: Ensemble, iens: int, iter: int) -> dict[str, Any]:
@@ -130,6 +131,7 @@ def _manifest_to_json(ensemble: Ensemble, iens: int, iter: int) -> dict[str, Any
             ExtParamConfig | GenKwConfig | Field | SurfaceConfig,
         )
         if param_config.forward_init and ensemble.iteration == 0:
+            assert not isinstance(param_config, GenKwConfig)
             assert param_config.forward_init_file is not None
             file_path = substitute_runpath_name(
                 param_config.forward_init_file, iens, iter
@@ -225,6 +227,15 @@ def create_run_path(
         run_path = Path(run_arg.runpath)
         if run_arg.active:
             run_path.mkdir(parents=True, exist_ok=True)
+            param_data = _generate_parameter_files(
+                ensemble.experiment.parameter_configuration.values(),
+                parameters_file,
+                run_path,
+                run_arg.iens,
+                ensemble,
+                ensemble.iteration,
+            )
+
             for (
                 source_file_content,
                 target_file,
@@ -237,6 +248,11 @@ def create_run_path(
                     run_arg.iens,
                     ensemble.iteration,
                 )
+                result = substitutions.substitute_parameters(
+                    result,
+                    param_data,
+                )
+
                 target = run_path / target_file
                 if not target.parent.exists():
                     os.makedirs(
@@ -245,15 +261,6 @@ def create_run_path(
                     )
                 target.write_text(result)
 
-            _generate_parameter_files(
-                ensemble.experiment.parameter_configuration.values(),
-                parameters_file,
-                run_path,
-                run_arg.iens,
-                ensemble,
-                ensemble.iteration,
-            )
-
             path = run_path / "jobs.json"
             _backup_if_existing(path)
 

src/ert/storage/migration/to10.py

@@ -4,13 +4,24 @@
 
 from ert.storage.local_storage import local_storage_get_ert_config
 
-info = "Internalizes run templates in to the storage"
+info = "Internalizes run templates in to the storage and migrate template GEN_KW feature into RUN_TEMPLATE"
 
 
 def migrate(path: Path) -> None:
     ert_config = local_storage_get_ert_config()
     if ert_config.ert_templates:
         for experiment in path.glob("experiments/*"):
+            with open(experiment / "parameter.json", encoding="utf-8") as fin:
+                parameters_json = json.load(fin)
+
+            with open(experiment / "parameter.json", "w", encoding="utf-8") as fout:
+                for param in parameters_json.values():
+                    if param["_ert_kind"] == "GenKwConfig":
+                        param.pop("template_file", None)
+                        param.pop("output_file", None)
+                        param.pop("forward_init_file", None)
+                fout.write(json.dumps(parameters_json, indent=4))
+
             templates_abs: list[tuple[str, str]] = []
             templates_path = experiment / "templates"
             templates_path.mkdir(parents=True, exist_ok=True)

src/ert/substitutions.py

@@ -30,6 +30,14 @@ def substitute(
         """
         return _substitute(self, to_substitute, context, max_iterations, warn_max_iter)
 
+    def substitute_parameters(
+        self, to_substitute: str, data: dict[str, dict[str, float]]
+    ) -> str:
+        for values in data.values():
+            for key, value in values.items():
+                to_substitute = to_substitute.replace(f"<{key}>", f"{value:.6g}")
+        return to_substitute
+
     def substitute_real_iter(
         self, to_substitute: str, realization: int, iteration: int
     ) -> str: