Move GenKw template feature into run_template functionality

This includes adding parameter substition feature into substitute
class. GenKwConfig.templates_from_config is reponsible for extracting tmpl
and output files. In case an absolute path is provided as an output file
it raises ConfigValidationError.

To prevent multiple storage migrations this also remove forward_init
from gen_kw.

Author: xjules

src/ert/config/design_matrix.py

@@ -168,8 +168,6 @@ def merge_with_existing_parameters(
                     )
 
                 design_parameter_group.name = parameter_group.name
-                design_parameter_group.template_file = parameter_group.template_file
-                design_parameter_group.output_file = parameter_group.output_file
                 design_group_added = True
             elif set(design_keys) & set(existing_keys):
                 raise ConfigValidationError(
@@ -249,8 +247,6 @@ def read_and_validate_design_matrix(
         parameter_configuration = GenKwConfig(
             name=DESIGN_MATRIX_GROUP,
             forward_init=False,
-            template_file=None,
-            output_file=None,
             transform_function_definitions=transform_function_definitions,
             update=False,
         )

src/ert/config/ensemble_config.py

@@ -60,10 +60,6 @@ def __post_init__(self) -> None:
             [p for p in self.parameter_configs.values() if isinstance(p, GenKwConfig)]
         )
 
-        self._check_for_forward_init_in_gen_kw(
-            [p for p in self.parameter_configs.values() if isinstance(p, GenKwConfig)]
-        )
-
         self.grid_file = _get_abs_path(self.grid_file)
 
     @staticmethod
@@ -97,11 +93,15 @@ def _check_for_duplicate_gen_kw_param_names(gen_kw_list: list[GenKwConfig]) -> N
                 f"Found duplicate GEN_KW parameter names: {duplicates_formatted}"
             )
 
+    @no_type_check
     @staticmethod
-    def _check_for_forward_init_in_gen_kw(gen_kw_list: list[GenKwConfig]) -> None:
-        for gen_kw in gen_kw_list:
-            if gen_kw.forward_init_file is not None:
-                logger.info(f"GEN_KW uses FORWARD_INIT: {gen_kw}")
+    def get_gen_kw_templates(config_dict: ConfigDict) -> list[tuple[str, str]]:
+        gen_kw_list = config_dict.get(ConfigKeys.GEN_KW, [])
+        return [
+            template
+            for g in gen_kw_list
+            if (template := GenKwConfig.templates_from_config(g)) is not None
+        ]
 
     @no_type_check
     @classmethod

src/ert/config/ert_config.py

@@ -58,9 +58,7 @@
     ObservationConfigError,
     SummaryValues,
 )
-from .parsing.observations_parser import (
-    parse_content as parse_observations,
-)
+from .parsing.observations_parser import parse_content as parse_observations
 from .queue_config import QueueConfig
 from .workflow import Workflow
 from .workflow_job import (
@@ -322,6 +320,7 @@ def read_templates(config_dict) -> list[tuple[str, str]]:
                 "it is synced with your DATA file."
             )
         templates.append(template)
+    templates.extend(EnsembleConfig.get_gen_kw_templates(config_dict))
     return templates
 
 

src/ert/config/gen_kw_config.py

@@ -2,7 +2,6 @@
 
 import math
 import os
-import shutil
 import warnings
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -16,8 +15,6 @@
 from scipy.stats import norm
 from typing_extensions import TypedDict
 
-from ert.substitutions import substitute_runpath_name
-
 from ._str_to_bool import str_to_bool
 from .parameter_config import ParameterConfig
 from .parsing import ConfigValidationError, ConfigWarning, ErrorInfo
@@ -59,10 +56,7 @@ class TransformFunctionDefinition:
 
 @dataclass
 class GenKwConfig(ParameterConfig):
-    template_file: str | None
-    output_file: str | None
     transform_function_definitions: list[TransformFunctionDefinition]
-    forward_init_file: str | None = None
 
     def __post_init__(self) -> None:
         self.transform_functions: list[TransformFunction] = []
@@ -86,31 +80,19 @@ def __len__(self) -> int:
         return len(self.transform_functions)
 
     @classmethod
-    def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
+    def templates_from_config(
+        cls, gen_kw: list[str | dict[str, str]]
+    ) -> tuple[str, str] | None:
         gen_kw_key = cast(str, gen_kw[0])
-
-        options = cast(dict[str, str], gen_kw[-1])
         positional_args = cast(list[str], gen_kw[:-1])
-        forward_init = str_to_bool(options.get("FORWARD_INIT", "FALSE"))
-        init_file = _get_abs_path(options.get("INIT_FILES"))
-        update_parameter = str_to_bool(options.get("UPDATE", "TRUE"))
-        errors = []
 
-        if len(positional_args) == 2:
-            parameter_file_contents = positional_args[1][1]
-            parameter_file_context = positional_args[1][0]
-            template_file = None
-            output_file = None
-        elif len(positional_args) == 4:
+        if len(positional_args) == 4:
             output_file = positional_args[2]
-            parameter_file_contents = positional_args[3][1]
             parameter_file_context = positional_args[3][0]
             template_file = _get_abs_path(positional_args[1][0])
             if not os.path.isfile(template_file):
-                errors.append(
-                    ConfigValidationError.with_context(
-                        f"No such template file: {template_file}", positional_args[1]
-                    )
+                raise ConfigValidationError.with_context(
+                    f"No such template file: {template_file}", positional_args[1]
                 )
             elif Path(template_file).stat().st_size == 0:
                 token = getattr(parameter_file_context, "token", parameter_file_context)
@@ -121,31 +103,35 @@ def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
                     f"instead: GEN_KW {gen_kw_key} {token}",
                     positional_args[1],
                 )
+            if output_file.startswith("/"):
+                raise ConfigValidationError.with_context(
+                    f"Output file cannot have an absolute path {output_file}",
+                    positional_args[2],
+                )
+            return template_file, output_file
+        return None
+
+    @classmethod
+    def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
+        gen_kw_key = cast(str, gen_kw[0])
+
+        options = cast(dict[str, str], gen_kw[-1])
+        positional_args = cast(list[str], gen_kw[:-1])
+        update_parameter = str_to_bool(options.get("UPDATE", "TRUE"))
+        errors = []
+
+        if len(positional_args) == 2:
+            parameter_file_contents = positional_args[1][1]
+            parameter_file_context = positional_args[1][0]
+        elif len(positional_args) == 4:
+            parameter_file_contents = positional_args[3][1]
+            parameter_file_context = positional_args[3][0]
 
         else:
             raise ConfigValidationError(
                 f"Unexpected positional arguments: {positional_args}"
             )
 
-        if forward_init:
-            errors.append(
-                ConfigValidationError.with_context(
-                    "Loading GEN_KW from files created by the forward "
-                    "model is not supported.",
-                    gen_kw,
-                )
-            )
-
-        if init_file and "%" not in init_file:
-            errors.append(
-                ConfigValidationError.with_context(
-                    "Loading GEN_KW from files requires %d in file format", gen_kw
-                )
-            )
-
-        if errors:
-            raise ConfigValidationError.from_collected(errors)
-
         transform_function_definitions: list[TransformFunctionDefinition] = []
         for line_number, item in enumerate(parameter_file_contents.splitlines()):
             item = item.split("--")[0]  # remove comments
@@ -187,10 +173,7 @@ def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
             )
         return cls(
             name=gen_kw_key,
-            forward_init=forward_init,
-            template_file=template_file,
-            output_file=output_file,
-            forward_init_file=init_file,
+            forward_init=False,
             transform_function_definitions=transform_function_definitions,
             update=update_parameter,
         )
@@ -281,9 +264,6 @@ def _check_valid_derrf_parameters(prior: PriorDict) -> None:
     def sample_or_load(
         self, real_nr: int, random_seed: int, ensemble_size: int
     ) -> xr.Dataset:
-        if self.forward_init_file:
-            return self.read_from_runpath(Path(), real_nr, 0)
-
         keys = [e.name for e in self.transform_functions]
         parameter_value = self._sample_value(
             self.name,
@@ -306,22 +286,7 @@ def read_from_runpath(
         real_nr: int,
         iteration: int,
     ) -> xr.Dataset:
-        keys = [e.name for e in self.transform_functions]
-        if not self.forward_init_file:
-            raise ValueError("loading gen_kw values requires forward_init_file")
-
-        parameter_value = self._values_from_file(
-            substitute_runpath_name(self.forward_init_file, real_nr, iteration),
-            keys,
-        )
-
-        return xr.Dataset(
-            {
-                "values": ("names", parameter_value),
-                "transformed_values": ("names", self.transform(parameter_value)),
-                "names": keys,
-            }
-        )
+        return xr.Dataset()
 
     def write_to_runpath(
         self,
@@ -362,21 +327,6 @@ def parse_value(value: float | int | str) -> float | int | str:
             if tf.use_log
         }
 
-        if self.template_file is not None and self.output_file is not None:
-            target_file = substitute_runpath_name(
-                self.output_file, real_nr, ensemble.iteration
-            )
-            target_file = target_file.removeprefix("/")
-            (run_path / target_file).parent.mkdir(exist_ok=True, parents=True)
-            template_file_path = (
-                ensemble.experiment.mount_point / Path(self.template_file).name
-            )
-            with open(template_file_path, encoding="utf-8") as f:
-                template = f.read()
-            for key, value in data.items():
-                template = template.replace(f"<{key}>", f"{value:.6g}")
-            with open(run_path / target_file, "w", encoding="utf-8") as f:
-                f.write(template)
         if log10_data:
             return {self.name: data, f"LOG10_{self.name}": log10_data}
         else:
@@ -553,14 +503,6 @@ def _parse_transform_function_definition(
             calc_func=PRIOR_FUNCTIONS[t.param_name],
         )
 
-    def save_experiment_data(self, experiment_path: Path) -> None:
-        if self.template_file:
-            incoming_template_file_path = Path(self.template_file)
-            template_file_path = Path(
-                experiment_path / incoming_template_file_path.name
-            )
-            shutil.copyfile(incoming_template_file_path, template_file_path)
-
 
 @dataclass
 class TransformFunction:

src/ert/enkf_main.py

@@ -90,7 +90,7 @@ def _generate_parameter_files(
     iens: int,
     fs: Ensemble,
     iteration: int,
-) -> None:
+) -> dict[str, dict[str, float]]:
     """
     Generate parameter files that are placed in each runtime directory for
     forward-model jobs to consume.
@@ -119,6 +119,7 @@ def _generate_parameter_files(
 
     _value_export_txt(run_path, export_base_name, exports)
     _value_export_json(run_path, export_base_name, exports)
+    return exports
 
 
 def _manifest_to_json(ensemble: Ensemble, iens: int, iter: int) -> dict[str, Any]:
@@ -130,6 +131,7 @@ def _manifest_to_json(ensemble: Ensemble, iens: int, iter: int) -> dict[str, Any
             ExtParamConfig | GenKwConfig | Field | SurfaceConfig,
         )
         if param_config.forward_init and ensemble.iteration == 0:
+            assert not isinstance(param_config, GenKwConfig)
             assert param_config.forward_init_file is not None
             file_path = substitute_runpath_name(
                 param_config.forward_init_file, iens, iter
@@ -225,6 +227,15 @@ def create_run_path(
         run_path = Path(run_arg.runpath)
         if run_arg.active:
             run_path.mkdir(parents=True, exist_ok=True)
+            param_data = _generate_parameter_files(
+                ensemble.experiment.parameter_configuration.values(),
+                parameters_file,
+                run_path,
+                run_arg.iens,
+                ensemble,
+                ensemble.iteration,
+            )
+
             for (
                 source_file_content,
                 target_file,
@@ -237,6 +248,11 @@ def create_run_path(
                     run_arg.iens,
                     ensemble.iteration,
                 )
+                result = substitutions.substitute_parameters(
+                    result,
+                    param_data,
+                )
+
                 target = run_path / target_file
                 if not target.parent.exists():
                     os.makedirs(
@@ -245,15 +261,6 @@ def create_run_path(
                     )
                 target.write_text(result)
 
-            _generate_parameter_files(
-                ensemble.experiment.parameter_configuration.values(),
-                parameters_file,
-                run_path,
-                run_arg.iens,
-                ensemble,
-                ensemble.iteration,
-            )
-
             path = run_path / "jobs.json"
             _backup_if_existing(path)
 

src/ert/storage/local_experiment.py

@@ -272,24 +272,23 @@ def parameter_info(self) -> dict[str, Any]:
 
     @property
     def templates_configuration(self) -> list[tuple[str, str]]:
+        templates: list[tuple[str, str]] = []
         try:
-            templates: list[tuple[str, str]] = []
             with open(self.mount_point / self._templates_file, encoding="utf-8") as f:
                 templates = json.load(f)
-            templates_with_content: list[tuple[str, str]] = []
-            for source_file, target_file in templates:
-                try:
-                    file_content = (self.mount_point / source_file).read_text("utf-8")
-                    templates_with_content.append((file_content, target_file))
-                except UnicodeDecodeError as e:
-                    raise ValueError(
-                        f"Unsupported non UTF-8 character found in file: {source_file}"
-                    ) from e
-                return templates_with_content
         except (FileNotFoundError, json.JSONDecodeError):
-            pass
             # If the file is missing or broken, we return an empty list
-        return []
+            pass
+        templates_with_content: list[tuple[str, str]] = []
+        for source_file, target_file in templates:
+            try:
+                file_content = (self.mount_point / source_file).read_text("utf-8")
+                templates_with_content.append((file_content, target_file))
+            except UnicodeDecodeError as e:
+                raise ValueError(
+                    f"Unsupported non UTF-8 character found in file: {source_file}"
+                ) from e
+        return templates_with_content
 
     @property
     def response_info(self) -> dict[str, Any]: