Move GenKw template feature into run_template functionality

 - Add parameter substition feature into substitute
 - Raise ConfigValidationError when absolute path is provided as output file

Author: xjules

src/ert/config/design_matrix.py

@@ -238,8 +238,6 @@ def read_and_validate_design_matrix(
         parameter_configuration = GenKwConfig(
             name=DESIGN_MATRIX_GROUP,
             forward_init=False,
-            template_file=None,
-            output_file=None,
             transform_function_definitions=transform_function_definitions,
             update=False,
         )

src/ert/config/ensemble_config.py

@@ -103,6 +103,16 @@ def _check_for_forward_init_in_gen_kw(gen_kw_list: list[GenKwConfig]) -> None:
             if gen_kw.forward_init_file is not None:
                 logger.info(f"GEN_KW uses FORWARD_INIT: {gen_kw}")
 
+    @no_type_check
+    @staticmethod
+    def get_gen_kw_templates(config_dict: ConfigDict) -> list[tuple[str, str]]:
+        gen_kw_list = config_dict.get(ConfigKeys.GEN_KW, [])
+        return [
+            template
+            for g in gen_kw_list
+            if (template := GenKwConfig.templates_from_config(g)) is not None
+        ]
+
     @no_type_check
     @classmethod
     def from_dict(cls, config_dict: ConfigDict) -> EnsembleConfig:

src/ert/config/ert_config.py

@@ -57,9 +57,7 @@
     ObservationConfigError,
     SummaryValues,
 )
-from .parsing.observations_parser import (
-    parse_content as parse_observations,
-)
+from .parsing.observations_parser import parse_content as parse_observations
 from .queue_config import QueueConfig
 from .workflow import Workflow
 from .workflow_job import (
@@ -299,6 +297,7 @@ def read_templates(config_dict) -> list[tuple[str, str]]:
                 "it is synced with your DATA file."
             )
         templates.append(template)
+    templates.extend(EnsembleConfig.get_gen_kw_templates(config_dict))
     return templates
 
 

src/ert/config/gen_kw_config.py

@@ -2,7 +2,6 @@
 
 import math
 import os
-import shutil
 import warnings
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -59,8 +58,6 @@ class TransformFunctionDefinition:
 
 @dataclass
 class GenKwConfig(ParameterConfig):
-    template_file: str | None
-    output_file: str | None
     transform_function_definitions: list[TransformFunctionDefinition]
     forward_init_file: str | None = None
 
@@ -85,6 +82,36 @@ def __contains__(self, item: str) -> bool:
     def __len__(self) -> int:
         return len(self.transform_functions)
 
+    @classmethod
+    def templates_from_config(
+        cls, gen_kw: list[str | dict[str, str]]
+    ) -> tuple[str, str] | None:
+        gen_kw_key = cast(str, gen_kw[0])
+        positional_args = cast(list[str], gen_kw[:-1])
+
+        if len(positional_args) == 4:
+            output_file = positional_args[2]
+            parameter_file_context = positional_args[3][0]
+            template_file = _get_abs_path(positional_args[1][0])
+            if not os.path.isfile(template_file):
+                raise ConfigValidationError.with_context(
+                    f"No such template file: {template_file}", positional_args[1]
+                )
+            elif Path(template_file).stat().st_size == 0:
+                token = getattr(parameter_file_context, "token", parameter_file_context)
+                ConfigWarning.deprecation_warn(
+                    f"The template file for GEN_KW ({gen_kw_key}) is empty. If templating is not needed, you "
+                    f"can use GEN_KW with just the distribution file instead: GEN_KW {gen_kw_key} {token}",
+                    positional_args[1],
+                )
+            if output_file.startswith("/"):
+                raise ConfigValidationError.with_context(
+                    f"Output file cannot have an absolute path {output_file}",
+                    positional_args[2],
+                )
+            return template_file, output_file
+        return None
+
     @classmethod
     def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
         gen_kw_key = cast(str, gen_kw[0])
@@ -99,26 +126,9 @@ def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
         if len(positional_args) == 2:
             parameter_file_contents = positional_args[1][1]
             parameter_file_context = positional_args[1][0]
-            template_file = None
-            output_file = None
         elif len(positional_args) == 4:
-            output_file = positional_args[2]
             parameter_file_contents = positional_args[3][1]
             parameter_file_context = positional_args[3][0]
-            template_file = _get_abs_path(positional_args[1][0])
-            if not os.path.isfile(template_file):
-                errors.append(
-                    ConfigValidationError.with_context(
-                        f"No such template file: {template_file}", positional_args[1]
-                    )
-                )
-            elif Path(template_file).stat().st_size == 0:
-                token = getattr(parameter_file_context, "token", parameter_file_context)
-                ConfigWarning.deprecation_warn(
-                    f"The template file for GEN_KW ({gen_kw_key}) is empty. If templating is not needed, you "
-                    f"can use GEN_KW with just the distribution file instead: GEN_KW {gen_kw_key} {token}",
-                    positional_args[1],
-                )
 
         else:
             raise ConfigValidationError(
@@ -185,8 +195,6 @@ def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
         return cls(
             name=gen_kw_key,
             forward_init=forward_init,
-            template_file=template_file,
-            output_file=output_file,
             forward_init_file=init_file,
             transform_function_definitions=transform_function_definitions,
             update=update_parameter,
@@ -358,21 +366,21 @@ def parse_value(value: float | int | str) -> float | int | str:
             if tf.use_log
         }
 
-        if self.template_file is not None and self.output_file is not None:
-            target_file = substitute_runpath_name(
-                self.output_file, real_nr, ensemble.iteration
-            )
-            target_file = target_file.removeprefix("/")
-            (run_path / target_file).parent.mkdir(exist_ok=True, parents=True)
-            template_file_path = (
-                ensemble.experiment.mount_point / Path(self.template_file).name
-            )
-            with open(template_file_path, encoding="utf-8") as f:
-                template = f.read()
-            for key, value in data.items():
-                template = template.replace(f"<{key}>", f"{value:.6g}")
-            with open(run_path / target_file, "w", encoding="utf-8") as f:
-                f.write(template)
+        # if self.template_file is not None and self.output_file is not None:
+        #     target_file = substitute_runpath_name(
+        #         self.output_file, real_nr, ensemble.iteration
+        #     )
+        #     target_file = target_file.removeprefix("/")
+        #     (run_path / target_file).parent.mkdir(exist_ok=True, parents=True)
+        #     template_file_path = (
+        #         ensemble.experiment.mount_point / Path(self.template_file).name
+        #     )
+        #     with open(template_file_path, encoding="utf-8") as f:
+        #         template = f.read()
+        #     for key, value in data.items():
+        #         template = template.replace(f"<{key}>", f"{value:.6g}")
+        #     with open(run_path / target_file, "w", encoding="utf-8") as f:
+        #         f.write(template)
         if log10_data:
             return {self.name: data, f"LOG10_{self.name}": log10_data}
         else:
@@ -542,13 +550,13 @@ def _parse_transform_function_definition(
             calc_func=PRIOR_FUNCTIONS[t.param_name],
         )
 
-    def save_experiment_data(self, experiment_path: Path) -> None:
-        if self.template_file:
-            incoming_template_file_path = Path(self.template_file)
-            template_file_path = Path(
-                experiment_path / incoming_template_file_path.name
-            )
-            shutil.copyfile(incoming_template_file_path, template_file_path)
+    # def save_experiment_data(self, experiment_path: Path) -> None:
+    #     if self.template_file:
+    #         incoming_template_file_path = Path(self.template_file)
+    #         template_file_path = Path(
+    #             experiment_path / incoming_template_file_path.name
+    #         )
+    #         shutil.copyfile(incoming_template_file_path, template_file_path)
 
 
 @dataclass

src/ert/enkf_main.py

@@ -91,7 +91,7 @@ def _generate_parameter_files(
     iens: int,
     fs: Ensemble,
     iteration: int,
-) -> None:
+) -> dict[str, dict[str, float]]:
     """
     Generate parameter files that are placed in each runtime directory for
     forward-model jobs to consume.
@@ -120,6 +120,7 @@ def _generate_parameter_files(
 
     _value_export_txt(run_path, export_base_name, exports)
     _value_export_json(run_path, export_base_name, exports)
+    return exports
 
 
 def _manifest_to_json(ensemble: Ensemble, iens: int, iter: int) -> dict[str, Any]:
@@ -242,6 +243,15 @@ def create_run_path(
         run_path = Path(run_arg.runpath)
         if run_arg.active:
             run_path.mkdir(parents=True, exist_ok=True)
+            param_data = _generate_parameter_files(
+                ensemble.experiment.parameter_configuration.values(),
+                parameters_file,
+                run_path,
+                run_arg.iens,
+                ensemble,
+                ensemble.iteration,
+            )
+
             for source_file, target_file in templates:
                 target_file = substitutions.substitute_real_iter(
                     target_file, run_arg.iens, ensemble.iteration
@@ -258,6 +268,11 @@ def create_run_path(
                     run_arg.iens,
                     ensemble.iteration,
                 )
+                result = substitutions.substitute_parameters(
+                    result,
+                    param_data,
+                )
+
                 target = run_path / target_file
                 if not target.parent.exists():
                     os.makedirs(
@@ -266,15 +281,6 @@ def create_run_path(
                     )
                 target.write_text(result)
 
-            _generate_parameter_files(
-                ensemble.experiment.parameter_configuration.values(),
-                parameters_file,
-                run_path,
-                run_arg.iens,
-                ensemble,
-                ensemble.iteration,
-            )
-
             path = run_path / "jobs.json"
             _backup_if_existing(path)
 

src/ert/substitutions.py

@@ -32,6 +32,14 @@ def substitute(
         """
         return _substitute(self, to_substitute, context, max_iterations, warn_max_iter)
 
+    def substitute_parameters(
+        self, to_substitute: str, data: dict[str, dict[str, float]]
+    ) -> str:
+        for values in data.values():
+            for key, value in values.items():
+                to_substitute = to_substitute.replace(f"<{key}>", f"{value:.6g}")
+        return to_substitute
+
     def substitute_real_iter(
         self, to_substitute: str, realization: int, iteration: int
     ) -> str:

tests/ert/ui_tests/cli/analysis/test_es_update.py

@@ -24,11 +24,9 @@ def uniform_parameter():
     return GenKwConfig(
         name="PARAMETER",
         forward_init=False,
-        template_file="",
         transform_function_definitions=[
             TransformFunctionDefinition("KEY1", "UNIFORM", [0, 1]),
         ],
-        output_file="kw.txt",
         update=True,
     )
 

tests/ert/unit_tests/analysis/test_es_update.py

@@ -8,11 +8,7 @@
 import xtgeo
 from tabulate import tabulate
 
-from ert.analysis import (
-    ErtAnalysisError,
-    ObservationStatus,
-    smoother_update,
-)
+from ert.analysis import ErtAnalysisError, ObservationStatus, smoother_update
 from ert.analysis._es_update import (
     _load_observations_and_responses,
     _load_param_ensemble_array,
@@ -30,11 +26,9 @@ def uniform_parameter():
     return GenKwConfig(
         name="PARAMETER",
         forward_init=False,
-        template_file="",
         transform_function_definitions=[
             TransformFunctionDefinition("KEY1", "UNIFORM", [0, 1]),
         ],
-        output_file="kw.txt",
         update=True,
     )
 
@@ -208,8 +202,6 @@ def test_update_handles_precision_loss_in_std_dev(tmp_path):
         name="COEFFS",
         forward_init=False,
         update=True,
-        template_file=None,
-        output_file=None,
         transform_function_definitions=[
             TransformFunctionDefinition(
                 name="coeff_0", param_name="CONST", values=["0.1"]
@@ -331,8 +323,6 @@ def test_update_raises_on_singular_matrix(tmp_path):
         name="COEFFS",
         forward_init=False,
         update=True,
-        template_file=None,
-        output_file=None,
         transform_function_definitions=[
             TransformFunctionDefinition(
                 name="coeff_0", param_name="CONST", values=["0.1"]
@@ -1002,11 +992,9 @@ def test_update_subset_parameters(storage, uniform_parameter, obs):
     no_update_param = GenKwConfig(
         name="EXTRA_PARAMETER",
         forward_init=False,
-        template_file="",
         transform_function_definitions=[
             TransformFunctionDefinition("KEY1", "UNIFORM", [0, 1]),
         ],
-        output_file=None,
         update=False,
     )
     resp = GenDataConfig(keys=["RESPONSE"])