Move GenKw template feature into run_template functionality

 - Add parameter substition feature into substitute
 - Raise ConfigValidationError when absolute path is provided as output file
 - Storage migration

Author: xjules

src/ert/config/design_matrix.py

@@ -158,8 +158,6 @@ def merge_with_existing_parameters(
                     )
 
                 design_parameter_group.name = parameter_group.name
-                design_parameter_group.template_file = parameter_group.template_file
-                design_parameter_group.output_file = parameter_group.output_file
                 design_group_added = True
             elif set(design_keys) & set(existing_keys):
                 raise ConfigValidationError(
@@ -237,8 +235,6 @@ def read_and_validate_design_matrix(
         parameter_configuration = GenKwConfig(
             name=DESIGN_MATRIX_GROUP,
             forward_init=False,
-            template_file=None,
-            output_file=None,
             transform_function_definitions=transform_function_definitions,
             update=False,
         )

src/ert/config/ensemble_config.py

@@ -103,6 +103,16 @@ def _check_for_forward_init_in_gen_kw(gen_kw_list: list[GenKwConfig]) -> None:
             if gen_kw.forward_init_file is not None:
                 logger.info(f"GEN_KW uses FORWARD_INIT: {gen_kw}")
 
+    @no_type_check
+    @staticmethod
+    def get_gen_kw_templates(config_dict: ConfigDict) -> list[tuple[str, str]]:
+        gen_kw_list = config_dict.get(ConfigKeys.GEN_KW, [])
+        return [
+            template
+            for g in gen_kw_list
+            if (template := GenKwConfig.templates_from_config(g)) is not None
+        ]
+
     @no_type_check
     @classmethod
     def from_dict(cls, config_dict: ConfigDict) -> EnsembleConfig:

src/ert/config/ert_config.py

@@ -57,9 +57,7 @@
     ObservationConfigError,
     SummaryValues,
 )
-from .parsing.observations_parser import (
-    parse_content as parse_observations,
-)
+from .parsing.observations_parser import parse_content as parse_observations
 from .queue_config import QueueConfig
 from .workflow import Workflow
 from .workflow_job import (
@@ -299,6 +297,7 @@ def read_templates(config_dict) -> list[tuple[str, str]]:
                 "it is synced with your DATA file."
             )
         templates.append(template)
+    templates.extend(EnsembleConfig.get_gen_kw_templates(config_dict))
     return templates
 
 

src/ert/config/gen_kw_config.py

@@ -2,7 +2,6 @@
 
 import math
 import os
-import shutil
 import warnings
 from collections.abc import Callable
 from dataclasses import dataclass
@@ -59,8 +58,6 @@ class TransformFunctionDefinition:
 
 @dataclass
 class GenKwConfig(ParameterConfig):
-    template_file: str | None
-    output_file: str | None
     transform_function_definitions: list[TransformFunctionDefinition]
     forward_init_file: str | None = None
 
@@ -85,6 +82,36 @@ def __contains__(self, item: str) -> bool:
     def __len__(self) -> int:
         return len(self.transform_functions)
 
+    @classmethod
+    def templates_from_config(
+        cls, gen_kw: list[str | dict[str, str]]
+    ) -> tuple[str, str] | None:
+        gen_kw_key = cast(str, gen_kw[0])
+        positional_args = cast(list[str], gen_kw[:-1])
+
+        if len(positional_args) == 4:
+            output_file = positional_args[2]
+            parameter_file_context = positional_args[3][0]
+            template_file = _get_abs_path(positional_args[1][0])
+            if not os.path.isfile(template_file):
+                raise ConfigValidationError.with_context(
+                    f"No such template file: {template_file}", positional_args[1]
+                )
+            elif Path(template_file).stat().st_size == 0:
+                token = getattr(parameter_file_context, "token", parameter_file_context)
+                ConfigWarning.deprecation_warn(
+                    f"The template file for GEN_KW ({gen_kw_key}) is empty. If templating is not needed, you "
+                    f"can use GEN_KW with just the distribution file instead: GEN_KW {gen_kw_key} {token}",
+                    positional_args[1],
+                )
+            if output_file.startswith("/"):
+                raise ConfigValidationError.with_context(
+                    f"Output file cannot have an absolute path {output_file}",
+                    positional_args[2],
+                )
+            return template_file, output_file
+        return None
+
     @classmethod
     def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
         gen_kw_key = cast(str, gen_kw[0])
@@ -99,26 +126,9 @@ def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
         if len(positional_args) == 2:
             parameter_file_contents = positional_args[1][1]
             parameter_file_context = positional_args[1][0]
-            template_file = None
-            output_file = None
         elif len(positional_args) == 4:
-            output_file = positional_args[2]
             parameter_file_contents = positional_args[3][1]
             parameter_file_context = positional_args[3][0]
-            template_file = _get_abs_path(positional_args[1][0])
-            if not os.path.isfile(template_file):
-                errors.append(
-                    ConfigValidationError.with_context(
-                        f"No such template file: {template_file}", positional_args[1]
-                    )
-                )
-            elif Path(template_file).stat().st_size == 0:
-                token = getattr(parameter_file_context, "token", parameter_file_context)
-                ConfigWarning.deprecation_warn(
-                    f"The template file for GEN_KW ({gen_kw_key}) is empty. If templating is not needed, you "
-                    f"can use GEN_KW with just the distribution file instead: GEN_KW {gen_kw_key} {token}",
-                    positional_args[1],
-                )
 
         else:
             raise ConfigValidationError(
@@ -185,8 +195,6 @@ def from_config_list(cls, gen_kw: list[str | dict[str, str]]) -> Self:
         return cls(
             name=gen_kw_key,
             forward_init=forward_init,
-            template_file=template_file,
-            output_file=output_file,
             forward_init_file=init_file,
             transform_function_definitions=transform_function_definitions,
             update=update_parameter,
@@ -358,21 +366,21 @@ def parse_value(value: float | int | str) -> float | int | str:
             if tf.use_log
         }
 
-        if self.template_file is not None and self.output_file is not None:
-            target_file = substitute_runpath_name(
-                self.output_file, real_nr, ensemble.iteration
-            )
-            target_file = target_file.removeprefix("/")
-            (run_path / target_file).parent.mkdir(exist_ok=True, parents=True)
-            template_file_path = (
-                ensemble.experiment.mount_point / Path(self.template_file).name
-            )
-            with open(template_file_path, encoding="utf-8") as f:
-                template = f.read()
-            for key, value in data.items():
-                template = template.replace(f"<{key}>", f"{value:.6g}")
-            with open(run_path / target_file, "w", encoding="utf-8") as f:
-                f.write(template)
+        # if self.template_file is not None and self.output_file is not None:
+        #     target_file = substitute_runpath_name(
+        #         self.output_file, real_nr, ensemble.iteration
+        #     )
+        #     target_file = target_file.removeprefix("/")
+        #     (run_path / target_file).parent.mkdir(exist_ok=True, parents=True)
+        #     template_file_path = (
+        #         ensemble.experiment.mount_point / Path(self.template_file).name
+        #     )
+        #     with open(template_file_path, encoding="utf-8") as f:
+        #         template = f.read()
+        #     for key, value in data.items():
+        #         template = template.replace(f"<{key}>", f"{value:.6g}")
+        #     with open(run_path / target_file, "w", encoding="utf-8") as f:
+        #         f.write(template)
         if log10_data:
             return {self.name: data, f"LOG10_{self.name}": log10_data}
         else:
@@ -542,13 +550,13 @@ def _parse_transform_function_definition(
             calc_func=PRIOR_FUNCTIONS[t.param_name],
         )
 
-    def save_experiment_data(self, experiment_path: Path) -> None:
-        if self.template_file:
-            incoming_template_file_path = Path(self.template_file)
-            template_file_path = Path(
-                experiment_path / incoming_template_file_path.name
-            )
-            shutil.copyfile(incoming_template_file_path, template_file_path)
+    # def save_experiment_data(self, experiment_path: Path) -> None:
+    #     if self.template_file:
+    #         incoming_template_file_path = Path(self.template_file)
+    #         template_file_path = Path(
+    #             experiment_path / incoming_template_file_path.name
+    #         )
+    #         shutil.copyfile(incoming_template_file_path, template_file_path)
 
 
 @dataclass

src/ert/enkf_main.py

@@ -91,7 +91,7 @@ def _generate_parameter_files(
     iens: int,
     fs: Ensemble,
     iteration: int,
-) -> None:
+) -> dict[str, dict[str, float]]:
     """
     Generate parameter files that are placed in each runtime directory for
     forward-model jobs to consume.
@@ -120,6 +120,7 @@ def _generate_parameter_files(
 
     _value_export_txt(run_path, export_base_name, exports)
     _value_export_json(run_path, export_base_name, exports)
+    return exports
 
 
 def _manifest_to_json(ensemble: Ensemble, iens: int, iter: int) -> dict[str, Any]:
@@ -242,6 +243,15 @@ def create_run_path(
         run_path = Path(run_arg.runpath)
         if run_arg.active:
             run_path.mkdir(parents=True, exist_ok=True)
+            param_data = _generate_parameter_files(
+                ensemble.experiment.parameter_configuration.values(),
+                parameters_file,
+                run_path,
+                run_arg.iens,
+                ensemble,
+                ensemble.iteration,
+            )
+
             for source_file, target_file in templates:
                 target_file = substitutions.substitute_real_iter(
                     target_file, run_arg.iens, ensemble.iteration
@@ -258,6 +268,11 @@ def create_run_path(
                     run_arg.iens,
                     ensemble.iteration,
                 )
+                result = substitutions.substitute_parameters(
+                    result,
+                    param_data,
+                )
+
                 target = run_path / target_file
                 if not target.parent.exists():
                     os.makedirs(
@@ -266,15 +281,6 @@ def create_run_path(
                     )
                 target.write_text(result)
 
-            _generate_parameter_files(
-                ensemble.experiment.parameter_configuration.values(),
-                parameters_file,
-                run_path,
-                run_arg.iens,
-                ensemble,
-                ensemble.iteration,
-            )
-
             path = run_path / "jobs.json"
             _backup_if_existing(path)
 

src/ert/storage/local_storage.py

@@ -456,6 +456,7 @@ def _migrate(self, version: int) -> None:
             to7,
             to8,
             to9,
+            to10,
         )
 
         try:
@@ -475,7 +476,8 @@ def _migrate(self, version: int) -> None:
 
                 logger.info("Blockfs storage backed up")
                 print(
-                    dedent(f"""
+                    dedent(
+                        f"""
                     Detected outdated storage (blockfs), which is no longer supported
                     by ERT. Its contents are copied to:
 
@@ -494,13 +496,14 @@ def _migrate(self, version: int) -> None:
                     This is not guaranteed to work. Other than setting the custom
                     ENSPATH, the ERT config should ideally be the same as it was
                     when the old blockfs storage was created.
-                """)
+                """
+                    )
                 )
                 return None
 
             elif version < _LOCAL_STORAGE_VERSION:
                 migrations = list(
-                    enumerate([to2, to3, to4, to5, to6, to7, to8, to9], start=1)
+                    enumerate([to2, to3, to4, to5, to6, to7, to8, to9, to10], start=1)
                 )
                 for from_version, migration in migrations[version - 1 :]:
                     print(f"* Updating storage to version: {from_version + 1}")

src/ert/storage/migration/to10.py

@@ -0,0 +1,17 @@
+import json
+from pathlib import Path
+
+info = "Migrate template GEN_KW feature into RUN_TEMPLATE"
+
+
+def migrate(path: Path) -> None:
+    for experiment in path.glob("experiments/*"):
+        with open(experiment / "parameter.json", encoding="utf-8") as fin:
+            parameters_json = json.load(fin)
+
+        with open(experiment / "parameter.json", "w", encoding="utf-8") as fout:
+            for param in parameters_json.values():
+                if param["_ert_kind"] == "GenKwConfig":
+                    param.pop("template_file", None)
+                    param.pop("output_file", None)
+            fout.write(json.dumps(parameters_json, indent=4))

src/ert/substitutions.py

@@ -30,6 +30,14 @@ def substitute(
         """
         return _substitute(self, to_substitute, context, max_iterations, warn_max_iter)
 
+    def substitute_parameters(
+        self, to_substitute: str, data: dict[str, dict[str, float]]
+    ) -> str:
+        for values in data.values():
+            for key, value in values.items():
+                to_substitute = to_substitute.replace(f"<{key}>", f"{value:.6g}")
+        return to_substitute
+
     def substitute_real_iter(
         self, to_substitute: str, realization: int, iteration: int
     ) -> str:

tests/ert/performance_tests/test_obs_and_responses_performance.py

@@ -57,8 +57,6 @@ def create_experiment_args(
 ) -> ExperimentInfo:
     gen_kw_config = GenKwConfig(
         name="all_my_parameters_live_here",
-        template_file=None,
-        output_file=None,
         forward_init=False,
         update=True,
         transform_function_definitions=[

tests/ert/ui_tests/cli/analysis/test_es_update.py

@@ -24,11 +24,9 @@ def uniform_parameter():
     return GenKwConfig(
         name="PARAMETER",
         forward_init=False,
-        template_file="",
         transform_function_definitions=[
             TransformFunctionDefinition("KEY1", "UNIFORM", [0, 1]),
         ],
-        output_file="kw.txt",
         update=True,
     )