Add yaml obs converter

The purpose of this converter class is to support conversion of
observation configurations to a format supported by webviz.

For now, only summary observations are of interest. RFTs are
manually loaded through other workflows and other observations
are not of interest as of now.

Localization is not supported in webviz, so those attributes are
left out.

Author: SAKavli

src/ert/observation_converters/dispatcher.py

@@ -9,18 +9,21 @@
 from .summary_to_bulk import (
     convert_summary_to_bulk,
 )
+from .summary_to_yaml import convert_summary_to_yaml
 
 
 class SupportedFormat(StrEnum):
     SUMMARY = "summary"
     BULK = "bulk"
+    YAML = "yaml"
 
 
 ConverterFunction = Callable[[str], None]
 
 _SUPPORTED_CONVERSIONS: dict[SupportedFormat, ConverterFunction] = {
     SupportedFormat.BULK: convert_summary_to_bulk,
     SupportedFormat.SUMMARY: convert_history_to_summary,
+    SupportedFormat.YAML: convert_summary_to_yaml,
 }
 
 

src/ert/observation_converters/summary_to_yaml.py

@@ -0,0 +1,83 @@
+from pathlib import Path
+from typing import Literal, TypedDict
+
+from ruamel.yaml import YAML
+
+from ert.cli.main import ErtCliError
+from ert.config import ErtConfig, ObservationType
+from ert.config.parsing import ObservationDict, read_file
+
+
+class YamlObservation(TypedDict):
+    date: str
+    value: float
+    error: float
+
+
+class SummaryDict(TypedDict):
+    key: str
+    observations: list[YamlObservation]
+
+
+YamlDict = dict[Literal["smry"], list[SummaryDict]]
+
+
+class YamlConverter:
+    target_file = "summary_observations.yaml"
+
+    def __init__(self, observations: list[ObservationDict]) -> None:
+        summary_observations = [
+            o for o in observations if o["type"] == ObservationType.SUMMARY
+        ]
+        if not summary_observations:
+            raise ErtCliError("No summary observations in configuration.\nExiting ...")
+
+        self.summary_observations = summary_observations
+
+    def _summary_to_yaml_dict(self) -> YamlDict:
+        summary_keys: set[str] = {str(o["KEY"]) for o in self.summary_observations}
+        summary_list: list[SummaryDict] = []
+        for key in summary_keys:
+            observations_with_key = [
+                o for o in self.summary_observations if o["KEY"] == key
+            ]
+            obs_dicts: list[YamlObservation] = [
+                {
+                    "date": str(o["DATE"]),
+                    "value": float(o["VALUE"]),
+                    "error": float(o["ERROR"]),
+                }
+                for o in observations_with_key
+            ]
+            summary_dict: SummaryDict = {"key": key, "observations": obs_dicts}
+            summary_list.append(summary_dict)
+        return {"smry": summary_list}
+
+    def export_yaml(self) -> None:
+        yaml = YAML()
+        yaml_dict = self._summary_to_yaml_dict()
+        if Path(self.target_file).is_file():
+            raise ErtCliError(
+                f"A file with name '{self.target_file}' already exists. "
+                f"Will not overwrite it and exit instead."
+            )
+        with Path(self.target_file).open("w", encoding="utf-8") as f:
+            yaml.dump(yaml_dict, f)
+        print(f"Successfully wrote summary observations to '{self.target_file}'.")
+
+
+def convert_summary_to_yaml(config: str) -> None:
+    user_config_contents = read_file(config)
+    config_dict = ErtConfig._config_dict_from_contents(
+        user_config_contents,
+        config,
+    )
+    file, obs_config = config_dict.get("OBS_CONFIG", (None, None))
+
+    if file is None or config_dict is None:
+        raise ErtCliError("No observation configuration found.\nExiting ...")
+
+    yaml_exporter = YamlConverter(
+        observations=obs_config,
+    )
+    yaml_exporter.export_yaml()

tests/ert/unit_tests/cli/test_summary_to_bulk.py

@@ -112,7 +112,7 @@ def _make_summary_obs(
     key += f":{well}" if well else ""
     return SummaryObservation(
         name="foo",
-        key=f"{key}:{well}",
+        key=key,
         value=0.5,
         error=0.02,
         date=date,

tests/ert/unit_tests/cli/test_summary_to_yaml.py

@@ -0,0 +1,162 @@
+import logging
+import warnings
+from pathlib import Path
+from textwrap import dedent
+from unittest.mock import MagicMock
+
+import pytest
+
+from ert.cli.main import ErtCliError
+from ert.config import ObservationType
+from ert.config.parsing import ObservationDict
+from ert.observation_converters import convert_observations
+from ert.observation_converters.summary_to_yaml import YamlConverter
+
+
+def _make_obs_dict(
+    obs_type: ObservationType = ObservationType.SUMMARY,
+    key: str = "WOPR",
+    well: str = "OP1",
+) -> ObservationDict:
+    key = f"{key}:{well}" if well else key
+    return ObservationDict(
+        {
+            "type": obs_type,
+            "KEY": key,
+            "ERROR": 5,
+            "VALUE": 10,
+            "DATE": "2010-10-10",
+        },
+        context=MagicMock(),
+    )
+
+
+@pytest.mark.usefixtures("snake_oil_case")
+def test_that_happy_path_on_snake_oil_produces_yaml_and_stdout(capsys):
+    args = MagicMock(format="yaml", config="snake_oil.ert")
+    convert_observations(args)
+
+    expected_yaml_content = dedent("""\
+    smry:
+    - key: WOPR:OP1
+      observations:
+      - date: '2010-03-31'
+        value: 0.1
+        error: 0.05
+      - date: '2010-12-26'
+        value: 0.7
+        error: 0.07
+      - date: '2011-12-21'
+        value: 0.5
+        error: 0.05
+      - date: '2012-12-15'
+        value: 0.3
+        error: 0.075
+      - date: '2013-12-10'
+        value: 0.2
+        error: 0.035
+      - date: '2015-03-15'
+        value: 0.015
+        error: 0.01
+    """)
+    yaml_content = Path("summary_observations.yaml").read_text(encoding="utf-8")
+    assert yaml_content == expected_yaml_content
+
+    expected_stdout = (
+        f"Successfully wrote summary observations to '{YamlConverter.target_file}'."
+    )
+    stdout = capsys.readouterr().out
+    assert expected_stdout in stdout
+
+
+def test_that_empty_observations_raises_ert_cli_error():
+    with pytest.raises(ErtCliError, match="No summary observations in configuration"):
+        YamlConverter([])
+
+
+def test_that_no_summary_observations_raises_ert_cli_error():
+    gen_obs = _make_obs_dict(obs_type=ObservationType.GENERAL)
+    with pytest.raises(ErtCliError, match="No summary observations in configuration"):
+        YamlConverter([gen_obs])
+
+
+def test_that_observations_with_same_summary_key_are_gathered_in_yaml_dict(use_tmpdir):
+    k1, k2 = "foo", "bar"
+    observations = 2 * [
+        _make_obs_dict(key=k1, well=""),
+        _make_obs_dict(key=k2, well=""),
+    ]
+
+    converter = YamlConverter(observations=observations)
+    summary_dicts = converter._summary_to_yaml_dict()
+
+    keys = [entry["key"] for entry in summary_dicts["smry"]]
+    assert len(keys) == 2
+    assert set(keys) == {k1, k2}
+
+    observations = [s_d["observations"] for s_d in summary_dicts["smry"]]
+    assert all(len(o) == 2 for o in observations)
+
+
+def test_that_observations_with_different_summary_keys_are_separated_in_yaml_dict():
+    k1, k2 = "foo", "bar"
+    observations = [
+        _make_obs_dict(key=k1, well=""),
+        _make_obs_dict(key=k2, well=""),
+    ]
+
+    converter = YamlConverter(observations=observations)
+    result = converter._summary_to_yaml_dict()
+
+    keys = [entry["key"] for entry in result["smry"]]
+    assert len(keys) == 2
+    assert set(keys) == {k1, k2}
+
+
+def test_that_dumping_to_yaml_is_skipped_when_file_already_exists(use_tmpdir):
+    observations = [_make_obs_dict()]
+
+    Path("summary_observations.yaml").write_text("existing", encoding="utf-8")
+    assert Path("summary_observations.yaml").is_file()
+
+    converter = YamlConverter(observations=observations)
+    with pytest.raises(
+        ErtCliError,
+        match=(
+            r"A file with name 'summary_observations.yaml' already exists. "
+            "Will not overwrite it and exit instead."
+        ),
+    ):
+        converter.export_yaml()
+
+
+def test_that_config_warnings_are_caught_instead_of_printed_to_terminal(
+    caplog, use_tmpdir
+):
+    caplog.set_level(logging.INFO)
+    config = "config.ert"
+    obs_config = "obs.txt"
+    # This setup expects the warning:
+    # 'Config contains a SUMMARY key but no forward model steps'
+    # to be raised
+    config_content = f"""\
+    NUM_REALIZATIONS 5
+    SUMMARY *
+    ECLBASE FOO
+    OBS_CONFIG {obs_config}
+    """
+    obs_config_content = """\
+    SUMMARY_OBSERVATION {
+        KEY=FOPR;
+        VALUE=10;
+        ERROR=5;
+        DATE=2010-10-10;
+    };"""
+    Path(config).write_text(config_content, encoding="utf-8")
+    Path(obs_config).write_text(obs_config_content, encoding="utf-8")
+
+    args = MagicMock(format="yaml", config=config)
+    with warnings.catch_warnings(record=True) as w:
+        convert_observations(args)
+
+    assert len(w) == 0