Unify convert and gather obs endpoints

Converter functionality is moved to an own module:
observation_converters

The module contains a dispatcher which calls the appropriate
converter based on arguments given to the cli command.

This will also simplify the functionality of BulkConfigConverter
such that it no longer spins up an ert API but simply reads the 
observations from the ert config directly. 
This will match the behavior of existing history observation 
converter.

We currently do not wish to expose the specific observation types
to the public interface of ert.config, therefore there is a bunch
of custom handling for the Observation types in the bulk config
converter - as we are sure we are working with SummaryObservations
and BreakthroughObservations. This could be revisited later, if we
wish to expose these classes to the interface.

This commit also contains minor changes to the formatting of the
bulk config converter.

Author: SAKavli

src/ert/__main__.py

@@ -8,12 +8,10 @@
 import os
 import re
 import resource
-import shutil
 import sys
 import warnings
 from argparse import ArgumentParser, ArgumentTypeError
 from collections.abc import Sequence
-from datetime import datetime
 from pathlib import Path
 from typing import Any
 from uuid import UUID
@@ -26,10 +24,6 @@
 from ert.base_model_context import use_runtime_plugins
 from ert.cli.main import ErtCliError, run_cli
 from ert.config import ConfigValidationError, ErtConfig, lint_file
-from ert.config.observation_config_migrations import (
-    remove_refcase_and_time_map_dependence_from_obs_config,
-)
-from ert.export_observations import export_observations
 from ert.logging import LOGGING_CONFIG
 from ert.mode_definitions import (
     ENIF_MODE,
@@ -40,6 +34,7 @@
     WORKFLOW_MODE,
 )
 from ert.namespace import Namespace
+from ert.observation_converters import SupportedFormat, convert_observations
 from ert.plugins import ErtRuntimePlugins, get_site_plugins, setup_site_logging
 from ert.services import ErtServerController
 from ert.services._storage_main import add_parser_options as ert_api_add_parser_options
@@ -58,66 +53,6 @@
 logger = logging.getLogger(__name__)
 
 
-def run_convert_observations(
-    args: Namespace, _: ErtRuntimePlugins | None = None
-) -> None:
-    changes = remove_refcase_and_time_map_dependence_from_obs_config(args.config)
-
-    if changes is None or changes.is_empty():
-        logger.info("convert_observations did not make any changes")
-        print(
-            "No observations dependent on TIME_MAP / REFCASE found, you can "
-            "safely remove TIME_MAP / REFCASE and the "
-            "corresponding files from ERT config."
-        )
-        return
-
-    obs_config_to_edit_path = changes.obs_config_path + ".updated"
-    print(
-        f"Making copy of obs config "
-        f"@ {changes.obs_config_path} -> {obs_config_to_edit_path}"
-    )
-
-    shutil.copy(changes.obs_config_path, obs_config_to_edit_path)
-    print(f"Applying change to obs config @ {obs_config_to_edit_path}...")
-    changes.apply_to_file(Path(obs_config_to_edit_path))
-    convert_observations_trace = ""
-    for history_change in changes.history_changes:
-        convert_observations_trace += (
-            f"History obs {history_change.source_observation.name} "
-            f"-> {len(history_change.summary_obs_declarations)} summary observations\n"
-        )
-
-    for gen_obs_change in changes.general_obs_changes:
-        convert_observations_trace += (
-            f"General obs {gen_obs_change.source_observation.name}, changing "
-            f"DATE {gen_obs_change.source_observation.date} "
-            f"to RESTART={gen_obs_change.restart}\n"
-        )
-    for summary_change in changes.summary_obs_changes:
-        convert_observations_trace += (
-            f"Summary obs {summary_change.source_observation.name}, changing "
-            f"RESTART {summary_change.source_observation.restart} "
-            f"to DATE={summary_change.date}\n"
-        )
-
-    logger.info(f"convert_observations trace: \n {convert_observations_trace}")
-    print(convert_observations_trace)
-
-    os.rename(
-        changes.obs_config_path,
-        f"{changes.obs_config_path}-{datetime.now().astimezone().strftime('%Y-%m-%d-%H-%M-%S')}.old",
-    )
-    os.rename(obs_config_to_edit_path, changes.obs_config_path)
-    msg = (
-        f"Observation changes applied to {changes.obs_config_path}. The old "
-        f"observations file is now at {changes.obs_config_path}.old and can be "
-        f"safely deleted if the new one works."
-    )
-    print(msg)
-    logger.info(msg)
-
-
 def run_ert_storage(args: Namespace, _: ErtRuntimePlugins | None = None) -> None:
     with ErtServerController.start_server(
         verbose=True,
@@ -554,71 +489,39 @@ def get_ert_parser(parser: ArgumentParser | None = None) -> ArgumentParser:
     )
 
     # convert_observations_parser
+    help_text = (
+        "Convert observations to an observation format. "
+        "Must be provided with an ERT configuration file. "
+        "The default behaviour is to convert from history "
+        "observation format to summary, but this can be "
+        "configured using the --format flag to specify "
+        "which format to convert to."
+        "Valid formats are: bulk, summary"
+    )
     convert_obs_parser = subparsers.add_parser(
         "convert_observations",
-        help=(
-            "Convert HISTORY_OBSERVATION to SUMMARY_OBSERVATION and "
-            "remove REFCASE and TIME_MAP from ERT config."
-        ),
-        description=(
-            "Convert HISTORY_OBSERVATION to SUMMARY_OBSERVATION, "
-            "and embed REFCASE and TIME_MAP into observations"
-        ),
-    )
-    convert_obs_parser.set_defaults(func=run_convert_observations)
+        help=help_text,
+        description=help_text,
+    )
+    convert_obs_parser.set_defaults(func=convert_observations)
     convert_obs_parser.add_argument(
         "config",
         type=valid_file,
         help="Path to ERT config file",
     )
+
+    convert_obs_parser.add_argument(
+        "--format",
+        dest="format",
+        required=False,
+        default=SupportedFormat.SUMMARY,
+        choices=SupportedFormat,
+        help="Observation format to convert to.",
+    )
     convert_obs_parser.add_argument(
         "--verbose", action="store_true", help="Show verbose output.", default=False
     )
 
-    # Experimental feature
-    if os.environ.get("ERT_FEATURE_GATHER_OBS"):
-        extract_obs_summary_keys_parser = subparsers.add_parser(
-            "export_observations",
-            description=(
-                "Identify all summary and breakthrough observations and output them "
-                "in a bulk config format. A CSV file containing the observation values "
-                "is written to disk while the bulk config is printed to terminal to be "
-                "copied into the observation configuration.\n"
-                "Given no experiment argument, will prompt for which experiment to "
-                "extract observations from if there are multiple to choose from in "
-                "storage.\n"
-                "This workflow will not overwrite any existing configuration files, so "
-                "the output will have to be manually integrated.\n"
-                "This command requires the path to an ert config file as first "
-                "argument and optionally an experiment ID as second argument. The name "
-                "of the produced CSV filename is defaulted to "
-                "'summary_observations.csv', but can be manually set using the keyword "
-                "argument:\n"
-                "--output-csv-file <filename>"
-            ),
-            help="This command requires the path to an ert config file as first "
-            "argument and optionally an experiment ID as second argument. The name "
-            "of the produced CSV filename is defaulted to 'summary_observations.csv', "
-            "but can be manually set using the keyword argument:\n"
-            "--output-csv-file <filename>",
-        )
-        extract_obs_summary_keys_parser.set_defaults(func=export_observations)
-        extract_obs_summary_keys_parser.add_argument(
-            "config", type=valid_file, help="Path to ERT config file"
-        )
-        extract_obs_summary_keys_parser.add_argument(
-            "experiment_id", nargs="?", type=str, default=None, help="Experiment ID"
-        )
-        extract_obs_summary_keys_parser.add_argument(
-            "--output-csv-file",
-            default="summary_observations.csv",
-            type=str,
-            help="Output CSV file name",
-        )
-        extract_obs_summary_keys_parser.add_argument(
-            "--verbose", action="store_true", help="Show verbose output.", default=False
-        )
-
     # Common arguments/defaults for all non-gui modes
     for cli_parser in [
         test_run_parser,