Remove derived_response from storage

All responses are now considered equal in the eyes of storage and
general operations.

Author: achaikou

src/ert/config/__init__.py

@@ -31,7 +31,6 @@
 )
 from .gen_data_config import GenDataConfig
 from .gen_kw_config import DataSource, GenKwConfig, PriorDict
-from .known_derived_response_types import KnownDerivedResponseTypes
 from .known_response_types import KnownResponseTypes
 from .lint_file import lint_file
 from .model_config import ModelConfig
@@ -121,7 +120,6 @@
     "HookedWorkflowFixtures",
     "InvalidResponseFile",
     "InversionTypeES",
-    "KnownDerivedResponseTypes",
     "KnownQueueOptions",
     "KnownQueueOptionsAdapter",
     "KnownResponseTypes",

src/ert/config/ensemble_config.py

@@ -7,30 +7,23 @@
 
 from pydantic import BaseModel, Field, model_validator
 
-from .breakthrough_config import BreakthroughConfig
 from .everest_control import EverestControl
 from .field import Field as FieldConfig
 from .gen_kw_config import GenKwConfig
-from .known_derived_response_types import KnownDerivedResponseTypes
 from .known_response_types import (
     KNOWN_ERT_SIMULATION_RESPONSE_TYPES,
-    KnownErtSimulationResponseTypes,
+    KnownErtResponseTypes,
 )
 from .parameter_config import ParameterConfig
 from .parsing import ConfigDict, ConfigKeys, ConfigValidationError
-from .response_config import SimulationResponseConfig
+from .response_config import ResponseConfig
 from .surface_config import SurfaceConfig
 
 logger = logging.getLogger(__name__)
 
 
 class EnsembleConfig(BaseModel):
-    response_configs: dict[str, KnownErtSimulationResponseTypes] = Field(
-        default_factory=dict
-    )
-    derived_response_configs: dict[str, KnownDerivedResponseTypes] = Field(
-        default_factory=dict
-    )
+    response_configs: dict[str, KnownErtResponseTypes] = Field(default_factory=dict)
     parameter_configs: dict[
         str, GenKwConfig | FieldConfig | SurfaceConfig | EverestControl
     ] = Field(default_factory=dict)
@@ -137,7 +130,7 @@ def make_field(field_list: list[str | dict[str, str]]) -> FieldConfig:
             + [make_field(f) for f in field_list]
         )
         EnsembleConfig._check_for_duplicate_gen_kw_param_names(gen_kw_cfgs)
-        response_configs: list[KnownErtSimulationResponseTypes] = []
+        response_configs: list[KnownErtResponseTypes] = []
 
         for config_cls in KNOWN_ERT_SIMULATION_RESPONSE_TYPES:
             instance = config_cls.from_config_dict(config_dict)
@@ -152,7 +145,7 @@ def make_field(field_list: list[str | dict[str, str]]) -> FieldConfig:
             },
         )
 
-    def __getitem__(self, key: str) -> ParameterConfig | SimulationResponseConfig:
+    def __getitem__(self, key: str) -> ParameterConfig | ResponseConfig:
         if key in self.parameter_configs:
             return self.parameter_configs[key]
         if key in self.response_configs:
@@ -202,9 +195,5 @@ def parameter_configuration(self) -> list[ParameterConfig]:
         return list(self.parameter_configs.values())
 
     @property
-    def response_configuration(self) -> list[SimulationResponseConfig]:
+    def response_configuration(self) -> list[ResponseConfig]:
         return list(self.response_configs.values())
-
-    @property
-    def derived_response_configuration(self) -> list[BreakthroughConfig]:
-        return list(self.derived_response_configs.values())

src/ert/config/ert_config.py

@@ -1287,14 +1287,12 @@ def derive_breakthrough_response_input_from_observations(self) -> None:
 
         bt_obs = [o for o in observations if isinstance(o, BreakthroughObservation)]
 
-        if "breakthrough" not in ensemble_config.derived_response_configs and bt_obs:
-            ensemble_config.derived_response_configs["breakthrough"] = (
-                BreakthroughConfig(
-                    keys=[f"BREAKTHROUGH:{o.key}" for o in bt_obs],
-                    summary_keys=[o.key for o in bt_obs],
-                    thresholds=[o.threshold for o in bt_obs],
-                    observed_dates=[o.date for o in bt_obs],
-                )
+        if "breakthrough" not in ensemble_config.response_configs and bt_obs:
+            ensemble_config.response_configs["breakthrough"] = BreakthroughConfig(
+                keys=[f"BREAKTHROUGH:{o.key}" for o in bt_obs],
+                summary_keys=[o.key for o in bt_obs],
+                thresholds=[o.threshold for o in bt_obs],
+                observed_dates=[o.date for o in bt_obs],
             )
 
     @classmethod

src/ert/config/known_derived_response_types.py

@@ -1,5 +1,6 @@
 from ert.config.seismic_config import SeismicConfig
 
+from .breakthrough_config import BreakthroughConfig
 from .everest_response import EverestConstraintsConfig, EverestObjectivesConfig
 from .gen_data_config import GenDataConfig
 from .rft_config import RFTConfig
@@ -8,12 +9,18 @@
 KnownErtSimulationResponseTypes = (
     SummaryConfig | GenDataConfig | RFTConfig | SeismicConfig
 )
+KnownErtDerivedResponseTypes = BreakthroughConfig
+KnownErtResponseTypes = KnownErtSimulationResponseTypes | KnownErtDerivedResponseTypes
 KNOWN_ERT_SIMULATION_RESPONSE_TYPES = (
     SummaryConfig,
     GenDataConfig,
     RFTConfig,
     SeismicConfig,
 )
+KNOWN_ERT_DERIVED_RESPONSE_TYPES = (BreakthroughConfig,)
+KNOWN_ERT_RESPONSE_TYPES = (
+    KNOWN_ERT_SIMULATION_RESPONSE_TYPES + KNOWN_ERT_DERIVED_RESPONSE_TYPES
+)
 KnownResponseTypes = (
-    KnownErtSimulationResponseTypes | EverestConstraintsConfig | EverestObjectivesConfig
+    KnownErtResponseTypes | EverestConstraintsConfig | EverestObjectivesConfig
 )

src/ert/config/known_response_types.py

@@ -32,11 +32,7 @@ def get_experiments(
             },
             responses={
                 response_type: config.model_dump()
-                for response_type, config in experiment.simulation_response_configuration.items()  # noqa: E501
-            },
-            derived_responses={
-                response_type: config.model_dump()
-                for response_type, config in experiment.derived_response_configuration.items()  # noqa: E501
+                for response_type, config in experiment.response_configuration.items()
             },
             observations=experiment.response_key_to_observation_key,
         )

src/ert/dark_storage/endpoints/experiments.py

@@ -25,5 +25,4 @@ class ExperimentOut(_Experiment):
     userdata: Mapping[str, Any]
     parameters: Mapping[str, dict[str, Any]]
     responses: Mapping[str, dict[str, Any]]
-    derived_responses: Mapping[str, dict[str, Any]]
     observations: Mapping[str, dict[str, list[str]]]

src/ert/dark_storage/json_schema/experiment.py

@@ -18,8 +18,9 @@
 from resfo_utilities import history_key
 
 from ert.config import ParameterConfig
-from ert.config.known_derived_response_types import KnownDerivedResponseTypes
-from ert.config.known_response_types import KnownResponseTypes
+from ert.config.known_response_types import (
+    KnownResponseTypes,
+)
 from ert.config.response_config import ResponseConfig
 from ert.services import create_ertserver_client
 from ert.storage.local_experiment import _parameters_adapter as parameter_config_adapter
@@ -189,10 +190,8 @@ def update_keydef(plot_key_def: PlotApiKeyDefinition) -> None:
                     key_defs[plot_key_def.key] = plot_key_def
 
             for experiment in http_response.json():
-                for response_type, metadata in (
-                    experiment["responses"] | experiment["derived_responses"]
-                ).items():
-                    response_config: KnownResponseTypes | KnownDerivedResponseTypes = (
+                for response_type, metadata in experiment["responses"].items():
+                    response_config: KnownResponseTypes = (
                         response_config_adapter.validate_python(metadata)
                     )
                     keys = response_config.response_keys()

src/ert/gui/plotting/plot_api.py

@@ -152,7 +152,6 @@ def _setup_single_test_run(
         design_matrix=design_matrix,
         parameter_configuration=parameter_configs,
         response_configuration=config.ensemble_config.response_configuration,
-        derived_response_configuration=config.ensemble_config.derived_response_configuration,
         ert_templates=config.ert_templates,
         user_config_file=Path(config.user_config_file),
         env_vars=config.env_vars,
@@ -213,7 +212,6 @@ def _setup_ensemble_experiment(
         design_matrix=design_matrix,
         parameter_configuration=parameter_configs,
         response_configuration=config.ensemble_config.response_configuration,
-        derived_response_configuration=config.ensemble_config.derived_response_configuration,
         ert_templates=config.ert_templates,
         user_config_file=Path(config.user_config_file),
         env_vars=config.env_vars,
@@ -349,7 +347,6 @@ def _setup_manual_update_enif(
         design_matrix=config.analysis_config.design_matrix,
         parameter_configuration=config.ensemble_config.parameter_configuration,
         response_configuration=config.ensemble_config.response_configuration,
-        derived_response_configuration=config.ensemble_config.derived_response_configuration,
         ert_templates=config.ert_templates,
         user_config_file=Path(config.user_config_file),
         env_vars=config.env_vars,
@@ -397,7 +394,6 @@ def _setup_ensemble_smoother(
         design_matrix=design_matrix,
         parameter_configuration=parameter_configs,
         response_configuration=config.ensemble_config.response_configuration,
-        derived_response_configuration=config.ensemble_config.derived_response_configuration,
         ert_templates=config.ert_templates,
         user_config_file=Path(config.user_config_file),
         env_vars=config.env_vars,
@@ -445,7 +441,6 @@ def _setup_ensemble_information_filter(
         design_matrix=design_matrix,
         parameter_configuration=parameter_configs,
         response_configuration=config.ensemble_config.response_configuration,
-        derived_response_configuration=config.ensemble_config.derived_response_configuration,
         ert_templates=config.ert_templates,
         user_config_file=Path(config.user_config_file),
         env_vars=config.env_vars,
@@ -519,7 +514,6 @@ def _setup_multiple_data_assimilation(
         design_matrix=design_matrix,
         parameter_configuration=parameter_configs,
         response_configuration=config.ensemble_config.response_configuration,
-        derived_response_configuration=config.ensemble_config.derived_response_configuration,
         ert_templates=config.ert_templates,
         user_config_file=Path(config.user_config_file),
         env_vars=config.env_vars,

src/ert/run_models/model_factory.py

@@ -19,7 +19,6 @@
     GenDataConfig,
     GenKwConfig,
     HookRuntime,
-    KnownDerivedResponseTypes,
     KnownResponseTypes,
     ModelConfig,
     Observation,
@@ -153,12 +152,6 @@ class InitialEnsembleRunModelConfig(RunModelConfig):
             Field(discriminator="type"),
         ]
     ]
-    derived_response_configuration: list[
-        Annotated[
-            (KnownDerivedResponseTypes),
-            Field(discriminator="type"),
-        ]
-    ]
     ert_templates: list[tuple[str, str]]
     observations: list[Observation] | None = None
     shape_registry: ShapeRegistry | None = None
@@ -172,10 +165,6 @@ def _initial_ensemble_experiment_config(self) -> ExperimentConfig:
             "response_configuration": [
                 resp.model_dump(mode="json") for resp in self.response_configuration
             ],
-            "derived_response_configuration": [
-                resp.model_dump(mode="json")
-                for resp in self.derived_response_configuration
-            ],
             "experiment_name": self.experiment_name,
         }
 
@@ -321,9 +310,6 @@ def to_experiment_config(
             "response_configuration": prior_experiment_config.get(
                 "response_configuration", []
             ),
-            "derived_response_configuration": prior_experiment_config.get(
-                "derived_response_configuration", []
-            ),
             "observations": prior_experiment_config.get("observations", []),
             **self._common_fields(),
             "experiment_type": ExperimentType.MANUAL_UPDATE,