Use objectives & constraints from ert storage

Author: Yngve S. Kristiansen

src/ert/run_models/everest_run_model.py

@@ -720,8 +720,6 @@ def run_experiment(
         )
 
         self._ever_storage.init(
-            objective_functions=self.objectives_config,
-            output_constraints=self.output_constraints_config,
             realizations=self.model.realizations,
         )
         optimizer.set_results_callback(self._handle_optimizer_results)

src/everest/api/everest_data_api.py

@@ -30,11 +30,7 @@ def accepted_batches(self) -> list[int]:
 
     @property
     def objective_function_names(self) -> list[str]:
-        if self._ever_storage.objective_functions is None:
-            return []
-        return sorted(
-            self._ever_storage.objective_functions["objective_name"].unique().to_list()
-        )
+        return self._ever_storage.objective_functions.keys
 
     @property
     def output_constraint_names(self) -> list[str]:
@@ -87,26 +83,22 @@ def control_values(self) -> list[dict[str, Any]]:
     @property
     def objective_values(self) -> list[dict[str, Any]]:
         obj_values = []
+
+        objectives = self._ever_storage.objective_functions
         for b in self._ever_storage.batches_with_function_results:
             for (
                 model_realization,
                 simulation_id,
             ), df in b.realization_objectives.sort(
                 ["realization", "simulation_id"]
             ).group_by(["realization", "simulation_id"], maintain_order=True):
-                for obj_dict in (
-                    self._ever_storage.objective_functions.sort(
-                        ["objective_name"]
-                    ).to_dicts()
-                    if self._ever_storage.objective_functions is not None
-                    else []
+                for key, scale, weight in zip(
+                    objectives.keys, objectives.scales, objectives.weights, strict=False
                 ):
-                    obj_name = obj_dict["objective_name"]
-                    obj_value = df[obj_name].item()
-
+                    obj_value = float(df[key].item())
                     if obj_value is None:
                         logger.error(
-                            f"Objective {obj_name} has no value for "
+                            f"Objective {key} has no value for "
                             f"batch {b.batch_id}, "
                             f"model realization {model_realization},"
                             f"simulation id {simulation_id}. "
@@ -119,10 +111,10 @@ def objective_values(self) -> list[dict[str, Any]]:
                             "batch": int(b.batch_id),
                             "realization": int(model_realization),
                             "simulation": int(simulation_id),
-                            "function": obj_name,
-                            "scale": float(obj_dict["scale"]),
-                            "value": float(obj_value),
-                            "weight": float(obj_dict["weight"]),
+                            "function": key,
+                            "scale": float(scale) if scale is not None else None,
+                            "value": obj_value,
+                            "weight": float(weight) if weight is not None else None,
                         }
                     )
 
@@ -140,18 +132,16 @@ def single_objective_values(self) -> list[dict[str, Any]]:
         )
         objectives = self._ever_storage.objective_functions
         assert objectives is not None
-        objective_names = objectives["objective_name"].unique().to_list()
-
-        for o in objectives.to_dicts():
-            batch_datas = batch_datas.with_columns(
-                pl.col(o["objective_name"]) * o["weight"] / o["scale"]
-            )
 
+        for name, weight, scale in zip(
+            objectives.keys, objectives.scales, objectives.weights, strict=False
+        ):
+            batch_datas = batch_datas.with_columns(pl.col(name) * weight / scale)
         columns = [
             "batch",
             "objective",
             "accepted",
-            *(objective_names if len(objective_names) > 1 else []),
+            *(objectives.keys if len(objectives.keys) > 1 else []),
         ]
 
         return (

src/everest/everest_storage.py

@@ -5,14 +5,14 @@
 import traceback
 from functools import cached_property
 from pathlib import Path
-from typing import Any, ClassVar, TypedDict
+from typing import Any, ClassVar, TypedDict, cast
 from uuid import UUID
 
 import numpy as np
 import polars as pl
 from ropt.results import FunctionResults, GradientResults, Results
 
-from ert.config import EverestConstraintsConfig, EverestObjectivesConfig
+from ert.config import EverestObjectivesConfig
 from ert.storage import LocalEnsemble, LocalExperiment, LocalStorage, open_storage
 from ert.storage.local_ensemble import BatchDataframes
 from everest.strings import EVEREST
@@ -25,8 +25,6 @@ def try_read_df(path: Path) -> pl.DataFrame | None:
 
 
 class OptimizationDataframes(TypedDict, total=False):
-    objective_functions: pl.DataFrame | None
-    nonlinear_constraints: pl.DataFrame | None
     realization_weights: pl.DataFrame | None
 
 
@@ -260,8 +258,6 @@ class _GradientResults(TypedDict):
 
 class EverestStorage:
     EXPERIMENT_DATAFRAMES: ClassVar[list[str]] = [
-        "objective_functions",
-        "nonlinear_constraints",
         "realization_weights",
     ]
 
@@ -319,8 +315,12 @@ def control_names(self) -> list[str]:
         return self.experiment.parameter_keys
 
     @property
-    def objective_functions(self) -> pl.DataFrame | None:
-        return pl.read_parquet(self.experiment._path / "objective_functions.parquet")
+    def objective_functions(self) -> EverestObjectivesConfig:
+        objectives_config = self.experiment.response_configuration.get(
+            "everest_objectives"
+        )
+        assert objectives_config is not None
+        return cast(EverestObjectivesConfig, objectives_config)
 
     @property
     def nonlinear_constraints(self) -> list[str]:
@@ -483,38 +483,8 @@ def batches(self) -> list[BatchStorageData]:
 
     def init(
         self,
-        objective_functions: EverestObjectivesConfig,
-        output_constraints: EverestConstraintsConfig | None,
         realizations: list[int],
     ) -> None:
-        weights = np.fromiter(
-            (
-                1.0 if weight is None else weight
-                for weight in objective_functions.weights
-            ),
-            dtype=np.float64,
-        )
-
-        objective_functions_dataframe = pl.DataFrame(
-            {
-                "objective_name": objective_functions.keys,
-                "weight": pl.Series(weights / sum(weights), dtype=pl.Float64),
-                "scale": pl.Series(
-                    [
-                        1.0 if scale is None else scale
-                        for scale in objective_functions.scales
-                    ],
-                    dtype=pl.Float64,
-                ),
-            }
-        )
-
-        nonlinear_constraints = (
-            pl.DataFrame({"constraint_name": output_constraints.keys})
-            if output_constraints
-            else None
-        )
-
         realization_weights = pl.DataFrame(
             {
                 "realization": pl.Series(realizations, dtype=pl.UInt32),
@@ -523,15 +493,9 @@ def init(
 
         self.save_experiment_dataframes(
             dataframes={
-                "objective_functions": objective_functions_dataframe,
-                "nonlinear_constraints": nonlinear_constraints,
                 "realization_weights": realization_weights,  # Store in metadata
             },
             experiment_path=self.experiment._path,
-            # Note: Write storage is held by ERT runmodel, hence we need to bypass
-            # the read/write, this should/could be synced better up between ERT /
-            # everest storage. Ideally Everest would have its own "write" priviliege
-            # for dumping optimization results.
         )
 
     @classmethod

tests/everest/snapshots/test_api_snapshots/test_api_snapshots/config_multiobj.yml/snapshot.json

@@ -111,7 +111,7 @@
       "scale": 0.66666667,
       "simulation": 0,
       "value": -0.75,
-      "weight": 0.66666667
+      "weight": 0.5
     },
     {
       "batch": 0,
@@ -120,7 +120,7 @@
       "scale": 1.0,
       "simulation": 0,
       "value": -4.75,
-      "weight": 0.33333333
+      "weight": 0.25
     },
     {
       "batch": 1,
@@ -129,7 +129,7 @@
       "scale": 0.66666667,
       "simulation": 0,
       "value": -0.76564598,
-      "weight": 0.66666667
+      "weight": 0.5
     },
     {
       "batch": 1,
@@ -138,7 +138,7 @@
       "scale": 1.0,
       "simulation": 0,
       "value": -4.70363998,
-      "weight": 0.33333333
+      "weight": 0.25
     },
     {
       "batch": 2,
@@ -147,7 +147,7 @@
       "scale": 0.66666667,
       "simulation": 0,
       "value": -0.50778502,
-      "weight": 0.66666667
+      "weight": 0.5
     },
     {
       "batch": 2,
@@ -156,7 +156,7 @@
       "scale": 1.0,
       "simulation": 0,
       "value": -4.47678995,
-      "weight": 0.33333333
+      "weight": 0.25
     }
   ],
   "optimal_result_json": {
@@ -179,22 +179,22 @@
     {
       "accepted": 1,
       "batch": 0,
-      "distance_p": -0.75,
-      "distance_q": -1.58333333,
+      "distance_p": -1.0,
+      "distance_q": -19.0,
       "objective": -2.08333333
     },
     {
       "accepted": 1,
       "batch": 1,
-      "distance_p": -0.76564598,
-      "distance_q": -1.56787999,
+      "distance_p": -1.02086131,
+      "distance_q": -18.81455994,
       "objective": -2.07831065
     },
     {
       "accepted": 1,
       "batch": 2,
-      "distance_p": -0.50778502,
-      "distance_q": -1.49226332,
+      "distance_p": -0.6770467,
+      "distance_q": -17.90715981,
       "objective": -1.83078667
     }
   ]

tests/everest/test_everest_storage.py

@@ -34,8 +34,6 @@ def test_csv_export(config_file, cached_example, snapshot):
         storage_path=config.storage_dir,
     )
     ever_storage.init(
-        objective_functions=config.create_ert_objectives_config(),
-        output_constraints=config.create_ert_output_constraints_config(),
         realizations=config.model.realizations,
     )
     combined_df, pert_real_df, batch_df = ever_storage.export_dataframes()