Remove EverestDataAPI and is_improvement flag from everest storage (#14084)

* Remove unused EverestDataApi

* Remove is_improvement flag from everest storage

* Move utilities to utils.py

Author: eqbech

src/ert/dark_storage/endpoints/responses.py

@@ -19,6 +19,7 @@
     serialize_dataframe_to_response,
 )
 from ert.storage import Ensemble, Storage
+from everest.config.utils import CONSTRAINT_TOLERANCE, constraint_violation_check
 
 router = APIRouter(tags=["responses"])
 logger = logging.getLogger(__name__)
@@ -160,12 +161,6 @@ def data_for_response(
         if ensemble.batch_objectives is None:
             return pd.DataFrame()
 
-        def constraint_violation_check(violation: pl.DataFrame | None) -> float:
-            if violation is None:
-                return 0.0
-            return violation.drop("batch_id").to_numpy().max().item()
-
-        CONSTRAINT_TOL = 1e-6
         accepted_batches: list[tuple[Ensemble, float]] = []
         rejected_batches_with_value_and_reason: list[
             tuple[Ensemble, float, RejectionReason]
@@ -196,7 +191,7 @@ def constraint_violation_check(violation: pl.DataFrame | None) -> float:
                     input_violation,
                     output_violation,
                 )
-                < CONSTRAINT_TOL
+                < CONSTRAINT_TOLERANCE
                 and total_objective < max_total_objective
             ):
                 accepted_batches.append(

src/ert/run_models/everest_run_model.py

@@ -624,7 +624,6 @@ def _handle_optimizer_results(self, results: tuple[Results, ...]) -> None:
         for batch_id, batch_dict in batch_dataframes.items():
             target_ensemble = self._experiment.get_ensemble_by_name(f"batch_{batch_id}")
             target_ensemble.save_batch_dataframes(dataframes=batch_dict)
-            target_ensemble.update_improvement_flag(is_improvement=False)
 
         for r in results:
             batches = (
@@ -753,59 +752,6 @@ def objectives_config(self) -> EverestObjectivesConfig:
         assert isinstance(obj_config, EverestObjectivesConfig)
         return obj_config
 
-    def _update_ensemble_improvement_flags(self) -> None:
-        assert self._experiment is not None
-
-        # This a somewhat arbitrary threshold, this should be a user choice
-        # during visualization:
-        CONSTRAINT_TOL = 1e-6
-
-        max_total_objective = np.inf
-        for ensemble in self._experiment.ensembles_with_function_results:
-            assert ensemble.batch_objectives is not None
-            total_objective = ensemble.batch_objectives["total_objective_value"].item()
-            bound_constraint_violation = (
-                0.0
-                if ensemble.batch_bound_constraint_violations is None
-                else (
-                    ensemble.batch_bound_constraint_violations.drop("batch_id")
-                    .to_numpy()
-                    .max()
-                    .item()
-                )
-            )
-            input_constraint_violation = (
-                0.0
-                if ensemble.batch_input_constraint_violations is None
-                else (
-                    ensemble.batch_input_constraint_violations.drop("batch_id")
-                    .to_numpy()
-                    .max()
-                    .item()
-                )
-            )
-            output_constraint_violation = (
-                0.0
-                if ensemble.batch_output_constraint_violations is None
-                else (
-                    ensemble.batch_output_constraint_violations.drop("batch_id")
-                    .to_numpy()
-                    .max()
-                    .item()
-                )
-            )
-            if (
-                max(
-                    bound_constraint_violation,
-                    input_constraint_violation,
-                    output_constraint_violation,
-                )
-                < CONSTRAINT_TOL
-                and total_objective < max_total_objective
-            ):
-                ensemble.update_improvement_flag(is_improvement=True)
-                max_total_objective = total_objective
-
     def _create_experiment_storage(self) -> LocalExperiment:
         return self._storage.create_experiment(
             name=self.experiment_name, experiment_config=self.to_experiment_config()
@@ -836,8 +782,6 @@ def run_experiment(
         # Run the optimization:
         optimizer_exit_code = optimizer.run(initial_guesses)
 
-        # Store some final results.
-        self._update_ensemble_improvement_flags()
         if (
             optimizer_exit_code is not RoptExitCode.UNKNOWN
             and optimizer_exit_code is not RoptExitCode.TOO_FEW_REALIZATIONS

src/ert/storage/local_ensemble.py

@@ -101,7 +101,6 @@ class _Index(BaseModel, extra="forbid"):
     prior_ensemble_id: UUID | None
     started_at: datetime
     everest_realization_info: dict[int, EverestRealizationInfo] | None = None
-    is_improvement: bool | None = None
 
 
 class _Failure(BaseModel, extra="forbid"):
@@ -1727,17 +1726,6 @@ def perturbation_constraints(self) -> pl.DataFrame | None:
             )
         )
 
-    @property
-    def is_improvement(self) -> bool:
-        return bool(self._index.is_improvement)
-
-    def update_improvement_flag(self, is_improvement: bool) -> None:
-        self._index.is_improvement = is_improvement
-        self._storage._write_transaction(
-            self._path / "index.json",
-            self._index.model_dump_json(indent=2).encode("utf-8"),
-        )
-
     def load_all_misfit_data(self) -> pl.DataFrame:
         """Loads all misfit data for a given ensemble.