fix(PST regulation): Objective function reevaluation after PST regulation (#1561)

* do not regulate skipped perimeters
* add log when no PST regulated
* fix bug on reported cost

Signed-off-by: Thomas Bouquet <thomas.bouquet@rte-france.com>
Co-authored-by: Godelaine <godelaine.demontmorillon@rte-france.com>

Author: bqth29

ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/castor/algorithm/CastorFullOptimization.java

@@ -10,7 +10,6 @@
 import com.powsybl.openrao.commons.OpenRaoException;
 import com.powsybl.openrao.data.crac.api.Crac;
 import com.powsybl.openrao.data.crac.api.Instant;
-import com.powsybl.openrao.data.crac.api.InstantKind;
 import com.powsybl.openrao.data.crac.api.State;
 import com.powsybl.openrao.data.crac.api.cnec.FlowCnec;
 import com.powsybl.openrao.data.crac.api.networkaction.NetworkAction;
@@ -46,6 +45,7 @@
 import java.time.temporal.ChronoUnit;
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.stream.Collectors;
 
 import static com.powsybl.openrao.commons.logs.OpenRaoLoggerProvider.*;
 import static com.powsybl.openrao.searchtreerao.commons.HvdcUtils.getHvdcRangeActionsOnHvdcLineInAcEmulation;
@@ -200,28 +200,34 @@ public CompletableFuture<RaoResult> run() {
             boolean logFinalResultsOutsideOfSecondPreventive = true;
             // Run second preventive when necessary
             CastorSecondPreventive castorSecondPreventive = new CastorSecondPreventive(crac, raoParameters, network, stateTree, toolProvider, targetEndInstant);
-            PostPerimeterResult secondPreventiveResult = postPreventiveResult;
+
+            // define variables to set with second preventive results only if it improves first
+            PostPerimeterResult finalSecondPreventiveResult = postPreventiveResult;
+            PostPerimeterResult intermediateSecondPreventiveResult = postPreventiveResult;
+            Map<State, PostPerimeterResult> finalPostContingencyResults = new HashMap<>(postContingencyResults);
+
             if (castorSecondPreventive.shouldRunSecondPreventiveRao(preventiveResult, postContingencyResults.values(), mergedRaoResults, preventiveRaoTime)) {
                 CastorSecondPreventive.SecondPreventiveRaoResultsHolder secondPreventiveRaoResultsHolder = castorSecondPreventive.runSecondPreventiveAndAutoRao(castorContingencyScenarios, prePerimeterSensitivityAnalysis, initialOutput, postPreventiveResult, postContingencyResults);
                 RaoResult secondPreventiveRaoResults;
                 if (secondPreventiveRaoResultsHolder.hasFailed()) {
                     secondPreventiveRaoResults = new FailedRaoResultImpl(secondPreventiveRaoResultsHolder.errorMessage());
                 } else {
-                    secondPreventiveResult = new PostPerimeterResult(
+                    intermediateSecondPreventiveResult = new PostPerimeterResult(
                         secondPreventiveRaoResultsHolder.secondPreventiveRaoResult().perimeterResult(),
                         secondPreventiveRaoResultsHolder.secondPreventiveRaoResult().postPraSensitivityAnalysisOutput()
                     );
                     secondPreventiveRaoResults = new PreventiveAndCurativesRaoResultImpl(
                         stateTree,
                         initialOutput,
                         postPreventiveResult,
-                        secondPreventiveResult,
+                        intermediateSecondPreventiveResult,
                         secondPreventiveRaoResultsHolder.postContingencyResults(),
                         crac,
                         raoParameters);
                 }
                 if (secondPreventiveImprovesResults(secondPreventiveRaoResults, mergedRaoResults)) {
-                    postContingencyResults = secondPreventiveRaoResultsHolder.postContingencyResults();
+                    finalSecondPreventiveResult = intermediateSecondPreventiveResult;
+                    finalPostContingencyResults = new HashMap<>(secondPreventiveRaoResultsHolder.postContingencyResults());
                     mergedRaoResults = secondPreventiveRaoResults;
                     mergedRaoResults.setExecutionDetails(OptimizationStepsExecuted.SECOND_PREVENTIVE_IMPROVED_FIRST);
                     logFinalResultsOutsideOfSecondPreventive = false;
@@ -232,8 +238,8 @@ public CompletableFuture<RaoResult> run() {
             // Log final results
             if (logFinalResultsOutsideOfSecondPreventive) {
                 BUSINESS_LOGS.info("Merging preventive and post-contingency RAO results:");
-                RaoLogger.logMostLimitingElementsResults(BUSINESS_LOGS, stateTree.getBasecaseScenario(), preventiveResult, stateTree.getContingencyScenarios(), postContingencyResults, raoParameters.getObjectiveFunctionParameters().getType(), raoParameters.getObjectiveFunctionParameters().getUnit(), NUMBER_LOGGED_ELEMENTS_END_RAO);
-                RaoLogger.checkIfMostLimitingElementIsFictional(BUSINESS_LOGS, stateTree.getBasecaseScenario(), preventiveResult, stateTree.getContingencyScenarios(), postContingencyResults, raoParameters.getObjectiveFunctionParameters().getType(), raoParameters.getObjectiveFunctionParameters().getUnit());
+                RaoLogger.logMostLimitingElementsResults(BUSINESS_LOGS, stateTree.getBasecaseScenario(), finalSecondPreventiveResult.optimizationResult(), stateTree.getContingencyScenarios(), finalPostContingencyResults, raoParameters.getObjectiveFunctionParameters().getType(), raoParameters.getObjectiveFunctionParameters().getUnit(), NUMBER_LOGGED_ELEMENTS_END_RAO);
+                RaoLogger.checkIfMostLimitingElementIsFictional(BUSINESS_LOGS, stateTree.getBasecaseScenario(), finalSecondPreventiveResult.optimizationResult(), stateTree.getContingencyScenarios(), finalPostContingencyResults, raoParameters.getObjectiveFunctionParameters().getType(), raoParameters.getObjectiveFunctionParameters().getUnit());
             }
 
             CompletableFuture<RaoResult> raoResult = postCheckResults(mergedRaoResults, initialOutput, raoParameters.getObjectiveFunctionParameters(), true);
@@ -244,20 +250,21 @@ public CompletableFuture<RaoResult> run() {
                 BUSINESS_LOGS.info("----- PST regulation [start]");
                 network.getVariantManager().cloneVariant(INITIAL_SCENARIO, PST_REGULATION);
                 network.getVariantManager().setWorkingVariant(PST_REGULATION);
-                Set<PstRegulationResult> pstRegulationResults = CastorPstRegulation.regulatePsts(pstsToRegulate, postContingencyResults, network, crac, raoParameters, raoResult.join());
-                postContingencyResults = mergeRaoAndPstRegulationResults(pstRegulationResults, secondPreventiveResult, postContingencyResults, prePerimeterSensitivityAnalysis, initialOutput, toolProvider);
+                Set<PstRegulationResult> pstRegulationResults = CastorPstRegulation.regulatePsts(pstsToRegulate, finalPostContingencyResults, network, crac, raoParameters, mergedRaoResults);
+                Map<State, PostPerimeterResult> postRegulationResults = mergeRaoAndPstRegulationResults(pstRegulationResults, finalSecondPreventiveResult, finalPostContingencyResults, prePerimeterSensitivityAnalysis, initialOutput, toolProvider);
                 RaoResult raoResultWithRegulation = new PreventiveAndCurativesRaoResultImpl(
                     stateTree,
                     initialOutput,
                     postPreventiveResult,
-                    secondPreventiveResult,
-                    postContingencyResults,
+                    finalSecondPreventiveResult,
+                    postRegulationResults,
                     crac,
                     raoParameters);
+                raoResultWithRegulation.setExecutionDetails(mergedRaoResults.getExecutionDetails());
                 BUSINESS_LOGS.info("----- PST regulation [end]");
                 BUSINESS_LOGS.info("Merging RAO and PST regulation results:");
-                RaoLogger.logMostLimitingElementsResults(BUSINESS_LOGS, stateTree.getBasecaseScenario(), preventiveResult, stateTree.getContingencyScenarios(), postContingencyResults, raoParameters.getObjectiveFunctionParameters().getType(), raoParameters.getObjectiveFunctionParameters().getUnit(), NUMBER_LOGGED_ELEMENTS_END_RAO);
-                RaoLogger.checkIfMostLimitingElementIsFictional(BUSINESS_LOGS, stateTree.getBasecaseScenario(), preventiveResult, stateTree.getContingencyScenarios(), postContingencyResults, raoParameters.getObjectiveFunctionParameters().getType(), raoParameters.getObjectiveFunctionParameters().getUnit());
+                RaoLogger.logMostLimitingElementsResults(BUSINESS_LOGS, stateTree.getBasecaseScenario(), finalSecondPreventiveResult.optimizationResult(), stateTree.getContingencyScenarios(), postRegulationResults, raoParameters.getObjectiveFunctionParameters().getType(), raoParameters.getObjectiveFunctionParameters().getUnit(), NUMBER_LOGGED_ELEMENTS_END_RAO);
+                RaoLogger.checkIfMostLimitingElementIsFictional(BUSINESS_LOGS, stateTree.getBasecaseScenario(), finalSecondPreventiveResult.optimizationResult(), stateTree.getContingencyScenarios(), postRegulationResults, raoParameters.getObjectiveFunctionParameters().getType(), raoParameters.getObjectiveFunctionParameters().getUnit());
                 return postCheckResults(raoResultWithRegulation, initialOutput, raoParameters.getObjectiveFunctionParameters(), false);
             }
 
@@ -394,59 +401,46 @@ private Pair<OptimizationResult, Set<FlowCnec>> optimizePreventivePerimeter(Stat
     }
 
     private Map<State, PostPerimeterResult> mergeRaoAndPstRegulationResults(Set<PstRegulationResult> pstRegulationResults, PostPerimeterResult postPraResult, Map<State, PostPerimeterResult> postContingencyResults, PrePerimeterSensitivityAnalysis prePerimeterSensitivityAnalysis, FlowResult initialFlowResult, ToolProvider toolProvider) {
-        network.getVariantManager().setWorkingVariant(PREVENTIVE_SCENARIO);
-        Map<State, PostPerimeterResult> postRegulationPostContingencyResults = new HashMap<>(postContingencyResults);
-        for (PstRegulationResult pstRegulationResult : pstRegulationResults) {
-            State curativeState = crac.getState(pstRegulationResult.contingency().getId(), crac.getLastInstant());
-            if (!pstRegulationResult.regulatedTapPerPst().isEmpty()) {
-                String postRegulationVariantName = "PostPstRegulation_Contingency_%s".formatted(pstRegulationResult.contingency().getId());
-                network.getVariantManager().cloneVariant(PREVENTIVE_SCENARIO, postRegulationVariantName);
-                network.getVariantManager().setWorkingVariant(postRegulationVariantName);
-
-                AppliedRemedialActions appliedArasAndCras = new AppliedRemedialActions();
-                Map<State, Set<NetworkAction>> appliedNetworkActions = new HashMap<>();
-                List<State> previousStates = new ArrayList<>();
-                if (crac.hasAutoInstant()) {
-                    State autoState = crac.getState(pstRegulationResult.contingency().getId(), crac.getInstant(InstantKind.AUTO));
-                    if (autoState != null) {
-                        previousStates.add(autoState);
-                        appliedArasAndCras.addAppliedNetworkActions(autoState, postContingencyResults.get(autoState).optimizationResult().getActivatedNetworkActions());
-                        appliedArasAndCras.addAppliedRangeActions(autoState, getAppliedRangeActionsAndSetPoint(autoState, postContingencyResults.get(autoState).optimizationResult()));
-                        appliedNetworkActions.put(autoState, appliedArasAndCras.getAppliedNetworkActions(autoState));
-                    }
-                }
-                crac.getInstants(InstantKind.CURATIVE).stream().map(instant -> crac.getState(pstRegulationResult.contingency().getId(), instant))
-                    .filter(Objects::nonNull)
-                    .forEach(cState -> {
-                        previousStates.add(cState);
-                        appliedArasAndCras.addAppliedNetworkActions(cState, postContingencyResults.get(cState).optimizationResult().getActivatedNetworkActions());
-                        appliedArasAndCras.addAppliedRangeActions(cState, getAppliedRangeActionsAndSetPoint(cState, postContingencyResults.get(cState).optimizationResult()));
-                        appliedNetworkActions.put(cState, appliedArasAndCras.getAppliedNetworkActions(cState));
-                    });
-                pstRegulationResult.regulatedTapPerPst().forEach((pstRangeAction, regulatedTap) -> appliedArasAndCras.addAppliedRangeAction(curativeState, pstRangeAction, pstRangeAction.convertTapToAngle(regulatedTap)));
-
-                previousStates.forEach(s -> appliedArasAndCras.applyOnNetwork(s, network));
-
-                // retrieve pre-perimeter result before optimization and PST regulation of last curative state
-                PrePerimeterResult preLastCurativePerimeterResult = getPreLastCurativePerimeterResult(previousStates, postPraResult, postContingencyResults);
-                RangeActionActivationResultImpl postRegulationRangeActionActivationResult = new RangeActionActivationResultImpl(preLastCurativePerimeterResult.getRangeActionSetpointResult());
-                appliedArasAndCras.getAppliedRangeActions(curativeState).forEach((rangeAction, setPoint) -> postRegulationRangeActionActivationResult.putResult(rangeAction, curativeState, setPoint));
-
-                // compute pre-perimeter result including CRAs and PST regulation
-                PrePerimeterResult postCraSensitivityAnalysisOutput = prePerimeterSensitivityAnalysis.runBasedOnInitialResults(network, initialFlowResult, Collections.emptySet(), new AppliedRemedialActions());
-                OptimizationResult optimizationResult = new OptimizationResultImpl(postCraSensitivityAnalysisOutput, postCraSensitivityAnalysisOutput, postCraSensitivityAnalysisOutput, new NetworkActionsResultImpl(appliedNetworkActions), postRegulationRangeActionActivationResult);
-
-                try {
-                    PostPerimeterResult postRegulationResult = new PostPerimeterSensitivityAnalysis(crac, crac.getFlowCnecs(curativeState), crac.getRangeActions(curativeState), raoParameters, toolProvider)
-                        .runBasedOnInitialPreviousAndOptimizationResults(network, initialFlowResult, CompletableFuture.completedFuture(preLastCurativePerimeterResult), Collections.emptySet(), optimizationResult, null)
-                        .get();
-                    postRegulationPostContingencyResults.put(curativeState, postRegulationResult);
-                } catch (InterruptedException | ExecutionException e) {
-                    Thread.currentThread().interrupt();
-                    throw new OpenRaoException("Exception during post PST regulation sensitivity analysis for state %s".formatted(curativeState.getId()), e);
-                }
-            }
+        // create a new network variant from initial variant for performing the results merging
+        String variantName = "PSTRegulationResultsMerging";
+        network.getVariantManager().setWorkingVariant(INITIAL_SCENARIO);
+        network.getVariantManager().cloneVariant(INITIAL_SCENARIO, variantName);
+        network.getVariantManager().setWorkingVariant(variantName);
+
+        // apply PRAs
+        applyRemedialActions(network, postPraResult.optimizationResult(), crac.getPreventiveState());
+
+        Set<State> regulatedStates = pstRegulationResults.stream().map(pstRegulationResult -> crac.getState(pstRegulationResult.contingency().getId(), crac.getLastInstant())).collect(Collectors.toSet());
+        Map<State, Set<NetworkAction>> appliedNetworkActions = new HashMap<>();
+
+        // gather all applied ARAs and CRAs
+        AppliedRemedialActions appliedRemedialActions = new AppliedRemedialActions();
+        postContingencyResults.forEach((state, postPerimeterResult) -> {
+            appliedNetworkActions.put(state, postPerimeterResult.optimizationResult().getActivatedNetworkActions());
+            appliedRemedialActions.addAppliedNetworkActions(state, postPerimeterResult.optimizationResult().getActivatedNetworkActions());
+            appliedRemedialActions.addAppliedRangeActions(state, getAppliedRangeActionsAndSetPoint(state, postPraResult.optimizationResult()));
+            appliedRemedialActions.addAppliedRangeActions(state, getAppliedRangeActionsAndSetPoint(state, postPerimeterResult.optimizationResult()));
+        });
+
+        // overwrite PST range action results for regulated PSTs
+        pstRegulationResults.forEach(pstRegulationResult -> pstRegulationResult.regulatedTapPerPst().forEach((pstRangeAction, regulatedTap) -> appliedRemedialActions.addAppliedRangeAction(crac.getState(pstRegulationResult.contingency().getId(), crac.getLastInstant()), pstRangeAction, pstRangeAction.convertTapToAngle(regulatedTap))));
+
+        PrePerimeterResult postCraSensitivityAnalysisOutput = prePerimeterSensitivityAnalysis.runBasedOnInitialResults(network, initialFlowResult, Collections.emptySet(), appliedRemedialActions);
+
+        Map<State, PostPerimeterResult> postRegulationPostContingencyResults = new HashMap<>();
+
+        // override optimization result
+        RangeActionActivationResultImpl postRegulationRangeActionActivationResult = new RangeActionActivationResultImpl(postCraSensitivityAnalysisOutput);
+        postContingencyResults.keySet().forEach(state -> appliedRemedialActions.getAppliedRangeActions(state).forEach((rangeAction, setPoint) -> postRegulationRangeActionActivationResult.putResult(rangeAction, state, setPoint)));
+        OptimizationResult newOptimizationResult = new OptimizationResultImpl(postCraSensitivityAnalysisOutput, postCraSensitivityAnalysisOutput, postCraSensitivityAnalysisOutput, new NetworkActionsResultImpl(appliedNetworkActions), postRegulationRangeActionActivationResult);
+
+        for (State state : postContingencyResults.keySet()) {
+            postRegulationPostContingencyResults.put(state, new PostPerimeterResult(
+                regulatedStates.contains(state) ? newOptimizationResult : postContingencyResults.get(state).optimizationResult(),
+                postCraSensitivityAnalysisOutput
+            ));
         }
+
         return postRegulationPostContingencyResults;
     }
 
@@ -455,16 +449,4 @@ private static Map<RangeAction<?>, Double> getAppliedRangeActionsAndSetPoint(Sta
         optimizationResult.getActivatedRangeActions(state).forEach(rangeAction -> optimizedRangeActions.put(rangeAction, optimizationResult.getOptimizedSetpoint(rangeAction, state)));
         return optimizedRangeActions;
     }
-
-    /**
-     * Returns the PrePerimeterResult corresponding to the situation just before the optimization of the last instant of
-     * the CRAC. All the remedial actions from the previous states are thus applied.
-     *
-     * @param previousStates : sorted set containing all the post-outage optimized states that share a common contingency (the final element always is the last CRAC instant's state)
-     */
-    private static PrePerimeterResult getPreLastCurativePerimeterResult(List<State> previousStates, PostPerimeterResult postPraResult, Map<State, PostPerimeterResult> postContingencyResults) {
-        // if previousStates is of size 1, then it only contains the final instant's state so the pre-perimeter result corresponds to the post-PRAs result
-        // otherwise, the penultimate pre-perimeter result must be retrieved
-        return previousStates.size() == 1 ? postPraResult.prePerimeterResultForAllFollowingStates() : postContingencyResults.get(previousStates.get(previousStates.size() - 2)).prePerimeterResultForAllFollowingStates();
-    }
 }