diff --git a/commons/src/main/java/com/powsybl/openrao/commons/TemporalData.java b/commons/src/main/java/com/powsybl/openrao/commons/TemporalData.java index 239c235caa..a057a0d4fc 100644 --- a/commons/src/main/java/com/powsybl/openrao/commons/TemporalData.java +++ b/commons/src/main/java/com/powsybl/openrao/commons/TemporalData.java @@ -30,4 +30,6 @@ default List getTimestamps() { void put(OffsetDateTime timestamp, T data); TemporalData map(Function function); + + TemporalData mapMultiThreading(Function function, int parallelism); } diff --git a/commons/src/main/java/com/powsybl/openrao/commons/TemporalDataImpl.java b/commons/src/main/java/com/powsybl/openrao/commons/TemporalDataImpl.java index b9895b8d38..d25e12f1d3 100644 --- a/commons/src/main/java/com/powsybl/openrao/commons/TemporalDataImpl.java +++ b/commons/src/main/java/com/powsybl/openrao/commons/TemporalDataImpl.java @@ -8,8 +8,15 @@ package com.powsybl.openrao.commons; import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.function.Function; import java.util.stream.Collectors; @@ -20,15 +27,15 @@ public class TemporalDataImpl implements TemporalData { private final Map dataPerTimestamp; public TemporalDataImpl() { - this(new TreeMap<>()); + this(new ConcurrentHashMap<>()); } public TemporalDataImpl(Map dataPerTimestamp) { - this.dataPerTimestamp = new TreeMap<>(dataPerTimestamp); + this.dataPerTimestamp = new ConcurrentHashMap<>(dataPerTimestamp); } public Map getDataPerTimestamp() { - return new TreeMap<>(dataPerTimestamp); + return new ConcurrentHashMap<>(dataPerTimestamp); } public void put(OffsetDateTime timestamp, T data) { @@ -38,4 +45,33 @@ public void put(OffsetDateTime timestamp, T data) { public TemporalData map(Function function) { return new TemporalDataImpl<>(dataPerTimestamp.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> function.apply(entry.getValue())))); } + + public TemporalData mapMultiThreading(Function function, int parallelism) { + try (ExecutorService executor = Executors.newFixedThreadPool(parallelism)) { + try { + List>> futures = new ArrayList<>(); + + for (Map.Entry entry : dataPerTimestamp.entrySet()) { + futures.add(executor.submit(() -> + Map.entry(entry.getKey(), function.apply(entry.getValue())) + )); + } + + Map result = new HashMap<>(); + + for (Future> future : futures) { + Map.Entry e = future.get(); + result.put(e.getKey(), e.getValue()); + } + + return new TemporalDataImpl<>(result); + + } catch (InterruptedException | ExecutionException e) { + Thread.currentThread().interrupt(); + throw new OpenRaoException(e); + } finally { + executor.shutdown(); + } + } + } } diff --git a/data/crac/crac-util/pom.xml b/data/crac/crac-util/pom.xml index 00038cba74..8f39fc3aba 100644 --- a/data/crac/crac-util/pom.xml +++ b/data/crac/crac-util/pom.xml @@ -41,6 +41,11 @@ + + ch.qos.logback + logback-classic + test + org.junit.jupiter junit-jupiter diff --git a/data/crac/crac-util/src/test/resources/logback-test.xml b/data/crac/crac-util/src/test/resources/logback-test.xml new file mode 100644 index 0000000000..dc4a8872fc --- /dev/null +++ b/data/crac/crac-util/src/test/resources/logback-test.xml @@ -0,0 +1,15 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + diff --git a/data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsData.java b/data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsData.java index 6f01616808..8d3b443560 100644 --- a/data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsData.java +++ b/data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsData.java @@ -150,7 +150,7 @@ public static Set createGeneratorConstraints(String raId, * @return A map associating each node identifier to its corresponding generator identifier. * Returns an empty map if the process is aborted due to missing network components. */ - public static Map createGeneratorAndLoadAndUpdateNetworks(TemporalData initialNetworksToModify, + public static Map createGeneratorAndLoadAndUpdateNetworks(TemporalData initialNetworksToModify, String raId) { Map networkElementPerGskElement = new HashMap<>(); @@ -163,7 +163,7 @@ public static Map createGeneratorAndLoadAndUpdateNetworks(Tempor Double shiftKey = entry.getValue(); String generatorId = getGeneratorIdFromRaIdAndNodeId(raId, nodeId); - for (Map.Entry networkEntry : initialNetworksToModify.getDataPerTimestamp().entrySet()) { + for (Map.Entry networkEntry : initialNetworksToModify.getDataPerTimestamp().entrySet()) { OffsetDateTime dateTime = networkEntry.getKey(); Network network = networkEntry.getValue(); @@ -256,11 +256,18 @@ public TimeCoupledRaoInput processAllRedispatchingActions(TimeCoupledRaoInput ti // Update nominal voltage in network // TODO: More of a IDCC focused special processing ? Move elsewhere ? - TemporalData modifiedInitialNetworks = new TemporalDataImpl<>(); + TemporalData modifiedInitialNetworks = new TemporalDataImpl<>(); timeCoupledRaoInput.getRaoInputs().getDataPerTimestamp().forEach((dateTime, raoInput) -> { Network network = raoInput.getNetwork(); updateNominalVoltage(network); - modifiedInitialNetworks.put(dateTime, network); + modifiedInitialNetworks.put(dateTime, new LazyNetwork(network)); + if (network instanceof LazyNetwork lazyNetwork) { + try { + lazyNetwork.close(); + } catch (Exception e) { + throw new OpenRaoException(e); + } + } }); TemporalData cracToModify = new TemporalDataImpl<>(); @@ -291,7 +298,16 @@ public TimeCoupledRaoInput processAllRedispatchingActions(TimeCoupledRaoInput ti modifiedInitialNetworks.getDataPerTimestamp().forEach((dateTime, initialNetwork) -> { String exportedNetworkPath = exportDirectory + dateTime.format(DateTimeFormatter.ofPattern("%y%m%d_%H%M%S")) + ".jiidm"; initialNetwork.write("JIIDM", new Properties(), Path.of(exportedNetworkPath)); - postIcsRaoInputs.put(dateTime, RaoInput.build(new LazyNetwork(exportedNetworkPath), timeCoupledRaoInput.getRaoInputs().getData(dateTime).orElseThrow().getCrac()).build()); + try (LazyNetwork postIcsNetwork = new LazyNetwork(exportedNetworkPath)) { + postIcsRaoInputs.put(dateTime, RaoInput.build(postIcsNetwork, timeCoupledRaoInput.getRaoInputs().getData(dateTime).orElseThrow().getCrac()).build()); + } catch (Exception e) { + throw new OpenRaoException(e); + } + try { + initialNetwork.close(); + } catch (Exception e) { + throw new RuntimeException(e); + } }); return new TimeCoupledRaoInput(postIcsRaoInputs, timeCoupledRaoInput.getTimestampsToRun(), timeCoupledRaoInput.getTimeCoupledConstraints()); diff --git a/data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsDataImporter.java b/data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsDataImporter.java index f00cc4399c..0aed5a4b58 100644 --- a/data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsDataImporter.java +++ b/data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsDataImporter.java @@ -217,8 +217,8 @@ private static boolean shouldBeImported(CSVRecord staticRecord, List dateTimes) { + private static boolean p0RespectsConstraints(CSVRecord staticRecord, Map seriesRecord, List dateTimes) { + // Generatcr constraints varaibles + double timestampDuration = IcsUtil.computeTimestampDuration(dateTimes); + Boolean shutDownAllowed = Boolean.parseBoolean(staticRecord.get(SHUTDOWN_ALLOWED)); + Boolean startUpAllowed = Boolean.parseBoolean(staticRecord.get(STARTUP_ALLOWED)); + Optional lead = Optional.empty(); + Optional lagAndLead = Optional.empty(); + Optional parsedLead = Optional.empty(); + if (!staticRecord.get(LEAD_TIME).isEmpty()) { + parsedLead = Optional.of(parseDoubleWithPossibleCommas(staticRecord.get(LEAD_TIME))); + lead = Optional.of((int) Math.ceil(parsedLead.get() / timestampDuration)); + } + if (!staticRecord.get(LAG_TIME).isEmpty()) { + double parsedLag = parseDoubleWithPossibleCommas(staticRecord.get(LAG_TIME)); + double parsedLagAndLead = parsedLead.map(aDouble -> aDouble + parsedLag).orElse(parsedLag); + lagAndLead = Optional.of((int) Math.ceil(parsedLagAndLead / timestampDuration)); + } double maxGradient = staticRecord.get(MAXIMUM_POSITIVE_POWER_GRADIENT).isEmpty() ? - MAX_GRADIENT : parseDoubleWithPossibleCommas(staticRecord.get(MAXIMUM_POSITIVE_POWER_GRADIENT)); + MAX_GRADIENT : parseDoubleWithPossibleCommas(staticRecord.get(MAXIMUM_POSITIVE_POWER_GRADIENT)); double minGradient = staticRecord.get(MAXIMUM_NEGATIVE_POWER_GRADIENT).isEmpty() ? - -MAX_GRADIENT : -parseDoubleWithPossibleCommas(staticRecord.get(MAXIMUM_NEGATIVE_POWER_GRADIENT)); + -MAX_GRADIENT : -parseDoubleWithPossibleCommas(staticRecord.get(MAXIMUM_NEGATIVE_POWER_GRADIENT)); + + // Intermediate variables + boolean countLag = false; + int countConsecutiveNullValues = 0; Iterator dateTimeIterator = dateTimes.iterator(); OffsetDateTime currentDateTime = dateTimeIterator.next(); + boolean allValuesAreNullSinceFirstP0 = parseDoubleWithPossibleCommas(seriesRecord.get(P0).get(currentDateTime.getHour() + OFFSET)) < ON_POWER_THRESHOLD; + while (dateTimeIterator.hasNext()) { OffsetDateTime nextDateTime = dateTimeIterator.next(); - double diff = parseDoubleWithPossibleCommas(p0record.get(nextDateTime.getHour() + OFFSET)) - parseDoubleWithPossibleCommas(p0record.get(currentDateTime.getHour() + OFFSET)); - if (diff > maxGradient || diff < minGradient) { - BUSINESS_WARNS.warn( - "Redispatching action {} is not imported: does not respect power gradients : min/max/diff = {} / {} / {}", - staticRecord.get(0), minGradient, maxGradient, diff - ); + double nextP0 = parseDoubleWithPossibleCommas(seriesRecord.get(P0).get(nextDateTime.getHour() + OFFSET)); + double currentP0 = parseDoubleWithPossibleCommas(seriesRecord.get(P0).get(currentDateTime.getHour() + OFFSET)); + Optional pMinRD = parseValue(seriesRecord, P_MIN_RD, currentDateTime, 1); + double pMin = pMinRD.orElse(ON_POWER_THRESHOLD); + Optional nextPminPD = parseValue(seriesRecord, P_MIN_RD, nextDateTime, 1); + double nextPmin = nextPminPD.orElse(ON_POWER_THRESHOLD); + + // 1- Check gradients + if (!areGradientsRespected(staticRecord, nextP0, currentP0, pMin, nextPmin, maxGradient, minGradient, currentDateTime)) { + return false; + } + + // 2 - Check Pmin is respected + if (!isPminRespected(staticRecord, currentP0, pMin, currentDateTime)) { return false; } + + // 3 - Starting up next timestamp + // a) Check start up is allowed + // b) Check lead time is respected + // c) Check lag time + lead time is respected + if (currentP0 < pMin) { + countConsecutiveNullValues += 1; + } else { + allValuesAreNullSinceFirstP0 = false; + } + if (currentP0 < pMin && nextP0 >= pMin) { + if (!isStartUpRespected(staticRecord, startUpAllowed, currentDateTime)) { + return false; + } + if (!isLeadTimeRespected(staticRecord, lead, allValuesAreNullSinceFirstP0, countConsecutiveNullValues, currentDateTime)) { + return false; + } + if (countLag) { + if (!isLeadTimeAndLagTimeRespected(staticRecord, countConsecutiveNullValues, lagAndLead, currentDateTime)) { + return false; + } + // Re-initialize + countLag = false; + } + } + // 4 - Shutting down next timestamp + // a) Check shut down is allowed + // activate lag time + lead time checking + if (currentP0 >= pMin && nextP0 < pMin) { + if (!isShutDownRespected(staticRecord, shutDownAllowed, currentDateTime)) { + return false; + } + if (lagAndLead.isPresent()) { + countLag = true; + } + } + + // Re-init countConsecutiveNullValues + if (currentP0 >= pMin) { + countConsecutiveNullValues = 0; + } currentDateTime = nextDateTime; } + + // Last timestamp + double currentP0 = parseDoubleWithPossibleCommas(seriesRecord.get(P0).get(currentDateTime.getHour() + OFFSET)); + Optional pMinRD = parseValue(seriesRecord, P_MIN_RD, currentDateTime, 1); + double pMin = pMinRD.orElse(ON_POWER_THRESHOLD); + return isPminRespected(staticRecord, currentP0, pMin, currentDateTime); + } + + private static boolean isShutDownRespected(CSVRecord staticRecord, Boolean shutDownAllowed, OffsetDateTime currentDateTime) { + if (!shutDownAllowed) { + BUSINESS_WARNS.warn("Redispatching action {} is not imported (hour {}): shut down prohibited", + staticRecord.get(0), currentDateTime.getHour()); + return false; + } + return true; + } + + private static boolean isLeadTimeAndLagTimeRespected(CSVRecord staticRecord, int countConsecutiveNullValues, Optional lagAndLead, OffsetDateTime currentDateTime) { + if (lagAndLead.isPresent() && countConsecutiveNullValues < lagAndLead.get()) { + BUSINESS_WARNS.warn("Redispatching action {} is not imported (hour {}): lagTime + leadTime ({}) not respected. RA was OFF after shut down for only {} timestamps", + staticRecord.get(0), currentDateTime.getHour(), lagAndLead.get(), countConsecutiveNullValues); + return false; + } + return true; + } + + private static boolean isLeadTimeRespected(CSVRecord staticRecord, Optional lead, boolean allValuesAreNullSinceFirstP0, int countConsecutiveNullValues, OffsetDateTime currentDateTime) { + // The generator is initially oFF and we don't know when it was last shut down> no lead check. + if (allValuesAreNullSinceFirstP0) { + return true; + } + if (lead.isPresent() && countConsecutiveNullValues < lead.get()) { + BUSINESS_WARNS.warn("Redispatching action {} is not imported (hour {}): leadTime ({}) not respected. RA was OFF before start up for only {} timestamps", + staticRecord.get(0), currentDateTime.getHour(), lead.get(), countConsecutiveNullValues); + return false; + } + return true; + } + + private static boolean isStartUpRespected(CSVRecord staticRecord, Boolean startUpAllowed, OffsetDateTime currentDateTime) { + if (!startUpAllowed) { + BUSINESS_WARNS.warn("Redispatching action {} is not imported (hour {}): start up prohibited", + staticRecord.get(0), currentDateTime.getHour()); + return false; + } + return true; + } + + private static boolean areGradientsRespected(CSVRecord staticRecord, double nextP0, double currentP0, double pMin, double nextPmin, double maxGradient, double minGradient, OffsetDateTime currentDateTime) { + double diff = nextP0 - currentP0; + if (currentP0 >= pMin && nextP0 >= nextPmin && (diff > maxGradient || diff < minGradient)) { + BUSINESS_WARNS.warn( + "Redispatching action {} is not imported (hour {}): does not respect power gradients : min/max/diff = {} / {} / {}", + staticRecord.get(0), currentDateTime.getHour(), minGradient, maxGradient, diff + ); + return false; + } + return true; + } + + private static boolean isPminRespected(CSVRecord staticRecord, double currentP0, double pMin, OffsetDateTime currentDateTime) { + if (currentP0 < pMin && currentP0 > ON_POWER_THRESHOLD) { + BUSINESS_WARNS.warn("Redispatching action {} is not imported (hour {}): does not respect Pmin : P0 is {} and Pmin at {}", + staticRecord.get(0), currentDateTime.getHour(), currentP0, pMin); + return false; + } return true; } @@ -293,7 +437,7 @@ private static boolean rangeIsOkay(Map seriesPerType, List timestamps) { + if (timestamps.size() < 2) { + throw new OpenRaoException("There must be at least two timestamps."); + } + double referenceTimestampDuration = computeTimeGap(timestamps.getFirst(), timestamps.get(1)); + for (int timestampIndex = 1; timestampIndex < timestamps.size() - 1; timestampIndex++) { + double timestampDuration = computeTimeGap(timestamps.get(timestampIndex), timestamps.get(timestampIndex + 1)); + if (timestampDuration != referenceTimestampDuration) { + throw new OpenRaoException("All timestamps are not evenly spread."); + } + } + return referenceTimestampDuration; + } } diff --git a/data/ics-importer/src/test/java/com/powsybl/openrao/data/icsimporter/IcsDataImporterTest.java b/data/ics-importer/src/test/java/com/powsybl/openrao/data/icsimporter/IcsDataImporterTest.java index 293331e952..aa1f2c80f5 100644 --- a/data/ics-importer/src/test/java/com/powsybl/openrao/data/icsimporter/IcsDataImporterTest.java +++ b/data/ics-importer/src/test/java/com/powsybl/openrao/data/icsimporter/IcsDataImporterTest.java @@ -101,8 +101,8 @@ void testParseStaticCsv() throws IOException { assertEquals("Node", record.get("RD description mode")); assertEquals("BBE1AA1", record.get("UCT Node or GSK ID")); assertEquals("50", record.get("Minimum Redispatch [MW]")); - assertEquals("FALSE", record.get("Startup allowed")); - assertEquals("FALSE", record.get("Shutdown allowed")); + assertEquals("TRUE", record.get("Startup allowed")); + assertEquals("TRUE", record.get("Shutdown allowed")); } @Test @@ -267,15 +267,15 @@ private static Stream rangeIsNotOkayCases() { return Stream.of( Arguments.of(negativeRdpPlusCsv, - "Redispatching action Redispatching_RA is not imported: RDP+ -1.0 or RDP- 35.0 is negative for datetime 2025-02-13T01:30Z"), + "Redispatching action Redispatching_RA is not imported (hour 1): RDP+ -1.0 or RDP- 35.0 is negative for datetime 2025-02-13T01:30Z"), Arguments.of(negativeRdpMinusCsv, - "Redispatching action Redispatching_RA is not imported: RDP+ 1.0 or RDP- -35.0 is negative for datetime 2025-02-13T01:30Z"), + "Redispatching action Redispatching_RA is not imported (hour 1): RDP+ 1.0 or RDP- -35.0 is negative for datetime 2025-02-13T01:30Z"), Arguments.of(tooSmallRangeCsv, "Redispatching action Redispatching_RA is not imported: max range in the day 0.0 MW is too small") ); } - private static Stream gradientNotRespectCsvCases() { + private static Stream gradientNotRespectedCsvCases() { String header = """ RA RD ID;Type of timeseries;00:30;01:30;02:30 """; @@ -284,7 +284,7 @@ private static Stream gradientNotRespectCsvCases() { String tooHighGradientCsv = header + """ Redispatching_RA;RDP-;35;35;35 Redispatching_RA;RDP+;43;43;43 - Redispatching_RA;P0;116;150;117 + Redispatching_RA;P0;116;150;132 Redispatching_RA;Pmin_RD;10;15;20 """; @@ -299,25 +299,148 @@ private static Stream gradientNotRespectCsvCases() { return Stream.of( Arguments.of( tooHighGradientCsv, - "Redispatching action Redispatching_RA is not imported: does not respect power gradients : min/max/diff = -20.0 / 20.0 / 34.0" + "Redispatching action Redispatching_RA is not imported (hour 0): does not respect power gradients : min/max/diff = -20.0 / 20.0 / 34.0" ), Arguments.of( tooLowGradientCsv, - "Redispatching action Redispatching_RA is not imported: does not respect power gradients : min/max/diff = -20.0 / 20.0 / -36.0" + "Redispatching action Redispatching_RA is not imported (hour 0): does not respect power gradients : min/max/diff = -20.0 / 20.0 / -36.0" ) ); } + private static Stream pMinNotRespectedCsvCases() { + String baseHeader = """ + RA RD ID;Type of timeseries;00:30;01:30;02:30 + """; + + String pMinNotRespectedsv = baseHeader + """ + Redispatching_RA;RDP-;7;7;7 + Redispatching_RA;RDP+;7;7;7 + Redispatching_RA;P0;15;5;15 + Redispatching_RA;Pmin_RD;10;10;10 + """; + return Stream.of( + Arguments.of(pMinNotRespectedsv, + "Redispatching action Redispatching_RA is not imported (hour 1): does not respect Pmin : P0 is 5.0 and Pmin at 10.0") + ); + } + + private static Stream startUpShutDownNotRespectedCsvCases() { + String baseHeader = """ + RA RD ID;Type of timeseries;00:30;01:30;02:30 + """; + + String startUpNotRespectedCsv = baseHeader + """ + Redispatching_RA;RDP-;7;7;7 + Redispatching_RA;RDP+;7;7;7 + Redispatching_RA;P0;0;15;15 + Redispatching_RA;Pmin_RD;10;10;10 + """; + + String shutDownNotRespectedCsv = baseHeader + """ + Redispatching_RA;RDP-;7;7;7 + Redispatching_RA;RDP+;7;7;7 + Redispatching_RA;P0;15;15;0 + Redispatching_RA;Pmin_RD;10;10;10 + """; + return Stream.of( + Arguments.of(startUpNotRespectedCsv, + "Redispatching action Redispatching_RA is not imported (hour 0): start up prohibited"), + Arguments.of(shutDownNotRespectedCsv, + "Redispatching action Redispatching_RA is not imported (hour 1): shut down prohibited") + + ); + } + + private static Stream leadAndLagNotRespectedCsvCases() { + String header = """ + RA RD ID;Type of timeseries;00:30;01:30;02:30 + """; + + String leadNotRespected = header + """ + Redispatching_RA;RDP-;35;35;35;35 + Redispatching_RA;RDP+;43;43;43;43 + Redispatching_RA;P0;11;0;21;21 + Redispatching_RA;Pmin_RD;10;15;20;20 + """; + + String leadAndLagNotRespected = header + """ + Redispatching_RA;RDP-;35;35;35;35 + Redispatching_RA;RDP+;43;43;43;43 + Redispatching_RA;P0;11;0;0;21 + Redispatching_RA;Pmin_RD;10;15;20;20 + """; + + return Stream.of( + Arguments.of( + leadNotRespected, + "Redispatching action Redispatching_RA is not imported (hour 1): leadTime (2) not respected. RA was OFF before start up for only 1 timestamps" + ), + Arguments.of( + leadAndLagNotRespected, + "Redispatching action Redispatching_RA is not imported (hour 2): lagTime + leadTime (3) not respected. RA was OFF after shut down for only 2 timestamps" + ) + ); + } + @ParameterizedTest - @MethodSource("gradientNotRespectCsvCases") + @MethodSource("gradientNotRespectedCsvCases") @MethodSource("rangeIsNotOkayCases") @MethodSource("seriesCsvWithMissingSeriesTypeCases") + @MethodSource("pMinNotRespectedCsvCases") void testSeriesCsv(String seriesCsv, String expectedLogMessage) throws IOException { IcsData icsData = IcsDataImporter.read( - getClass().getResourceAsStream("/ics/static.csv"), - new ByteArrayInputStream(seriesCsv.getBytes(StandardCharsets.UTF_8)), - getClass().getResourceAsStream("/glsk/gsk.csv"), - generateOffsetDateTimeList(3)); + getClass().getResourceAsStream("/ics/static.csv"), + new ByteArrayInputStream(seriesCsv.getBytes(StandardCharsets.UTF_8)), + getClass().getResourceAsStream("/glsk/gsk.csv"), + generateOffsetDateTimeList(3)); + + assertEquals(0, icsData.getRedispatchingActions().size()); + assertEquals(expectedLogMessage, logsList.get(0).getFormattedMessage()); + } + + @ParameterizedTest + @MethodSource("startUpShutDownNotRespectedCsvCases") + void testSeriesWithStatic2Csv(String seriesCsv, String expectedLogMessage) throws IOException { + IcsData icsData = IcsDataImporter.read( + getClass().getResourceAsStream("/ics/static2.csv"), + new ByteArrayInputStream(seriesCsv.getBytes(StandardCharsets.UTF_8)), + getClass().getResourceAsStream("/glsk/gsk.csv"), + generateOffsetDateTimeList(3)); + + assertEquals(0, icsData.getRedispatchingActions().size()); + assertEquals(expectedLogMessage, logsList.get(0).getFormattedMessage()); + } + + @Test + void testGradientNotTakenIntoAccountBelowPmin() throws IOException { + String header = """ + RA RD ID;Type of timeseries;00:30;01:30;02:30 + """; + + String belowPminCsv = header + """ + Redispatching_RA;RDP-;35;35;35 + Redispatching_RA;RDP+;43;43;43 + Redispatching_RA;P0;0;150;132 + Redispatching_RA;Pmin_RD;10;15;20 + """; + IcsData icsData = IcsDataImporter.read( + getClass().getResourceAsStream("/ics/static.csv"), + new ByteArrayInputStream(belowPminCsv.getBytes(StandardCharsets.UTF_8)), + getClass().getResourceAsStream("/glsk/gsk.csv"), + generateOffsetDateTimeList(3)); + + assertEquals(1, icsData.getRedispatchingActions().size()); + } + + @ParameterizedTest + @MethodSource("leadAndLagNotRespectedCsvCases") + void testLongSeriesCsv(String seriesCsv, String expectedLogMessage) throws IOException { + IcsData icsData = IcsDataImporter.read( + getClass().getResourceAsStream("/ics/static.csv"), + new ByteArrayInputStream(seriesCsv.getBytes(StandardCharsets.UTF_8)), + getClass().getResourceAsStream("/glsk/gsk.csv"), + generateOffsetDateTimeList(4)); assertEquals(0, icsData.getRedispatchingActions().size()); assertEquals(expectedLogMessage, logsList.get(0).getFormattedMessage()); @@ -344,7 +467,7 @@ void testMissingGradientInStaticCsv() throws IOException { generateOffsetDateTimeList(3)); assertEquals(0, icsData.getRedispatchingActions().size()); - assertEquals("Redispatching action Redispatching_RA is not imported: does not respect power gradients : min/max/diff = -1000.0 / 1000.0 / 1964.0", logsList.get(0).getFormattedMessage()); + assertEquals("Redispatching action Redispatching_RA is not imported (hour 0): does not respect power gradients : min/max/diff = -1000.0 / 1000.0 / 1964.0", logsList.get(0).getFormattedMessage()); } @Test diff --git a/data/ics-importer/src/test/java/com/powsybl/openrao/data/icsimporter/IcsDataTest.java b/data/ics-importer/src/test/java/com/powsybl/openrao/data/icsimporter/IcsDataTest.java index 8fc7c8ef4c..359d793f1a 100644 --- a/data/ics-importer/src/test/java/com/powsybl/openrao/data/icsimporter/IcsDataTest.java +++ b/data/ics-importer/src/test/java/com/powsybl/openrao/data/icsimporter/IcsDataTest.java @@ -50,9 +50,9 @@ public class IcsDataTest { private static final double DOUBLE_EPSILON = 1e-6; private Crac crac1; private Crac crac2; - private Network network1; - private Network network2; - private TemporalData networkTemporalData; + private LazyNetwork network1; + private LazyNetwork network2; + private TemporalData networkTemporalData; private TemporalData cracTemporalData; List logsList; private final OffsetDateTime timestamp1 = OffsetDateTime.of(2025, 2, 13, 0, 30, 0, 0, ZoneOffset.UTC); @@ -63,8 +63,8 @@ void setUp() throws IOException { // we need to import twice the network to avoid variant names conflicts on the same network object String networkFilePath1 = "2Nodes2ParallelLinesPST_0030.uct"; String networkFilePath2 = "2Nodes2ParallelLinesPST_0130.uct"; - network1 = Network.read(networkFilePath1, IcsDataTest.class.getResourceAsStream("/network/" + networkFilePath1)); - network2 = Network.read(networkFilePath2, IcsDataTest.class.getResourceAsStream("/network/" + networkFilePath2)); + network1 = new LazyNetwork(getResourcePath("/network/" + networkFilePath1)); + network2 = new LazyNetwork(getResourcePath("/network/" + networkFilePath2)); crac1 = Crac.read("/crac/crac-0030.json", getClass().getResourceAsStream("/crac/crac-0030.json"), network1); crac2 = Crac.read("/crac/crac-0130.json", getClass().getResourceAsStream("/crac/crac-0130.json"), network2); @@ -116,11 +116,11 @@ void testCreateGeneratorConstraintRaDefinedOnANode() throws IOException { assertTrue(generatorConstraints.getUpwardPowerGradient().isPresent()); assertEquals(20., generatorConstraints.getUpwardPowerGradient().get(), DOUBLE_EPSILON); assertTrue(generatorConstraints.getLeadTime().isPresent()); - assertEquals(1.0, generatorConstraints.getLeadTime().get(), DOUBLE_EPSILON); + assertEquals(2.0, generatorConstraints.getLeadTime().get(), DOUBLE_EPSILON); assertTrue(generatorConstraints.getLagTime().isPresent()); assertEquals(1.0, generatorConstraints.getLagTime().get(), DOUBLE_EPSILON); - assertFalse(generatorConstraints.isShutDownAllowed()); - assertFalse(generatorConstraints.isStartUpAllowed()); + assertTrue(generatorConstraints.isShutDownAllowed()); + assertTrue(generatorConstraints.isStartUpAllowed()); } @Test diff --git a/data/ics-importer/src/test/resources/ics/static.csv b/data/ics-importer/src/test/resources/ics/static.csv index 4ec750bbc7..acb66851aa 100644 --- a/data/ics-importer/src/test/resources/ics/static.csv +++ b/data/ics-importer/src/test/resources/ics/static.csv @@ -1,2 +1,2 @@ RA RD ID;TSO;Preventive;Curative;Time From;Time To;Generator Name;RD description mode;UCT Node or GSK ID;Minimum Redispatch [MW];Fuel type;Minimum up-time [h];Minimum down-time [h];Maximum positive power gradient [MW/h];Maximum negative power gradient [MW/h];Lead time [h];Lag time [h];Startup allowed;Shutdown allowed -Redispatching_RA;FR;TRUE;FALSE;00:00;24:00:00;Generator_Name;Node;BBE1AA1;50;Coal;2;2;20;20;1;1;FALSE;FALSE \ No newline at end of file +Redispatching_RA;FR;TRUE;FALSE;00:00;24:00:00;Generator_Name;Node;BBE1AA1;50;Coal;2;2;20;20;2;1;TRUE;TRUE \ No newline at end of file diff --git a/data/ics-importer/src/test/resources/ics/static2.csv b/data/ics-importer/src/test/resources/ics/static2.csv new file mode 100644 index 0000000000..c3b3686ecf --- /dev/null +++ b/data/ics-importer/src/test/resources/ics/static2.csv @@ -0,0 +1,2 @@ +RA RD ID;TSO;Preventive;Curative;Time From;Time To;Generator Name;RD description mode;UCT Node or GSK ID;Minimum Redispatch [MW];Fuel type;Minimum up-time [h];Minimum down-time [h];Maximum positive power gradient [MW/h];Maximum negative power gradient [MW/h];Lead time [h];Lag time [h];Startup allowed;Shutdown allowed +Redispatching_RA;FR;TRUE;FALSE;00:00;24:00:00;Generator_Name;Node;BBE1AA1;50;Coal;2;2;20;20;2;1;FALSE;FALSE \ No newline at end of file diff --git a/ra-optimisation/rao-api/src/main/java/com/powsybl/openrao/raoapi/LazyNetwork.java b/ra-optimisation/rao-api/src/main/java/com/powsybl/openrao/raoapi/LazyNetwork.java index a2f3d3d01d..ca387b6fed 100644 --- a/ra-optimisation/rao-api/src/main/java/com/powsybl/openrao/raoapi/LazyNetwork.java +++ b/ra-optimisation/rao-api/src/main/java/com/powsybl/openrao/raoapi/LazyNetwork.java @@ -7,6 +7,7 @@ package com.powsybl.openrao.raoapi; +import com.google.common.annotations.Beta; import com.powsybl.commons.datasource.DataSource; import com.powsybl.commons.datasource.ReadOnlyDataSource; import com.powsybl.commons.extensions.Extension; @@ -16,14 +17,14 @@ import com.powsybl.iidm.network.Area; import com.powsybl.iidm.network.AreaAdder; import com.powsybl.iidm.network.Battery; +import com.powsybl.iidm.network.BoundaryLine; +import com.powsybl.iidm.network.BoundaryLineFilter; import com.powsybl.iidm.network.Branch; import com.powsybl.iidm.network.BusbarSection; import com.powsybl.iidm.network.Component; import com.powsybl.iidm.network.Connectable; import com.powsybl.iidm.network.ContainerType; import com.powsybl.iidm.network.Country; -import com.powsybl.iidm.network.BoundaryLine; -import com.powsybl.iidm.network.BoundaryLineFilter; import com.powsybl.iidm.network.DcBus; import com.powsybl.iidm.network.DcConnectable; import com.powsybl.iidm.network.DcGround; @@ -71,12 +72,14 @@ import com.powsybl.iidm.network.VoltageSourceConverter; import com.powsybl.iidm.network.VscConverterStation; +import java.io.File; import java.nio.file.Path; import java.time.ZonedDateTime; import java.util.Collection; import java.util.Optional; import java.util.Properties; import java.util.Set; +import java.util.UUID; import java.util.stream.Stream; /** @@ -85,7 +88,9 @@ * * @author Thomas Bouquet {@literal } */ -public class LazyNetwork implements Network { +@Beta +public class LazyNetwork implements Network, AutoCloseable { + private static final String TEMP_DIR = System.getProperty("java.io.tmpdir") + File.separator; private final String networkPath; private boolean isLoaded; private Network network; @@ -95,6 +100,14 @@ public LazyNetwork(String networkPath) { this.isLoaded = false; } + public LazyNetwork(Network network) { + String networkName = TEMP_DIR + UUID.randomUUID() + ".jiidm"; + // TODO serialize to BIIDM, not stabilized for the moment (04/2026) + network.write("JIIDM", new Properties(), Path.of(networkName)); + this.networkPath = networkName; + this.isLoaded = false; + } + private void load() { if (!isLoaded) { network = Network.read(networkPath); @@ -102,6 +115,14 @@ private void load() { } } + @Override + public void close() throws Exception { + // TODO: currently modifications on the network will not be saved -> perhaps override networkPath with a UUID and write content to it + network = null; + isLoaded = false; + System.gc(); + } + @Override public Collection getSubnetworks() { load(); diff --git a/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/castor/algorithm/AbstractMultiPerimeterSensitivityAnalysis.java b/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/castor/algorithm/AbstractMultiPerimeterSensitivityAnalysis.java index 90bc13f9f3..f3ec6405b3 100644 --- a/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/castor/algorithm/AbstractMultiPerimeterSensitivityAnalysis.java +++ b/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/castor/algorithm/AbstractMultiPerimeterSensitivityAnalysis.java @@ -88,7 +88,8 @@ protected int setNewThreadCountAndGetOldValue() { } OpenSensitivityAnalysisParameters openSensitivityAnalysisParameters = sensitivityAnalysisParameters.getExtension(OpenSensitivityAnalysisParameters.class); - int oldThreadCount = openSensitivityAnalysisParameters.getThreadCount(); + int oldThreadCount = + openSensitivityAnalysisParameters.getThreadCount(); if (multiThreadedSensitivities) { openSensitivityAnalysisParameters.setThreadCount(MultithreadingParameters.getAvailableCPUs(raoParameters)); } diff --git a/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/linearoptimisation/algorithms/fillers/GeneratorConstraintsFiller.java b/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/linearoptimisation/algorithms/fillers/GeneratorConstraintsFiller.java index feed27e8ca..44515c0f30 100644 --- a/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/linearoptimisation/algorithms/fillers/GeneratorConstraintsFiller.java +++ b/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/linearoptimisation/algorithms/fillers/GeneratorConstraintsFiller.java @@ -143,11 +143,11 @@ public void fill(LinearProblem linearProblem, FlowResult flowResult, Sensitivity } addPowerToInjectionConstraint( - linearProblem, - generatorId, - timestamp, - associatedInjections.get().getData(timestamps.get(timestampIndex)).orElseThrow(), - preventiveStates.getData(timestamp).orElseThrow() + linearProblem, + generatorId, + timestamp, + associatedInjections.get().getData(timestamps.get(timestampIndex)).orElseThrow(), + preventiveStates.getData(timestamp).orElseThrow() ); } // Specific first timestamp constraints @@ -294,27 +294,27 @@ private void addPowerVariationConstraints(LinearProblem linearProblem, double pMin = getMinP(generatorConstraints.getGeneratorId(), networks.getData(timestamp).orElseThrow()); OpenRaoMPConstraint powerTransitionConstraintInf = linearProblem.addGeneratorPowerTransitionConstraint( - generatorConstraints.getGeneratorId(), 0, linearProblem.infinity(), timestamp, LinearProblem.AbsExtension.POSITIVE + generatorConstraints.getGeneratorId(), 0, linearProblem.infinity(), timestamp, LinearProblem.AbsExtension.POSITIVE ); powerTransitionConstraintInf.setCoefficient(linearProblem.getGeneratorPowerVariable(generatorConstraints.getGeneratorId(), nextTimestamp), 1.0); powerTransitionConstraintInf.setCoefficient(linearProblem.getGeneratorPowerVariable(generatorConstraints.getGeneratorId(), timestamp), -1.0); OpenRaoMPConstraint powerTransitionConstraintSup = linearProblem.addGeneratorPowerTransitionConstraint( - generatorConstraints.getGeneratorId(), -linearProblem.infinity(), 0, timestamp, LinearProblem.AbsExtension.NEGATIVE + generatorConstraints.getGeneratorId(), -linearProblem.infinity(), 0, timestamp, LinearProblem.AbsExtension.NEGATIVE ); powerTransitionConstraintSup.setCoefficient(linearProblem.getGeneratorPowerVariable(generatorConstraints.getGeneratorId(), nextTimestamp), 1.0); powerTransitionConstraintSup.setCoefficient(linearProblem.getGeneratorPowerVariable(generatorConstraints.getGeneratorId(), timestamp), -1.0); // ON -> ON OpenRaoMPVariable onOnTransitionVariable = linearProblem.getGeneratorStateTransitionVariable( - generatorConstraints.getGeneratorId(), timestamp, LinearProblem.GeneratorState.ON, LinearProblem.GeneratorState.ON + generatorConstraints.getGeneratorId(), timestamp, LinearProblem.GeneratorState.ON, LinearProblem.GeneratorState.ON ); powerTransitionConstraintInf.setCoefficient(onOnTransitionVariable, -downwardPowerGradient * timestampDuration); powerTransitionConstraintSup.setCoefficient(onOnTransitionVariable, -upwardPowerGradient * timestampDuration); // OFF -> OFF OpenRaoMPVariable offOffTransitionVariable = linearProblem.getGeneratorStateTransitionVariable( - generatorConstraints.getGeneratorId(), timestamp, LinearProblem.GeneratorState.OFF, LinearProblem.GeneratorState.OFF + generatorConstraints.getGeneratorId(), timestamp, LinearProblem.GeneratorState.OFF, LinearProblem.GeneratorState.OFF ); powerTransitionConstraintInf.setCoefficient(offOffTransitionVariable, OFF_POWER_THRESHOLD); powerTransitionConstraintSup.setCoefficient(offOffTransitionVariable, -OFF_POWER_THRESHOLD); @@ -322,14 +322,14 @@ private void addPowerVariationConstraints(LinearProblem linearProblem, // OFF -> ON double nextPMin = getMinP(generatorConstraints.getGeneratorId(), networks.getData(nextTimestamp).orElseThrow()); OpenRaoMPVariable offOnTransitionVariable = linearProblem.getGeneratorStateTransitionVariable( - generatorConstraints.getGeneratorId(), timestamp, LinearProblem.GeneratorState.OFF, LinearProblem.GeneratorState.ON + generatorConstraints.getGeneratorId(), timestamp, LinearProblem.GeneratorState.OFF, LinearProblem.GeneratorState.ON ); powerTransitionConstraintInf.setCoefficient(offOnTransitionVariable, -(nextPMin - OFF_POWER_THRESHOLD)); powerTransitionConstraintSup.setCoefficient(offOnTransitionVariable, -nextPMin - upwardPowerGradient * timestampDuration); // ON -> OFF OpenRaoMPVariable onOffTransitionVariable = linearProblem.getGeneratorStateTransitionVariable( - generatorConstraints.getGeneratorId(), timestamp, LinearProblem.GeneratorState.ON, LinearProblem.GeneratorState.OFF + generatorConstraints.getGeneratorId(), timestamp, LinearProblem.GeneratorState.ON, LinearProblem.GeneratorState.OFF ); powerTransitionConstraintInf.setCoefficient(onOffTransitionVariable, pMin - downwardPowerGradient * timestampDuration); @@ -422,14 +422,14 @@ private Optional> getInjectionRangeActionOfGe private static Optional getInjectionRangeActionOfGenerator(String generatorId, Set allInjectionRangeActions) { return allInjectionRangeActions.stream() - .filter(injectionRangeAction -> injectionRangeAction.getNetworkElements().stream().map(NetworkElement::getId).anyMatch(generatorId::equals)) - .min(Comparator.comparing(Identifiable::getId)); + .filter(injectionRangeAction -> injectionRangeAction.getNetworkElements().stream().map(NetworkElement::getId).anyMatch(generatorId::equals)) + .min(Comparator.comparing(Identifiable::getId)); } private static double getDistributionKey(String generatorId, InjectionRangeAction injectionRangeAction) { NetworkElement networkElement = injectionRangeAction.getNetworkElements().stream() - .filter(element -> element.getId().equals(generatorId)) - .findFirst().orElseThrow(); + .filter(element -> element.getId().equals(generatorId)) + .findFirst().orElseThrow(); return injectionRangeAction.getInjectionDistributionKeys().get(networkElement); } diff --git a/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/marmot/Marmot.java b/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/marmot/Marmot.java index 4b4c469daf..12f09e2db6 100644 --- a/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/marmot/Marmot.java +++ b/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/marmot/Marmot.java @@ -8,6 +8,7 @@ package com.powsybl.openrao.searchtreerao.marmot; import com.google.auto.service.AutoService; +import com.powsybl.openrao.commons.OpenRaoException; import com.powsybl.openrao.commons.TemporalData; import com.powsybl.openrao.commons.TemporalDataImpl; import com.powsybl.openrao.commons.Unit; @@ -18,6 +19,7 @@ import com.powsybl.openrao.data.crac.api.rangeaction.RangeAction; import com.powsybl.openrao.data.raoresult.api.RaoResult; import com.powsybl.openrao.data.raoresult.api.TimeCoupledRaoResult; +import com.powsybl.openrao.raoapi.LazyNetwork; import com.powsybl.openrao.raoapi.RaoInput; import com.powsybl.openrao.raoapi.TimeCoupledRaoInput; import com.powsybl.openrao.raoapi.TimeCoupledRaoProvider; @@ -86,12 +88,30 @@ public class Marmot implements TimeCoupledRaoProvider { private static final String MIP_SCENARIO = "MipScenario"; private static final String MIN_MARGIN_VIOLATION_EVALUATOR = "min-margin-violation-evaluator"; + private static final int PARALLELISM = 1; // TODO: configure as a parameter in MARMOT extension + @Override public CompletableFuture run(TimeCoupledRaoInput timeCoupledRaoInput, RaoParameters raoParameters) { + // Initiate lazy networks + TemporalData cracs = timeCoupledRaoInput.getRaoInputs().map(RaoInput::getCrac); + TemporalData initialNetworks = MarmotUtils.cloneNetworks(timeCoupledRaoInput.getRaoInputs().map(RaoInput::getNetwork)); + MarmotUtils.releaseAll(timeCoupledRaoInput.getRaoInputs().map(RaoInput::getNetwork)); + + TemporalData initialInputs = MarmotUtils.merge(initialNetworks, cracs); + + // custom settings for XPRESS optimization + // RaoParametes are stored in a TemporalData. They're the same for every timestamp but this prevents concurrent access + // when multi threading is activated + raoParameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getLinearOptimizationSolver().setSolverSpecificParameters("MAXTIME 15"); + TemporalData raoParametersDuplicates = new TemporalDataImpl<>(); + timeCoupledRaoInput.getTimestampsToRun().forEach(timestamp -> raoParametersDuplicates.put(timestamp, MarmotUtils.cloneParameters(raoParameters))); + + int parallelism = Math.min(PARALLELISM, timeCoupledRaoInput.getTimestampsToRun().size()); + // 1. Run independent RAOs to compute optimal preventive topological remedial actions TECHNICAL_LOGS.info("[MARMOT] ----- Topological optimization [start]"); TemporalData> consideredCnecs = new TemporalDataImpl<>(); - TemporalData topologicalOptimizationResults = runTopologicalOptimization(timeCoupledRaoInput.getRaoInputs(), consideredCnecs, raoParameters); + TemporalData topologicalOptimizationResults = runTopologicalOptimization(initialInputs, consideredCnecs, raoParametersDuplicates, parallelism); TECHNICAL_LOGS.info("[MARMOT] ----- Topological optimization [end]"); // 2. Get the initial results from the various independent results to avoid recomputing them @@ -101,22 +121,24 @@ public CompletableFuture run(TimeCoupledRaoInput timeCoupl boolean noTimeCoupledConstraints = timeCoupledRaoInput.getTimeCoupledConstraints().getGeneratorConstraints().isEmpty(); // 3. Apply independent topological remedial actions (and preventive range actions if there are no time-coupled constraints) + // TODO pq construit on la global objective function avant d'appliquer les topos TECHNICAL_LOGS.info("[MARMOT] Applying optimal topological actions on networks"); - ObjectiveFunction fullObjectiveFunction = buildGlobalObjectiveFunction(timeCoupledRaoInput.getRaoInputs().map(RaoInput::getCrac), new GlobalFlowResult(initialResults), raoParameters); + ObjectiveFunction fullObjectiveFunction = buildGlobalObjectiveFunction(cracs, new GlobalFlowResult(initialResults), raoParameters); LinearOptimizationResult initialObjectiveFunctionResult = getInitialObjectiveFunctionResult(initialResults, fullObjectiveFunction); // 4. Evaluate objective function after independent optimizations TECHNICAL_LOGS.info("[MARMOT] Evaluating global result after independent optimizations"); TemporalData postTopologicalActionsResults = topologicalOptimizationResults.map( - raoResult -> ((FastRaoResultImpl) raoResult).getFinalResult() + raoResult -> ((FastRaoResultImpl) raoResult).getFinalResult() ); - TemporalData initialSetpointResults = getInitialSetpointResults(topologicalOptimizationResults, timeCoupledRaoInput.getRaoInputs()); + TemporalData initialSetpointResults = getInitialSetpointResults(topologicalOptimizationResults, cracs, parallelism); LinearOptimizationResult postTopologicalOptimizationResult = getPostTopologicalOptimizationResult( - initialSetpointResults, - postTopologicalActionsResults, - fullObjectiveFunction, - topologicalOptimizationResults, - timeCoupledRaoInput.getRaoInputs().map(individualRaoInput -> individualRaoInput.getCrac().getPreventiveState())); + initialSetpointResults, + postTopologicalActionsResults, + fullObjectiveFunction, + topologicalOptimizationResults, + cracs.map(Crac::getPreventiveState), + parallelism); // if no time-coupled constraints are defined, the results can be returned // TODO @@ -126,94 +148,96 @@ public CompletableFuture run(TimeCoupledRaoInput timeCoupl // } // 5. Get and apply topological actions applied in independent optimizations - TemporalData preventiveTopologicalActions = getPreventiveTopologicalActions( - timeCoupledRaoInput.getRaoInputs().map(RaoInput::getCrac), - topologicalOptimizationResults - ); - applyPreventiveTopologicalActionsOnNetworks(timeCoupledRaoInput.getRaoInputs(), preventiveTopologicalActions); + TemporalData preventiveTopologicalActions = getPreventiveTopologicalActions(cracs, topologicalOptimizationResults, parallelism); + + TemporalData postTopologicalActionsInputs = applyPreventiveTopologicalActionsOnNetworks(initialInputs, preventiveTopologicalActions, parallelism); + // Work on new input : work on networks with applied topological actions. Convert them to LazyNetworks. + // TODO this clone is probably a clone of a clone, rather try using postTopologicalActionsInputs's networks. + TemporalData postTopoNetworks = MarmotUtils.cloneNetworks(postTopologicalActionsInputs.map(RaoInput::getNetwork)); + MarmotUtils.releaseAll(postTopologicalActionsInputs.map(RaoInput::getNetwork)); // 6. Create and iteratively solve MIP to find optimal range actions' set-points // Get the curative actions applied in the individual results to be able to apply them during sensitivity computations - TemporalData curativeRemedialActions = MarmotUtils.getAppliedRemedialActionsInCurative(timeCoupledRaoInput.getRaoInputs(), topologicalOptimizationResults); + TemporalData curativeRemedialActions = MarmotUtils.getAppliedRemedialActionsInCurative(cracs, topologicalOptimizationResults); TECHNICAL_LOGS.info("[MARMOT] ----- Global range actions optimization [start]"); // make fast rao result lighter by keeping only initial flow result and filtered rao result for actions replaceFastRaoResultsWithLightVersions(topologicalOptimizationResults); - //TODO: loop - TemporalData loadFlowResults; + TemporalData sensiResults; GlobalLinearOptimizationResult linearOptimizationResults; GlobalLinearOptimizationResult fullResults; int counter = 1; do { - // Clone the PostTopoScenario variant to make sure we work on a clean variant every time - timeCoupledRaoInput.getRaoInputs().getDataPerTimestamp().values().forEach(raoInput -> { - raoInput.getNetwork().getVariantManager().cloneVariant(POST_TOPO_SCENARIO, MIP_SCENARIO, true); - raoInput.getNetwork().getVariantManager().setWorkingVariant(MIP_SCENARIO); - }); + // TODO : this is probably useless : no more variant handling with lazyNetworks + // Clone the post-topological actions networks to make sure we work on a clean variant every time + TemporalData inputsForMip = MarmotUtils.merge(postTopoNetworks, cracs); // Run post topo sensitivity analysis on all timestamps ON CONSIDERED CNECS ONLY (which is why we do it every loop) TECHNICAL_LOGS.info("[MARMOT] Systematic time-coupled sensitivity analysis [start]"); TemporalData postTopoResults = runAllSensitivityAnalysesBasedOnInitialResult( - timeCoupledRaoInput.getRaoInputs(), - curativeRemedialActions, - initialResults, - raoParameters, - consideredCnecs + inputsForMip, + curativeRemedialActions, + initialResults, + raoParameters, + consideredCnecs, + parallelism ); TECHNICAL_LOGS.info("[MARMOT] Systematic time-coupled sensitivity analysis [end]"); // Build objective function with ONLY THE CONSIDERED CNECS ObjectiveFunction filteredObjectiveFunction = buildFilteredObjectiveFunction( - timeCoupledRaoInput.getRaoInputs().map(RaoInput::getCrac), - new GlobalFlowResult(initialResults), - raoParameters, - consideredCnecs + cracs, + new GlobalFlowResult(initialResults), + raoParameters, + consideredCnecs ); // Create and iteratively solve MIP to find optimal range actions' set-points FOR THE CONSIDERED CNECS TECHNICAL_LOGS.info("[MARMOT] ----- Global range actions optimization [start] for iteration {}", counter); linearOptimizationResults = optimizeLinearRemedialActions( - timeCoupledRaoInput, - initialResults, - initialSetpointResults, - postTopoResults, - raoParameters, - preventiveTopologicalActions, - curativeRemedialActions, - consideredCnecs, - filteredObjectiveFunction + new TimeCoupledRaoInput(inputsForMip, timeCoupledRaoInput.getTimestampsToRun(), timeCoupledRaoInput.getTimeCoupledConstraints()), + initialResults, + initialSetpointResults, + postTopoResults, + raoParameters, + preventiveTopologicalActions, + curativeRemedialActions, + consideredCnecs, + filteredObjectiveFunction, + parallelism ); + MarmotUtils.releaseAll(inputsForMip.map(RaoInput::getNetwork)); TECHNICAL_LOGS.info("[MARMOT] ----- Global range actions optimization [end] for iteration {}", counter); // Compute the flows on ALL the cnecs to check if the worst cnecs have changed and were considered in the MIP or not - loadFlowResults = applyActionsAndRunFullLoadflow(timeCoupledRaoInput.getRaoInputs(), curativeRemedialActions, linearOptimizationResults, initialResults, raoParameters); + sensiResults = applyActionsAndRunFullSensitivityAnalysis(postTopologicalActionsInputs, curativeRemedialActions, linearOptimizationResults, initialResults, raoParameters, parallelism); // Create a global result with the flows on ALL cnecs and the actions applied during MIP TemporalData rangeActionActivationResultTemporalData = linearOptimizationResults.getRangeActionActivationResultTemporalData(); fullResults = new GlobalLinearOptimizationResult( - loadFlowResults, - loadFlowResults.map(PrePerimeterResult::getSensitivityResult), - rangeActionActivationResultTemporalData, - preventiveTopologicalActions, - fullObjectiveFunction, - LinearProblemStatus.OPTIMAL + sensiResults, + sensiResults.map(PrePerimeterResult::getSensitivityResult), + rangeActionActivationResultTemporalData, + preventiveTopologicalActions, + fullObjectiveFunction, + LinearProblemStatus.OPTIMAL ); logCost("[MARMOT] next iteration of MIP: ", fullResults, raoParameters, 10); counter++; - } while (shouldContinueAndAddCnecs(loadFlowResults, consideredCnecs, getFlowUnit(raoParameters)) && counter < 10); // Stop if the worst element of each TS has been considered during MIP + } while (shouldContinueAndAddCnecs(sensiResults, consideredCnecs, getFlowUnit(raoParameters)) && counter < 10); // Stop if the worst element of each TS has been considered during MIP TECHNICAL_LOGS.info("[MARMOT] ----- Global range actions optimization [end]"); // 7. Merge topological and linear result TECHNICAL_LOGS.info("[MARMOT] Merging topological and linear remedial action results"); TimeCoupledRaoResultImpl timeCoupledRaoResult = mergeTopologicalAndLinearOptimizationResults( - timeCoupledRaoInput.getRaoInputs(), - initialResults, - initialObjectiveFunctionResult, - fullResults, - topologicalOptimizationResults, - raoParameters + postTopologicalActionsInputs, + initialResults, + initialObjectiveFunctionResult, + fullResults, + topologicalOptimizationResults, + raoParameters ); // 8. Log initial and final results @@ -224,20 +248,20 @@ public CompletableFuture run(TimeCoupledRaoInput timeCoupl return CompletableFuture.completedFuture(timeCoupledRaoResult); } - private TemporalData getInitialSetpointResults(TemporalData postTopologicalActionsResults, TemporalData raoInputs) { - TemporalData initialSetpointResults = new TemporalDataImpl<>(); - raoInputs.getDataPerTimestamp().forEach((timestamp, raoInput) -> { - Map, Double> setPointMap = new HashMap<>(); - raoInput.getCrac().getRangeActions().forEach(rangeAction -> - setPointMap.put(rangeAction, postTopologicalActionsResults.getData(timestamp).orElseThrow() - .getPreOptimizationSetPointOnState(raoInput.getCrac().getPreventiveState(), rangeAction)) - ); - RangeActionSetpointResult rangeActionSetpointResult = new RangeActionSetpointResultImpl( - setPointMap - ); - initialSetpointResults.put(timestamp, rangeActionSetpointResult); - }); - return initialSetpointResults; + private TemporalData getInitialSetpointResults(TemporalData postTopologicalActionsResults, TemporalData cracs, int parallelism) { + return MarmotUtils.smartMap( + cracs, + crac -> { + OffsetDateTime timestamp = crac.getTimestamp().orElseThrow(); + Map, Double> setPointMap = new HashMap<>(); + crac.getRangeActions().forEach(rangeAction -> + setPointMap.put(rangeAction, postTopologicalActionsResults.getData(timestamp).orElseThrow() + .getPreOptimizationSetPointOnState(crac.getPreventiveState(), rangeAction)) + ); + return new RangeActionSetpointResultImpl(setPointMap); + }, + parallelism + ); } private boolean shouldContinueAndAddCnecs(TemporalData loadFlowResults, TemporalData> consideredCnecs, Unit flowUnit) { @@ -266,18 +290,18 @@ private static void updateShouldContinue(TemporalData loadFl // for margin violation - need to compare to min improvement on margin // ordered list of cnecs with an overload List worstCnecsForMarginViolation = loadFlowResult.getCostlyElements( - MIN_MARGIN_VIOLATION_EVALUATOR, - Integer.MAX_VALUE + MIN_MARGIN_VIOLATION_EVALUATOR, + Integer.MAX_VALUE ); double worstConsideredMargin = worstCnecsForMarginViolation.stream() - .filter(previousCnecs::contains) - .findFirst() - .map(cnec -> loadFlowResult.getMargin(cnec, flowUnit)) - .orElse(0.); + .filter(previousCnecs::contains) + .findFirst() + .map(cnec -> loadFlowResult.getMargin(cnec, flowUnit)) + .orElse(0.); double worstMarginOfAll = worstCnecsForMarginViolation.stream() - .findFirst() - .map(cnec -> loadFlowResult.getMargin(cnec, flowUnit)) - .orElse(0.); + .findFirst() + .map(cnec -> loadFlowResult.getMargin(cnec, flowUnit)) + .orElse(0.); // if worst overload > worst considered overload *( 1 + minImprovementOnLoad) if (worstMarginOfAll < worstConsideredMargin * (1 + minRelativeImprovementOnMargin) - 1e-6) { shouldContinue.set(true); @@ -285,13 +309,13 @@ private static void updateShouldContinue(TemporalData loadFl // for other violations - just check if cnec was considered loadFlowResult.getVirtualCostNames().stream() - .filter(vcName -> !vcName.equals(MIN_MARGIN_VIOLATION_EVALUATOR)) - .forEach(vcName -> { - Optional worstCnec = loadFlowResult.getCostlyElements(vcName, 1).stream().findFirst(); - if (worstCnec.isPresent() && !previousCnecs.contains(worstCnec.get())) { - shouldContinue.set(true); - } - }); + .filter(vcName -> !vcName.equals(MIN_MARGIN_VIOLATION_EVALUATOR)) + .forEach(vcName -> { + Optional worstCnec = loadFlowResult.getCostlyElements(vcName, 1).stream().findFirst(); + if (worstCnec.isPresent() && !previousCnecs.contains(worstCnec.get())) { + shouldContinue.set(true); + } + }); } } @@ -307,11 +331,11 @@ private static void updateConsideredCnecs(TemporalData loadF Set nextIterationCnecs = new HashSet<>(previousIterationCnecs); double worstConsideredMargin = loadFlowResult.getCostlyElements(MIN_MARGIN_VIOLATION_EVALUATOR, Integer.MAX_VALUE) - .stream() - .filter(previousIterationCnecs::contains) - .findFirst() - .map(cnec -> loadFlowResult.getMargin(cnec, flowUnit)) - .orElse(0.); + .stream() + .filter(previousIterationCnecs::contains) + .findFirst() + .map(cnec -> loadFlowResult.getMargin(cnec, flowUnit)) + .orElse(0.); for (String vcName : loadFlowResult.getVirtualCostNames()) { LoggingAddedCnecs currentLoggingAddedCnecs = new LoggingAddedCnecs(timestamp, vcName, new ArrayList<>(), new HashMap<>()); @@ -321,8 +345,8 @@ private static void updateConsideredCnecs(TemporalData loadF if (vcName.equals(MIN_MARGIN_VIOLATION_EVALUATOR)) { for (FlowCnec cnec : loadFlowResult.getCostlyElements(vcName, Integer.MAX_VALUE)) { if (loadFlowResult.getMargin( - cnec, - flowUnit + cnec, + flowUnit ) > worstConsideredMargin + marginWindowToConsider && addedCnecsForVcName > cnecsToAddPerVirtualCostName) { // stop if out of window and already added enough break; @@ -355,8 +379,8 @@ private static void logCnecs(List addedCnecsForLogging) { logMessage.append(" for timestamp ").append(loggingAddedCnecs.offsetDateTime().toString()).append(" and virtual cost ").append(loggingAddedCnecs.vcName()).append(" "); for (String cnec : loggingAddedCnecs.addedCnecs()) { String cnecString = loggingAddedCnecs.vcName().equals(MIN_MARGIN_VIOLATION_EVALUATOR) ? - cnec + "(" + loggingAddedCnecs.margins().get(cnec) + ")" + "," : - cnec + ","; + cnec + "(" + loggingAddedCnecs.margins().get(cnec) + ")" + "," : + cnec + ","; logMessage.append(cnecString); } } @@ -364,7 +388,8 @@ private static void logCnecs(List addedCnecsForLogging) { TECHNICAL_LOGS.info(logMessage.toString()); } - record LoggingAddedCnecs(OffsetDateTime offsetDateTime, String vcName, List addedCnecs, Map margins) { + record LoggingAddedCnecs(OffsetDateTime offsetDateTime, String vcName, List addedCnecs, + Map margins) { private void addCnec(String cnec) { addedCnecs.add(cnec); } @@ -375,64 +400,77 @@ private void addCnec(String cnec, double margin) { } } - private static TemporalData applyActionsAndRunFullLoadflow(TemporalData raoInputs, + private static TemporalData applyActionsAndRunFullSensitivityAnalysis(TemporalData postTopoInputs, TemporalData curativeRemedialActions, LinearOptimizationResult filteredResult, TemporalData initialResults, - RaoParameters raoParameters) { - TemporalData prePerimeterResults = new TemporalDataImpl<>(); - raoInputs.getDataPerTimestamp().forEach((timestamp, raoInput) -> { - // duplicate the postTopoScenario variant and switch to the new clone - raoInput.getNetwork().getVariantManager().cloneVariant(POST_TOPO_SCENARIO, "PostPreventiveScenario", true); - raoInput.getNetwork().getVariantManager().setWorkingVariant("PostPreventiveScenario"); - State preventiveState = raoInput.getCrac().getPreventiveState(); - raoInput.getCrac().getRangeActions(preventiveState).forEach(rangeAction -> - rangeAction.apply(raoInput.getNetwork(), filteredResult.getOptimizedSetpoint(rangeAction, preventiveState)) - ); - prePerimeterResults.put(timestamp, runInitialPrePerimeterSensitivityAnalysisWithoutRangeActions( - raoInputs.getData(timestamp).orElseThrow(), - curativeRemedialActions.getData(timestamp).orElseThrow(), - initialResults.getData(timestamp).orElseThrow(), - raoParameters)); - // switch back to the postTopoScenario to avoid keeping applied range actions when entering the MIP - raoInput.getNetwork().getVariantManager().setWorkingVariant(POST_TOPO_SCENARIO); - }); - return prePerimeterResults; + RaoParameters raoParameters, int parallelism) { + return MarmotUtils.smartMap( + postTopoInputs, + raoInput -> { + OffsetDateTime timestamp = MarmotUtils.getTimestamp(raoInput); + State preventiveState = raoInput.getCrac().getPreventiveState(); + raoInput.getCrac().getRangeActions(preventiveState).forEach(rangeAction -> + rangeAction.apply(raoInput.getNetwork(), filteredResult.getOptimizedSetpoint(rangeAction, preventiveState)) + ); + PrePerimeterResult sensitivityAnalysisResults = runInitialPrePerimeterSensitivityAnalysisWithoutRangeActions( + postTopoInputs.getData(timestamp).orElseThrow(), + curativeRemedialActions.getData(timestamp).orElseThrow(), + initialResults.getData(timestamp).orElseThrow(), + raoParameters); + MarmotUtils.releaseNetwork(raoInput.getNetwork()); + return sensitivityAnalysisResults; + }, + parallelism + ); } private void replaceFastRaoResultsWithLightVersions(TemporalData topologicalOptimizationResults) { topologicalOptimizationResults.getDataPerTimestamp().forEach((timestamp, raoResult) -> topologicalOptimizationResults.put( - timestamp, new LightFastRaoResultImpl((FastRaoResultImpl) raoResult))); + timestamp, new LightFastRaoResultImpl((FastRaoResultImpl) raoResult))); } - private static TemporalData runTopologicalOptimization(TemporalData raoInputs, TemporalData> consideredCnecs, RaoParameters raoParameters) { - raoParameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getLinearOptimizationSolver().setSolverSpecificParameters("MAXTIME 15"); + private static TemporalData runTopologicalOptimization(TemporalData raoInputs, TemporalData> consideredCnecs, TemporalData raoParameters, int parallelism) { + return MarmotUtils.smartMap(raoInputs, raoInput -> runSingleTopologicalOptimization(raoInput, consideredCnecs, raoParameters.getData(MarmotUtils.getTimestamp(raoInput)).orElseThrow()), parallelism); + } - TemporalData individualResults = new TemporalDataImpl<>(); - raoInputs.getDataPerTimestamp().forEach((datetime, raoInput) -> { + private static RaoResult runSingleTopologicalOptimization(RaoInput raoInput, TemporalData> consideredCnecs, RaoParameters raoParameters) { + try (LazyNetwork ignored = (LazyNetwork) raoInput.getNetwork()) { Set cnecs = new HashSet<>(); - String logMessage = "[MARMOT] Running RAO for timestamp %s [{}]".formatted(raoInput.getCrac().getTimestamp().orElseThrow()); + OffsetDateTime timestamp = MarmotUtils.getTimestamp(raoInput); + String logMessage = "[MARMOT] Running RAO for timestamp %s [{}]".formatted(timestamp); TECHNICAL_LOGS.info(logMessage, "start"); RaoResult raoResult = FastRao.launchFastRaoOptimization(raoInput, raoParameters, null, cnecs); TECHNICAL_LOGS.info(logMessage, "end"); - consideredCnecs.put(datetime, cnecs); - individualResults.put(datetime, raoResult); - }); - return individualResults; + consideredCnecs.put(timestamp, cnecs); + return raoResult; + } catch (Exception e) { + throw new OpenRaoException(e); + } } - private static void applyPreventiveTopologicalActionsOnNetworks(TemporalData raoInputs, TemporalData preventiveTopologicalActionsResults) { - raoInputs.getTimestamps().forEach(timestamp -> { - RaoInput raoInput = raoInputs.getData(timestamp).orElseThrow(); - NetworkActionsResult networkActionsResult = preventiveTopologicalActionsResults.getData(timestamp).orElseThrow(); - MarmotUtils.applyPreventiveRemedialActions(raoInput, networkActionsResult, INITIAL_SCENARIO, POST_TOPO_SCENARIO); - }); + private static TemporalData applyPreventiveTopologicalActionsOnNetworks(TemporalData raoInputs, TemporalData preventiveTopologicalActionsResults, int parallelism) { + return MarmotUtils.smartMap( + raoInputs, + raoInput -> { + OffsetDateTime timestamp = MarmotUtils.getTimestamp(raoInput); + NetworkActionsResult networkActionsResult = preventiveTopologicalActionsResults.getData(timestamp).orElseThrow(); + MarmotUtils.applyPreventiveRemedialActions(raoInput, networkActionsResult, INITIAL_SCENARIO, POST_TOPO_SCENARIO); + try (LazyNetwork lazyNetwork = new LazyNetwork(raoInput.getNetwork())) { + MarmotUtils.releaseNetwork(raoInput.getNetwork()); + return RaoInput.build(lazyNetwork, raoInput.getCrac()).build(); + } catch (Exception e) { + throw new OpenRaoException(e); + } + }, + parallelism + ); } private TemporalData buildInitialResults(TemporalData topologicalOptimizationResults) { TemporalData initialResults = new TemporalDataImpl<>(); topologicalOptimizationResults.getDataPerTimestamp().forEach((timestamp, raoResult) -> - initialResults.put(timestamp, ((FastRaoResultImpl) raoResult).getInitialResult())); + initialResults.put(timestamp, ((FastRaoResultImpl) raoResult).getInitialResult())); return initialResults; } @@ -440,32 +478,38 @@ private static TemporalData runAllSensitivityAnalysesBasedOn TemporalData curativeRemedialActions, TemporalData initialFlowResults, RaoParameters raoParameters, - TemporalData> consideredCnecs) { - TemporalData prePerimeterResults = new TemporalDataImpl<>(); - raoInputs.getTimestamps().forEach(timestamp -> - prePerimeterResults.put(timestamp, runSensitivityAnalysisBasedOnInitialResult( - raoInputs.getData(timestamp).orElseThrow(), - curativeRemedialActions.getData(timestamp).orElseThrow(), - initialFlowResults.getData(timestamp).orElseThrow(), - raoParameters, - consideredCnecs.getData(timestamp).orElseThrow() - ))); - return prePerimeterResults; + TemporalData> consideredCnecs, int parallelism) { + return MarmotUtils.smartMap( + raoInputs, + raoInput -> { + OffsetDateTime timestamp = MarmotUtils.getTimestamp(raoInput); + PrePerimeterResult sensitivityAnalysisResult = runSensitivityAnalysisBasedOnInitialResult( + raoInput, + curativeRemedialActions.getData(timestamp).orElseThrow(), + initialFlowResults.getData(timestamp).orElseThrow(), + raoParameters, + consideredCnecs.getData(timestamp).orElseThrow() + ); + MarmotUtils.releaseNetwork(raoInput.getNetwork()); + return sensitivityAnalysisResult; + }, + parallelism + ); } - private static TemporalData getPreventiveTopologicalActions(TemporalData cracs, TemporalData raoResults) { - Map preventiveTopologicalActions = new HashMap<>(); - cracs.getTimestamps().forEach(timestamp -> { - State preventiveState = cracs.getData(timestamp).orElseThrow().getPreventiveState(); - preventiveTopologicalActions.put( - timestamp, - new NetworkActionsResultImpl(Map.of( - preventiveState, - raoResults.getData(timestamp).orElseThrow().getActivatedNetworkActionsDuringState(preventiveState) - )) - ); - }); - return new TemporalDataImpl<>(preventiveTopologicalActions); + private static TemporalData getPreventiveTopologicalActions(TemporalData cracs, TemporalData raoResults, int parallelism) { + return MarmotUtils.smartMap( + cracs, + crac -> { + OffsetDateTime timestamp = crac.getTimestamp().orElseThrow(); + return new NetworkActionsResultImpl( + Map.of( + crac.getPreventiveState(), + raoResults.getData(timestamp).orElseThrow().getActivatedNetworkActionsDuringState(crac.getPreventiveState()) + )); + }, + parallelism + ); } private static GlobalLinearOptimizationResult optimizeLinearRemedialActions(TimeCoupledRaoInput raoInput, @@ -476,46 +520,63 @@ private static GlobalLinearOptimizationResult optimizeLinearRemedialActions(Time TemporalData preventiveTopologicalActions, TemporalData curativeRemedialActions, TemporalData> consideredCnecs, - ObjectiveFunction objectiveFunction) { + ObjectiveFunction objectiveFunction, + int parallelism) { // -- Build IteratingLinearOptimizertimeCoupledInput - TemporalData optimizationPerimeterPerTimestamp = computeOptimizationPerimetersPerTimestamp(raoInput.getRaoInputs().map(RaoInput::getCrac), consideredCnecs); + TemporalData optimizationPerimeterPerTimestamp = computeOptimizationPerimetersPerTimestamp(raoInput.getRaoInputs().map(RaoInput::getCrac), consideredCnecs, parallelism); // no objective function defined in individual IteratingLinearOptimizerInputs as it is global - Map linearOptimizerInputPerTimestamp = new HashMap<>(); - raoInput.getRaoInputs().getTimestamps().forEach(timestamp -> linearOptimizerInputPerTimestamp.put(timestamp, - IteratingLinearOptimizerInput.create() - .withNetwork(raoInput.getRaoInputs().getData(timestamp).orElseThrow().getNetwork()) - .withOptimizationPerimeter(optimizationPerimeterPerTimestamp.getData(timestamp).orElseThrow() - .copyWithFilteredAvailableHvdcRangeAction(raoInput.getRaoInputs().getData(timestamp).orElseThrow().getNetwork())) - .withInitialFlowResult(initialResults.getData(timestamp).orElseThrow()) - .withPrePerimeterFlowResult(initialResults.getData(timestamp).orElseThrow()) - .withPreOptimizationFlowResult(postTopologicalActionsResults.getData(timestamp).orElseThrow()) - .withPrePerimeterSetpoints(initialSetpoints.getData(timestamp).orElseThrow()) - .withPreOptimizationSensitivityResult(postTopologicalActionsResults.getData(timestamp).orElseThrow()) - .withPreOptimizationAppliedRemedialActions(curativeRemedialActions.getData(timestamp).orElseThrow()) - .withToolProvider(ToolProvider.buildFromRaoInputAndParameters(raoInput.getRaoInputs().getData(timestamp).orElseThrow(), parameters)) - .withOutageInstant(raoInput.getRaoInputs().getData(timestamp).orElseThrow().getCrac().getOutageInstant()) - .withAppliedNetworkActionsInPrimaryState(preventiveTopologicalActions.getData(timestamp).orElseThrow()) - .build())); + TemporalData linearOptimizerInputs = MarmotUtils.smartMap( + raoInput.getRaoInputs(), + individualRaoInput -> { + OffsetDateTime timestamp = MarmotUtils.getTimestamp(individualRaoInput); + try (LazyNetwork lazyNetwork = new LazyNetwork(individualRaoInput.getNetwork())) { + IteratingLinearOptimizerInput iteratingLinearOptimizerInput = IteratingLinearOptimizerInput.create() + .withNetwork(lazyNetwork) + .withOptimizationPerimeter(optimizationPerimeterPerTimestamp.getData(timestamp).orElseThrow() + .copyWithFilteredAvailableHvdcRangeAction(individualRaoInput.getNetwork())) + .withInitialFlowResult(initialResults.getData(timestamp).orElseThrow()) + .withPrePerimeterFlowResult(initialResults.getData(timestamp).orElseThrow()) + .withPreOptimizationFlowResult(postTopologicalActionsResults.getData(timestamp).orElseThrow()) + .withPrePerimeterSetpoints(initialSetpoints.getData(timestamp).orElseThrow()) + .withPreOptimizationSensitivityResult(postTopologicalActionsResults.getData(timestamp).orElseThrow()) + .withPreOptimizationAppliedRemedialActions(curativeRemedialActions.getData(timestamp).orElseThrow()) + .withToolProvider(ToolProvider.buildFromRaoInputAndParameters(raoInput.getRaoInputs().getData(timestamp).orElseThrow(), parameters)) + .withOutageInstant(individualRaoInput.getCrac().getOutageInstant()) + .withAppliedNetworkActionsInPrimaryState(preventiveTopologicalActions.getData(timestamp).orElseThrow()) + .build(); + MarmotUtils.releaseNetwork(individualRaoInput.getNetwork()); + return iteratingLinearOptimizerInput; + } catch (Exception e) { + throw new OpenRaoException(e); + } + }, + parallelism + ); + TimeCoupledIteratingLinearOptimizerInput timeCoupledLinearOptimizerInput = new TimeCoupledIteratingLinearOptimizerInput( - new TemporalDataImpl<>(linearOptimizerInputPerTimestamp), objectiveFunction, raoInput.getTimeCoupledConstraints()); + linearOptimizerInputs, objectiveFunction, raoInput.getTimeCoupledConstraints()); + + // TODO : a priori ce release all ne devrait pas être utile MAIS il semblerait qu'il y ait des réseaux pas fermés en arrivant ici, + // à investiguer + MarmotUtils.releaseAll(timeCoupledLinearOptimizerInput.iteratingLinearOptimizerInputs().map(IteratingLinearOptimizerInput::network)); // Build parameters // Unoptimized cnec parameters ignored because only PRAs // TODO: define static method to define Ra Limitation Parameters from crac and topos (mutualize with search tree) : SearchTreeParameters::decreaseRemedialActionsUsageLimits IteratingLinearOptimizerParameters.LinearOptimizerParametersBuilder linearOptimizerParametersBuilder = IteratingLinearOptimizerParameters.create() - .withObjectiveFunction(parameters.getObjectiveFunctionParameters().getType()) - .withFlowUnit(getFlowUnit(parameters)) - .withRangeActionParameters(parameters.getRangeActionsOptimizationParameters()) - .withRangeActionParametersExtension(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters()) - .withMaxNumberOfIterations(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getMaxMipIterations()) - .withRaRangeShrinking(ENABLED.equals(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getRaRangeShrinking()) - || ENABLED_IN_FIRST_PRAO_AND_CRAO.equals(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getRaRangeShrinking())) - .withSolverParameters(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getLinearOptimizationSolver()) - .withMaxMinRelativeMarginParameters(parameters.getExtension(SearchTreeRaoRelativeMarginsParameters.class)) - .withRaLimitationParameters(new RangeActionLimitationParameters()) - .withMinMarginParameters(parameters.getExtension(OpenRaoSearchTreeParameters.class).getMinMarginsParameters().orElse(new SearchTreeRaoCostlyMinMarginParameters())); + .withObjectiveFunction(parameters.getObjectiveFunctionParameters().getType()) + .withFlowUnit(getFlowUnit(parameters)) + .withRangeActionParameters(parameters.getRangeActionsOptimizationParameters()) + .withRangeActionParametersExtension(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters()) + .withMaxNumberOfIterations(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getMaxMipIterations()) + .withRaRangeShrinking(ENABLED.equals(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getRaRangeShrinking()) + || ENABLED_IN_FIRST_PRAO_AND_CRAO.equals(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getRaRangeShrinking())) + .withSolverParameters(parameters.getExtension(OpenRaoSearchTreeParameters.class).getRangeActionsOptimizationParameters().getLinearOptimizationSolver()) + .withMaxMinRelativeMarginParameters(parameters.getExtension(SearchTreeRaoRelativeMarginsParameters.class)) + .withRaLimitationParameters(new RangeActionLimitationParameters()) + .withMinMarginParameters(parameters.getExtension(OpenRaoSearchTreeParameters.class).getMinMarginsParameters().orElse(new SearchTreeRaoCostlyMinMarginParameters())); parameters.getMnecParameters().ifPresent(linearOptimizerParametersBuilder::withMnecParameters); parameters.getExtension(OpenRaoSearchTreeParameters.class).getMnecParameters().ifPresent(linearOptimizerParametersBuilder::withMnecParametersExtension); parameters.getLoopFlowParameters().ifPresent(linearOptimizerParametersBuilder::withLoopFlowParameters); @@ -525,19 +586,21 @@ private static GlobalLinearOptimizationResult optimizeLinearRemedialActions(Time return TimeCoupledIteratingLinearOptimizer.optimize(timeCoupledLinearOptimizerInput, linearOptimizerParameters); } - private static TemporalData computeOptimizationPerimetersPerTimestamp(TemporalData cracs, TemporalData> consideredCnecs) { - TemporalData optimizationPerimeters = new TemporalDataImpl<>(); - cracs.getTimestamps().forEach(timestamp -> { - Crac crac = cracs.getData(timestamp).orElseThrow(); - optimizationPerimeters.put(timestamp, new PreventiveOptimizationPerimeter( - crac.getPreventiveState(), - consideredCnecs.getData(timestamp).orElseThrow(), - new HashSet<>(), // no loopflows for now - new HashSet<>(), // don't re-optimize topological actions in Marmot - crac.getRangeActions(crac.getPreventiveState()) - )); - }); - return optimizationPerimeters; + private static TemporalData computeOptimizationPerimetersPerTimestamp(TemporalData cracs, TemporalData> consideredCnecs, int parallelism) { + return MarmotUtils.smartMap( + cracs, + crac -> { + OffsetDateTime timestamp = crac.getTimestamp().orElseThrow(); + return new PreventiveOptimizationPerimeter( + crac.getPreventiveState(), + consideredCnecs.getData(timestamp).orElseThrow(), + new HashSet<>(), // no loopflows for now + new HashSet<>(), // don't re-optimize topological actions in Marmot + crac.getRangeActions(crac.getPreventiveState()) + ); + }, + parallelism + ); } private static TimeCoupledRaoResultImpl mergeTopologicalAndLinearOptimizationResults(TemporalData raoInputs, @@ -547,14 +610,14 @@ private static TimeCoupledRaoResultImpl mergeTopologicalAndLinearOptimizationRes TemporalData topologicalOptimizationResults, RaoParameters raoParameters) { return new TimeCoupledRaoResultImpl( - initialLinearOptimizationResult, - globalLinearOptimizationResult, - getPostOptimizationResults( - raoInputs, - initialResults, + initialLinearOptimizationResult, globalLinearOptimizationResult, - topologicalOptimizationResults, - raoParameters)); + getPostOptimizationResults( + raoInputs, + initialResults, + globalLinearOptimizationResult, + topologicalOptimizationResults, + raoParameters)); } private static ObjectiveFunction buildGlobalObjectiveFunction(TemporalData cracs, FlowResult globalInitialFlowResult, RaoParameters raoParameters) { @@ -562,12 +625,12 @@ private static ObjectiveFunction buildGlobalObjectiveFunction(TemporalData cracs.map(Crac::getFlowCnecs).getDataPerTimestamp().values().forEach(allFlowCnecs::addAll); Set allOptimizedStates = new HashSet<>(cracs.map(Crac::getPreventiveState).getDataPerTimestamp().values()); return ObjectiveFunction.build(allFlowCnecs, - new HashSet<>(), // no loop flows for now - globalInitialFlowResult, - globalInitialFlowResult, // always building from preventive so prePerimeter = initial - Collections.emptySet(), - raoParameters, - allOptimizedStates); + new HashSet<>(), // no loop flows for now + globalInitialFlowResult, + globalInitialFlowResult, // always building from preventive so prePerimeter = initial + Collections.emptySet(), + raoParameters, + allOptimizedStates); } private static ObjectiveFunction buildFilteredObjectiveFunction(TemporalData cracs, @@ -579,25 +642,25 @@ private static ObjectiveFunction buildFilteredObjectiveFunction(TemporalData allOptimizedStates = new HashSet<>(cracs.map(Crac::getPreventiveState).getDataPerTimestamp().values()); return ObjectiveFunction.build( - flatConsideredCnecs, - new HashSet<>(), // no loop flows for now - globalInitialFlowResult, - globalInitialFlowResult, // always building from preventive so prePerimeter = initial - Collections.emptySet(), - raoParameters, - allOptimizedStates); + flatConsideredCnecs, + new HashSet<>(), // no loop flows for now + globalInitialFlowResult, + globalInitialFlowResult, // always building from preventive so prePerimeter = initial + Collections.emptySet(), + raoParameters, + allOptimizedStates); } private LinearOptimizationResult getInitialObjectiveFunctionResult(TemporalData prePerimeterResults, ObjectiveFunction objectiveFunction) { TemporalData rangeActionActivationResults = prePerimeterResults.map(RangeActionActivationResultImpl::new); TemporalData networkActionsResults = new TemporalDataImpl<>(); return new GlobalLinearOptimizationResult( - prePerimeterResults.map(PrePerimeterResult::getFlowResult), - prePerimeterResults.map(PrePerimeterResult::getSensitivityResult), - rangeActionActivationResults, - networkActionsResults, - objectiveFunction, - LinearProblemStatus.OPTIMAL + prePerimeterResults.map(PrePerimeterResult::getFlowResult), + prePerimeterResults.map(PrePerimeterResult::getSensitivityResult), + rangeActionActivationResults, + networkActionsResults, + objectiveFunction, + LinearProblemStatus.OPTIMAL ); } @@ -605,47 +668,48 @@ private LinearOptimizationResult getPostTopologicalOptimizationResult(TemporalDa TemporalData prePerimeterResults, ObjectiveFunction objectiveFunction, TemporalData topologicalOptimizationResults, - TemporalData preventiveStates) { - TemporalData rangeActionActivationResults = getRangeActionActivationResults(allInitialSetPoints, topologicalOptimizationResults, preventiveStates); - TemporalData networkActionsResults = getNetworkActionActivationResults(topologicalOptimizationResults, preventiveStates); + TemporalData preventiveStates, + int parallelism) { + TemporalData rangeActionActivationResults = getRangeActionActivationResults(allInitialSetPoints, topologicalOptimizationResults, preventiveStates, parallelism); + TemporalData networkActionsResults = getNetworkActionActivationResults(topologicalOptimizationResults, preventiveStates, parallelism); return new GlobalLinearOptimizationResult( - prePerimeterResults.map(PrePerimeterResult::getFlowResult), - prePerimeterResults.map(PrePerimeterResult::getSensitivityResult), - rangeActionActivationResults, - networkActionsResults, - objectiveFunction, - LinearProblemStatus.OPTIMAL + prePerimeterResults.map(PrePerimeterResult::getFlowResult), + prePerimeterResults.map(PrePerimeterResult::getSensitivityResult), + rangeActionActivationResults, + networkActionsResults, + objectiveFunction, + LinearProblemStatus.OPTIMAL ); } private static TemporalData getRangeActionActivationResults(TemporalData allInitialSetPoints, TemporalData topologicalOptimizationResults, - TemporalData preventiveStates) { - Map rangeActionsResults = new HashMap<>(); - topologicalOptimizationResults.getTimestamps().forEach( - timestamp -> { - State preventiveState = preventiveStates.getData(timestamp).orElseThrow(); - RangeActionSetpointResult initialSetPoints = allInitialSetPoints.getData(timestamp).orElseThrow(); - RangeActionSetpointResult optimizedSetPoints = new RangeActionSetpointResultImpl( - topologicalOptimizationResults.getData(timestamp).orElseThrow().getOptimizedSetPointsOnState(preventiveState)); - RangeActionActivationResultImpl rangeActionActivationResult = new RangeActionActivationResultImpl(initialSetPoints); - optimizedSetPoints.getRangeActions().forEach(rangeAction -> rangeActionActivationResult.putResult(rangeAction, preventiveState, optimizedSetPoints.getSetpoint(rangeAction))); - rangeActionsResults.put(timestamp, rangeActionActivationResult); - } + TemporalData preventiveStates, int parallelism) { + return MarmotUtils.smartMap( + preventiveStates, + preventiveState -> { + OffsetDateTime timestamp = preventiveState.getTimestamp().orElseThrow(); + RangeActionSetpointResult initialSetPoints = allInitialSetPoints.getData(timestamp).orElseThrow(); + RangeActionSetpointResult optimizedSetPoints = new RangeActionSetpointResultImpl( + topologicalOptimizationResults.getData(timestamp).orElseThrow().getOptimizedSetPointsOnState(preventiveState)); + RangeActionActivationResultImpl rangeActionActivationResult = new RangeActionActivationResultImpl(initialSetPoints); + optimizedSetPoints.getRangeActions().forEach(rangeAction -> rangeActionActivationResult.putResult(rangeAction, preventiveState, optimizedSetPoints.getSetpoint(rangeAction))); + return rangeActionActivationResult; + }, + parallelism ); - return new TemporalDataImpl<>(rangeActionsResults); } - private static TemporalData getNetworkActionActivationResults(TemporalData topologicalOptimizationResults, TemporalData preventiveStates) { - Map networkActionsResults = new HashMap<>(); - topologicalOptimizationResults.getTimestamps().forEach( - timestamp -> { - State preventiveState = preventiveStates.getData(timestamp).orElseThrow(); - Set activatedNetworkActions = topologicalOptimizationResults.getData(timestamp).orElseThrow().getActivatedNetworkActionsDuringState(preventiveState); - networkActionsResults.put(timestamp, new NetworkActionsResultImpl(Map.of(preventiveState, activatedNetworkActions))); - } + private static TemporalData getNetworkActionActivationResults(TemporalData topologicalOptimizationResults, TemporalData preventiveStates, int parallelism) { + return MarmotUtils.smartMap( + preventiveStates, + preventiveState -> { + OffsetDateTime timestamp = preventiveState.getTimestamp().orElseThrow(); + Set activatedNetworkActions = topologicalOptimizationResults.getData(timestamp).orElseThrow().getActivatedNetworkActionsDuringState(preventiveState); + return new NetworkActionsResultImpl(Map.of(preventiveState, activatedNetworkActions)); + }, + parallelism ); - return new TemporalDataImpl<>(networkActionsResults); } @Override diff --git a/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/marmot/MarmotUtils.java b/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/marmot/MarmotUtils.java index ad3a7e97d1..c1e6bb00bc 100644 --- a/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/marmot/MarmotUtils.java +++ b/ra-optimisation/search-tree-rao/src/main/java/com/powsybl/openrao/searchtreerao/marmot/MarmotUtils.java @@ -19,7 +19,9 @@ import com.powsybl.openrao.data.crac.api.rangeaction.RangeAction; import com.powsybl.openrao.data.raoresult.api.ComputationStatus; import com.powsybl.openrao.data.raoresult.api.RaoResult; +import com.powsybl.openrao.raoapi.LazyNetwork; import com.powsybl.openrao.raoapi.RaoInput; +import com.powsybl.openrao.raoapi.json.JsonRaoParameters; import com.powsybl.openrao.raoapi.parameters.RaoParameters; import com.powsybl.openrao.searchtreerao.castor.algorithm.PrePerimeterSensitivityAnalysis; import com.powsybl.openrao.searchtreerao.commons.ToolProvider; @@ -29,6 +31,9 @@ import com.powsybl.openrao.searchtreerao.result.api.PrePerimeterResult; import com.powsybl.openrao.sensitivityanalysis.AppliedRemedialActions; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.time.OffsetDateTime; import java.util.HashMap; import java.util.HashSet; @@ -73,10 +78,10 @@ public static PrePerimeterResult runInitialPrePerimeterSensitivityAnalysisWithou ).runBasedOnInitialResults(network, initialResult, null, curativeRemedialActions); } - public static TemporalData getAppliedRemedialActionsInCurative(TemporalData inputs, TemporalData raoResults) { + public static TemporalData getAppliedRemedialActionsInCurative(TemporalData cracs, TemporalData raoResults) { TemporalData curativeRemedialActions = new TemporalDataImpl<>(); - inputs.getTimestamps().forEach(timestamp -> { - Crac crac = inputs.getData(timestamp).orElseThrow().getCrac(); + cracs.getTimestamps().forEach(timestamp -> { + Crac crac = cracs.getData(timestamp).orElseThrow(); RaoResult raoResult = raoResults.getData(timestamp).orElseThrow(); AppliedRemedialActions appliedRemedialActions = new AppliedRemedialActions(); // TODO: maybe check it is indeed curative @@ -153,6 +158,7 @@ public static void applyPreventiveRemedialActions(RaoInput raoInput, NetworkActi Crac crac = raoInput.getCrac(); State preventiveState = crac.getPreventiveState(); + network.getVariantManager().cloneVariant(network.getVariantManager().getWorkingVariantId(), initialVariantId); network.getVariantManager().setWorkingVariant(initialVariantId); network.getVariantManager().cloneVariant(initialVariantId, newVariantId); network.getVariantManager().setWorkingVariant(newVariantId); @@ -160,7 +166,69 @@ public static void applyPreventiveRemedialActions(RaoInput raoInput, NetworkActi if (networkActionsToBeApplied.isEmpty()) { OpenRaoLoggerProvider.TECHNICAL_LOGS.info("[MARMOT] No preventive topological actions applied for timestamp {}", crac.getTimestamp().orElseThrow()); } else { + // TODO: close systematically networkActionsToBeApplied.forEach(networkAction -> networkAction.apply(network)); } } + + public static TemporalData cloneNetworks(TemporalData networks) { + TemporalData lazyNetworks = new TemporalDataImpl<>(); + networks.getDataPerTimestamp().forEach((timestamp, network) -> { + lazyNetworks.put(timestamp, new LazyNetwork(network)); + MarmotUtils.releaseNetwork(network); + }); + return lazyNetworks; + } + + public static TemporalData merge(TemporalData networks, TemporalData cracs) { + Map raoInputs = new HashMap<>(); + for (OffsetDateTime timestamp : networks.getTimestamps()) { + try (LazyNetwork lazyNetwork = networks.getData(timestamp).orElseThrow()) { + raoInputs.put(timestamp, RaoInput.build(lazyNetwork, cracs.getData(timestamp).orElseThrow()).build()); + } catch (Exception e) { + throw new OpenRaoException(e); + } + } + return new TemporalDataImpl<>(raoInputs); + } + + public static void releaseAll(TemporalData networks) { + networks.getDataPerTimestamp().values().forEach(MarmotUtils::releaseNetwork); + } + + public static void releaseNetwork(N network) { + if (network instanceof LazyNetwork lazyNetwork) { + try { + lazyNetwork.close(); + } catch (Exception e) { + throw new OpenRaoException(e); + } + } + } + + public static OffsetDateTime getTimestamp(RaoInput raoInput) { + return raoInput.getCrac().getTimestamp().orElseThrow(); + } + + public static RaoParameters cloneParameters(RaoParameters raoParameters) { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + JsonRaoParameters.write(raoParameters, outputStream); + try (InputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray())) { + return JsonRaoParameters.read(inputStream); + } + } catch (Exception e) { + throw new OpenRaoException(e); + } + } + + /** + * Select the best TemporalData mapping strategy based on the number of threads. + * Necessary not to create a pool for only one thread. + */ + public static TemporalData smartMap(TemporalData temporalData, Function function, int threads) { + if (threads == 1) { + return temporalData.map(function); + } + return temporalData.mapMultiThreading(function, threads); + } } diff --git a/tests/src/test/java/com/powsybl/openrao/tests/steps/TimeCoupledRaoSteps.java b/tests/src/test/java/com/powsybl/openrao/tests/steps/TimeCoupledRaoSteps.java index 29ebff3866..0a186f1979 100644 --- a/tests/src/test/java/com/powsybl/openrao/tests/steps/TimeCoupledRaoSteps.java +++ b/tests/src/test/java/com/powsybl/openrao/tests/steps/TimeCoupledRaoSteps.java @@ -10,6 +10,7 @@ import com.powsybl.iidm.network.Bus; import com.powsybl.iidm.network.Generator; import com.powsybl.iidm.network.Network; +import com.powsybl.openrao.commons.OpenRaoException; import com.powsybl.openrao.commons.PhysicalParameter; import com.powsybl.openrao.commons.TemporalData; import com.powsybl.openrao.commons.TemporalDataImpl; @@ -40,6 +41,7 @@ import com.powsybl.openrao.raoapi.RaoInput; import com.powsybl.openrao.raoapi.TimeCoupledRao; import com.powsybl.openrao.raoapi.TimeCoupledRaoInput; +import com.powsybl.openrao.searchtreerao.marmot.MarmotUtils; import com.powsybl.openrao.tests.utils.CoreCcPreprocessor; import io.cucumber.datatable.DataTable; import io.cucumber.java.After; @@ -197,9 +199,12 @@ public static void loadDataForTimeCoupledRao(String timeCoupledConstraintsPath, List> inputs = arg1.asMaps(String.class, String.class); for (Map tsInput : inputs) { OffsetDateTime offsetDateTime = getOffsetDateTimeFromBrusselsTimestamp(tsInput.get("Timestamp")); - Network network = importNetwork(getFile(networkFolderPath.concat(tsInput.get("Network"))), false); - Crac crac = importCrac(getFile(cracFolderPath.concat(tsInput.get("CRAC"))), network, null).getLeft(); - raoInputs.put(offsetDateTime, RaoInput.build(LazyNetwork.of(networkFolderPath.concat(tsInput.get("Network"))), crac).build()); + try (LazyNetwork lazyNetwork = new LazyNetwork(importNetwork(getFile(networkFolderPath.concat(tsInput.get("Network"))), false))) { + Crac crac = importCrac(getFile(cracFolderPath.concat(tsInput.get("CRAC"))), lazyNetwork, null).getLeft(); + raoInputs.put(offsetDateTime, RaoInput.build(lazyNetwork, crac).build()); + } catch (Exception e) { + throw new OpenRaoException(e); + } } TimeCoupledConstraints timeCoupledConstraints = JsonTimeCoupledConstraints.read(new FileInputStream(timeCoupledConstraintsFolderPath.concat(timeCoupledConstraintsPath))); @@ -209,6 +214,7 @@ public static void loadDataForTimeCoupledRao(String timeCoupledConstraintsPath, @Given("time-coupled rao inputs for CORE are:") public static void coreTimeCoupledRaoInputsAre(DataTable arg1) throws IOException { loadDataForCoreTimeCoupledRao(arg1); + System.gc(); } public static void loadDataForCoreTimeCoupledRao(DataTable arg1) throws IOException { @@ -233,23 +239,25 @@ public static void loadDataForCoreTimeCoupledRao(DataTable arg1) throws IOExcept for (Map tsInput : inputs) { OffsetDateTime offsetDateTime = getOffsetDateTimeFromBrusselsTimestamp(tsInput.get("Timestamp")); TECHNICAL_LOGS.info("**** Loading data for TS {} ****", offsetDateTime); - // Network - String postIcsNetworkPath = networkFolderPathPostIcsImport.concat(tsInput.get("Network")).split(".uct")[0].concat(".jiidm"); Network network = importNetwork(getFile(networkFolderPath.concat(tsInput.get("Network"))), false); CoreCcPreprocessor.applyCoreCcNetworkPreprocessing(network); - // Crac - Pair cracImportResult; - if (useIndividualCracs) { // only works with json - cracImportResult = importCrac(getFile(cracFolderPath.concat(tsInput.get("Crac"))), network, null); - } else { - cracCreationParameters.getExtension(FbConstraintCracCreationParameters.class).setTimestamp(offsetDateTime); - cracImportResult = importCrac(cracFile, network, cracCreationParameters); + try (LazyNetwork lazyNetwork = new LazyNetwork(network)) { + // Crac + Pair cracImportResult; + if (useIndividualCracs) { // only works with json + cracImportResult = importCrac(getFile(cracFolderPath.concat(tsInput.get("Crac"))), lazyNetwork, null); + } else { + cracCreationParameters.getExtension(FbConstraintCracCreationParameters.class).setTimestamp(offsetDateTime); + cracImportResult = importCrac(cracFile, lazyNetwork, cracCreationParameters); + } + RaoInput raoInput = RaoInput + .build(lazyNetwork, cracImportResult.getLeft()) + .build(); + raoInputs.put(offsetDateTime, raoInput); + cracCreationContexts.put(offsetDateTime, cracImportResult.getRight()); + } catch (Exception e) { + throw new OpenRaoException(e); } - RaoInput raoInput = RaoInput - .build(network, cracImportResult.getLeft()) - .build(); - raoInputs.put(offsetDateTime, raoInput); - cracCreationContexts.put(offsetDateTime, cracImportResult.getRight()); } InputStream gskInputStream = icsGskPath == null ? null : new FileInputStream(getFile(icsGskPath)); @@ -270,6 +278,8 @@ public static void loadDataForCoreTimeCoupledRao(DataTable arg1) throws IOExcept fbConstraintParameters.getIcsCostUp(), fbConstraintParameters.getIcsCostDown(), networkFolderPathPostIcsImport.concat(inputs.getFirst().get("Network")).split(".uct")[0]); + MarmotUtils.releaseAll(raoInputs.map(RaoInput::getNetwork)); + MarmotUtils.releaseAll(timeCoupledRaoInput.getRaoInputs().map(RaoInput::getNetwork)); } @When("I launch marmot")