Marmot melting pot: Marmot multithreading, p0RespectsConstraints in ICSImporter, LazyNetwork closing in Marmot

commons/src/main/java/com/powsybl/openrao/commons/TemporalData.java

@@ -30,4 +30,6 @@ default List<OffsetDateTime> getTimestamps() {
     void put(OffsetDateTime timestamp, T data);
 
     <U> TemporalData<U> map(Function<T, U> function);
+
+    <U> TemporalData<U> mapMultiThreading(Function<T, U> function, int parallelism);
 }

commons/src/main/java/com/powsybl/openrao/commons/TemporalDataImpl.java

@@ -8,8 +8,15 @@
 package com.powsybl.openrao.commons;
 
 import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
-import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -20,15 +27,15 @@ public class TemporalDataImpl<T> implements TemporalData<T> {
     private final Map<OffsetDateTime, T> dataPerTimestamp;
 
     public TemporalDataImpl() {
-        this(new TreeMap<>());
+        this(new ConcurrentHashMap<>());
     }
 
     public TemporalDataImpl(Map<OffsetDateTime, ? extends T> dataPerTimestamp) {
-        this.dataPerTimestamp = new TreeMap<>(dataPerTimestamp);
+        this.dataPerTimestamp = new ConcurrentHashMap<>(dataPerTimestamp);
     }
 
     public Map<OffsetDateTime, T> getDataPerTimestamp() {
-        return new TreeMap<>(dataPerTimestamp);
+        return new ConcurrentHashMap<>(dataPerTimestamp);
     }
 
     public void put(OffsetDateTime timestamp, T data) {
@@ -38,4 +45,33 @@ public void put(OffsetDateTime timestamp, T data) {
     public <U> TemporalData<U> map(Function<T, U> function) {
         return new TemporalDataImpl<>(dataPerTimestamp.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, entry -> function.apply(entry.getValue()))));
     }
+
+    public <U> TemporalData<U> mapMultiThreading(Function<T, U> function, int parallelism) {
+        try (ExecutorService executor = Executors.newFixedThreadPool(parallelism)) {
+            try {
+                List<Future<Map.Entry<OffsetDateTime, U>>> futures = new ArrayList<>();
+
+                for (Map.Entry<OffsetDateTime, T> entry : dataPerTimestamp.entrySet()) {
+                    futures.add(executor.submit(() ->
+                        Map.entry(entry.getKey(), function.apply(entry.getValue()))
+                    ));
+                }
+
+                Map<OffsetDateTime, U> result = new HashMap<>();
+
+                for (Future<Map.Entry<OffsetDateTime, U>> future : futures) {
+                    Map.Entry<OffsetDateTime, U> e = future.get();
+                    result.put(e.getKey(), e.getValue());
+                }
+
+                return new TemporalDataImpl<>(result);
+
+            } catch (InterruptedException | ExecutionException e) {
+                Thread.currentThread().interrupt();
+                throw new OpenRaoException(e);
+            } finally {
+                executor.shutdown();
+            }
+        }
+    }
 }

data/crac/crac-util/pom.xml

@@ -41,6 +41,11 @@
         </dependency>
 
         <!-- Test scope -->
+        <dependency>
+            <groupId>ch.qos.logback</groupId>
+            <artifactId>logback-classic</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>

data/crac/crac-util/src/test/resources/logback-test.xml

@@ -0,0 +1,15 @@
+<configuration>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="ERROR">
+        <appender-ref ref="STDOUT" />
+    </root>
+    <logger name="com.powsybl.openloadflow" level="ERROR"/>
+    <logger name="com.powsybl.math" level="ERROR"/>
+    <logger name="com.powsybl.openrao" level="DEBUG"/>
+    <logger name="com.powsybl.ucte.network.UcteLogger" level="OFF"/>
+</configuration>

data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsData.java

@@ -150,7 +150,7 @@
      * @return A map associating each node identifier to its corresponding generator identifier.
      *         Returns an empty map if the process is aborted due to missing network components.
      */
-    public static Map<String, String> createGeneratorAndLoadAndUpdateNetworks(TemporalData<Network> initialNetworksToModify,
+    public static Map<String, String> createGeneratorAndLoadAndUpdateNetworks(TemporalData<LazyNetwork> initialNetworksToModify,
                                                                               String raId) {
 
         Map<String, String> networkElementPerGskElement = new HashMap<>();
@@ -163,7 +163,7 @@
             Double shiftKey = entry.getValue();
             String generatorId = getGeneratorIdFromRaIdAndNodeId(raId, nodeId);
 
-            for (Map.Entry<OffsetDateTime, Network> networkEntry : initialNetworksToModify.getDataPerTimestamp().entrySet()) {
+            for (Map.Entry<OffsetDateTime, LazyNetwork> networkEntry : initialNetworksToModify.getDataPerTimestamp().entrySet()) {
                 OffsetDateTime dateTime = networkEntry.getKey();
                 Network network = networkEntry.getValue();
 
@@ -256,11 +256,18 @@
 
         // Update nominal voltage in network
         // TODO: More of a IDCC focused special processing ? Move elsewhere ?
-        TemporalData<Network> modifiedInitialNetworks = new TemporalDataImpl<>();
+        TemporalData<LazyNetwork> modifiedInitialNetworks = new TemporalDataImpl<>();
         timeCoupledRaoInput.getRaoInputs().getDataPerTimestamp().forEach((dateTime, raoInput) -> {
             Network network = raoInput.getNetwork();
             updateNominalVoltage(network);
-            modifiedInitialNetworks.put(dateTime, network);
+            modifiedInitialNetworks.put(dateTime, new LazyNetwork(network));
+            if (network instanceof LazyNetwork lazyNetwork) {
+                try {
+                    lazyNetwork.close();
+                } catch (Exception e) {
+                    throw new OpenRaoException(e);
+                }
+            }
         });
 
         TemporalData<Crac> cracToModify = new TemporalDataImpl<>();
@@ -291,7 +298,16 @@
         modifiedInitialNetworks.getDataPerTimestamp().forEach((dateTime, initialNetwork) -> {
             String exportedNetworkPath = exportDirectory + dateTime.format(DateTimeFormatter.ofPattern("%y%m%d_%H%M%S")) + ".jiidm";
             initialNetwork.write("JIIDM", new Properties(), Path.of(exportedNetworkPath));
-            postIcsRaoInputs.put(dateTime, RaoInput.build(new LazyNetwork(exportedNetworkPath), timeCoupledRaoInput.getRaoInputs().getData(dateTime).orElseThrow().getCrac()).build());
+            try (LazyNetwork postIcsNetwork = new LazyNetwork(exportedNetworkPath)) {
+                postIcsRaoInputs.put(dateTime, RaoInput.build(postIcsNetwork, timeCoupledRaoInput.getRaoInputs().getData(dateTime).orElseThrow().getCrac()).build());
+            } catch (Exception e) {
+                throw new OpenRaoException(e);
+            }
+            try {
+                initialNetwork.close();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
         });
 
         return new TimeCoupledRaoInput(postIcsRaoInputs, timeCoupledRaoInput.getTimestampsToRun(), timeCoupledRaoInput.getTimeCoupledConstraints());