feat(ics data): add additional checks for ics importer (#1680)

* add additional checks with TUs

---------

Signed-off-by: CHEN Roxane <roxane.chen@rte-france.com>

Author: RoxaneChen02

data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsData.java

@@ -113,6 +113,11 @@ public static Set<GeneratorConstraints> createGeneratorConstraints(String raId,
             Double shiftKey = entry.getValue();
             CSVRecord staticRecord = staticConstraintPerId.get(raId);
             GeneratorConstraints.GeneratorConstraintsBuilder builder = GeneratorConstraints.create().withGeneratorId(networkElementIdPerNodeId.get(nodeId));
+
+            // Shutdown allowed and startup allowed are mandatory fields
+            builder.withShutDownAllowed(Boolean.parseBoolean(staticRecord.get(SHUTDOWN_ALLOWED)));
+            builder.withStartUpAllowed(Boolean.parseBoolean(staticRecord.get(STARTUP_ALLOWED)));
+
             if (!staticRecord.get(MAXIMUM_POSITIVE_POWER_GRADIENT).isEmpty()) {
                 builder.withUpwardPowerGradient(shiftKey * parseDoubleWithPossibleCommas(staticRecord.get(MAXIMUM_POSITIVE_POWER_GRADIENT)));
             } else {
@@ -129,20 +134,7 @@ public static Set<GeneratorConstraints> createGeneratorConstraints(String raId,
             if (!staticRecord.get(LAG_TIME).isEmpty()) {
                 builder.withLagTime(parseDoubleWithPossibleCommas(staticRecord.get(LAG_TIME)));
             }
-            // TODO: instead of throwing an error, just ignore the RA + move the check in the import
-            if (staticRecord.get(SHUTDOWN_ALLOWED).isEmpty() ||
-                !staticRecord.get(SHUTDOWN_ALLOWED).equalsIgnoreCase(TRUE) && !staticRecord.get(SHUTDOWN_ALLOWED).equalsIgnoreCase(FALSE)) {
-                throw new OpenRaoException("Could not parse shutDownAllowed value for raId " + raId + ": " + staticRecord.get(SHUTDOWN_ALLOWED));
-            } else {
-                builder.withShutDownAllowed(Boolean.parseBoolean(staticRecord.get(SHUTDOWN_ALLOWED)));
-            }
-            // TODO: instead of throwing an error, just ignore the RA + move the check in the import
-            if (staticRecord.get(STARTUP_ALLOWED).isEmpty() ||
-                !staticRecord.get(STARTUP_ALLOWED).equalsIgnoreCase(TRUE) && !staticRecord.get(STARTUP_ALLOWED).equalsIgnoreCase(FALSE)) {
-                throw new OpenRaoException("Could not parse startUpAllowed value for raId " + raId + ": " + staticRecord.get(STARTUP_ALLOWED));
-            } else {
-                builder.withStartUpAllowed(Boolean.parseBoolean(staticRecord.get(STARTUP_ALLOWED)));
-            }
+
             GeneratorConstraints generatorConstraints = builder.build();
             generatorConstraintsSet.add(generatorConstraints);
         }

data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsDataImporter.java

@@ -128,46 +128,87 @@ static Map<String, Map<String, Double>> parseGskCsv(InputStream gskInputStream)
         return weightPerNodePerGsk;
     }
 
+    private static boolean isValidBooleanValue(String value) {
+        return value.equalsIgnoreCase(TRUE) || value.equalsIgnoreCase(FALSE);
+    }
+
     // Consistency check functions
     private static boolean shouldBeImported(CSVRecord staticRecord, List<OffsetDateTime> sortedTimestampToRun, Map<String, Map<String, Double>> weightPerNodePerGsk, Map<String, Map<String, CSVRecord>> timeseriesPerIdAndType) {
         String raId = staticRecord.get(RA_RD_ID);
 
-        // TODO: checks that all the mandatory fields are defined for all timestamps
+        // Check static record mandatory fields : Preventive, curative, Generator Name, RD Description mode, UCT Node or GSK ID, Startup allowed and Shutdown allowed
+        if (staticRecord.get(PREVENTIVE).isEmpty() || staticRecord.get(CURATIVE).isEmpty() ||
+            staticRecord.get(GENERATOR_NAME).isEmpty() || staticRecord.get(RD_DESCRIPTION_MODE).isEmpty() ||
+            staticRecord.get(UCT_NODE_OR_GSK_ID).isEmpty() || staticRecord.get(STARTUP_ALLOWED).isEmpty() ||
+            staticRecord.get(SHUTDOWN_ALLOWED).isEmpty()) {
+            BUSINESS_WARNS.warn("Redispatching action {} is not imported: missing mandatory static data", raId);
+            return false;
+        }
+
+        // Check that boolean fields are either "TRUE" or "FALSE"
+        List<String> booleanFields = List.of(PREVENTIVE, CURATIVE, STARTUP_ALLOWED, SHUTDOWN_ALLOWED);
+        for (String field : booleanFields) {
+            String value = staticRecord.get(field);
+            if (!isValidBooleanValue(value)) {
+                BUSINESS_WARNS.warn("Redispatching action {} is not imported: invalid '{}' value '{}' (expected TRUE or FALSE)", raId, field, value);
+                return false;
+            }
+        }
 
         // Check that remedial action is defined in series csv and gsk (if defined on a gsk)
         if (!timeseriesPerIdAndType.containsKey(raId)) {
-            BUSINESS_WARNS.warn("Redispatching action {} is not defined in the time series csv", raId);
+            BUSINESS_WARNS.warn("Redispatching action {} is not imported: not defined in the time series csv", raId);
+            return false;
+        }
+
+        // Check that mandatory timeseries type (P0, RDP_DOWN, RDP_UP) are defined in the time series csv
+        Map<String, CSVRecord> seriesPerType = timeseriesPerIdAndType.get(raId);
+        boolean isDefinedInSeriesCsv = seriesPerType.containsKey(P0) &&
+            seriesPerType.containsKey(RDP_DOWN) &&
+            seriesPerType.containsKey(RDP_UP);
+
+        if (!isDefinedInSeriesCsv) {
+            BUSINESS_WARNS.warn("Redispatching action {} is not imported: missing one or several mandatory timeseries type (P0, RDP_DOWN, RDP_UP).", raId);
             return false;
         }
+
+        // Check that data exists for all timestamps to run
+        List<String> mandatorySeriesTypes = List.of(P0, RDP_DOWN, RDP_UP);
+        for (OffsetDateTime timestamp : sortedTimestampToRun) {
+            int columnName = timestamp.getHour() + OFFSET;
+            for (String seriesType : mandatorySeriesTypes) {
+                String value = seriesPerType.get(seriesType).get(columnName);
+                if (value == null || value.isEmpty()) {
+                    BUSINESS_WARNS.warn("Redispatching action {} is not imported: missing {} data for timestamp {}", raId, seriesType, timestamp);
+                    return false;
+                }
+            }
+        }
+
         // If remedial action is defined on a gsk
         if (staticRecord.get(RD_DESCRIPTION_MODE).equalsIgnoreCase(GSK)) {
             // Check that the gsk is defined in the gsk csv
             if (!weightPerNodePerGsk.containsKey(staticRecord.get(UCT_NODE_OR_GSK_ID))) {
-                BUSINESS_WARNS.warn("Redispatching action {} is defined on a gsk {} but the gsk is not defined in the gsk csv", raId, staticRecord.get(UCT_NODE_OR_GSK_ID));
+                BUSINESS_WARNS.warn("Redispatching action {} is not imported: defined on a gsk {} but the gsk is not defined in the gsk csv", raId, staticRecord.get(UCT_NODE_OR_GSK_ID));
+                return false;
+            }
+
+            // Check that the sum of weight if RA is defined on GSK equals to 1
+            if (!sumOfGskEqualsOne(staticRecord.get(UCT_NODE_OR_GSK_ID), weightPerNodePerGsk)) {
+                BUSINESS_WARNS.warn("Redispatching action {} is not imported: defined on a GSK but sum of weights is not equal to 1", raId);
                 return false;
             }
         }
 
         // Check that remedial action should at least be defined on preventive instant
         if (!staticRecord.get(PREVENTIVE).equalsIgnoreCase(TRUE)) {
-            BUSINESS_WARNS.warn("Redispatching action {} is not defined on preventive instant", raId);
+            BUSINESS_WARNS.warn("Redispatching action {} is not imported: not defined on preventive instant", raId);
             return false;
         }
 
         // Check that the remedial action is defined on a node or a gsk
         if (!staticRecord.get(RD_DESCRIPTION_MODE).equalsIgnoreCase(NODE) && !staticRecord.get(RD_DESCRIPTION_MODE).equalsIgnoreCase(GSK)) {
-            BUSINESS_WARNS.warn("Redispatching action {} is not defined on a node or a gsk but on a {}", raId, staticRecord.get(RD_DESCRIPTION_MODE));
-            return false;
-        }
-
-        // Check that the RA is correctly defined in series csv
-        Map<String, CSVRecord> seriesPerType = timeseriesPerIdAndType.get(raId);
-        boolean isDefinedInSeriesCsv = seriesPerType.containsKey(P0) &&
-            seriesPerType.containsKey(RDP_DOWN) &&
-            seriesPerType.containsKey(RDP_UP);
-
-        if (!isDefinedInSeriesCsv) {
-            BUSINESS_WARNS.warn("Redispatching action {} is not defined in the time series csv. Missing one or several timeseries type (P0, RDP_DOWN, RDP_UP or P_MIN_RD).", raId);
+            BUSINESS_WARNS.warn("Redispatching action {} is not imported: not defined on a node or a gsk but on a {}", raId, staticRecord.get(RD_DESCRIPTION_MODE));
             return false;
         }
 
@@ -184,6 +225,22 @@ private static boolean shouldBeImported(CSVRecord staticRecord, List<OffsetDateT
         return true;
     }
 
+    /**
+     * Checks if the sum of the generation shift key (GSK) weights associated with a specific GSK ID equals 1
+     *
+     * @param gskId
+     * @param weightPerNodePerGsk
+     * @return {@code true} if the sum of the GSK weights equals 1 within a small tolerance;
+     *         {@code false} otherwise.
+     */
+    private static boolean sumOfGskEqualsOne(String gskId, Map<String, Map<String, Double>> weightPerNodePerGsk) {
+        double sumOfGsk = 0.;
+        for (Map.Entry<String, Double> entry : weightPerNodePerGsk.get(gskId).entrySet()) {
+            sumOfGsk += entry.getValue();
+        }
+        return Math.abs(sumOfGsk - 1.) < 1e-6;
+    }
+
     /**
      * Determines whether the P0 record values respect the specified power gradients for each time interval.
      * It checks the difference in values between consecutive timestamps and ensures that the differences
@@ -209,7 +266,7 @@ private static boolean p0RespectsGradients(CSVRecord staticRecord, CSVRecord p0r
             double diff = parseDoubleWithPossibleCommas(p0record.get(nextDateTime.getHour() + OFFSET)) - parseDoubleWithPossibleCommas(p0record.get(currentDateTime.getHour() + OFFSET));
             if (diff > maxGradient || diff < minGradient) {
                 BUSINESS_WARNS.warn(
-                    "Redispatching action {} will not be imported because it does not respect power gradients : min/max/diff = {} / {} / {}",
+                    "Redispatching action {} is not imported: does not respect power gradients : min/max/diff = {} / {} / {}",
                     staticRecord.get(0), minGradient, maxGradient, diff
                 );
                 return false;
@@ -236,12 +293,12 @@ private static boolean rangeIsOkay(Map<String, CSVRecord> seriesPerType, List<Of
             double rdpMinus = parseDoubleWithPossibleCommas(seriesPerType.get(RDP_DOWN).get(dateTime.getHour() + OFFSET));
             maxRange = Math.max(maxRange, rdpPlus + rdpMinus);
             if (rdpPlus < -1e-6 || rdpMinus < -1e-6) {
-                BUSINESS_WARNS.warn("Redispatching action {} will not be imported because of RDP+ {} or RDP- {} is negative for datetime {}", seriesPerType.get(P0).get(RA_RD_ID), rdpPlus, rdpMinus, dateTime);
+                BUSINESS_WARNS.warn("Redispatching action {} is not imported: RDP+ {} or RDP- {} is negative for datetime {}", seriesPerType.get(P0).get(RA_RD_ID), rdpPlus, rdpMinus, dateTime);
                 return false;
             }
         }
         if (maxRange < 1) {
-            BUSINESS_WARNS.warn("Redispatching action {} will not be imported because max range in the day {} MW is too small", seriesPerType.get(P0).get(RA_RD_ID), maxRange);
+            BUSINESS_WARNS.warn("Redispatching action {} is not imported: max range in the day {} MW is too small", seriesPerType.get(P0).get(RA_RD_ID), maxRange);
             return false;
         }
         return true;

data/ics-importer/src/main/java/com/powsybl/openrao/data/icsimporter/IcsUtil.java

@@ -16,6 +16,8 @@
 import java.util.Map;
 import java.util.Optional;
 
+import static com.powsybl.openrao.commons.logs.OpenRaoLoggerProvider.BUSINESS_WARNS;
+
 /**
  * @author Roxane Chen {@literal <roxane.chen at rte-france.com>}
  */
@@ -61,8 +63,11 @@ static Optional<Double> parseValue(Map<String, CSVRecord> seriesPerType, String
         if (seriesPerType.containsKey(key)) {
             CSVRecord series = seriesPerType.get(key);
             String value = series.get(timestamp.getHour() + OFFSET);
-            if (value != null) {
+            if (value != null && !value.isEmpty()) {
                 return Optional.of(parseDoubleWithPossibleCommas(value) * shiftKey);
+            } else {
+                BUSINESS_WARNS.warn("Redispatching action {} is missing {} value for datetime {}", series.get(RA_RD_ID), key, timestamp);
+                return Optional.empty();
             }
         }
         return Optional.empty();