Skip to content

Create DataTableGroovyScriptExtension #201

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 26, 2025
Merged

Create DataTableGroovyScriptExtension #201

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
208455d
create DataTableGroovyScriptExtension and use it
rolnico Feb 5, 2025
f574599
add dataTableStore and scriptLogBufferedWriter in contextObjects map …
rolnico Feb 28, 2025
40f8a95
Little revert on toList()
marifunf Mar 17, 2025
126e68b
update powsybl-core
rolnico Mar 25, 2025
13ca55d
Merge branch 'main' into nro/datatables_everywhere
rolnico Mar 25, 2025
9e9c12e
fix issue
rolnico Mar 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 36 additions & 5 deletions metrix-integration/src/main/groovy/com/powsybl/metrix/integration/MetrixDslDataLoader.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@ package com.powsybl.metrix.integration


import com.powsybl.iidm.network.Network
import com.powsybl.metrix.mapping.DataTableStore
import com.powsybl.metrix.mapping.LogDslLoader
import com.powsybl.metrix.mapping.TimeSeriesMappingConfig
import com.powsybl.metrix.mapping.TimeSeriesMappingConfigLoader
import com.powsybl.scripting.groovy.GroovyScriptExtension
import com.powsybl.scripting.groovy.GroovyScripts
import com.powsybl.timeseries.ReadOnlyTimeSeriesStore
import com.powsybl.timeseries.TimeSeriesFilter
import com.powsybl.timeseries.dsl.CalculatedTimeSeriesGroovyDslLoader
Expand All @@ -31,10 +34,10 @@ import static GeneratorsBindingData.generatorsBindingData
import static HvdcData.hvdcData
import static LoadData.loadData
import static LoadsBindingData.loadsBindingData
import static LossesData.lossesData
import static ParametersData.parametersData
import static PhaseShifterData.phaseShifterData
import static SectionMonitoringData.sectionMonitoringData
import static LossesData.lossesData

/**
* @author Paul Bui-Quang {@literal <paul.buiquang at rte-france.com>}
Expand Down Expand Up @@ -121,8 +124,18 @@ class MetrixDslDataLoader {
shell.evaluate(dslSrc)
}

static void bind(Binding binding, Network network, ReadOnlyTimeSeriesStore store, MetrixParameters parameters, TimeSeriesMappingConfig mappingConfig, MetrixDslData data, LogDslLoader logDslLoader) {
static void bind(Binding binding, Network network, ReadOnlyTimeSeriesStore store, DataTableStore dataTableStore, MetrixParameters parameters, TimeSeriesMappingConfig mappingConfig, MetrixDslData data, LogDslLoader logDslLoader) {
// External bindings
CalculatedTimeSeriesGroovyDslLoader.bind(binding, store, mappingConfig.getTimeSeriesNodes())

// Context objects
Map<Class<?>, Object> contextObjects = new HashMap<>()
contextObjects.put(DataTableStore.class, dataTableStore)

// Bindings through extensions
Iterable<GroovyScriptExtension> extensions = ServiceLoader.load(GroovyScriptExtension.class, GroovyScripts.class.getClassLoader())
extensions.forEach { it.load(binding, contextObjects) }

TimeSeriesMappingConfigLoader loader = new TimeSeriesMappingConfigLoader(mappingConfig, store.getTimeSeriesNames(new TimeSeriesFilter()))

// map the base case to network variable
Expand Down Expand Up @@ -186,15 +199,25 @@ class MetrixDslDataLoader {

static MetrixDslData load(Reader reader, Network network, MetrixParameters parameters, ReadOnlyTimeSeriesStore store,
TimeSeriesMappingConfig mappingConfig, Writer out) {
load(reader, network, parameters, store, new DataTableStore(), mappingConfig, out)
}

static MetrixDslData load(Reader reader, Network network, MetrixParameters parameters, ReadOnlyTimeSeriesStore store,
DataTableStore dataTableStore, TimeSeriesMappingConfig mappingConfig, Writer out) {
MetrixDslDataLoader dslLoader = new MetrixDslDataLoader(reader, "metrixDsl.groovy")
dslLoader.load(network, parameters, store, mappingConfig, out)
dslLoader.load(network, parameters, store, dataTableStore, mappingConfig, out)
}

static MetrixDslData load(Path metrixDslFile, Network network, MetrixParameters parameters, ReadOnlyTimeSeriesStore store,
TimeSeriesMappingConfig mappingConfig) {
load(metrixDslFile, network, parameters, store, mappingConfig, null)
}

static MetrixDslData load(Path metrixDslFile, Network network, MetrixParameters parameters, ReadOnlyTimeSeriesStore store,
DataTableStore dataTableStore, TimeSeriesMappingConfig mappingConfig) {
load(metrixDslFile, network, parameters, store, dataTableStore, mappingConfig, null)
}

static MetrixDslData load(Path metrixDslFile, Network network, MetrixParameters parameters, ReadOnlyTimeSeriesStore store,
TimeSeriesMappingConfig mappingConfig, Writer out) {

Expand All @@ -203,13 +226,21 @@ class MetrixDslDataLoader {
}
}

static MetrixDslData load(Path metrixDslFile, Network network, MetrixParameters parameters, ReadOnlyTimeSeriesStore store,
DataTableStore dataTableStore, TimeSeriesMappingConfig mappingConfig, Writer out) {

Files.newBufferedReader(metrixDslFile, StandardCharsets.UTF_8).withReader { Reader reader ->
load(reader, network, parameters, store, dataTableStore, mappingConfig, out)
}
}

MetrixDslData load(Network network, MetrixParameters parameters, ReadOnlyTimeSeriesStore store,
TimeSeriesMappingConfig mappingConfig, Writer out) {
DataTableStore dataTableStore, TimeSeriesMappingConfig mappingConfig, Writer out) {

MetrixDslData data = new MetrixDslData()
Binding binding = new Binding()
LogDslLoader logDslLoader = LogDslLoader.create(binding, out, METRIX_SCRIPT_SECTION)
bind(binding, network, store, parameters, mappingConfig, data, logDslLoader)
bind(binding, network, store, dataTableStore, parameters, mappingConfig, data, logDslLoader)

evaluate(dslSrc, binding)

Expand Down
4 changes: 2 additions & 2 deletions metrix-integration/src/main/java/com/powsybl/metrix/integration/metrix/MetrixAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ public MetrixAnalysisResult runAnalysis(String id) {
metrixDslData = loadMetrixDslData(metrixDslReader, network, metrixParameters, mappingConfig, scriptLogBufferedWriter, id);
timeSeriesNodesAfterMetrix = new HashMap<>(mappingConfig.getTimeSeriesNodes());
}
MetrixInputAnalysisResult inputs = new MetrixInputAnalysis(remedialActionsReader, contingenciesProvider, network, metrixDslData, inputLogBufferedWriter).runAnalysis();
MetrixInputAnalysisResult inputs = new MetrixInputAnalysis(remedialActionsReader, contingenciesProvider, network, metrixDslData, dataTableStore, inputLogBufferedWriter, scriptLogBufferedWriter).runAnalysis();
MetrixConfigResult metrixConfigResult = new MetrixConfigResult(timeSeriesNodesAfterMapping, timeSeriesNodesAfterMetrix);
return new MetrixAnalysisResult(metrixDslData, mappingConfig, network, metrixParameters, mappingParameters, metrixConfigResult, inputs.getContingencies(), inputs.getRemedials());
} catch (IOException e) {
Expand Down Expand Up @@ -159,7 +159,7 @@ private MetrixDslData loadMetrixDslData(Reader metrixDslReader, Network network,
boolean inError = false;
try {
CompletableFuture<MetrixDslData> metrixFuture = new LocalThreadExecutor<MetrixDslData>("Script_M_" + id)
.supplyAsync(() -> MetrixDslDataLoader.load(metrixDslReader, network, metrixParameters, store, mappingConfig, writer));
.supplyAsync(() -> MetrixDslDataLoader.load(metrixDslReader, network, metrixParameters, store, dataTableStore, mappingConfig, writer));
updateTask.accept(metrixFuture);
MetrixDslData metrixDslData = metrixFuture.get();
metrixDslData.setComputationType(metrixParameters.getComputationType());
Expand Down
51 changes: 45 additions & 6 deletions ...-integration/src/main/java/com/powsybl/metrix/integration/metrix/MetrixInputAnalysis.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,32 @@
import com.powsybl.contingency.Contingency;
import com.powsybl.contingency.ContingencyElement;
import com.powsybl.contingency.ContingencyElementType;
import com.powsybl.iidm.network.*;
import com.powsybl.iidm.network.Branch;
import com.powsybl.iidm.network.Identifiable;
import com.powsybl.iidm.network.IdentifiableType;
import com.powsybl.iidm.network.Network;
import com.powsybl.iidm.network.Switch;
import com.powsybl.metrix.integration.MetrixDslData;
import com.powsybl.metrix.integration.exceptions.ContingenciesScriptLoadingException;
import com.powsybl.metrix.integration.remedials.Remedial;
import com.powsybl.metrix.integration.remedials.RemedialReader;

import java.io.*;
import java.util.*;
import com.powsybl.metrix.mapping.DataTableStore;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.UncheckedIOException;
import java.io.Writer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.ResourceBundle;
import java.util.Set;
import java.util.stream.Collectors;

import static com.powsybl.metrix.integration.remedials.RemedialReader.rTrim;
Expand Down Expand Up @@ -52,16 +70,26 @@ enum LogType {
private final ContingenciesProvider contingenciesProvider;
private final Network network;
private final MetrixDslData metrixDslData;
private final DataTableStore dataTableStore;
private final BufferedWriter writer;
private final BufferedWriter scriptLogBufferedWriter;

public MetrixInputAnalysis(Reader remedialActionsReader, ContingenciesProvider contingenciesProvider, Network network,
MetrixDslData metrixDslData, DataTableStore dataTableStore, BufferedWriter writer) {
this(remedialActionsReader, contingenciesProvider, network, metrixDslData, dataTableStore, writer, null);
}

public MetrixInputAnalysis(Reader remedialActionsReader, ContingenciesProvider contingenciesProvider, Network network, MetrixDslData metrixDslData, BufferedWriter writer) {
public MetrixInputAnalysis(Reader remedialActionsReader, ContingenciesProvider contingenciesProvider, Network network,
MetrixDslData metrixDslData, DataTableStore dataTableStore, BufferedWriter writer, BufferedWriter scriptLogBufferedWriter) {
Objects.requireNonNull(contingenciesProvider);
Objects.requireNonNull(network);
this.remedialActionsReader = remedialActionsReader;
this.contingenciesProvider = contingenciesProvider;
this.network = network;
this.metrixDslData = metrixDslData;
this.dataTableStore = dataTableStore;
this.writer = writer;
this.scriptLogBufferedWriter = scriptLogBufferedWriter;
}

public MetrixInputAnalysisResult runAnalysis() throws IOException {
Expand Down Expand Up @@ -130,7 +158,7 @@ private void writeRemedialFileLog(int line, String messageReason) {
private List<Contingency> loadContingencies() {
List<Contingency> allContingencies;
try {
allContingencies = contingenciesProvider.getContingencies(network);
allContingencies = contingenciesProvider.getContingencies(network, getContextObjects());
} catch (RuntimeException e) {
throw new ContingenciesScriptLoadingException(e);
}
Expand All @@ -143,6 +171,17 @@ private List<Contingency> loadContingencies() {
return contingencies;
}

private Map<Class<?>, Object> getContextObjects() {
Map<Class<?>, Object> contextObjects = new HashMap<>();
if (dataTableStore != null) {
contextObjects.put(DataTableStore.class, dataTableStore);
}
if (scriptLogBufferedWriter != null) {
contextObjects.put(Writer.class, scriptLogBufferedWriter);
}
return contextObjects;
}

private static boolean isValidContingencyType(ContingencyElementType elementType) {
return elementType == ContingencyElementType.LINE ||
elementType == ContingencyElementType.TWO_WINDINGS_TRANSFORMER ||
Expand Down
2 changes: 1 addition & 1 deletion ...rc/test/java/com/powsybl/metrix/integration/MetrixBranchPostProcessingTimeSeriesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,7 @@ void postProcessingTimeSeriesTest() {
TimeSeriesDslLoader timeSeriesDslLoader = new TimeSeriesDslLoader(mappingScript);
TimeSeriesMappingConfig mappingConfig = timeSeriesDslLoader.load(network, mappingParameters, store, new DataTableStore(), null);
MetrixDslDataLoader metrixDslDataLoader = new MetrixDslDataLoader(metrixConfigurationScript);
MetrixDslData dslData = metrixDslDataLoader.load(network, parameters, store, mappingConfig, null);
MetrixDslData dslData = metrixDslDataLoader.load(network, parameters, store, new DataTableStore(), mappingConfig, null);

MetrixBranchPostProcessingTimeSeries branchProcessing = new MetrixBranchPostProcessingTimeSeries(dslData, mappingConfig, metrixResultTimeSeries.getTimeSeriesNames(null), null);
postProcessingTimeSeries = branchProcessing.createPostProcessingTimeSeries();
Expand Down
17 changes: 11 additions & 6 deletions ...tegration/src/test/java/com/powsybl/metrix/integration/MetrixContingencyAnalysisTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,12 @@
package com.powsybl.metrix.integration;

import com.google.common.collect.ImmutableList;
import com.powsybl.contingency.*;
import com.powsybl.contingency.BranchContingency;
import com.powsybl.contingency.BusbarSectionContingency;
import com.powsybl.contingency.ContingenciesProvider;
import com.powsybl.contingency.Contingency;
import com.powsybl.contingency.ContingencyElement;
import com.powsybl.contingency.EmptyContingencyListProvider;
import com.powsybl.iidm.network.Network;
import com.powsybl.iidm.serde.NetworkSerDe;
import com.powsybl.metrix.integration.metrix.MetrixInputAnalysis;
Expand All @@ -28,14 +33,14 @@
/**
* @author Marianne Funfrock {@literal <marianne.funfrock at rte-france.com>}
*/
public class MetrixContingencyAnalysisTest {
class MetrixContingencyAnalysisTest {

private static final ResourceBundle RESOURCE_BUNDLE = ResourceBundle.getBundle("lang.MetrixAnalysis");

private Network network;

@BeforeEach
public void setUp() {
void setUp() {
network = NetworkSerDe.read(getClass().getResourceAsStream("/simpleNetwork.xml"));
}

Expand Down Expand Up @@ -63,7 +68,7 @@ private String getWarningInvalidMetrixDslDataContingency(String section, String
private void metrixDslDataContingencyAnalysisTest(MetrixDslData metrixDslData, String expected) throws IOException {
StringWriter writer = new StringWriter();
try (BufferedWriter bufferedWriter = new BufferedWriter(writer)) {
MetrixInputAnalysis metrixInputAnalysis = new MetrixInputAnalysis(new StringReader(""), new EmptyContingencyListProvider(), network, metrixDslData, bufferedWriter);
MetrixInputAnalysis metrixInputAnalysis = new MetrixInputAnalysis(new StringReader(""), new EmptyContingencyListProvider(), network, metrixDslData, null, bufferedWriter);
metrixInputAnalysis.runAnalysis();
bufferedWriter.flush();
String actual = writer.toString();
Expand All @@ -78,7 +83,7 @@ private void loadContingencyTest(Contingency contingency, String expected) throw

StringWriter writer = new StringWriter();
try (BufferedWriter bufferedWriter = new BufferedWriter(writer)) {
MetrixInputAnalysis metrixInputAnalysis = new MetrixInputAnalysis(new StringReader(""), provider, network, new MetrixDslData(), bufferedWriter);
MetrixInputAnalysis metrixInputAnalysis = new MetrixInputAnalysis(new StringReader(""), provider, network, new MetrixDslData(), null, bufferedWriter);
metrixInputAnalysis.runAnalysis();
bufferedWriter.flush();
String actual = writer.toString();
Expand Down Expand Up @@ -166,7 +171,7 @@ void metrixRemedialContingencyAnalysisTest() throws IOException {
String.join(System.lineSeparator(),
"NB;1;",
"ctyId;1;FP.AND1 FVERGE1 1;")
), new EmptyContingencyListProvider(), network, new MetrixDslData(), bufferedWriter);
), new EmptyContingencyListProvider(), network, new MetrixDslData(), null, bufferedWriter);
metrixInputAnalysis.runAnalysis();
bufferedWriter.flush();
String actual = writer.toString();
Expand Down
8 changes: 4 additions & 4 deletions ...ion/src/test/java/com/powsybl/metrix/integration/MetrixDslDataLoaderDoctrineCostTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ void testLoadShedding() throws IOException {

ReadOnlyTimeSeriesStore store = new ReadOnlyTimeSeriesStoreCache();
TimeSeriesMappingConfig mappingConfig = new TimeSeriesDslLoader(mappingFile).load(network, mappingParameters, store, new DataTableStore(), null);
MetrixDslDataLoader.load(dslFile, network, parameters, store, mappingConfig);
MetrixDslDataLoader.load(dslFile, network, parameters, store, new DataTableStore(), mappingConfig);

assertEquals("12000", mappingConfig.getEquipmentToTimeSeries().get(new MappingKey(MetrixVariable.PREVENTIVE_DOCTRINE_COST_DOWN, "FSSV.O11_L")));
assertEquals("ts1", mappingConfig.getEquipmentToTimeSeries().get(new MappingKey(MetrixVariable.PREVENTIVE_DOCTRINE_COST_DOWN, "FVALDI11_L2")));
Expand Down Expand Up @@ -132,7 +132,7 @@ void testGeneratorRedispatching() throws IOException {

ReadOnlyTimeSeriesStore store = new ReadOnlyTimeSeriesStoreCache();
TimeSeriesMappingConfig mappingConfig = new TimeSeriesDslLoader(mappingFile).load(network, mappingParameters, store, new DataTableStore(), null);
MetrixDslDataLoader.load(dslFile, network, parameters, store, mappingConfig);
MetrixDslDataLoader.load(dslFile, network, parameters, store, new DataTableStore(), mappingConfig);

assertEquals("50", mappingConfig.getEquipmentToTimeSeries().get(new MappingKey(MetrixVariable.ON_GRID_DOCTRINE_COST_UP, "FSSV.O11_G")));
assertEquals("200", mappingConfig.getEquipmentToTimeSeries().get(new MappingKey(MetrixVariable.ON_GRID_DOCTRINE_COST_DOWN, "FSSV.O11_G")));
Expand All @@ -157,7 +157,7 @@ void testLosses() throws IOException {

ReadOnlyTimeSeriesStore store = new ReadOnlyTimeSeriesStoreCache();
TimeSeriesMappingConfig mappingConfig = new TimeSeriesDslLoader(mappingFile).load(network, mappingParameters, store, new DataTableStore(), null);
MetrixDslDataLoader.load(dslFile, network, parameters, store, mappingConfig);
MetrixDslDataLoader.load(dslFile, network, parameters, store, new DataTableStore(), mappingConfig);

assertEquals("ts1", mappingConfig.getEquipmentToTimeSeries().get(new MappingKey(MetrixVariable.LOSSES_DOCTRINE_COST, "SimpleNetworkWithReducedThresholds")));
assertEquals(Set.of(new MappingKey(MetrixVariable.LOSSES_DOCTRINE_COST, "SimpleNetworkWithReducedThresholds")), mappingConfig.getTimeSeriesToEquipment().get("ts1"));
Expand Down Expand Up @@ -196,7 +196,7 @@ void testDoctrineCostIsMissing() throws IOException {
ReadOnlyTimeSeriesStore store = new ReadOnlyTimeSeriesStoreCache();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
try (Writer out = new BufferedWriter(new OutputStreamWriter(outputStream))) {
MetrixDslDataLoader.load(dslFile, network, parameters, store, new TimeSeriesMappingConfig(), out);
MetrixDslDataLoader.load(dslFile, network, parameters, store, new DataTableStore(), new TimeSeriesMappingConfig(), out);
}

String output = outputStream.toString();
Expand Down
4 changes: 2 additions & 2 deletions ...ion/src/test/java/com/powsybl/metrix/integration/MetrixDslDataLoaderInterruptionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ void testCancelMetrixDslDataLoaderShort(boolean isDelayed) throws Exception {
}
ReadOnlyTimeSeriesStore store = new ReadOnlyTimeSeriesStoreCache();
TimeSeriesMappingConfig tsConfig = new TimeSeriesDslLoader(mappingFile).load(network, mappingParameters, store, new DataTableStore(), null);
testCancelShortTask(isDelayed, () -> MetrixDslDataLoader.load(dslFile, network, parameters, store, tsConfig));
testCancelShortTask(isDelayed, () -> MetrixDslDataLoader.load(dslFile, network, parameters, store, new DataTableStore(), tsConfig));
}

@ParameterizedTest
Expand Down Expand Up @@ -120,6 +120,6 @@ void testCancelMetrixDslDataLoaderLong(boolean isDelayed) throws IOException, In
}
ReadOnlyTimeSeriesStore store = new ReadOnlyTimeSeriesStoreCache();
TimeSeriesMappingConfig tsConfig = new TimeSeriesDslLoader(mappingFile).load(network, mappingParameters, store, new DataTableStore(), null);
testCancelLongTask(isDelayed, () -> MetrixDslDataLoader.load(dslFile, network, parameters, store, tsConfig));
testCancelLongTask(isDelayed, () -> MetrixDslDataLoader.load(dslFile, network, parameters, store, new DataTableStore(), tsConfig));
}
}
Loading
Loading