Skip to content

Commit 1edd738

Browse files
committed
read-input-tables and replace-output-tables APIs
1 parent 3749e4c commit 1edd738

File tree

2 files changed

+80
-20
lines changed

2 files changed

+80
-20
lines changed

CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1111
### Added
1212
- ScenarioDbManager - Edit cells in tables
1313
- ScenarioDbManager - Duplicate, Rename and Delete scenario
14+
- ScenarioDbManager.read_scenario_input_tables_from_db main API to read input for solve
15+
- ScenarioDbManager.update_scenario_output_tables_in_db main API to store solve output
1416

1517
## [0.5.3.1] - 2021-12-30
1618
### Changed

dse_do_utils/scenariodbmanager.py

Lines changed: 78 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,15 @@ def insert_table_in_db_bulk(self, df: pd.DataFrame, mgr, connection=None):
147147
print(f"DataFrame insert/append of table '{table_name}'")
148148
print(e)
149149

150+
def _delete_scenario_table_from_db(self, scenario_name, connection):
151+
"""Delete all rows associated with the scenario in the DB table.
152+
Beware: make sure this is done in the right 'inverse cascading' order to avoid FK violations.
153+
"""
154+
# sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
155+
t = self.get_sa_table() # A Table()
156+
sql = t.delete().where(t.c.scenario_name == scenario_name)
157+
connection.execute(sql)
158+
150159
@staticmethod
151160
def sqlcol(df: pd.DataFrame) -> Dict:
152161
dtypedict = {}
@@ -842,9 +851,14 @@ def _read_scenario_db_table_from_db_thread(self, scenario_table_name, db_table):
842851

843852
return inputs, outputs
844853

854+
def read_scenario_input_tables_from_db(self, scenario_name: str):
855+
"""Convenience method to load all input tables.
856+
Typically used at start if optimization model."""
857+
return self.read_scenario_tables_from_db(scenario_name, input_table_names=['*'])
858+
845859
def read_scenario_tables_from_db(self, scenario_name: str,
846-
input_table_names: List[str] = None,
847-
output_table_names: List[str] = None) -> (Inputs, Outputs):
860+
input_table_names: Optional[List[str]] = None,
861+
output_table_names: Optional[List[str]] = None) -> (Inputs, Outputs):
848862
"""Read selected set input and output tables from scenario.
849863
If input_table_names/output_table_names contains a '*', then all input/output tables will be read.
850864
If empty list or None, then no tables will be read.
@@ -948,6 +962,61 @@ def _update_cell_change_in_db(self, db_cell_update: DbCellUpdate, connection):
948962

949963
connection.execute(sql)
950964

965+
############################################################################################
966+
# Update/Replace tables in scenario
967+
############################################################################################
968+
def update_scenario_output_tables_in_db(self, scenario_name, outputs: Outputs):
969+
"""Main API to update output from a DO solve in the scenario.
970+
Deletes ALL output tables. Then inserts the given set of tables.
971+
Since this only touches the output tables, more efficient than replacing the whole scenario."""
972+
if self.enable_transactions:
973+
with self.engine.begin() as connection:
974+
self._update_scenario_output_tables_in_db(scenario_name, outputs, connection)
975+
else:
976+
self._update_scenario_output_tables_in_db(scenario_name, outputs, self.engine)
977+
978+
def _update_scenario_output_tables_in_db(self, scenario_name, outputs: Outputs, connection):
979+
"""Deletes ALL output tables. Then inserts the given set of tables.
980+
Note that if a defined output table is not included in the outputs, it will still be deleted from the scenario data."""
981+
# 1. Add scenario name to dfs:
982+
outputs = ScenarioDbManager.add_scenario_name_to_dfs(scenario_name, outputs)
983+
# 2. Delete all output tables
984+
for scenario_table_name, db_table in reversed(self.output_db_tables.items()): # Note this INCLUDES the SCENARIO table!
985+
if (scenario_table_name != 'Scenario'):
986+
db_table._delete_scenario_table_from_db()
987+
# 3. Insert new data
988+
for scenario_table_name, db_table in self.output_db_tables.items(): # Note this INCLUDES the SCENARIO table!
989+
if (scenario_table_name != 'Scenario') and db_table.db_table_name in outputs.keys(): # If in given set of tables to replace
990+
df = outputs[scenario_table_name]
991+
db_table.insert_table_in_db_bulk(df=df, mgr=self, connection=connection) # The scenario_name is a column in the df
992+
993+
def replace_scenario_tables_in_db(self, scenario_name, inputs={}, outputs={}):
994+
"""Untested"""
995+
if self.enable_transactions:
996+
with self.engine.begin() as connection:
997+
self._replace_scenario_tables_in_db(connection, scenario_name, inputs, outputs)
998+
else:
999+
self._replace_scenario_tables_in_db(self.engine, scenario_name, inputs, outputs)
1000+
1001+
def _replace_scenario_tables_in_db(self, connection, scenario_name, inputs={}, outputs={}):
1002+
"""Untested
1003+
Replace only the tables listed in the inputs and outputs. But leave all other tables untouched.
1004+
Will first delete all given tables (in reverse cascading order), then insert the new ones (in cascading order)"""
1005+
1006+
# Add scenario name to dfs:
1007+
inputs = ScenarioDbManager.add_scenario_name_to_dfs(scenario_name, inputs)
1008+
outputs = ScenarioDbManager.add_scenario_name_to_dfs(scenario_name, outputs)
1009+
dfs = {**inputs, **outputs}
1010+
# 1. Delete tables
1011+
for scenario_table_name, db_table in reversed(self.db_tables.items()): # Note this INCLUDES the SCENARIO table!
1012+
if (scenario_table_name != 'Scenario') and db_table.db_table_name in dfs.keys(): # If in given set of tables to replace
1013+
db_table._delete_scenario_table_from_db()
1014+
# 2. Insert new data
1015+
for scenario_table_name, db_table in self.db_tables.items(): # Note this INCLUDES the SCENARIO table!
1016+
if (scenario_table_name != 'Scenario') and db_table.db_table_name in dfs.keys(): # If in given set of tables to replace
1017+
df = dfs[scenario_table_name]
1018+
db_table.insert_table_in_db_bulk(df=df, mgr=self, connection=connection) # The scenario_name is a column in the df
1019+
9511020
############################################################################################
9521021
# CRUD operations on scenarios in DB:
9531022
# - Delete scenario
@@ -1117,28 +1186,17 @@ def _delete_scenario_from_db(self, scenario_name: str, connection):
11171186
TODO: check with 'auto-inserted' tables
11181187
TODO: batch all sql statements in single execute. Faster? And will that do the defer integrity checks?
11191188
"""
1120-
batch_sql=False # Batch=True does NOT work!
1189+
# batch_sql=False # Batch=True does NOT work!
11211190
insp = sqlalchemy.inspect(connection)
11221191
tables_in_db = insp.get_table_names(schema=self.schema)
1123-
sql_statements = []
1192+
# sql_statements = []
11241193
for scenario_table_name, db_table in reversed(self.db_tables.items()): # Note this INCLUDES the SCENARIO table!
11251194
if db_table.db_table_name in tables_in_db:
1126-
# sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
1127-
t = db_table.table_metadata # A Table()
1128-
sql = t.delete().where(t.c.scenario_name == scenario_name)
1129-
if batch_sql:
1130-
sql_statements.append(sql)
1131-
else:
1132-
connection.execute(sql)
1133-
1134-
# Because the scenario table has already been included in above loop, no need to do separately
1135-
# Delete scenario entry in scenario table:
1136-
# sql = f"DELETE FROM SCENARIO WHERE scenario_name = '{scenario_name}'"
1137-
# sql_statements.append(sql)
1138-
if batch_sql:
1139-
batch_sql = ";\n".join(sql_statements)
1140-
# print(batch_sql)
1141-
connection.execute(batch_sql)
1195+
# # sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
1196+
# t = db_table.table_metadata # A Table()
1197+
# sql = t.delete().where(t.c.scenario_name == scenario_name)
1198+
# connection.execute(sql)
1199+
db_table._delete_scenario_table_from_db(scenario_name, connection)
11421200

11431201
############################################################################################
11441202
# Old Read scenario APIs

0 commit comments

Comments
 (0)