diff --git a/CHANGELOG.md b/CHANGELOG.md
index cddcca0..1c7e835 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [Unreleased]## [0.5.4.1b]
+
+## [0.5.4.0] - 2022-01-11
+### Changed
+- ScenarioDbManager - Converted text SQL operations to SQLAlchemy operations to support any column-name (i.e. lower, upper, mixed, reserved words)
+- Updated ScenarioDbManager.read_scenario_tables_from_db to selectively read tables from a scenario
+### Added
+- ScenarioDbManager - Edit cells in tables
+- ScenarioDbManager - Duplicate, Rename and Delete scenario
+- ScenarioDbManager.read_scenario_input_tables_from_db main API to read input for solve
+- ScenarioDbManager.update_scenario_output_tables_in_db main API to store solve output
+
## [0.5.3.1] - 2021-12-30
### Changed
- (critical) ScenarioDbManager - Replaced OrderedDict with Dict as type. Was causing a syntax error.
diff --git a/docs/doc_build/doctrees/dse_do_utils.doctree b/docs/doc_build/doctrees/dse_do_utils.doctree
index 5d4dc5e..f87f1dd 100644
Binary files a/docs/doc_build/doctrees/dse_do_utils.doctree and b/docs/doc_build/doctrees/dse_do_utils.doctree differ
diff --git a/docs/doc_build/doctrees/environment.pickle b/docs/doc_build/doctrees/environment.pickle
index 04c41f5..fbcfc9b 100644
Binary files a/docs/doc_build/doctrees/environment.pickle and b/docs/doc_build/doctrees/environment.pickle differ
diff --git a/docs/doc_build/html/.buildinfo b/docs/doc_build/html/.buildinfo
index 7ff3c5f..e47c0e5 100644
--- a/docs/doc_build/html/.buildinfo
+++ b/docs/doc_build/html/.buildinfo
@@ -1,4 +1,4 @@
# Sphinx build info version 1
# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: effd2c6945fa34906f8d2c06e4196c89
+config: bd2b9a92071f1c40e294c4bb1fb76baa
tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/docs/doc_build/html/_modules/dse_do_utils.html b/docs/doc_build/html/_modules/dse_do_utils.html
index e8eb574..ccdd09f 100644
--- a/docs/doc_build/html/_modules/dse_do_utils.html
+++ b/docs/doc_build/html/_modules/dse_do_utils.html
@@ -6,7 +6,7 @@
- dse_do_utils — DSE DO Utils 0.5.2.0 documentation
+ dse_do_utils — DSE DO Utils 0.5.4.0 documentation
@@ -31,7 +31,7 @@
# - Make 'multi_scenario' the default option
# -----------------------------------------------------------------------------------fromabcimportABC
+frommultiprocessing.poolimportThreadPoolimportsqlalchemyimportpandasaspd
-fromtypingimportDict,List
+fromtypingimportDict,List,NamedTuple,Any,OptionalfromcollectionsimportOrderedDictimportrefromsqlalchemyimportexc
@@ -108,6 +109,7 @@
Source code for dse_do_utils.scenariodbmanager
reserved_table_names =['order','parameter']# TODO: add more reserved words for table namesifdb_table_nameinreserved_table_names:print(f"Warning: the db_table_name '{db_table_name}' is a reserved word. Do not use as table name.")
+ self._sa_column_by_name=None# Dict[str, sqlalchemy.Column] Will be generated dynamically first time it is needed.
[docs]defget_sa_table(self)->sqlalchemy.Table:
+ """Returns the SQLAlchemy Table"""
+ returnself.table_metadata
+
+
[docs]defget_sa_column(self,db_column_name)->Optional[sqlalchemy.Column]:
+ """Returns the SQLAlchemy column with the specified name.
+ Dynamically creates a dict/hashtable for more efficient access."""
+ # for c in self.columns_metadata:
+ # if isinstance(c, sqlalchemy.Column) and c.name == db_column_name:
+ # return c
+ ifself._sa_column_by_nameisNone:
+ self._sa_column_by_name={c.name:cforcinself.columns_metadataifisinstance(c,sqlalchemy.Column)}
+ returnself._sa_column_by_name.get(db_column_name)# returns None if npt found (?)
+
+
[docs]defcreate_table_metadata(self,metadata,multi_scenario:bool=False)->sqlalchemy.Table:"""If multi_scenario, then add a primary key 'scenario_name'."""columns_metadata=self.columns_metadataconstraints_metadata=self.constraints_metadata
@@ -177,6 +193,15 @@
Source code for dse_do_utils.scenariodbmanager
print(f"DataFrame insert/append of table '{table_name}'")print(e)
+ def_delete_scenario_table_from_db(self,scenario_name,connection):
+ """Delete all rows associated with the scenario in the DB table.
+ Beware: make sure this is done in the right 'inverse cascading' order to avoid FK violations.
+ """
+ # sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
+ t=self.get_sa_table()# A Table()
+ sql=t.delete().where(t.c.scenario_name==scenario_name)
+ connection.execute(sql)
+
[docs]classDbCellUpdate(NamedTuple):
+ scenario_name:str
+ table_name:str
+ row_index:List[Dict[str,Any]]# e.g. [{'column': 'col1', 'value': 1}, {'column': 'col2', 'value': 'pear'}]
+ column_name:str
+ current_value:Any
+ previous_value:Any# Not used for DB operation
+ row_idx:int# Not used for DB operation
print("Warning: the `Scenario` table should be the first in the input tables")returninput_db_tables
+
[docs]defget_scenario_db_table(self)->ScenarioDbTable:
+ """Scenario table must be the first in self.input_db_tables"""
+ db_table:ScenarioTable=list(self.input_db_tables.values())[0]
+ returndb_table
+
def_create_database_engine(self,credentials=None,schema:str=None,echo:bool=False):"""Creates a SQLAlchemy engine at initialization. If no credentials, creates an in-memory SQLite DB. Which can be used for schema validation of the data.
@@ -458,9 +498,9 @@
Source code for dse_do_utils.scenariodbmanager
with self.engine.begin()asconnection:self._create_schema_transaction(connection=connection)else:
- self._create_schema_transaction()
+ self._create_schema_transaction(self.engine)
- def_create_schema_transaction(self,connection=None):
+ def_create_schema_transaction(self,connection):"""(Re)creates a schema, optionally using a transaction Drops all tables and re-creates the schema in the DB."""# if self.schema is None:
@@ -469,10 +509,7 @@
Source code for dse_do_utils.scenariodbmanager
# self.drop_schema_transaction(self.schema)
# DROP SCHEMA isn't working properly, so back to dropping all tablesself._drop_all_tables_transaction(connection=connection)
- ifconnectionisNone:
- self.metadata.create_all(self.engine,checkfirst=True)
- else:
- self.metadata.create_all(connection,checkfirst=True)
+ self.metadata.create_all(connection,checkfirst=True)
[docs]defdrop_all_tables(self):"""Drops all tables in the current schema."""
@@ -480,9 +517,9 @@
Source code for dse_do_utils.scenariodbmanager
with self.engine.begin()asconnection:self._drop_all_tables_transaction(connection=connection)else:
- self._drop_all_tables_transaction()
+ self._drop_all_tables_transaction(self.engine)
- def_drop_all_tables_transaction(self,connection=None):
+ def_drop_all_tables_transaction(self,connection):"""Drops all tables as defined in db_tables (if exists) TODO: loop over tables as they exist in the DB. This will make sure that however the schema definition has changed, all tables will be cleared.
@@ -493,15 +530,18 @@
Source code for dse_do_utils.scenariodbmanager
However, the order is alphabetically, which causes FK constraint violation
Weirdly, this happens in SQLite, not in DB2! With or without transactions
+
+ TODO:
+ 1. Use SQLAlchemy to drop table, avoid text SQL
+ 2. Drop all tables without having to loop and know all tables
+ See: https://stackoverflow.com/questions/35918605/how-to-delete-a-table-in-sqlalchemy)
+ See https://docs.sqlalchemy.org/en/14/core/metadata.html#sqlalchemy.schema.MetaData.drop_all """forscenario_table_name,db_tableinreversed(self.db_tables.items()):db_table_name=db_table.db_table_namesql=f"DROP TABLE IF EXISTS {db_table_name}"# print(f"Dropping table {db_table_name}")
- ifconnectionisNone:
- r=self.engine.execute(sql)
- else:
- r=connection.execute(sql)
+ connection.execute(sql)def_drop_schema_transaction(self,schema:str,connection=None):"""NOT USED. Not working in DB2 Cloud.
@@ -509,6 +549,7 @@
Source code for dse_do_utils.scenariodbmanager
See: https://www.ibm.com/docs/en/db2/11.5?topic=procedure-admin-drop-schema-drop-schema
However, this doesn't work on DB2 cloud. TODO: find out if and how we can get this to work.
+ See https://docs.sqlalchemy.org/en/14/core/metadata.html#sqlalchemy.schema.MetaData.drop_all """# sql = f"DROP SCHEMA {schema} CASCADE" # Not allowed in DB2!sql=f"CALL SYSPROC.ADMIN_DROP_SCHEMA('{schema}', NULL, 'ERRORSCHEMA', 'ERRORTABLE')"
@@ -561,18 +602,20 @@
Source code for dse_do_utils.scenariodbmanager
if self.enable_transactions:print("Replacing scenario within transaction")withself.engine.begin()asconnection:
- self._replace_scenario_in_db_transaction(scenario_name=scenario_name,inputs=inputs,outputs=outputs,bulk=bulk,connection=connection)
+ self._replace_scenario_in_db_transaction(connection,scenario_name=scenario_name,inputs=inputs,outputs=outputs,bulk=bulk)else:
- self._replace_scenario_in_db_transaction(scenario_name=scenario_name,inputs=inputs,outputs=outputs,bulk=bulk)
- def_replace_scenario_in_db_transaction(self,scenario_name:str,inputs:Inputs={},outputs:Outputs={},
- bulk:bool=True,connection=None):
+ def_replace_scenario_in_db_transaction(self,connection,scenario_name:str,inputs:Inputs={},outputs:Outputs={},
+ bulk:bool=True):"""Replace a single full scenario in the DB. If doesn't exist, will insert. Only inserts tables with an entry defined in self.db_tables (i.e. no `auto_insert`). Will first delete all rows associated with a scenario_name. Will set/overwrite the scenario_name in all dfs, so no need to add in advance. Assumes schema has been created. Note: there is no difference between dfs in inputs or outputs, i.e. they are inserted the same way.
+
+ TODO: break-out in a delete and an insert. Then we can re-use the insert for the duplicate API """# Step 1: delete scenario if existsself._delete_scenario_from_db(scenario_name,connection=connection)
@@ -580,11 +623,10 @@
Source code for dse_do_utils.scenariodbmanager
inputs =ScenarioDbManager.add_scenario_name_to_dfs(scenario_name,inputs)outputs=ScenarioDbManager.add_scenario_name_to_dfs(scenario_name,outputs)# Step 3: insert scenario_name in scenario table
- sql=f"INSERT INTO SCENARIO (scenario_name) VALUES ('{scenario_name}')"
- ifconnectionisNone:
- self.engine.execute(sql)
- else:
- connection.execute(sql)
+ # sql = f"INSERT INTO SCENARIO (scenario_name) VALUES ('{scenario_name}')"
+ sa_scenario_table=self.get_scenario_db_table().get_sa_table()
+ sql_insert=sa_scenario_table.insert().values(scenario_name=scenario_name)
+ connection.execute(sql_insert)# Step 4: (bulk) insert scenarionum_caught_exceptions=self._insert_single_scenario_tables_in_db(inputs=inputs,outputs=outputs,bulk=bulk,connection=connection)# Throw exception if any exceptions caught in 'non-bulk' mode
@@ -592,34 +634,38 @@
Source code for dse_do_utils.scenariodbmanager
if num_caught_exceptions>0:raiseRuntimeError(f"Multiple ({num_caught_exceptions}) Integrity and/or Statement errors caught. See log. Raising exception to allow for rollback.")
- def_delete_scenario_from_db(self,scenario_name:str,connection=None):
- """Deletes all rows associated with a given scenario.
- Note that it only deletes rows from tables defined in the self.db_tables, i.e. will NOT delete rows in 'auto-inserted' tables!
- Must do a 'cascading' delete to ensure not violating FK constraints. In reverse order of how they are inserted.
- Also deletes entry in scenario table
- TODO: do within one session/cursor, so we don't have to worry about the order of the delete?
- """
- insp=sqlalchemy.inspect(self.engine)
- forscenario_table_name,db_tableinreversed(self.db_tables.items()):
- ifinsp.has_table(db_table.db_table_name,schema=self.schema):
- sql=f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'"
- ifconnectionisNone:
- self.engine.execute(sql)
- else:
- connection.execute(sql)
-
- # Delete scenario entry in scenario table:
- sql=f"DELETE FROM SCENARIO WHERE scenario_name = '{scenario_name}'"
- ifconnectionisNone:
- self.engine.execute(sql)
- else:
- connection.execute(sql)
+ # def _delete_scenario_from_db(self, scenario_name: str, connection=None):
+ # """Deletes all rows associated with a given scenario.
+ # Note that it only deletes rows from tables defined in the self.db_tables, i.e. will NOT delete rows in 'auto-inserted' tables!
+ # Must do a 'cascading' delete to ensure not violating FK constraints. In reverse order of how they are inserted.
+ # Also deletes entry in scenario table
+ # """
+ # insp = sqlalchemy.inspect(self.engine)
+ # for scenario_table_name, db_table in reversed(self.db_tables.items()):
+ # if insp.has_table(db_table.db_table_name, schema=self.schema):
+ #
+ # # sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'"
+ # t: sqlalchemy.Table = db_table.get_sa_table() # A Table()
+ # sql = t.delete().where(t.c.scenario_name == scenario_name)
+ # if connection is None:
+ # self.engine.execute(sql)
+ # else:
+ # connection.execute(sql)
+ #
+ # # Delete scenario entry in scenario table:
+ # # sql = f"DELETE FROM SCENARIO WHERE scenario_name = '{scenario_name}'"
+ # t: sqlalchemy.Table = self.get_scenario_db_table().get_sa_table() # A Table()
+ # sql = t.delete().where(t.c.scenario_name == scenario_name)
+ # if connection is None:
+ # self.engine.execute(sql)
+ # else:
+ # connection.execute(sql)def_insert_single_scenario_tables_in_db(self,inputs:Inputs={},outputs:Outputs={},bulk:bool=True,connection=None)->int:"""Specifically for single scenario replace/insert. Does NOT insert into the `scenario` table.
- No `auto_insert`, i.e. only df matching db_tables.
+ No `auto_insert`, i.e. only df matching db_tables. TODO: verify if doesn't work with AutoScenarioDbTable """num_caught_exceptions=0dfs={**inputs,**outputs}# Combine all dfs in one dict
@@ -712,13 +758,19 @@
[docs]defget_scenarios_df(self)->pd.DataFrame:"""Return all scenarios in df. Result is indexed by `scenario_name`. Main API to get all scenarios. The API called by a cached procedure in the dse_do_dashboard.DoDashApp. """
- sql=f"SELECT * FROM SCENARIO"
- df=pd.read_sql(sql,con=self.engine).set_index(['scenario_name'])
+ # sql = f"SELECT * FROM SCENARIO"
+ sa_scenario_table=list(self.input_db_tables.values())[0].table_metadata
+ sql=sa_scenario_table.select()
+ ifself.enable_transactions:
+ withself.engine.begin()asconnection:
+ df=pd.read_sql(sql,con=connection).set_index(['scenario_name'])
+ else:
+ df=pd.read_sql(sql,con=self.engine).set_index(['scenario_name'])returndf
# error!
raiseValueError(f"Scenario table name '{scenario_table_name}' unknown. Cannot load data from DB.")
- df=self._read_scenario_db_table_from_db(scenario_name,db_table)
+ ifself.enable_transactions:
+ withself.engine.begin()asconnection:
+ df=self._read_scenario_db_table_from_db(scenario_name,db_table,connection)
+ else:
+ df=self._read_scenario_db_table_from_db(scenario_name,db_table,self.engine)returndf
-
[docs]defread_scenario_from_db(self,scenario_name:str)->(Inputs,Outputs):
+ # def read_scenario_from_db(self, scenario_name: str) -> (Inputs, Outputs):
+ # """Single scenario load.
+ # Main API to read a complete scenario.
+ # Reads all tables for a single scenario.
+ # Returns all tables in one dict"""
+ # inputs = {}
+ # for scenario_table_name, db_table in self.input_db_tables.items():
+ # inputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table)
+ #
+ # outputs = {}
+ # for scenario_table_name, db_table in self.output_db_tables.items():
+ # outputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table)
+ #
+ # return inputs, outputs
+
+
+ # def _read_scenario_from_db(self, scenario_name: str, connection) -> (Inputs, Outputs):
+ # """Single scenario load.
+ # Main API to read a complete scenario.
+ # Reads all tables for a single scenario.
+ # Returns all tables in one dict
+ # """
+ # inputs = {}
+ # for scenario_table_name, db_table in self.input_db_tables.items():
+ # # print(f"scenario_table_name = {scenario_table_name}")
+ # if scenario_table_name != 'Scenario': # Skip the Scenario table as an input
+ # inputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table, connection=connection)
+ #
+ # outputs = {}
+ # for scenario_table_name, db_table in self.output_db_tables.items():
+ # outputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table, connection=connection)
+ # # if scenario_table_name == 'kpis':
+ # # # print(f"kpis table columns = {outputs[scenario_table_name].columns}")
+ # # outputs[scenario_table_name] = outputs[scenario_table_name].rename(columns={'name': 'NAME'}) #HACK!!!!!
+ # return inputs, outputs
+
[docs]defread_scenario_from_db(self,scenario_name:str,multi_threaded:bool=False)->(Inputs,Outputs):"""Single scenario load. Main API to read a complete scenario. Reads all tables for a single scenario.
- Returns all tables in one dict"""
+ Returns all tables in one dict
+
+ Note: multi_threaded doesn't seem to lead to performance improvement.
+ Fixed: omit reading scenario table as an input.
+ """
+ # print(f"read_scenario_from_db.multi_threaded = {multi_threaded}")
+ ifmulti_threaded:
+ inputs,outputs=self._read_scenario_from_db_multi_threaded(scenario_name)
+ else:
+ ifself.enable_transactions:
+ withself.engine.begin()asconnection:
+ inputs,outputs=self._read_scenario_from_db(scenario_name,connection)
+ else:
+ inputs,outputs=self._read_scenario_from_db(scenario_name,self.engine)
+ returninputs,outputs
+
+ def_read_scenario_from_db(self,scenario_name:str,connection)->(Inputs,Outputs):
+ """Single scenario load.
+ Main API to read a complete scenario.
+ Reads all tables for a single scenario.
+ Returns all tables in one dict
+ """inputs={}forscenario_table_name,db_tableinself.input_db_tables.items():
- inputs[scenario_table_name]=self._read_scenario_db_table_from_db(scenario_name,db_table)
+ # print(f"scenario_table_name = {scenario_table_name}")
+ ifscenario_table_name!='Scenario':# Skip the Scenario table as an input
+ inputs[scenario_table_name]=self._read_scenario_db_table_from_db(scenario_name,db_table,connection=connection)outputs={}forscenario_table_name,db_tableinself.output_db_tables.items():
- outputs[scenario_table_name]=self._read_scenario_db_table_from_db(scenario_name,db_table)
+ outputs[scenario_table_name]=self._read_scenario_db_table_from_db(scenario_name,db_table,connection=connection)
+
+ returninputs,outputs
+
+ def_read_scenario_from_db_multi_threaded(self,scenario_name)->(Inputs,Outputs):
+ """Reads all tables from a scenario using multi-threading.
+ Does NOT seem to result in performance improvement!"""
+ classReadTableFunction(object):
+ def__init__(self,dbm):
+ self.dbm=dbm
+ def__call__(self,scenario_table_name,db_table):
+ returnself._read_scenario_db_table_from_db_thread(scenario_table_name,db_table)
+ def_read_scenario_db_table_from_db_thread(self,scenario_table_name,db_table):
+ withself.dbm.engine.begin()asconnection:
+ df=self.dbm._read_scenario_db_table_from_db(scenario_name,db_table,connection)
+ dict={scenario_table_name:df}
+ returndict
+
+ thread_number=8
+ pool=ThreadPool(thread_number)
+ thread_worker=ReadTableFunction(self)
+ # print("ThreadPool created")
+ all_tables=[(scenario_table_name,db_table)forscenario_table_name,db_tableinself.db_tables.items()ifscenario_table_name!='Scenario']
+ # print(all_tables)
+ all_results=pool.starmap(thread_worker,all_tables)
+ inputs={k:vforelementinall_resultsfork,vinelement.items()ifkinself.input_db_tables.keys()}
+ outputs={k:vforelementinall_resultsfork,vinelement.items()ifkinself.output_db_tables.keys()}
+ # print("All tables loaded")
+
+ returninputs,outputs
+
+
[docs]defread_scenario_input_tables_from_db(self,scenario_name:str):
+ """Convenience method to load all input tables.
+ Typically used at start if optimization model."""
+ returnself.read_scenario_tables_from_db(scenario_name,input_table_names=['*'])
+
[docs]defread_scenario_tables_from_db(self,scenario_name:str,
+ input_table_names:Optional[List[str]]=None,
+ output_table_names:Optional[List[str]]=None)->(Inputs,Outputs):
+ """Read selected set input and output tables from scenario.
+ If input_table_names/output_table_names contains a '*', then all input/output tables will be read.
+ If empty list or None, then no tables will be read.
+ """
+ ifself.enable_transactions:
+ withself.engine.begin()asconnection:
+ inputs,outputs=self._read_scenario_tables_from_db(connection,scenario_name,input_table_names,output_table_names)
+ else:
+ inputs,outputs=self._read_scenario_tables_from_db(self.engine,scenario_name,input_table_names,output_table_names)returninputs,outputs
- def_read_scenario_db_table_from_db(self,scenario_name:str,db_table:ScenarioDbTable)->pd.DataFrame:
+ def_read_scenario_tables_from_db(self,connection,scenario_name:str,
+ input_table_names:List[str]=None,
+ output_table_names:List[str]=None)->(Inputs,Outputs):
+ """Loads data for selected input and output tables.
+ If either list is names is ['*'], will load all tables as defined in db_tables configuration.
+ """
+ ifinput_table_namesisNone:# load no tables by default
+ input_table_names=[]
+ elif'*'ininput_table_names:
+ input_table_names=list(self.input_db_tables.keys())
+ if'Scenario'ininput_table_names:input_table_names.remove('Scenario')# Remove the scenario table
+
+ ifoutput_table_namesisNone:# load no tables by default
+ output_table_names=[]
+ elif'*'inoutput_table_names:
+ output_table_names=self.output_db_tables.keys()
+
+ inputs={}
+ forscenario_table_name,db_tableinself.input_db_tables.items():
+ ifscenario_table_nameininput_table_names:
+ inputs[scenario_table_name]=self._read_scenario_table_from_db(scenario_name,db_table,connection=connection)
+ outputs={}
+ forscenario_table_name,db_tableinself.output_db_tables.items():
+ ifscenario_table_nameinoutput_table_names:
+ outputs[scenario_table_name]=self._read_scenario_db_table_from_db(scenario_name,db_table,connection=connection)
+ returninputs,outputs
+
+ # def _read_scenario_db_table_from_db(self, scenario_name: str, db_table: ScenarioDbTable) -> pd.DataFrame:
+ # """Read one table from the DB.
+ # Removes the `scenario_name` column."""
+ # db_table_name = db_table.db_table_name
+ # sql = f"SELECT * FROM {db_table_name} WHERE scenario_name = '{scenario_name}'"
+ # df = pd.read_sql(sql, con=self.engine)
+ # if db_table_name != 'scenario':
+ # df = df.drop(columns=['scenario_name'])
+ #
+ # return df
+ def_read_scenario_db_table_from_db(self,scenario_name:str,db_table:ScenarioDbTable,connection)->pd.DataFrame:"""Read one table from the DB.
- Removes the `scenario_name` column."""
+ Removes the `scenario_name` column.
+
+ Modification: based on SQLAlchemy syntax. If doing the plain text SQL, then some column names not properly extracted
+ """db_table_name=db_table.db_table_name
- sql=f"SELECT * FROM {db_table_name} WHERE scenario_name = '{scenario_name}'"
- df=pd.read_sql(sql,con=self.engine)
+ # sql = f"SELECT * FROM {db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
+ # db_table.table_metadata is a Table()
+ t:sqlalchemy.Table=db_table.get_sa_table()#table_metadata
+ sql=t.select().where(t.c.scenario_name==scenario_name)# This is NOT a simple string!
+ df=pd.read_sql(sql,con=connection)ifdb_table_name!='scenario':df=df.drop(columns=['scenario_name'])returndf
+ ############################################################################################
+ # Update scenario
+ ############################################################################################
+
[docs]defupdate_cell_changes_in_db(self,db_cell_updates:List[DbCellUpdate]):
+ """Update a set of cells in the DB.
+
+ :param db_cell_updates:
+ :return:
+ """
+ ifself.enable_transactions:
+ print("Update cells with transaction")
+ withself.engine.begin()asconnection:
+ self._update_cell_changes_in_db(db_cell_updates,connection=connection)
+ else:
+ self._update_cell_changes_in_db(db_cell_updates)
+
+ def_update_cell_changes_in_db(self,db_cell_updates:List[DbCellUpdate],connection=None):
+ """Update an ordered list of single value changes (cell) in the DB."""
+ fordb_cell_changeindb_cell_updates:
+ self._update_cell_change_in_db(db_cell_change,connection)
+
+ def_update_cell_change_in_db(self,db_cell_update:DbCellUpdate,connection):
+ """Update a single value (cell) change in the DB."""
+ # db_table_name = self.db_tables[db_cell_update.table_name].db_table_name
+ # column_change = f"{db_cell_update.column_name} = '{db_cell_update.current_value}'"
+ # scenario_condition = f"scenario_name = '{db_cell_update.scenario_name}'"
+ # pk_conditions = ' AND '.join([f"{pk['column']} = '{pk['value']}'" for pk in db_cell_update.row_index])
+ # old_sql = f"UPDATE {db_table_name} SET {column_change} WHERE {pk_conditions} AND {scenario_condition};"
+
+ db_table:ScenarioDbTable=self.db_tables[db_cell_update.table_name]
+ t:sqlalchemy.Table=db_table.get_sa_table()
+ pk_conditions=[(db_table.get_sa_column(pk['column'])==pk['value'])forpkindb_cell_update.row_index]
+ target_col:sqlalchemy.Column=db_table.get_sa_column(db_cell_update.column_name)
+ sql=t.update().where(sqlalchemy.and_((t.c.scenario_name==db_cell_update.scenario_name),*pk_conditions)).values({target_col:db_cell_update.current_value})
+ # print(f"_update_cell_change_in_db = {sql}")
+
+ connection.execute(sql)
+
+ ############################################################################################
+ # Update/Replace tables in scenario
+ ############################################################################################
+
[docs]defupdate_scenario_output_tables_in_db(self,scenario_name,outputs:Outputs):
+ """Main API to update output from a DO solve in the scenario.
+ Deletes ALL output tables. Then inserts the given set of tables.
+ Since this only touches the output tables, more efficient than replacing the whole scenario."""
+ ifself.enable_transactions:
+ withself.engine.begin()asconnection:
+ self._update_scenario_output_tables_in_db(scenario_name,outputs,connection)
+ else:
+ self._update_scenario_output_tables_in_db(scenario_name,outputs,self.engine)
+
+ def_update_scenario_output_tables_in_db(self,scenario_name,outputs:Outputs,connection):
+ """Deletes ALL output tables. Then inserts the given set of tables.
+ Note that if a defined output table is not included in the outputs, it will still be deleted from the scenario data."""
+ # 1. Add scenario name to dfs:
+ outputs=ScenarioDbManager.add_scenario_name_to_dfs(scenario_name,outputs)
+ # 2. Delete all output tables
+ forscenario_table_name,db_tableinreversed(self.output_db_tables.items()):# Note this INCLUDES the SCENARIO table!
+ if(scenario_table_name!='Scenario'):
+ db_table._delete_scenario_table_from_db(scenario_name,connection)
+ # 3. Insert new data
+ forscenario_table_name,db_tableinself.output_db_tables.items():# Note this INCLUDES the SCENARIO table!
+ if(scenario_table_name!='Scenario')anddb_table.db_table_nameinoutputs.keys():# If in given set of tables to replace
+ df=outputs[scenario_table_name]
+ db_table.insert_table_in_db_bulk(df=df,mgr=self,connection=connection)# The scenario_name is a column in the df
+
+
+
+ def_replace_scenario_tables_in_db(self,connection,scenario_name,inputs={},outputs={}):
+ """Untested
+ Replace only the tables listed in the inputs and outputs. But leave all other tables untouched.
+ Will first delete all given tables (in reverse cascading order), then insert the new ones (in cascading order)"""
+
+ # Add scenario name to dfs:
+ inputs=ScenarioDbManager.add_scenario_name_to_dfs(scenario_name,inputs)
+ outputs=ScenarioDbManager.add_scenario_name_to_dfs(scenario_name,outputs)
+ dfs={**inputs,**outputs}
+ # 1. Delete tables
+ forscenario_table_name,db_tableinreversed(self.db_tables.items()):# Note this INCLUDES the SCENARIO table!
+ if(scenario_table_name!='Scenario')anddb_table.db_table_nameindfs.keys():# If in given set of tables to replace
+ db_table._delete_scenario_table_from_db()
+ # 2. Insert new data
+ forscenario_table_name,db_tableinself.db_tables.items():# Note this INCLUDES the SCENARIO table!
+ if(scenario_table_name!='Scenario')anddb_table.db_table_nameindfs.keys():# If in given set of tables to replace
+ df=dfs[scenario_table_name]
+ db_table.insert_table_in_db_bulk(df=df,mgr=self,connection=connection)# The scenario_name is a column in the df
+
+ ############################################################################################
+ # CRUD operations on scenarios in DB:
+ # - Delete scenario
+ # - Duplicate scenario
+ # - Rename scenario
+ ############################################################################################
+
[docs]defdelete_scenario_from_db(self,scenario_name:str):
+ """Delete a scenario. Uses a transaction (when enabled)."""
+ ifself.enable_transactions:
+ print("Delete scenario within a transaction")
+ withself.engine.begin()asconnection:
+ self._delete_scenario_from_db(scenario_name=scenario_name,connection=connection)
+ else:
+ self._delete_scenario_from_db(scenario_name=scenario_name,connection=self.engine)
[docs]defduplicate_scenario_in_db(self,source_scenario_name:str,target_scenario_name:str):
+ """Duplicate a scenario. Uses a transaction (when enabled)."""
+ ifself.enable_transactions:
+ print("Duplicate scenario within a transaction")
+ withself.engine.begin()asconnection:
+ self._duplicate_scenario_in_db(connection,source_scenario_name,target_scenario_name)
+ else:
+ self._duplicate_scenario_in_db(self.engine,source_scenario_name,target_scenario_name)
+
+ def_duplicate_scenario_in_db(self,connection,source_scenario_name:str,target_scenario_name:str=None):
+ """Is fully done in DB using SQL in one SQL execute statement
+ :param source_scenario_name:
+ :param target_scenario_name:
+ :param connection:
+ :return:
+ """
+ iftarget_scenario_nameisNone:
+ new_scenario_name=self._find_free_duplicate_scenario_name(source_scenario_name)
+ elifself._check_free_scenario_name(target_scenario_name):
+ new_scenario_name=target_scenario_name
+ else:
+ raiseValueError(f"Target name for duplicate scenario '{target_scenario_name}' already exists.")
+
+ # inputs, outputs = self.read_scenario_from_db(source_scenario_name)
+ # self._replace_scenario_in_db_transaction(scenario_name=new_scenario_name, inputs=inputs, outputs=outputs,
+ # bulk=True, connection=connection)
+ self._duplicate_scenario_in_db_sql(connection,source_scenario_name,new_scenario_name)
+
+ def_duplicate_scenario_in_db_sql(self,connection,source_scenario_name:str,target_scenario_name:str=None):
+ """
+ :param source_scenario_name:
+ :param target_scenario_name:
+ :param connection:
+ :return:
+
+ See https://stackoverflow.com/questions/9879830/select-modify-and-insert-into-the-same-table
+
+ Problem: the table Parameter/parameters has a column 'value' (lower-case).
+ Almost all of the column names in the DFs are lower-case, as are the column names in the ScenarioDbTable.
+ Typically, the DB schema converts that the upper-case column names in the DB.
+ But probably because 'VALUE' is a reserved word, it does NOT do this for 'value'. But that means in order to refer to this column in SQL,
+ one needs to put "value" between double quotes.
+ Problem is that you CANNOT do that for other columns, since these are in upper-case in the DB.
+ Note that the kpis table uses upper case 'VALUE' and that seems to work fine
+
+ Resolution: use SQLAlchemy to construct the SQL. Do NOT create SQL expressions by text manipulation.
+ SQLAlchemy has the smarts to properly deal with these complex names.
+ """
+ iftarget_scenario_nameisNone:
+ new_scenario_name=self._find_free_duplicate_scenario_name(source_scenario_name)
+ elifself._check_free_scenario_name(target_scenario_name):
+ new_scenario_name=target_scenario_name
+ else:
+ raiseValueError(f"Target name for duplicate scenario '{target_scenario_name}' already exists.")
+
+ batch_sql=False# BEWARE: batch = True does NOT work!
+ sql_statements=[]
+
+ # 1. Insert scenario in scenario table
+ # sql_insert = f"INSERT INTO SCENARIO (scenario_name) VALUES ('{new_scenario_name}')" # Old SQL
+ # sa_scenario_table = list(self.input_db_tables.values())[0].get_sa_table() # Scenario table must be the first
+ sa_scenario_table=self.get_scenario_db_table().get_sa_table()
+ sql_insert=sa_scenario_table.insert().values(scenario_name=new_scenario_name)
+ # print(f"_duplicate_scenario_in_db_sql - Insert SQL = {sql_insert}")
+ ifbatch_sql:
+ sql_statements.append(sql_insert)
+ else:
+ connection.execute(sql_insert)
+
+ # 2. Do 'insert into select' to duplicate rows in each table
+ forscenario_table_name,db_tableinself.db_tables.items():
+ ifscenario_table_name=='Scenario':
+ continue
+
+ t:sqlalchemy.table=db_table.table_metadata# The table at hand
+ s:sqlalchemy.table=sa_scenario_table# The scenario table
+ # print("+++++++++++SQLAlchemy insert-select")
+ select_columns=[s.c.scenario_nameifc.name=='scenario_name'elsecforcint.columns]# Replace the t.c.scenario_name with s.c.scenario_name, so we get the new value
+ # print(f"select columns = {select_columns}")
+ select_sql=(sqlalchemy.select(select_columns)
+ .where(sqlalchemy.and_(t.c.scenario_name==source_scenario_name,s.c.scenario_name==target_scenario_name)))
+ target_columns=[cforcint.columns]
+ sql_insert=t.insert().from_select(target_columns,select_sql)
+ # print(f"sql_insert = {sql_insert}")
+
+ # sql_insert = f"INSERT INTO {db_table.db_table_name} ({target_columns_txt}) SELECT '{target_scenario_name}',{other_source_columns_txt} FROM {db_table.db_table_name} WHERE scenario_name = '{source_scenario_name}'"
+ ifbatch_sql:
+ sql_statements.append(sql_insert)
+ else:
+ connection.execute(sql_insert)
+ ifbatch_sql:
+ batch_sql=";\n".join(sql_statements)
+ print(batch_sql)
+ connection.execute(batch_sql)
+
+ def_find_free_duplicate_scenario_name(self,scenario_name:str,scenarios_df=None)->Optional[str]:
+ """Finds next free scenario name based on pattern '{scenario_name}_copy_n'.
+ Will try at maximum 20 attempts.
+ """
+ max_num_attempts=20
+ foriinrange(1,max_num_attempts+1):
+ new_name=f"{scenario_name}({i})"
+ free=self._check_free_scenario_name(new_name,scenarios_df)
+ iffree:
+ returnnew_name
+ raiseValueError(f"Cannot find free name for duplicate scenario. Tried {max_num_attempts}. Last attempt = {new_name}. Rename scenarios.")
+ returnNone
+
+ def_check_free_scenario_name(self,scenario_name,scenarios_df=None)->bool:
+ ifscenarios_dfisNone:
+ scenarios_df=self.get_scenarios_df()
+ free=(Falseifscenario_nameinscenarios_df.indexelseTrue)
+ returnfree
+
+ ##############################################
+
[docs]defrename_scenario_in_db(self,source_scenario_name:str,target_scenario_name:str):
+ """Rename a scenario. Uses a transaction (when enabled)."""
+ ifself.enable_transactions:
+ print("Rename scenario within a transaction")
+ withself.engine.begin()asconnection:
+ # self._rename_scenario_in_db(source_scenario_name, target_scenario_name, connection=connection)
+ self._rename_scenario_in_db_sql(connection,source_scenario_name,target_scenario_name)
+ else:
+ # self._rename_scenario_in_db(source_scenario_name, target_scenario_name)
+ self._rename_scenario_in_db_sql(self.engine,source_scenario_name,target_scenario_name)
+
+ def_rename_scenario_in_db_sql(self,connection,source_scenario_name:str,target_scenario_name:str=None):
+ """Rename scenario.
+ Uses 2 steps:
+ 1. Duplicate scenario
+ 2. Delete source scenario.
+
+ Problem is that we use scenario_name as a primary key. You should not change the value of primary keys in a DB.
+ Instead, first copy the data using a new scenario_name, i.e. duplicate a scenario. Next, delete the original scenario.
+
+ Long-term solution: use a scenario_seq sequence key as the PK. With scenario_name as a ordinary column in the scenario table.
+
+ Use of 'insert into select': https://stackoverflow.com/questions/9879830/select-modify-and-insert-into-the-same-table
+ """
+ # 1. Duplicate scenario
+ self._duplicate_scenario_in_db_sql(connection,source_scenario_name,target_scenario_name)
+ # 2. Delete scenario
+ self._delete_scenario_from_db(source_scenario_name,connection=connection)
+
+ def_delete_scenario_from_db(self,scenario_name:str,connection):
+ """Deletes all rows associated with a given scenario.
+ Note that it only deletes rows from tables defined in the self.db_tables, i.e. will NOT delete rows in 'auto-inserted' tables!
+ Must do a 'cascading' delete to ensure not violating FK constraints. In reverse order of how they are inserted.
+ Also deletes entry in scenario table
+ Uses SQLAlchemy syntax to generate SQL
+ TODO: check with 'auto-inserted' tables
+ TODO: batch all sql statements in single execute. Faster? And will that do the defer integrity checks?
+ """
+ # batch_sql=False # Batch=True does NOT work!
+ insp=sqlalchemy.inspect(connection)
+ tables_in_db=insp.get_table_names(schema=self.schema)
+ # sql_statements = []
+ forscenario_table_name,db_tableinreversed(self.db_tables.items()):# Note this INCLUDES the SCENARIO table!
+ ifdb_table.db_table_nameintables_in_db:
+ # # sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
+ # t = db_table.table_metadata # A Table()
+ # sql = t.delete().where(t.c.scenario_name == scenario_name)
+ # connection.execute(sql)
+ db_table._delete_scenario_table_from_db(scenario_name,connection)############################################################################################# Old Read scenario APIs
@@ -912,29 +1388,13 @@
[docs]defread_scenario_tables_from_db(self,scenario_name:str,
- input_table_names:List[str]=None,
- output_table_names:List[str]=None)->(Inputs,Outputs):
- """Loads data for selected input and output tables.
- If either list is names is None, will load all tables as defined in db_tables configuration.
- """
- ifinput_table_namesisNone:# load all tables by default
- input_table_names=list(self.input_db_tables.keys())
- if'Scenario'ininput_table_names:input_table_names.remove('Scenario')# Remove the scenario table
- ifoutput_table_namesisNone:# load all tables by default
- output_table_names=self.output_db_tables.keys()
- inputs={}
- forscenario_table_nameininput_table_names:
- inputs[scenario_table_name]=self.read_scenario_table_from_db(scenario_name,scenario_table_name)
- outputs={}
- forscenario_table_nameinoutput_table_names:
- outputs[scenario_table_name]=self.read_scenario_table_from_db(scenario_name,scenario_table_name)
- returninputs,outputs
[docs]defread_scenarios_from_db(self,scenario_names:List[str]=[])->(Inputs,Outputs):"""Multi scenario load.
- Reads all tables from set of scenarios"""
+ Reads all tables from set of scenarios
+ TODO: avoid use of text SQL. Use SQLAlchemy sql generation.
+ """where_scenarios=','.join([f"'{n}'"forninscenario_names])inputs={}
@@ -1063,7 +1523,7 @@
Return all scenarios in df. Result is indexed by scenario_name.
Main API to get all scenarios.
The API called by a cached procedure in the dse_do_dashboard.DoDashApp.
Loads data for selected input and output tables.
-If either list is names is None, will load all tables as defined in db_tables configuration.
+
Read selected set input and output tables from scenario.
+If input_table_names/output_table_names contains a ‘*’, then all input/output tables will be read.
+If empty list or None, then no tables will be read.
Main API to update output from a DO solve in the scenario.
+Deletes ALL output tables. Then inserts the given set of tables.
+Since this only touches the output tables, more efficient than replacing the whole scenario.
diff --git a/docs/doc_build/html/searchindex.js b/docs/doc_build/html/searchindex.js
index ae215ad..97e8490 100644
--- a/docs/doc_build/html/searchindex.js
+++ b/docs/doc_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["dse_do_utils","index","modules","readme_link"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["dse_do_utils.rst","index.rst","modules.rst","readme_link.rst"],objects:{"":{dse_do_utils:[0,0,0,"-"]},"dse_do_utils.cpd25utilities":{add_file_as_data_asset_cpd25:[0,1,1,""],add_file_path_as_data_asset_cpd25:[0,1,1,""],add_file_path_as_data_asset_wsc:[0,1,1,""],write_data_asset_as_file_cpd25:[0,1,1,""],write_data_asset_as_file_wsc:[0,1,1,""]},"dse_do_utils.datamanager":{DataManager:[0,2,1,""]},"dse_do_utils.datamanager.DataManager":{apply_and_concat:[0,3,1,""],df_crossjoin_ai:[0,3,1,""],df_crossjoin_mi:[0,3,1,""],df_crossjoin_si:[0,3,1,""],extract_solution:[0,3,1,""],get_parameter_value:[0,3,1,""],get_raw_table_by_name:[0,3,1,""],prep_parameters:[0,3,1,""],prepare_data_frames:[0,3,1,""],prepare_input_data_frames:[0,3,1,""],prepare_output_data_frames:[0,3,1,""],print_hello:[0,3,1,""],print_inputs_outputs_summary:[0,3,1,""]},"dse_do_utils.deployeddomodel":{DeployedDOModel:[0,2,1,""]},"dse_do_utils.deployeddomodel.DeployedDOModel":{execute_model:[0,3,1,""],extract_solution:[0,3,1,""],get_deployment_id:[0,3,1,""],get_job_status:[0,3,1,""],get_outputs:[0,3,1,""],get_solve_details:[0,3,1,""],get_solve_details_objective:[0,3,1,""],get_solve_payload:[0,3,1,""],get_solve_status:[0,3,1,""],get_space_id:[0,3,1,""],monitor_execution:[0,3,1,""],solve:[0,3,1,""]},"dse_do_utils.deployeddomodelcpd21":{DeployedDOModel_CPD21:[0,2,1,""]},"dse_do_utils.deployeddomodelcpd21.DeployedDOModel_CPD21":{cleanup:[0,3,1,""],execute_model:[0,3,1,""],get_debug_dump_name_and_url:[0,3,1,""],get_debug_file_url:[0,3,1,""],get_execution_service_model_url:[0,3,1,""],get_execution_status:[0,3,1,""],get_headers:[0,3,1,""],get_input_files:[0,3,1,""],get_job_url:[0,3,1,""],get_kill_job_url:[0,3,1,""],get_log_file_name_and_url:[0,3,1,""],get_log_file_url:[0,3,1,""],get_objective:[0,3,1,""],get_solution_name_and_url:[0,3,1,""],get_solve_config:[0,3,1,""],get_solve_status:[0,3,1,""],get_solve_url:[0,3,1,""],get_stop_job_url:[0,3,1,""],kill_job:[0,3,1,""],monitor_execution:[0,3,1,""],post_process_container:[0,3,1,""],post_process_container_get_dataframe:[0,3,1,""],post_process_failed:[0,3,1,""],post_process_inline_table:[0,3,1,""],post_process_inline_table_get_dataframe:[0,3,1,""],post_process_interrupted:[0,3,1,""],post_process_processed:[0,3,1,""],retrieve_debug_materials:[0,3,1,""],retrieve_file:[0,3,1,""],retrieve_solution:[0,3,1,""],retrieve_solve_configuration:[0,3,1,""],set_output_settings_in_solve_configuration:[0,3,1,""],solve:[0,3,1,""],stop_job:[0,3,1,""]},"dse_do_utils.domodelexporter":{DOModelExporter:[0,2,1,""]},"dse_do_utils.domodelexporter.DOModelExporter":{export_do_models:[0,3,1,""],get_access_token_curl:[0,3,1,""],get_access_token_web:[0,3,1,""],get_do_model_export_curl:[0,3,1,""],get_do_model_export_web:[0,3,1,""],get_project_id:[0,3,1,""],write_do_model_to_file:[0,3,1,""]},"dse_do_utils.mapmanager":{MapManager:[0,2,1,""]},"dse_do_utils.mapmanager.MapManager":{add_full_screen:[0,3,1,""],add_layer_control:[0,3,1,""],create_blank_map:[0,3,1,""],get_arrows:[0,3,1,""],get_bearing:[0,3,1,""],get_html_table:[0,3,1,""],get_popup_table:[0,3,1,""],kansas_city_coord:[0,4,1,""]},"dse_do_utils.multiscenariomanager":{MultiScenarioManager:[0,2,1,""]},"dse_do_utils.multiscenariomanager.MultiScenarioManager":{add_data_file_to_project:[0,3,1,""],env_is_wscloud:[0,3,1,""],get_all_scenario_names:[0,3,1,""],get_data_directory:[0,3,1,""],get_dd_client:[0,3,1,""],get_multi_scenario_data:[0,3,1,""],get_root_directory:[0,3,1,""],get_scenarios_df:[0,3,1,""],load_data_from_scenario:[0,3,1,""],merge_scenario_data:[0,3,1,""],write_data_to_excel:[0,3,1,""]},"dse_do_utils.optimizationengine":{MyProgressListener:[0,2,1,""],OptimizationEngine:[0,2,1,""]},"dse_do_utils.optimizationengine.MyProgressListener":{notify_progress:[0,3,1,""]},"dse_do_utils.optimizationengine.OptimizationEngine":{add_mip_progress_kpis:[0,3,1,""],binary_var_series:[0,3,1,""],binary_var_series_s:[0,3,1,""],continuous_var_series:[0,3,1,""],continuous_var_series_s:[0,3,1,""],export_as_cpo:[0,3,1,""],export_as_cpo_s:[0,3,1,""],export_as_lp:[0,3,1,""],export_as_lp_s:[0,3,1,""],get_kpi_output_table:[0,3,1,""],integer_var_series:[0,3,1,""],integer_var_series_s:[0,3,1,""],solve:[0,3,1,""]},"dse_do_utils.plotlymanager":{PlotlyManager:[0,2,1,""]},"dse_do_utils.plotlymanager.PlotlyManager":{get_dash_tab_layout_m:[0,3,1,""],get_plotly_fig_m:[0,3,1,""]},"dse_do_utils.scenariodbmanager":{AutoScenarioDbTable:[0,2,1,""],BusinessKpiTable:[0,2,1,""],KpiTable:[0,2,1,""],ParameterTable:[0,2,1,""],ScenarioDbManager:[0,2,1,""],ScenarioDbTable:[0,2,1,""],ScenarioTable:[0,2,1,""]},"dse_do_utils.scenariodbmanager.AutoScenarioDbTable":{create_table_metadata:[0,3,1,""],insert_table_in_db_bulk:[0,3,1,""]},"dse_do_utils.scenariodbmanager.ScenarioDbManager":{add_scenario_name_to_dfs:[0,3,1,""],create_schema:[0,3,1,""],delete_scenario_name_column:[0,3,1,""],drop_all_tables:[0,3,1,""],get_scenarios_df:[0,3,1,""],insert_scenarios_in_db:[0,3,1,""],insert_tables_in_db:[0,3,1,""],read_scenario_from_db:[0,3,1,""],read_scenario_table_from_db:[0,3,1,""],read_scenario_table_from_db_cached:[0,3,1,""],read_scenario_tables_from_db:[0,3,1,""],read_scenario_tables_from_db_cached:[0,3,1,""],read_scenarios_from_db:[0,3,1,""],read_scenarios_table_from_db_cached:[0,3,1,""],replace_scenario_in_db:[0,3,1,""],set_scenarios_table_read_callback:[0,3,1,""],set_table_read_callback:[0,3,1,""]},"dse_do_utils.scenariodbmanager.ScenarioDbTable":{add_scenario_name_to_fk_constraint:[0,3,1,""],camel_case_to_snake_case:[0,3,1,""],create_table_metadata:[0,3,1,""],df_column_names_to_snake_case:[0,3,1,""],get_db_table_name:[0,3,1,""],get_df_column_names:[0,3,1,""],insert_table_in_db_bulk:[0,3,1,""],sqlcol:[0,3,1,""]},"dse_do_utils.scenariomanager":{Platform:[0,2,1,""],ScenarioManager:[0,2,1,""]},"dse_do_utils.scenariomanager.Platform":{CPD25:[0,4,1,""],CPD40:[0,4,1,""],CPDaaS:[0,4,1,""],Local:[0,4,1,""]},"dse_do_utils.scenariomanager.ScenarioManager":{add_data_file_to_project_s:[0,3,1,""],add_data_file_using_project_lib:[0,3,1,""],add_data_file_using_ws_lib:[0,3,1,""],add_data_file_using_ws_lib_s:[0,3,1,""],add_data_into_scenario:[0,3,1,""],add_data_into_scenario_s:[0,3,1,""],add_file_as_data_asset:[0,3,1,""],add_file_as_data_asset_s:[0,3,1,""],clear_scenario_data:[0,3,1,""],create_new_scenario:[0,3,1,""],detect_platform:[0,3,1,""],env_is_cpd25:[0,3,1,""],env_is_cpd40:[0,3,1,""],env_is_dsx:[0,3,1,""],env_is_wscloud:[0,3,1,""],export_model_as_lp:[0,3,1,""],get_data_directory:[0,3,1,""],get_dd_client:[0,3,1,""],get_do_scenario:[0,3,1,""],get_kpis_table_as_dataframe:[0,3,1,""],get_root_directory:[0,3,1,""],load_data:[0,3,1,""],load_data_from_csv:[0,3,1,""],load_data_from_csv_s:[0,3,1,""],load_data_from_excel:[0,3,1,""],load_data_from_excel_s:[0,3,1,""],load_data_from_scenario:[0,3,1,""],load_data_from_scenario_s:[0,3,1,""],print_table_names:[0,3,1,""],replace_data_in_scenario:[0,3,1,""],replace_data_into_scenario_s:[0,3,1,""],update_solve_output_into_scenario:[0,3,1,""],write_data_into_scenario:[0,3,1,""],write_data_into_scenario_s:[0,3,1,""],write_data_to_csv:[0,3,1,""],write_data_to_csv_s:[0,3,1,""],write_data_to_excel:[0,3,1,""],write_data_to_excel_s:[0,3,1,""]},"dse_do_utils.scenariopicker":{ScenarioPicker:[0,2,1,""]},"dse_do_utils.scenariopicker.ScenarioPicker":{ScenarioRefreshButton:[0,2,1,""],default_scenario:[0,4,1,""],get_dd_client:[0,3,1,""],get_scenario_picker_ui:[0,3,1,""],get_scenario_refresh_button:[0,3,1,""],get_scenario_select_drop_down:[0,3,1,""],get_selected_scenario:[0,3,1,""],load_selected_scenario_data:[0,3,1,""],widgets:[0,4,1,""]},"dse_do_utils.utilities":{add_sys_path:[0,1,1,""],list_file_hierarchy:[0,1,1,""]},dse_do_utils:{cpd25utilities:[0,0,0,"-"],datamanager:[0,0,0,"-"],deployeddomodel:[0,0,0,"-"],deployeddomodelcpd21:[0,0,0,"-"],domodelexporter:[0,0,0,"-"],mapmanager:[0,0,0,"-"],module_reload:[0,1,1,""],multiscenariomanager:[0,0,0,"-"],optimizationengine:[0,0,0,"-"],plotly_cpd_workaround:[0,0,0,"-"],plotlymanager:[0,0,0,"-"],scenariodbmanager:[0,0,0,"-"],scenariomanager:[0,0,0,"-"],scenariopicker:[0,0,0,"-"],utilities:[0,0,0,"-"],version:[0,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","function","Python function"],"2":["py","class","Python class"],"3":["py","method","Python method"],"4":["py","attribute","Python attribute"]},objtypes:{"0":"py:module","1":"py:function","2":"py:class","3":"py:method","4":"py:attribute"},terms:{"0":[0,1],"02ea6d480895":0,"04":0,"0596001673":0,"085594":0,"1":[0,3],"100":0,"1132":0,"16":0,"1993":0,"2":[0,3],"2005586":0,"2016":0,"21c8ac71":0,"23690284":0,"25a0fe88e4":0,"26c1":0,"3":[0,3],"30080":[],"31":0,"3810":0,"39":0,"4":[0,3],"41":0,"447a":0,"458550":0,"469":0,"49a5":0,"4bd2":0,"5":[0,1],"50":0,"5401":0,"585241":0,"58952002":[],"5de6560a1cfa":0,"6":0,"600":0,"7":3,"785":0,"8021":0,"8364":0,"94":0,"95":0,"96":0,"9f28":0,"abstract":0,"case":[0,3],"class":[0,1],"default":[0,3],"do":0,"enum":0,"export":[0,3],"float":0,"function":[0,3],"import":[0,1],"int":0,"new":0,"return":0,"static":0,"true":0,"while":0,A:[0,3],And:0,As:0,At:0,But:[0,3],By:0,FOR:0,For:[0,3],IF:[],IN:[],If:0,In:[0,3],Is:0,It:[0,3],NOT:[0,3],No:0,Not:0,On:0,One:0,Or:0,That:0,The:[0,3],Then:[0,3],These:0,To:[0,3],Will:0,With:0,_:0,__index__:0,__init__:0,_base:0,_export_:0,_multi_output:0,_output:0,_table_index_:0,_xlx:0,a567:0,a933:0,aa50:0,abbrevi:0,abc:0,abl:[0,3],about:0,abov:0,abspath:0,accept:0,access:[0,3],access_project_or_spac:0,access_tok:0,access_token:0,accesstoken:0,accordingli:0,ad:[0,3],adapt:[],add:[0,3],add_child:0,add_data_file_to_project:0,add_data_file_to_project_:0,add_data_file_using_project_lib:0,add_data_file_using_ws_lib:0,add_data_file_using_ws_lib_:0,add_data_into_scenario:0,add_data_into_scenario_:0,add_file_as_data_asset:0,add_file_as_data_asset_:0,add_file_as_data_asset_cpd25:0,add_file_path_as_data_asset_cpd25:0,add_file_path_as_data_asset_wsc:0,add_full_screen:0,add_layer_control:0,add_mip_progress_kpi:0,add_scenario_name_to_df:0,add_scenario_name_to_fk_constraint:0,add_sys_path:0,add_to:0,addit:0,advanc:0,advantag:0,advis:0,after:0,air:3,alia:0,all:[0,3],allow:[0,3],along:0,alreadi:0,also:[0,3],altern:0,alwai:0,an:[0,3],ani:0,anoth:0,api:[0,3],apicli:0,app:0,appdomain:[],appear:0,appli:[0,3],applic:0,apply_and_concat:0,ar:[0,3],arg:0,argument:0,around:[0,3],arrow:0,asset:[0,3],asset_nam:0,assign:0,associ:[],assum:0,attach:0,attachment_typ:0,attempt:0,attribut:0,auto:[],auto_insert:0,autodetect:0,automat:[0,3],autoreload:0,autoscenariodbt:0,avail:[0,3],avoid:[0,3],axi:0,b7bf7fd8:0,back:0,backward:3,base:0,bash:0,basic:3,bear:0,been:[0,3],befor:0,being:0,below:0,best:0,best_bound_kpi_nam:0,better:0,between:[0,3],bewar:0,binary_var_seri:0,binary_var_series_:0,binaryvartyp:0,blank:0,blob:0,bludb:[],blue:0,bobhaffn:0,bool:0,both:[0,3],bound:0,box:0,br:0,build:0,builder:[0,1],bulk:0,business_kpi:0,businesskpit:0,button:0,c:0,cach:0,call:0,callback:0,came:0,camel_case_to_snake_cas:0,camelcas:0,can:[0,3],cancel:0,cannot:[0,3],cartesian:0,cascad:[],categor:0,categori:0,caus:0,cell:[0,3],certain:0,certif:0,ch04s23:0,challeng:0,chang:0,channel:3,charact:0,chart:0,check:0,child:0,clean:0,cleanup:0,clear:0,clear_scenario_data:0,client:[0,3],cloud:[0,3],cluster4:0,cluster:0,cluster_nam:0,co:0,code:[0,3],cogno:0,collabor:0,collect:[],color:0,column:0,column_nam:0,columns_metadata:0,com:0,combin:[0,3],command:0,commun:0,compass:0,compat:[0,3],complet:0,complex:0,compon:0,comput:0,conda:3,condit:0,conf:0,configur:[0,3],connect:[0,3],constant:0,constraint:[],constraints_metadata:0,constructor:0,contain:[0,3],content:2,context:0,continuous_var_seri:0,continuous_var_series_:0,continuousvartyp:0,control:0,conveni:0,convers:0,convert:0,cookbook:0,coord:0,copi:0,copy_to_csv:0,core:0,corner:0,correctli:0,could:0,count:0,counti:0,cp4d25:0,cp4daa:0,cp4dv2:0,cp4dv4:0,cp:[0,3],cpd25:0,cpd25util:2,cpd2:0,cpd3:[0,3],cpd40:0,cpd4:3,cpd:[0,3],cpd_cluster_host:0,cpdaa:0,cpdv2:[0,1],cpdv3:[0,3],cpdv4:[0,1],cplex:[0,3],cpo:0,cpolab:0,cpsaa:0,creat:[0,3],create_blank_map:0,create_database_engin:[],create_db2_engin:[],create_new_scenario:0,create_schema:0,create_sqllite_engin:[],create_table_metadata:0,creation:0,credenti:0,cross:0,crossjoin:3,csv:[0,3],csv_directori:0,csv_name_pattern:0,curl:0,current:0,current_dir:0,cursor:[],custom:0,d4c69a0d8158:0,d:[0,3],dash:0,dashboard:0,dashenterpris:0,data:[0,3],data_asset:[0,3],data_by_scenario:0,data_id:0,data_manag:0,data_url:0,databas:[],datafram:0,datamanag:[2,3],datascienceelit:0,dataset:0,datetim:0,db2:[],db2_credenti:[],db:0,db_tabl:0,db_table_nam:0,dd:[0,3],dd_scenario:0,de:0,debug:0,debug_file_data_url:0,debug_file_dir:0,decis:3,decision_optimization_cli:0,def:0,default_max_oaas_time_limit_sec:0,default_max_run_time_sec:0,default_scenario:0,default_valu:0,defin:0,definit:0,delai:0,delet:0,delete_scenario_from_db:[],delete_scenario_name_column:0,depend:[0,3],deploi:0,deploy:[0,3],deployed_model_nam:0,deployeddomodel:2,deployeddomodel_cpd21:0,deployeddomodelcpd21:2,deployment_id:0,deprec:0,design:[0,3],despit:0,detail:3,detect_platform:0,develop:[0,3],df1:0,df2:0,df:0,df_column_names_to_snake_cas:0,df_crossjoin_ai:0,df_crossjoin_mi:0,df_crossjoin_si:0,df_dict:0,dict:0,dictionari:0,differ:[0,3],direct:0,directori:0,disadvantag:0,disk:0,dm:0,do4w:0,do_model_nam:0,do_util:0,doc:0,docplex:[0,3],document:[0,3],dodashapp:0,doe:0,doesn:0,domodeldeploy:2,domodelexport:2,don:[],done:0,down:[0,3],download:[0,3],driven:0,drop:[0,3],drop_all_t:0,drop_column_nam:0,dropdown:0,dse:[0,3],dse_do_dashboard:0,dse_do_util:1,dsx:0,dsx_project_dir:0,dsxuser:0,due:0,dump:0,dumpzipnam:0,dure:0,dvar:[0,3],e:[0,3],each:0,easi:3,easier:0,echo:0,ef365b2c:0,effect:0,eg:0,either:0,element:0,els:0,emb:0,empti:[0,3],en:0,enable_sqlite_fk:0,enable_transact:0,encod:[],end:0,engin:[0,3],ensur:0,entiti:0,entri:0,enumer:0,env_is_cpd25:0,env_is_cpd40:0,env_is_dsx:0,env_is_wscloud:0,environ:[0,1],error:0,establish:0,evalu:0,exampl:0,excel:[0,3],excel_file_nam:0,excel_output_file_nam:0,excel_test:0,excelfil:0,excelwrit:0,exclud:0,execut:0,execute_model:0,execution_result:0,execution_statu:0,execution_status_json:0,exist:[0,3],expect:0,experi:0,explicit:0,explicitli:0,export_as_cpo:0,export_as_cpo_:0,export_as_lp:0,export_as_lp_:0,export_do_model:0,export_model:0,export_model_as_lp:0,expos:0,express:0,extend:0,extended_columns_metadata:0,extens:0,extract:0,extract_dvar_nam:0,extract_solut:0,f:0,fail:0,fake:3,fals:0,faster:0,featur:3,featuregroup:0,field:0,fig:0,file:[0,3],file_nam:0,file_path:0,filenam:0,filter:0,find:0,firefox:0,first:0,fk:[],fkc:0,flask:0,folder:0,folium:[0,3],follow:[0,3],forc:0,foreign:0,foreignkeyconstraint:0,form:0,format:0,former:[0,3],found:0,frame:0,free:0,from:[0,3],full:0,full_project_nam:0,func:0,futur:[0,3],g:[0,3],gap:[0,3],gener:0,get:0,get_access_token_curl:0,get_access_token_web:0,get_all_scenario_nam:0,get_arrow:0,get_bear:0,get_dash_tab_layout_m:0,get_data_directori:0,get_db2_connection_str:[],get_db_table_nam:0,get_dd_client:0,get_debug_dump_name_and_url:0,get_debug_file_url:0,get_deployment_id:0,get_df_column_nam:0,get_do_model_export_curl:0,get_do_model_export_web:0,get_do_scenario:0,get_execution_service_model_url:0,get_execution_statu:0,get_head:0,get_html_tabl:0,get_input_fil:0,get_job_statu:0,get_job_url:0,get_kill_job_url:0,get_kpi_output_t:0,get_kpis_table_as_datafram:0,get_log_file_name_and_url:0,get_log_file_url:0,get_multi_scenario_data:0,get_object:0,get_output:0,get_parameter_valu:0,get_plotly_fig_m:0,get_popup_t:0,get_project_id:0,get_raw_table_by_nam:0,get_root_directori:0,get_scenario_picker_ui:0,get_scenario_refresh_button:0,get_scenario_select_drop_down:0,get_scenarios_df:0,get_selected_scenario:0,get_solution_name_and_url:0,get_solve_config:0,get_solve_detail:0,get_solve_details_object:0,get_solve_payload:0,get_solve_statu:0,get_solve_url:0,get_space_id:0,get_stop_job_url:0,get_tab_layout_:0,getcwd:0,gist:0,git:0,github:[0,3],githubpag:3,give:0,given:[],glob:0,grand:0,greatli:3,gsilabs_scnfo:0,h:0,ha:[0,3],hack:3,hand:0,happen:0,hard:0,have:[0,3],height:0,hello:0,here:0,hierarch:0,hold:0,home:0,host:[],hostnam:[],how:[0,3],howev:0,html:0,http:0,hypothet:0,i:0,ibm:[0,3],ibm_watson_machine_learn:0,ibm_watson_studio_lib:0,icon:0,icpd:3,id:0,ignor:[0,3],imp:0,impact:3,implement:0,improv:3,includ:0,inconsist:0,independ:0,index:[0,1],indic:0,individu:3,info:0,inform:0,init:0,initi:0,initialize_db_tables_metadata:[],inline_t:0,inner:0,input:0,input_csv_name_pattern:0,input_db_t:0,input_table_nam:0,insecurerequestwarn:0,insert:0,insert_scenarios_in_db:0,insert_single_scenario_tables_in_db:[],insert_table_in_db:[],insert_table_in_db_bulk:0,insert_tables_in_db:0,instal:[0,1],installationreadm:3,instanc:0,instead:0,integ:0,integer_var_list:0,integer_var_seri:0,integer_var_series_:0,integervartyp:0,intend:3,interact:[0,3],interfac:3,intermedi:0,intern:[0,3],internet:3,interrupt:0,interv:0,io:0,ipynb:0,ipywidget:0,issu:0,itself:3,jerom:0,job:0,job_config_json:0,job_detail:0,job_uid:0,join:0,json:0,jupyt:[0,3],jupyterlab:3,just:0,kansas_city_coord:0,karg:0,keep:0,kei:0,keyword:0,kill:0,kill_job:0,km:0,kpi:0,kpitabl:0,kwarg:0,lambda:0,last:0,lat:0,later:0,latest:0,layer_control_posit:0,leav:0,left:0,let:0,level:0,lib:0,librari:0,like:[0,3],limit:[0,3],line:0,list:0,list_file_hierarchi:0,lite:[],load:0,load_data:0,load_data_from_csv:0,load_data_from_csv_:0,load_data_from_excel:0,load_data_from_excel_:0,load_data_from_scenario:0,load_data_from_scenario_:0,load_from_excel:0,load_selected_scenario_data:0,local:[0,3],local_root:0,locat:0,log:0,log_file_nam:0,logic:0,lon:0,longer:[0,3],look:0,loop:0,lp:[0,3],m:0,machin:0,made:0,mai:0,main:[0,1],maintain:0,major:0,make:0,manag:0,mani:3,manipul:[0,3],manual:0,map:[0,3],mapmanag:[2,3],marker:0,master:0,match:0,max_oaas_time_limit_sec:0,max_run_time_sec:0,maximum:0,mb:3,md:3,mdl:0,me:[0,1],mean:0,medium:0,member:0,menu:[0,3],merg:0,merge_scenario_data:0,messag:0,metadata:0,method:0,mgr:0,might:3,minu:0,mip_gap_kpi_nam:0,miss:0,mix:0,mkonrad:0,mode:0,model1:0,model2:0,model:[0,1],model_build:0,model_nam:0,modelbuild:0,modifi:0,modul:[1,2],module_reload:0,moment:3,monitor:0,monitor_execut:0,monitor_loop_delay_sec:0,more:[0,3],most:0,mostli:3,move:3,mp:0,msm:0,multi:0,multi_scenario:0,multiindex:0,multipl:[0,3],multiscenariomanag:2,must:0,my:0,my_default_scenario:0,my_do_model:0,my_funct:0,my_input_column_nam:0,my_input_valu:0,my_output_column_name_1:0,my_output_column_name_2:0,my_output_value1:0,my_output_value_1:0,my_output_value_2:0,my_schema:[],my_tabl:0,myexcelfil:0,myexcelfileoutput:0,myfil:0,mymodel:0,myoptimizationengin:0,myprogresslisten:0,mytabl:0,n:0,n_arrow:0,name:0,namedtupl:0,nbviewer:0,necessari:0,need:0,neither:3,net:0,never:0,new_path:0,new_scenario_nam:0,next:0,nodefault:3,non:0,none:0,not_start:0,note:[0,3],notebook:[0,3],notify_progress:0,now:0,number:0,oaa:0,object:0,off:0,ok:0,onc:0,one:0,ones:0,onli:[0,3],open:0,oper:0,optim:[0,3],optimizationengin:[2,3],option:[0,3],order:0,ordereddict:0,ore:3,oreilli:0,org:0,organ:0,origin:0,os:0,other:[0,3],otherwis:0,out:0,output:0,output_csv_name_pattern:0,output_db_t:0,output_table_nam:0,outsid:3,overrid:0,overwrit:0,overwritten:0,p1:0,p2:0,packag:[1,2,3],page:[0,1],page_id:0,page_sourc:0,paid:[],pak:3,panda:0,panel:0,param:0,param_nam:0,param_typ:0,paramet:0,parametert:0,pardir:0,parent:0,parent_dir:0,parent_dir_2:0,pars:0,parse_html:0,part:0,particular:[0,3],pass:0,password1:[],password:0,past:0,path:0,pattern:0,payload:0,pd:0,per:0,perform:3,period:0,person:0,phase:0,pick:3,picker:0,pip:[0,3],place:0,placehold:0,plain:0,platform:0,plot:0,plotli:0,plotly_cpd_workaround:2,plotlymanag:2,plugin:0,point:0,poll:0,polylin:0,popul:0,popup:0,popup_t:0,port:[],possibl:0,post:[0,3],post_process_contain:0,post_process_container_get_datafram:0,post_process_fail:0,post_process_inline_t:0,post_process_inline_table_get_datafram:0,post_process_interrupt:0,post_process_process:0,practic:0,pre:[0,3],prefer:3,prep_paramet:0,prepar:0,prepare_data_fram:0,prepare_input_data_fram:0,prepare_output_data_fram:0,prevent:0,previou:3,primari:0,print:0,print_hello:0,print_inputs_outputs_summari:0,print_table_nam:0,problem:0,procedur:0,process:[0,3],product:0,productionplan:0,progress:0,progress_data:0,progressdata:0,project:[0,3],project_access_token:0,project_data:[0,3],project_id:0,project_lib:0,project_nam:0,properli:[],properti:0,property_1:0,property_2:0,provid:[0,3],put:3,pwd:0,py:0,pydata:0,pypi:3,python:[0,3],question:0,queu:0,quot:0,rais:0,raw:0,re:0,reach:0,read:[0,1],read_csv:0,read_scenario_from_db:0,read_scenario_table_from_db:0,read_scenario_table_from_db_cach:0,read_scenario_table_from_db_callback:0,read_scenario_tables_from_db:0,read_scenario_tables_from_db_cach:0,read_scenarios_from_db:0,read_scenarios_table_from_db_cach:0,read_scenarios_table_from_db_callback:0,readthedoc:0,reason:0,record:0,reduc:0,refactor:3,refer:0,refine_conflict:0,refresh:0,regist:0,regular:0,regularli:[0,3],rel:0,relat:0,relationship:0,releas:3,relev:0,reliabl:0,reload:0,remot:0,remov:0,renam:0,repeatedli:0,replac:0,replace_data_in_scenario:0,replace_data_into_scenario_:0,replace_scenario_in_db:0,repositori:3,repres:0,represent:0,request:0,requir:[0,1],rerun:0,resourc:0,respons:0,rest:0,restor:0,restrict:0,result:0,retreiv:0,retriev:0,retrieve_debug_materi:0,retrieve_fil:0,retrieve_solut:0,retrieve_solve_configur:0,revers:[],right:0,root:0,root_dir:0,rotat:0,round:0,rout:0,routin:0,row:0,run:[0,3],runtime_env_apsx_url:0,s:0,same:[0,3],save:0,scenario:[0,3],scenario_1:0,scenario_nam:0,scenario_table_nam:0,scenariodbmanag:2,scenariodbt:0,scenariomanag:[2,3],scenariopick:[2,3],scenariorefreshbutton:0,scenarios_table_read_callback:0,scenariot:0,schema:0,scnfo:0,scnfo_dash_pycharm_github:0,scope:1,screen:0,script:3,search:[0,1],second:0,secur:[],see:[0,3],seem:0,select:0,self:0,separ:[0,3],sequenc:0,seri:0,server:0,servic:0,service_configuration_json:0,service_nam:0,session:[],set:0,set_index:0,set_output_settings_in_solve_configur:0,set_scenarios_table_read_callback:0,set_table_read_callback:0,setup:0,share:3,sheet:0,should:[0,3],show:0,side:0,signific:3,similar:0,simpl:0,simplest:0,simpli:0,sinc:0,singl:0,site:0,size:0,skip:0,sm:0,small:0,snake_cas:0,so:0,solut:0,solution_count_kpi_nam:0,solutionlisten:0,solv:0,solve_config:0,solve_config_json:0,solve_payload:0,solve_phase_kpi_nam:0,solve_statu:0,solve_time_kpi_nam:0,solvesolut:0,some:[0,3],someth:0,somewhat:0,sourc:[0,3],sp:0,space:0,space_id:0,space_nam:0,spd:0,specif:[],specifi:0,speed:0,spreadhseet:3,spreadsheet:0,sql:0,sqlalchemi:0,sqlcol:0,ssl:0,stackoverflow:0,stamp:0,standard:0,start:0,startpath:0,state:0,statement:0,statu:0,step:0,stop:0,stop_job:0,storag:0,store:[0,3],str:0,strftime:0,string:0,strongli:0,structur:0,studio:[0,3],sub:0,subclass:[0,3],submiss:0,submodul:2,succesfulli:0,suffici:0,suggest:0,summari:0,support:0,suppress:0,suppress_warn:0,sure:0,sy:0,system:[0,3],t:0,tabl:[0,3],table_index_sheet:0,table_nam:0,table_read_callback:0,target:[0,1],task:0,templat:0,template_scenario_nam:0,temporari:0,termin:0,test:0,text:0,than:0,thei:0,them:0,therebi:0,therefor:[0,3],thi:[0,3],thing:0,those:0,thu:0,time:0,todo:0,token:0,tooltip:0,top:0,topleft:0,track:0,transact:0,translat:0,tree:0,truth:0,tupl:0,turn:0,two:0,txt:0,type:0,typic:[0,3],ugli:0,ui:0,un:0,under:0,unfortun:0,uniqu:0,unknown:0,unnam:0,unverifi:0,up:[0,3],updat:0,update_solve_output_into_scenario:0,upload:[0,3],url:0,urllib3:0,us:[0,3],usag:[0,3],user1:[],user:0,user_access_token:0,user_nam:0,usernam:[],util:[2,3],v0:3,v2:3,v4:3,valid:0,valu:0,value_1:0,value_2:0,value_format:0,valueerror:0,variabl:0,venv:0,veri:3,verif:0,verifi:0,verify_integr:0,version:[2,3],via:0,view:0,violat:[],visibl:0,visual:[0,3],w:0,wa:0,wai:0,want:0,warehous:[],warn:0,watson:[0,3],watsonmachinelearningapicli:0,we:0,web:0,well:0,what:0,whatev:0,wheel:3,when:[0,3],where:0,which:0,whole:0,widget:0,widget_button:0,widget_select:0,width:0,wil:0,window:0,within:[0,3],without:0,wml:1,wml_credenti:0,work:[0,3],workaround:[],world:0,worri:[],would:0,write:[0,3],write_data_asset_as_file_cpd25:0,write_data_asset_as_file_wsc:0,write_data_into_scenario:0,write_data_into_scenario_:0,write_data_to_csv:0,write_data_to_csv_:0,write_data_to_excel:0,write_data_to_excel_:0,write_do_model_to_fil:0,writer:0,written:0,ws:[0,3],wsl1:0,wsl:[0,3],wslib:0,wslv1:3,wsuser:0,www:0,xdvar:0,xl:0,xlsx:0,xlx:0,xxxxxx:0,y:0,yet:0,you:0,your:0,yyyymmdd_hhmm:0,zip:[0,3],zoom_start:0},titles:["dse_do_utils package","Welcome to DSE DO Utils documentation!","dse_do_utils","Read me"],titleterms:{"0":3,"5":3,"class":3,"do":[1,3],"import":3,builder:3,content:[0,1],cpd25util:0,cpdv2:3,cpdv4:3,custom:3,datamanag:0,deployeddomodel:0,deployeddomodelcpd21:0,document:1,domodeldeploy:0,domodelexport:0,dse:1,dse_do_util:[0,2,3],environ:3,indic:1,instal:3,main:3,mapmanag:0,me:3,model:3,modul:[0,3],multiscenariomanag:0,optimizationengin:0,packag:0,plotly_cpd_workaround:0,plotlymanag:0,read:3,requir:3,scenariodbmanag:0,scenariomanag:0,scenariopick:0,scope:3,submodul:0,tabl:1,target:3,util:[0,1],version:0,welcom:1,wml:3}})
\ No newline at end of file
+Search.setIndex({docnames:["dse_do_utils","index","modules","readme_link"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":4,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":3,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1,sphinx:56},filenames:["dse_do_utils.rst","index.rst","modules.rst","readme_link.rst"],objects:{"":{dse_do_utils:[0,0,0,"-"]},"dse_do_utils.cpd25utilities":{add_file_as_data_asset_cpd25:[0,1,1,""],add_file_path_as_data_asset_cpd25:[0,1,1,""],add_file_path_as_data_asset_wsc:[0,1,1,""],write_data_asset_as_file_cpd25:[0,1,1,""],write_data_asset_as_file_wsc:[0,1,1,""]},"dse_do_utils.datamanager":{DataManager:[0,2,1,""]},"dse_do_utils.datamanager.DataManager":{apply_and_concat:[0,3,1,""],df_crossjoin_ai:[0,3,1,""],df_crossjoin_mi:[0,3,1,""],df_crossjoin_si:[0,3,1,""],extract_solution:[0,3,1,""],get_parameter_value:[0,3,1,""],get_raw_table_by_name:[0,3,1,""],prep_parameters:[0,3,1,""],prepare_data_frames:[0,3,1,""],prepare_input_data_frames:[0,3,1,""],prepare_output_data_frames:[0,3,1,""],print_hello:[0,3,1,""],print_inputs_outputs_summary:[0,3,1,""]},"dse_do_utils.deployeddomodel":{DeployedDOModel:[0,2,1,""]},"dse_do_utils.deployeddomodel.DeployedDOModel":{execute_model:[0,3,1,""],extract_solution:[0,3,1,""],get_deployment_id:[0,3,1,""],get_job_status:[0,3,1,""],get_outputs:[0,3,1,""],get_solve_details:[0,3,1,""],get_solve_details_objective:[0,3,1,""],get_solve_payload:[0,3,1,""],get_solve_status:[0,3,1,""],get_space_id:[0,3,1,""],monitor_execution:[0,3,1,""],solve:[0,3,1,""]},"dse_do_utils.deployeddomodelcpd21":{DeployedDOModel_CPD21:[0,2,1,""]},"dse_do_utils.deployeddomodelcpd21.DeployedDOModel_CPD21":{cleanup:[0,3,1,""],execute_model:[0,3,1,""],get_debug_dump_name_and_url:[0,3,1,""],get_debug_file_url:[0,3,1,""],get_execution_service_model_url:[0,3,1,""],get_execution_status:[0,3,1,""],get_headers:[0,3,1,""],get_input_files:[0,3,1,""],get_job_url:[0,3,1,""],get_kill_job_url:[0,3,1,""],get_log_file_name_and_url:[0,3,1,""],get_log_file_url:[0,3,1,""],get_objective:[0,3,1,""],get_solution_name_and_url:[0,3,1,""],get_solve_config:[0,3,1,""],get_solve_status:[0,3,1,""],get_solve_url:[0,3,1,""],get_stop_job_url:[0,3,1,""],kill_job:[0,3,1,""],monitor_execution:[0,3,1,""],post_process_container:[0,3,1,""],post_process_container_get_dataframe:[0,3,1,""],post_process_failed:[0,3,1,""],post_process_inline_table:[0,3,1,""],post_process_inline_table_get_dataframe:[0,3,1,""],post_process_interrupted:[0,3,1,""],post_process_processed:[0,3,1,""],retrieve_debug_materials:[0,3,1,""],retrieve_file:[0,3,1,""],retrieve_solution:[0,3,1,""],retrieve_solve_configuration:[0,3,1,""],set_output_settings_in_solve_configuration:[0,3,1,""],solve:[0,3,1,""],stop_job:[0,3,1,""]},"dse_do_utils.domodelexporter":{DOModelExporter:[0,2,1,""]},"dse_do_utils.domodelexporter.DOModelExporter":{export_do_models:[0,3,1,""],get_access_token_curl:[0,3,1,""],get_access_token_web:[0,3,1,""],get_do_model_export_curl:[0,3,1,""],get_do_model_export_web:[0,3,1,""],get_project_id:[0,3,1,""],write_do_model_to_file:[0,3,1,""]},"dse_do_utils.mapmanager":{MapManager:[0,2,1,""]},"dse_do_utils.mapmanager.MapManager":{add_full_screen:[0,3,1,""],add_layer_control:[0,3,1,""],create_blank_map:[0,3,1,""],get_arrows:[0,3,1,""],get_bearing:[0,3,1,""],get_html_table:[0,3,1,""],get_popup_table:[0,3,1,""],kansas_city_coord:[0,4,1,""]},"dse_do_utils.multiscenariomanager":{MultiScenarioManager:[0,2,1,""]},"dse_do_utils.multiscenariomanager.MultiScenarioManager":{add_data_file_to_project:[0,3,1,""],env_is_wscloud:[0,3,1,""],get_all_scenario_names:[0,3,1,""],get_data_directory:[0,3,1,""],get_dd_client:[0,3,1,""],get_multi_scenario_data:[0,3,1,""],get_root_directory:[0,3,1,""],get_scenarios_df:[0,3,1,""],load_data_from_scenario:[0,3,1,""],merge_scenario_data:[0,3,1,""],write_data_to_excel:[0,3,1,""]},"dse_do_utils.optimizationengine":{MyProgressListener:[0,2,1,""],OptimizationEngine:[0,2,1,""]},"dse_do_utils.optimizationengine.MyProgressListener":{notify_progress:[0,3,1,""]},"dse_do_utils.optimizationengine.OptimizationEngine":{add_mip_progress_kpis:[0,3,1,""],binary_var_series:[0,3,1,""],binary_var_series_s:[0,3,1,""],continuous_var_series:[0,3,1,""],continuous_var_series_s:[0,3,1,""],export_as_cpo:[0,3,1,""],export_as_cpo_s:[0,3,1,""],export_as_lp:[0,3,1,""],export_as_lp_s:[0,3,1,""],get_kpi_output_table:[0,3,1,""],integer_var_series:[0,3,1,""],integer_var_series_s:[0,3,1,""],solve:[0,3,1,""]},"dse_do_utils.plotlymanager":{PlotlyManager:[0,2,1,""]},"dse_do_utils.plotlymanager.PlotlyManager":{get_dash_tab_layout_m:[0,3,1,""],get_plotly_fig_m:[0,3,1,""]},"dse_do_utils.scenariodbmanager":{AutoScenarioDbTable:[0,2,1,""],BusinessKpiTable:[0,2,1,""],DbCellUpdate:[0,2,1,""],KpiTable:[0,2,1,""],ParameterTable:[0,2,1,""],ScenarioDbManager:[0,2,1,""],ScenarioDbTable:[0,2,1,""],ScenarioTable:[0,2,1,""]},"dse_do_utils.scenariodbmanager.AutoScenarioDbTable":{create_table_metadata:[0,3,1,""],insert_table_in_db_bulk:[0,3,1,""]},"dse_do_utils.scenariodbmanager.DbCellUpdate":{column_name:[0,4,1,""],current_value:[0,4,1,""],previous_value:[0,4,1,""],row_idx:[0,4,1,""],row_index:[0,4,1,""],scenario_name:[0,4,1,""],table_name:[0,4,1,""]},"dse_do_utils.scenariodbmanager.ScenarioDbManager":{add_scenario_name_to_dfs:[0,3,1,""],create_schema:[0,3,1,""],delete_scenario_from_db:[0,3,1,""],delete_scenario_name_column:[0,3,1,""],drop_all_tables:[0,3,1,""],duplicate_scenario_in_db:[0,3,1,""],get_scenario_db_table:[0,3,1,""],get_scenarios_df:[0,3,1,""],insert_scenarios_in_db:[0,3,1,""],insert_tables_in_db:[0,3,1,""],read_scenario_from_db:[0,3,1,""],read_scenario_input_tables_from_db:[0,3,1,""],read_scenario_table_from_db:[0,3,1,""],read_scenario_table_from_db_cached:[0,3,1,""],read_scenario_tables_from_db:[0,3,1,""],read_scenario_tables_from_db_cached:[0,3,1,""],read_scenarios_from_db:[0,3,1,""],read_scenarios_table_from_db_cached:[0,3,1,""],rename_scenario_in_db:[0,3,1,""],replace_scenario_in_db:[0,3,1,""],replace_scenario_tables_in_db:[0,3,1,""],set_scenarios_table_read_callback:[0,3,1,""],set_table_read_callback:[0,3,1,""],update_cell_changes_in_db:[0,3,1,""],update_scenario_output_tables_in_db:[0,3,1,""]},"dse_do_utils.scenariodbmanager.ScenarioDbTable":{add_scenario_name_to_fk_constraint:[0,3,1,""],camel_case_to_snake_case:[0,3,1,""],create_table_metadata:[0,3,1,""],df_column_names_to_snake_case:[0,3,1,""],get_db_table_name:[0,3,1,""],get_df_column_names:[0,3,1,""],get_sa_column:[0,3,1,""],get_sa_table:[0,3,1,""],insert_table_in_db_bulk:[0,3,1,""],sqlcol:[0,3,1,""]},"dse_do_utils.scenariomanager":{Platform:[0,2,1,""],ScenarioManager:[0,2,1,""]},"dse_do_utils.scenariomanager.Platform":{CPD25:[0,4,1,""],CPD40:[0,4,1,""],CPDaaS:[0,4,1,""],Local:[0,4,1,""]},"dse_do_utils.scenariomanager.ScenarioManager":{add_data_file_to_project_s:[0,3,1,""],add_data_file_using_project_lib:[0,3,1,""],add_data_file_using_ws_lib:[0,3,1,""],add_data_file_using_ws_lib_s:[0,3,1,""],add_data_into_scenario:[0,3,1,""],add_data_into_scenario_s:[0,3,1,""],add_file_as_data_asset:[0,3,1,""],add_file_as_data_asset_s:[0,3,1,""],clear_scenario_data:[0,3,1,""],create_new_scenario:[0,3,1,""],detect_platform:[0,3,1,""],env_is_cpd25:[0,3,1,""],env_is_cpd40:[0,3,1,""],env_is_dsx:[0,3,1,""],env_is_wscloud:[0,3,1,""],export_model_as_lp:[0,3,1,""],get_data_directory:[0,3,1,""],get_dd_client:[0,3,1,""],get_do_scenario:[0,3,1,""],get_kpis_table_as_dataframe:[0,3,1,""],get_root_directory:[0,3,1,""],load_data:[0,3,1,""],load_data_from_csv:[0,3,1,""],load_data_from_csv_s:[0,3,1,""],load_data_from_excel:[0,3,1,""],load_data_from_excel_s:[0,3,1,""],load_data_from_scenario:[0,3,1,""],load_data_from_scenario_s:[0,3,1,""],print_table_names:[0,3,1,""],replace_data_in_scenario:[0,3,1,""],replace_data_into_scenario_s:[0,3,1,""],update_solve_output_into_scenario:[0,3,1,""],write_data_into_scenario:[0,3,1,""],write_data_into_scenario_s:[0,3,1,""],write_data_to_csv:[0,3,1,""],write_data_to_csv_s:[0,3,1,""],write_data_to_excel:[0,3,1,""],write_data_to_excel_s:[0,3,1,""]},"dse_do_utils.scenariopicker":{ScenarioPicker:[0,2,1,""]},"dse_do_utils.scenariopicker.ScenarioPicker":{ScenarioRefreshButton:[0,2,1,""],default_scenario:[0,4,1,""],get_dd_client:[0,3,1,""],get_scenario_picker_ui:[0,3,1,""],get_scenario_refresh_button:[0,3,1,""],get_scenario_select_drop_down:[0,3,1,""],get_selected_scenario:[0,3,1,""],load_selected_scenario_data:[0,3,1,""],widgets:[0,4,1,""]},"dse_do_utils.utilities":{add_sys_path:[0,1,1,""],list_file_hierarchy:[0,1,1,""]},dse_do_utils:{cpd25utilities:[0,0,0,"-"],datamanager:[0,0,0,"-"],deployeddomodel:[0,0,0,"-"],deployeddomodelcpd21:[0,0,0,"-"],domodelexporter:[0,0,0,"-"],mapmanager:[0,0,0,"-"],module_reload:[0,1,1,""],multiscenariomanager:[0,0,0,"-"],optimizationengine:[0,0,0,"-"],plotly_cpd_workaround:[0,0,0,"-"],plotlymanager:[0,0,0,"-"],scenariodbmanager:[0,0,0,"-"],scenariomanager:[0,0,0,"-"],scenariopicker:[0,0,0,"-"],utilities:[0,0,0,"-"],version:[0,0,0,"-"]}},objnames:{"0":["py","module","Python module"],"1":["py","function","Python function"],"2":["py","class","Python class"],"3":["py","method","Python method"],"4":["py","attribute","Python attribute"]},objtypes:{"0":"py:module","1":"py:function","2":"py:class","3":"py:method","4":"py:attribute"},terms:{"0":[0,1],"02ea6d480895":0,"04":0,"0596001673":0,"085594":0,"1":[0,3],"100":0,"1132":0,"16":0,"1993":0,"2":[0,3],"2005586":0,"2016":0,"21c8ac71":0,"23690284":0,"25a0fe88e4":0,"26c1":0,"3":[0,3],"31":0,"3810":0,"39":0,"4":[0,3],"41":0,"447a":0,"458550":0,"469":0,"49a5":0,"4bd2":0,"5":[0,1],"50":0,"5401":0,"585241":0,"5de6560a1cfa":0,"6":0,"600":0,"7":3,"785":0,"8021":0,"8364":0,"94":0,"95":0,"96":0,"9f28":0,"abstract":0,"case":[0,3],"class":[0,1],"default":[0,3],"do":0,"enum":0,"export":[0,3],"float":0,"function":[0,3],"import":[0,1],"int":0,"new":0,"return":0,"static":0,"true":0,"while":0,A:[0,3],And:0,As:0,At:0,But:[0,3],By:0,FOR:0,For:[0,3],If:0,In:[0,3],Is:0,It:[0,3],NOT:[0,3],No:0,Not:0,On:0,One:0,Or:0,That:0,The:[0,3],Then:[0,3],These:0,To:[0,3],Will:0,With:0,_:0,__index__:0,__init__:0,_base:0,_export_:0,_multi_output:0,_output:0,_table_index_:0,_xlx:0,a567:0,a933:0,aa50:0,abbrevi:0,abc:0,abl:[0,3],about:0,abov:0,abspath:0,accept:0,access:[0,3],access_project_or_spac:0,access_tok:0,access_token:0,accesstoken:0,accordingli:0,ad:[0,3],add:[0,3],add_child:0,add_data_file_to_project:0,add_data_file_to_project_:0,add_data_file_using_project_lib:0,add_data_file_using_ws_lib:0,add_data_file_using_ws_lib_:0,add_data_into_scenario:0,add_data_into_scenario_:0,add_file_as_data_asset:0,add_file_as_data_asset_:0,add_file_as_data_asset_cpd25:0,add_file_path_as_data_asset_cpd25:0,add_file_path_as_data_asset_wsc:0,add_full_screen:0,add_layer_control:0,add_mip_progress_kpi:0,add_scenario_name_to_df:0,add_scenario_name_to_fk_constraint:0,add_sys_path:0,add_to:0,addit:0,advanc:0,advantag:0,advis:0,after:0,air:3,alia:0,all:[0,3],allow:[0,3],along:0,alreadi:0,also:[0,3],altern:0,alwai:0,an:[0,3],ani:0,anoth:0,api:[0,3],apicli:0,app:0,appear:0,appli:[0,3],applic:0,apply_and_concat:0,ar:[0,3],arg:0,argument:0,around:[0,3],arrow:0,asset:[0,3],asset_nam:0,assign:0,assum:0,attach:0,attachment_typ:0,attempt:0,attribut:0,auto_insert:0,autodetect:0,automat:[0,3],autoreload:0,autoscenariodbt:0,avail:[0,3],avoid:[0,3],axi:0,b7bf7fd8:0,back:0,backward:3,base:0,bash:0,basic:3,bear:0,been:[0,3],befor:0,being:0,below:0,best:0,best_bound_kpi_nam:0,better:0,between:[0,3],bewar:0,binary_var_seri:0,binary_var_series_:0,binaryvartyp:0,blank:0,blob:0,blue:0,bobhaffn:0,bool:0,both:[0,3],bound:0,box:0,br:0,build:0,builder:[0,1],bulk:0,business_kpi:0,businesskpit:0,button:0,c:0,cach:0,call:0,callback:0,came:0,camel_case_to_snake_cas:0,camelcas:0,can:[0,3],cancel:0,cannot:[0,3],cartesian:0,categor:0,categori:0,caus:0,cell:[0,3],certain:0,certif:0,ch04s23:0,challeng:0,chang:0,channel:3,charact:0,chart:0,check:0,child:0,clean:0,cleanup:0,clear:0,clear_scenario_data:0,client:[0,3],cloud:[0,3],cluster4:0,cluster:0,cluster_nam:0,co:0,code:[0,3],cogno:0,collabor:0,color:0,column:0,column_nam:0,columns_metadata:0,com:0,combin:[0,3],command:0,commun:0,compass:0,compat:[0,3],complet:0,complex:0,compon:0,comput:0,conda:3,condit:0,conf:0,configur:[0,3],connect:[0,3],constant:0,constraints_metadata:0,constructor:0,contain:[0,3],content:2,context:0,continuous_var_seri:0,continuous_var_series_:0,continuousvartyp:0,control:0,conveni:0,convers:0,convert:0,cookbook:0,coord:0,copi:0,copy_to_csv:0,core:0,corner:0,correctli:0,could:0,count:0,counti:0,cp4d25:0,cp4daa:0,cp4dv2:0,cp4dv4:0,cp:[0,3],cpd25:0,cpd25util:2,cpd2:0,cpd3:[0,3],cpd40:0,cpd4:3,cpd:[0,3],cpd_cluster_host:0,cpdaa:0,cpdv2:[0,1],cpdv3:[0,3],cpdv4:[0,1],cplex:[0,3],cpo:0,cpolab:0,cpsaa:0,creat:[0,3],create_blank_map:0,create_new_scenario:0,create_schema:0,create_table_metadata:0,creation:0,credenti:0,cross:0,crossjoin:3,csv:[0,3],csv_directori:0,csv_name_pattern:0,curl:0,current:0,current_dir:0,current_valu:0,custom:0,d4c69a0d8158:0,d:[0,3],dash:0,dashboard:0,dashenterpris:0,data:[0,3],data_asset:[0,3],data_by_scenario:0,data_id:0,data_manag:0,data_url:0,datafram:0,datamanag:[2,3],datascienceelit:0,dataset:0,datetim:0,db:0,db_cell_upd:0,db_column_nam:0,db_table_nam:0,dbcellupd:0,dd:[0,3],dd_scenario:0,de:0,debug:0,debug_file_data_url:0,debug_file_dir:0,decis:3,decision_optimization_cli:0,def:0,default_max_oaas_time_limit_sec:0,default_max_run_time_sec:0,default_scenario:0,default_valu:0,defin:0,definit:0,delai:0,delet:0,delete_scenario_from_db:0,delete_scenario_name_column:0,depend:[0,3],deploi:0,deploy:[0,3],deployed_model_nam:0,deployeddomodel:2,deployeddomodel_cpd21:0,deployeddomodelcpd21:2,deployment_id:0,deprec:0,design:[0,3],despit:0,detail:3,detect_platform:0,develop:[0,3],df1:0,df2:0,df:0,df_column_names_to_snake_cas:0,df_crossjoin_ai:0,df_crossjoin_mi:0,df_crossjoin_si:0,df_dict:0,dict:0,dictionari:0,differ:[0,3],direct:0,directori:0,disadvantag:0,disk:0,dm:0,do4w:0,do_model_nam:0,do_util:0,doc:0,docplex:[0,3],document:[0,3],dodashapp:0,doe:0,doesn:0,domodeldeploy:2,domodelexport:2,done:0,down:[0,3],download:[0,3],driven:0,drop:[0,3],drop_all_t:0,drop_column_nam:0,dropdown:0,dse:[0,3],dse_do_dashboard:0,dse_do_util:1,dsx:0,dsx_project_dir:0,dsxuser:0,due:0,dump:0,dumpzipnam:0,duplic:0,duplicate_scenario_in_db:0,dure:0,dvar:[0,3],dynam:0,e:[0,3],each:0,easi:3,easier:0,echo:0,ef365b2c:0,effect:0,effici:0,eg:0,either:0,element:0,els:0,emb:0,empti:[0,3],en:0,enabl:0,enable_sqlite_fk:0,enable_transact:0,end:0,engin:[0,3],ensur:0,entiti:0,entri:0,enumer:0,env_is_cpd25:0,env_is_cpd40:0,env_is_dsx:0,env_is_wscloud:0,environ:[0,1],error:0,establish:0,evalu:0,exampl:0,excel:[0,3],excel_file_nam:0,excel_output_file_nam:0,excel_test:0,excelfil:0,excelwrit:0,exclud:0,execut:0,execute_model:0,execution_result:0,execution_statu:0,execution_status_json:0,exist:[0,3],expect:0,experi:0,explicit:0,explicitli:0,export_as_cpo:0,export_as_cpo_:0,export_as_lp:0,export_as_lp_:0,export_do_model:0,export_model:0,export_model_as_lp:0,expos:0,express:0,extend:0,extended_columns_metadata:0,extens:0,extract:0,extract_dvar_nam:0,extract_solut:0,f:0,fail:0,fake:3,fals:0,faster:0,featur:3,featuregroup:0,field:0,fig:0,file:[0,3],file_nam:0,file_path:0,filenam:0,filter:0,find:0,firefox:0,first:0,fix:0,fkc:0,flask:0,folder:0,folium:[0,3],follow:[0,3],forc:0,foreign:0,foreignkeyconstraint:0,form:0,format:0,former:[0,3],found:0,frame:0,free:0,from:[0,3],full:0,full_project_nam:0,func:0,futur:[0,3],g:[0,3],gap:[0,3],gener:0,get:0,get_access_token_curl:0,get_access_token_web:0,get_all_scenario_nam:0,get_arrow:0,get_bear:0,get_dash_tab_layout_m:0,get_data_directori:0,get_db_table_nam:0,get_dd_client:0,get_debug_dump_name_and_url:0,get_debug_file_url:0,get_deployment_id:0,get_df_column_nam:0,get_do_model_export_curl:0,get_do_model_export_web:0,get_do_scenario:0,get_execution_service_model_url:0,get_execution_statu:0,get_head:0,get_html_tabl:0,get_input_fil:0,get_job_statu:0,get_job_url:0,get_kill_job_url:0,get_kpi_output_t:0,get_kpis_table_as_datafram:0,get_log_file_name_and_url:0,get_log_file_url:0,get_multi_scenario_data:0,get_object:0,get_output:0,get_parameter_valu:0,get_plotly_fig_m:0,get_popup_t:0,get_project_id:0,get_raw_table_by_nam:0,get_root_directori:0,get_sa_column:0,get_sa_t:0,get_scenario_db_t:0,get_scenario_picker_ui:0,get_scenario_refresh_button:0,get_scenario_select_drop_down:0,get_scenarios_df:0,get_selected_scenario:0,get_solution_name_and_url:0,get_solve_config:0,get_solve_detail:0,get_solve_details_object:0,get_solve_payload:0,get_solve_statu:0,get_solve_url:0,get_space_id:0,get_stop_job_url:0,get_tab_layout_:0,getcwd:0,gist:0,git:0,github:[0,3],githubpag:3,give:0,given:0,glob:0,grand:0,greatli:3,gsilabs_scnfo:0,h:0,ha:[0,3],hack:3,hand:0,happen:0,hard:0,hashtabl:0,have:[0,3],height:0,hello:0,here:0,hierarch:0,hold:0,home:0,how:[0,3],howev:0,html:0,http:0,hypothet:0,i:0,ibm:[0,3],ibm_watson_machine_learn:0,ibm_watson_studio_lib:0,icon:0,icpd:3,id:0,ignor:[0,3],imp:0,impact:3,implement:0,improv:[0,3],includ:0,inconsist:0,independ:0,index:[0,1],indic:0,individu:3,info:0,inform:0,init:0,initi:0,inline_t:0,inner:0,input:0,input_csv_name_pattern:0,input_db_t:0,input_table_nam:0,insecurerequestwarn:0,insert:0,insert_scenarios_in_db:0,insert_table_in_db_bulk:0,insert_tables_in_db:0,instal:[0,1],installationreadm:3,instanc:0,instead:0,integ:0,integer_var_list:0,integer_var_seri:0,integer_var_series_:0,integervartyp:0,intend:3,interact:[0,3],interfac:3,intermedi:0,intern:[0,3],internet:3,interrupt:0,interv:0,io:0,ipynb:0,ipywidget:0,issu:0,itself:3,jerom:0,job:0,job_config_json:0,job_detail:0,job_uid:0,join:0,json:0,jupyt:[0,3],jupyterlab:3,just:0,kansas_city_coord:0,karg:0,keep:0,kei:0,keyword:0,kill:0,kill_job:0,km:0,kpi:0,kpitabl:0,kwarg:0,lambda:0,last:0,lat:0,later:0,latest:0,layer_control_posit:0,lead:0,leav:0,left:0,let:0,level:0,lib:0,librari:0,like:[0,3],limit:[0,3],line:0,list:0,list_file_hierarchi:0,load:0,load_data:0,load_data_from_csv:0,load_data_from_csv_:0,load_data_from_excel:0,load_data_from_excel_:0,load_data_from_scenario:0,load_data_from_scenario_:0,load_from_excel:0,load_selected_scenario_data:0,local:[0,3],local_root:0,locat:0,log:0,log_file_nam:0,logic:0,lon:0,longer:[0,3],look:0,loop:0,lp:[0,3],m:0,machin:0,made:0,mai:0,main:[0,1],maintain:0,major:0,make:0,manag:0,mani:3,manipul:[0,3],manual:0,map:[0,3],mapmanag:[2,3],marker:0,master:0,match:0,max_oaas_time_limit_sec:0,max_run_time_sec:0,maximum:0,mb:3,md:3,mdl:0,me:[0,1],mean:0,medium:0,member:0,menu:[0,3],merg:0,merge_scenario_data:0,messag:0,metadata:0,method:0,mgr:0,might:3,minu:0,mip_gap_kpi_nam:0,miss:0,mix:0,mkonrad:0,mode:0,model1:0,model2:0,model:[0,1],model_build:0,model_nam:0,modelbuild:0,modifi:0,modul:[1,2],module_reload:0,moment:3,monitor:0,monitor_execut:0,monitor_loop_delay_sec:0,more:[0,3],most:0,mostli:3,move:3,mp:0,msm:0,multi:0,multi_scenario:0,multi_thread:0,multiindex:0,multipl:[0,3],multiscenariomanag:2,must:0,my:0,my_default_scenario:0,my_do_model:0,my_funct:0,my_input_column_nam:0,my_input_valu:0,my_output_column_name_1:0,my_output_column_name_2:0,my_output_value1:0,my_output_value_1:0,my_output_value_2:0,my_tabl:0,myexcelfil:0,myexcelfileoutput:0,myfil:0,mymodel:0,myoptimizationengin:0,myprogresslisten:0,mytabl:0,n:0,n_arrow:0,name:0,namedtupl:0,nbviewer:0,necessari:0,need:0,neither:3,net:0,never:0,new_path:0,new_scenario_nam:0,next:0,nodefault:3,non:0,none:0,not_start:0,note:[0,3],notebook:[0,3],notify_progress:0,now:0,number:0,oaa:0,object:0,off:0,ok:0,omit:0,onc:0,one:0,ones:0,onli:[0,3],open:0,oper:0,optim:[0,3],optimizationengin:[2,3],option:[0,3],order:0,ordereddict:0,ore:3,oreilli:0,org:0,organ:0,origin:0,os:0,other:[0,3],otherwis:0,out:0,output:0,output_csv_name_pattern:0,output_db_t:0,output_table_nam:0,outsid:3,overrid:0,overwrit:0,overwritten:0,p1:0,p2:0,packag:[1,2,3],page:[0,1],page_id:0,page_sourc:0,pak:3,panda:0,panel:0,param:0,param_nam:0,param_typ:0,paramet:0,parametert:0,pardir:0,parent:0,parent_dir:0,parent_dir_2:0,pars:0,parse_html:0,part:0,particular:[0,3],pass:0,password:0,past:0,path:0,pattern:0,payload:0,pd:0,per:0,perform:[0,3],period:0,person:0,phase:0,pick:3,picker:0,pip:[0,3],place:0,placehold:0,plain:0,platform:0,plot:0,plotli:0,plotly_cpd_workaround:2,plotlymanag:2,plugin:0,point:0,poll:0,polylin:0,popul:0,popup:0,popup_t:0,possibl:0,post:[0,3],post_process_contain:0,post_process_container_get_datafram:0,post_process_fail:0,post_process_inline_t:0,post_process_inline_table_get_datafram:0,post_process_interrupt:0,post_process_process:0,practic:0,pre:[0,3],prefer:3,prep_paramet:0,prepar:0,prepare_data_fram:0,prepare_input_data_fram:0,prepare_output_data_fram:0,prevent:0,previou:3,previous_valu:0,primari:0,print:0,print_hello:0,print_inputs_outputs_summari:0,print_table_nam:0,problem:0,procedur:0,process:[0,3],product:0,productionplan:0,progress:0,progress_data:0,progressdata:0,project:[0,3],project_access_token:0,project_data:[0,3],project_id:0,project_lib:0,project_nam:0,properti:0,property_1:0,property_2:0,provid:[0,3],put:3,pwd:0,py:0,pydata:0,pypi:3,python:[0,3],question:0,queu:0,quot:0,rais:0,raw:0,re:0,reach:0,read:[0,1],read_csv:0,read_scenario_from_db:0,read_scenario_input_tables_from_db:0,read_scenario_table_from_db:0,read_scenario_table_from_db_cach:0,read_scenario_table_from_db_callback:0,read_scenario_tables_from_db:0,read_scenario_tables_from_db_cach:0,read_scenarios_from_db:0,read_scenarios_table_from_db_cach:0,read_scenarios_table_from_db_callback:0,readthedoc:0,reason:0,record:0,reduc:0,refactor:3,refer:0,refine_conflict:0,refresh:0,regist:0,regular:0,regularli:[0,3],rel:0,relat:0,relationship:0,releas:3,relev:0,reliabl:0,reload:0,remot:0,remov:0,renam:0,rename_scenario_in_db:0,repeatedli:0,replac:0,replace_data_in_scenario:0,replace_data_into_scenario_:0,replace_scenario_in_db:0,replace_scenario_tables_in_db:0,repositori:3,repres:0,represent:0,request:0,requir:[0,1],rerun:0,resourc:0,respons:0,rest:0,restor:0,restrict:0,result:0,retreiv:0,retriev:0,retrieve_debug_materi:0,retrieve_fil:0,retrieve_solut:0,retrieve_solve_configur:0,right:0,root:0,root_dir:0,rotat:0,round:0,rout:0,routin:0,row:0,row_idx:0,row_index:0,run:[0,3],runtime_env_apsx_url:0,s:0,same:[0,3],save:0,scenario:[0,3],scenario_1:0,scenario_nam:0,scenario_table_nam:0,scenariodbmanag:2,scenariodbt:0,scenariomanag:[2,3],scenariopick:[2,3],scenariorefreshbutton:0,scenarios_table_read_callback:0,scenariot:0,schema:0,scnfo:0,scnfo_dash_pycharm_github:0,scope:1,screen:0,script:3,search:[0,1],second:0,see:[0,3],seem:0,select:0,self:0,separ:[0,3],sequenc:0,seri:0,server:0,servic:0,service_configuration_json:0,service_nam:0,set:0,set_index:0,set_output_settings_in_solve_configur:0,set_scenarios_table_read_callback:0,set_table_read_callback:0,setup:0,share:3,sheet:0,should:[0,3],show:0,side:0,signific:3,similar:0,simpl:0,simplest:0,simpli:0,sinc:0,singl:0,site:0,size:0,skip:0,sm:0,small:0,snake_cas:0,so:0,solut:0,solution_count_kpi_nam:0,solutionlisten:0,solv:0,solve_config:0,solve_config_json:0,solve_payload:0,solve_phase_kpi_nam:0,solve_statu:0,solve_time_kpi_nam:0,solvesolut:0,some:[0,3],someth:0,somewhat:0,sourc:[0,3],source_scenario_nam:0,sp:0,space:0,space_id:0,space_nam:0,spd:0,specifi:0,speed:0,spreadhseet:3,spreadsheet:0,sql:0,sqlalchemi:0,sqlcol:0,ssl:0,stackoverflow:0,stamp:0,standard:0,start:0,startpath:0,state:0,statement:0,statu:0,step:0,stop:0,stop_job:0,storag:0,store:[0,3],str:0,strftime:0,string:0,strongli:0,structur:0,studio:[0,3],sub:0,subclass:[0,3],submiss:0,submodul:2,succesfulli:0,suffici:0,suggest:0,summari:0,support:0,suppress:0,suppress_warn:0,sure:0,sy:0,system:[0,3],t:0,tabl:[0,3],table_index_sheet:0,table_nam:0,table_read_callback:0,target:[0,1],target_scenario_nam:0,task:0,templat:0,template_scenario_nam:0,temporari:0,termin:0,test:0,text:0,than:0,thei:0,them:0,therebi:0,therefor:[0,3],thi:[0,3],thing:0,those:0,thu:0,time:0,todo:0,token:0,tooltip:0,top:0,topleft:0,touch:0,track:0,transact:0,translat:0,tree:0,truth:0,tupl:0,turn:0,two:0,txt:0,type:0,typic:[0,3],ugli:0,ui:0,un:0,under:0,unfortun:0,uniqu:0,unknown:0,unnam:0,untest:0,unverifi:0,up:[0,3],updat:0,update_cell_changes_in_db:0,update_scenario_output_tables_in_db:0,update_solve_output_into_scenario:0,upload:[0,3],url:0,urllib3:0,us:[0,3],usag:[0,3],user:0,user_access_token:0,user_nam:0,util:[2,3],v0:3,v2:3,v4:3,valid:0,valu:0,value_1:0,value_2:0,value_format:0,valueerror:0,variabl:0,venv:0,veri:3,verif:0,verifi:0,verify_integr:0,version:[2,3],via:0,view:0,visibl:0,visual:[0,3],w:0,wa:0,wai:0,want:0,warn:0,watson:[0,3],watsonmachinelearningapicli:0,we:0,web:0,well:0,what:0,whatev:0,wheel:3,when:[0,3],where:0,which:0,whole:0,widget:0,widget_button:0,widget_select:0,width:0,wil:0,window:0,within:[0,3],without:0,wml:1,wml_credenti:0,work:[0,3],world:0,would:0,write:[0,3],write_data_asset_as_file_cpd25:0,write_data_asset_as_file_wsc:0,write_data_into_scenario:0,write_data_into_scenario_:0,write_data_to_csv:0,write_data_to_csv_:0,write_data_to_excel:0,write_data_to_excel_:0,write_do_model_to_fil:0,writer:0,written:0,ws:[0,3],wsl1:0,wsl:[0,3],wslib:0,wslv1:3,wsuser:0,www:0,xdvar:0,xl:0,xlsx:0,xlx:0,xxxxxx:0,y:0,yet:0,you:0,your:0,yyyymmdd_hhmm:0,zip:[0,3],zoom_start:0},titles:["dse_do_utils package","Welcome to DSE DO Utils documentation!","dse_do_utils","Read me"],titleterms:{"0":3,"5":3,"class":3,"do":[1,3],"import":3,builder:3,content:[0,1],cpd25util:0,cpdv2:3,cpdv4:3,custom:3,datamanag:0,deployeddomodel:0,deployeddomodelcpd21:0,document:1,domodeldeploy:0,domodelexport:0,dse:1,dse_do_util:[0,2,3],environ:3,indic:1,instal:3,main:3,mapmanag:0,me:3,model:3,modul:[0,3],multiscenariomanag:0,optimizationengin:0,packag:0,plotly_cpd_workaround:0,plotlymanag:0,read:3,requir:3,scenariodbmanag:0,scenariomanag:0,scenariopick:0,scope:3,submodul:0,tabl:1,target:3,util:[0,1],version:0,welcom:1,wml:3}})
\ No newline at end of file
diff --git a/dse_do_utils/scenariodbmanager.py b/dse_do_utils/scenariodbmanager.py
index c59beee..472ed3c 100644
--- a/dse_do_utils/scenariodbmanager.py
+++ b/dse_do_utils/scenariodbmanager.py
@@ -19,10 +19,11 @@
# - Make 'multi_scenario' the default option
# -----------------------------------------------------------------------------------
from abc import ABC
+from multiprocessing.pool import ThreadPool
import sqlalchemy
import pandas as pd
-from typing import Dict, List
+from typing import Dict, List, NamedTuple, Any, Optional
from collections import OrderedDict
import re
from sqlalchemy import exc
@@ -62,6 +63,7 @@ def __init__(self, db_table_name: str, columns_metadata: List[sqlalchemy.Column]
reserved_table_names = ['order', 'parameter'] # TODO: add more reserved words for table names
if db_table_name in reserved_table_names:
print(f"Warning: the db_table_name '{db_table_name}' is a reserved word. Do not use as table name.")
+ self._sa_column_by_name = None # Dict[str, sqlalchemy.Column] Will be generated dynamically first time it is needed.
def get_db_table_name(self) -> str:
return self.db_table_name
@@ -74,7 +76,21 @@ def get_df_column_names(self) -> List[str]:
column_names.append(c.name)
return column_names
- def create_table_metadata(self, metadata, multi_scenario: bool = False):
+ def get_sa_table(self) -> sqlalchemy.Table:
+ """Returns the SQLAlchemy Table"""
+ return self.table_metadata
+
+ def get_sa_column(self, db_column_name) -> Optional[sqlalchemy.Column]:
+ """Returns the SQLAlchemy column with the specified name.
+ Dynamically creates a dict/hashtable for more efficient access."""
+ # for c in self.columns_metadata:
+ # if isinstance(c, sqlalchemy.Column) and c.name == db_column_name:
+ # return c
+ if self._sa_column_by_name is None:
+ self._sa_column_by_name = {c.name: c for c in self.columns_metadata if isinstance(c, sqlalchemy.Column)}
+ return self._sa_column_by_name.get(db_column_name) # returns None if npt found (?)
+
+ def create_table_metadata(self, metadata, multi_scenario: bool = False) -> sqlalchemy.Table:
"""If multi_scenario, then add a primary key 'scenario_name'."""
columns_metadata = self.columns_metadata
constraints_metadata = self.constraints_metadata
@@ -131,6 +147,15 @@ def insert_table_in_db_bulk(self, df: pd.DataFrame, mgr, connection=None):
print(f"DataFrame insert/append of table '{table_name}'")
print(e)
+ def _delete_scenario_table_from_db(self, scenario_name, connection):
+ """Delete all rows associated with the scenario in the DB table.
+ Beware: make sure this is done in the right 'inverse cascading' order to avoid FK violations.
+ """
+ # sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
+ t = self.get_sa_table() # A Table()
+ sql = t.delete().where(t.c.scenario_name == scenario_name)
+ connection.execute(sql)
+
@staticmethod
def sqlcol(df: pd.DataFrame) -> Dict:
dtypedict = {}
@@ -201,6 +226,16 @@ def insert_table_in_db_bulk(self, df, mgr, connection=None):
print(e)
+class DbCellUpdate(NamedTuple):
+ scenario_name: str
+ table_name: str
+ row_index: List[Dict[str, Any]] # e.g. [{'column': 'col1', 'value': 1}, {'column': 'col2', 'value': 'pear'}]
+ column_name: str
+ current_value: Any
+ previous_value: Any # Not used for DB operation
+ row_idx: int # Not used for DB operation
+
+
#########################################################################
# ScenarioDbManager
#########################################################################
@@ -256,6 +291,11 @@ def _add_scenario_db_table(self, input_db_tables: Dict[str, ScenarioDbTable]) ->
print("Warning: the `Scenario` table should be the first in the input tables")
return input_db_tables
+ def get_scenario_db_table(self) -> ScenarioDbTable:
+ """Scenario table must be the first in self.input_db_tables"""
+ db_table: ScenarioTable = list(self.input_db_tables.values())[0]
+ return db_table
+
def _create_database_engine(self, credentials=None, schema: str = None, echo: bool = False):
"""Creates a SQLAlchemy engine at initialization.
If no credentials, creates an in-memory SQLite DB. Which can be used for schema validation of the data.
@@ -412,9 +452,9 @@ def create_schema(self):
with self.engine.begin() as connection:
self._create_schema_transaction(connection=connection)
else:
- self._create_schema_transaction()
+ self._create_schema_transaction(self.engine)
- def _create_schema_transaction(self, connection=None):
+ def _create_schema_transaction(self, connection):
"""(Re)creates a schema, optionally using a transaction
Drops all tables and re-creates the schema in the DB."""
# if self.schema is None:
@@ -423,10 +463,7 @@ def _create_schema_transaction(self, connection=None):
# self.drop_schema_transaction(self.schema)
# DROP SCHEMA isn't working properly, so back to dropping all tables
self._drop_all_tables_transaction(connection=connection)
- if connection is None:
- self.metadata.create_all(self.engine, checkfirst=True)
- else:
- self.metadata.create_all(connection, checkfirst=True)
+ self.metadata.create_all(connection, checkfirst=True)
def drop_all_tables(self):
"""Drops all tables in the current schema."""
@@ -434,9 +471,9 @@ def drop_all_tables(self):
with self.engine.begin() as connection:
self._drop_all_tables_transaction(connection=connection)
else:
- self._drop_all_tables_transaction()
+ self._drop_all_tables_transaction(self.engine)
- def _drop_all_tables_transaction(self, connection=None):
+ def _drop_all_tables_transaction(self, connection):
"""Drops all tables as defined in db_tables (if exists)
TODO: loop over tables as they exist in the DB.
This will make sure that however the schema definition has changed, all tables will be cleared.
@@ -447,15 +484,18 @@ def _drop_all_tables_transaction(self, connection=None):
However, the order is alphabetically, which causes FK constraint violation
Weirdly, this happens in SQLite, not in DB2! With or without transactions
+
+ TODO:
+ 1. Use SQLAlchemy to drop table, avoid text SQL
+ 2. Drop all tables without having to loop and know all tables
+ See: https://stackoverflow.com/questions/35918605/how-to-delete-a-table-in-sqlalchemy)
+ See https://docs.sqlalchemy.org/en/14/core/metadata.html#sqlalchemy.schema.MetaData.drop_all
"""
for scenario_table_name, db_table in reversed(self.db_tables.items()):
db_table_name = db_table.db_table_name
sql = f"DROP TABLE IF EXISTS {db_table_name}"
# print(f"Dropping table {db_table_name}")
- if connection is None:
- r = self.engine.execute(sql)
- else:
- r = connection.execute(sql)
+ connection.execute(sql)
def _drop_schema_transaction(self, schema: str, connection=None):
"""NOT USED. Not working in DB2 Cloud.
@@ -463,6 +503,7 @@ def _drop_schema_transaction(self, schema: str, connection=None):
See: https://www.ibm.com/docs/en/db2/11.5?topic=procedure-admin-drop-schema-drop-schema
However, this doesn't work on DB2 cloud.
TODO: find out if and how we can get this to work.
+ See https://docs.sqlalchemy.org/en/14/core/metadata.html#sqlalchemy.schema.MetaData.drop_all
"""
# sql = f"DROP SCHEMA {schema} CASCADE" # Not allowed in DB2!
sql = f"CALL SYSPROC.ADMIN_DROP_SCHEMA('{schema}', NULL, 'ERRORSCHEMA', 'ERRORTABLE')"
@@ -515,18 +556,20 @@ def replace_scenario_in_db(self, scenario_name: str, inputs: Inputs = {}, output
if self.enable_transactions:
print("Replacing scenario within transaction")
with self.engine.begin() as connection:
- self._replace_scenario_in_db_transaction(scenario_name=scenario_name, inputs=inputs, outputs=outputs, bulk=bulk, connection=connection)
+ self._replace_scenario_in_db_transaction(connection, scenario_name=scenario_name, inputs=inputs, outputs=outputs, bulk=bulk)
else:
- self._replace_scenario_in_db_transaction(scenario_name=scenario_name, inputs=inputs, outputs=outputs, bulk=bulk)
+ self._replace_scenario_in_db_transaction(self.engine, scenario_name=scenario_name, inputs=inputs, outputs=outputs, bulk=bulk)
- def _replace_scenario_in_db_transaction(self, scenario_name: str, inputs: Inputs = {}, outputs: Outputs = {},
- bulk: bool = True, connection=None):
+ def _replace_scenario_in_db_transaction(self, connection, scenario_name: str, inputs: Inputs = {}, outputs: Outputs = {},
+ bulk: bool = True):
"""Replace a single full scenario in the DB. If doesn't exist, will insert.
Only inserts tables with an entry defined in self.db_tables (i.e. no `auto_insert`).
Will first delete all rows associated with a scenario_name.
Will set/overwrite the scenario_name in all dfs, so no need to add in advance.
Assumes schema has been created.
Note: there is no difference between dfs in inputs or outputs, i.e. they are inserted the same way.
+
+ TODO: break-out in a delete and an insert. Then we can re-use the insert for the duplicate API
"""
# Step 1: delete scenario if exists
self._delete_scenario_from_db(scenario_name, connection=connection)
@@ -534,11 +577,10 @@ def _replace_scenario_in_db_transaction(self, scenario_name: str, inputs: Inputs
inputs = ScenarioDbManager.add_scenario_name_to_dfs(scenario_name, inputs)
outputs = ScenarioDbManager.add_scenario_name_to_dfs(scenario_name, outputs)
# Step 3: insert scenario_name in scenario table
- sql = f"INSERT INTO SCENARIO (scenario_name) VALUES ('{scenario_name}')"
- if connection is None:
- self.engine.execute(sql)
- else:
- connection.execute(sql)
+ # sql = f"INSERT INTO SCENARIO (scenario_name) VALUES ('{scenario_name}')"
+ sa_scenario_table = self.get_scenario_db_table().get_sa_table()
+ sql_insert = sa_scenario_table.insert().values(scenario_name = scenario_name)
+ connection.execute(sql_insert)
# Step 4: (bulk) insert scenario
num_caught_exceptions = self._insert_single_scenario_tables_in_db(inputs=inputs, outputs=outputs, bulk=bulk, connection=connection)
# Throw exception if any exceptions caught in 'non-bulk' mode
@@ -546,34 +588,38 @@ def _replace_scenario_in_db_transaction(self, scenario_name: str, inputs: Inputs
if num_caught_exceptions > 0:
raise RuntimeError(f"Multiple ({num_caught_exceptions}) Integrity and/or Statement errors caught. See log. Raising exception to allow for rollback.")
- def _delete_scenario_from_db(self, scenario_name: str, connection=None):
- """Deletes all rows associated with a given scenario.
- Note that it only deletes rows from tables defined in the self.db_tables, i.e. will NOT delete rows in 'auto-inserted' tables!
- Must do a 'cascading' delete to ensure not violating FK constraints. In reverse order of how they are inserted.
- Also deletes entry in scenario table
- TODO: do within one session/cursor, so we don't have to worry about the order of the delete?
- """
- insp = sqlalchemy.inspect(self.engine)
- for scenario_table_name, db_table in reversed(self.db_tables.items()):
- if insp.has_table(db_table.db_table_name, schema=self.schema):
- sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'"
- if connection is None:
- self.engine.execute(sql)
- else:
- connection.execute(sql)
-
- # Delete scenario entry in scenario table:
- sql = f"DELETE FROM SCENARIO WHERE scenario_name = '{scenario_name}'"
- if connection is None:
- self.engine.execute(sql)
- else:
- connection.execute(sql)
+ # def _delete_scenario_from_db(self, scenario_name: str, connection=None):
+ # """Deletes all rows associated with a given scenario.
+ # Note that it only deletes rows from tables defined in the self.db_tables, i.e. will NOT delete rows in 'auto-inserted' tables!
+ # Must do a 'cascading' delete to ensure not violating FK constraints. In reverse order of how they are inserted.
+ # Also deletes entry in scenario table
+ # """
+ # insp = sqlalchemy.inspect(self.engine)
+ # for scenario_table_name, db_table in reversed(self.db_tables.items()):
+ # if insp.has_table(db_table.db_table_name, schema=self.schema):
+ #
+ # # sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'"
+ # t: sqlalchemy.Table = db_table.get_sa_table() # A Table()
+ # sql = t.delete().where(t.c.scenario_name == scenario_name)
+ # if connection is None:
+ # self.engine.execute(sql)
+ # else:
+ # connection.execute(sql)
+ #
+ # # Delete scenario entry in scenario table:
+ # # sql = f"DELETE FROM SCENARIO WHERE scenario_name = '{scenario_name}'"
+ # t: sqlalchemy.Table = self.get_scenario_db_table().get_sa_table() # A Table()
+ # sql = t.delete().where(t.c.scenario_name == scenario_name)
+ # if connection is None:
+ # self.engine.execute(sql)
+ # else:
+ # connection.execute(sql)
def _insert_single_scenario_tables_in_db(self, inputs: Inputs = {}, outputs: Outputs = {},
bulk: bool = True, connection=None) -> int:
"""Specifically for single scenario replace/insert.
Does NOT insert into the `scenario` table.
- No `auto_insert`, i.e. only df matching db_tables.
+ No `auto_insert`, i.e. only df matching db_tables. TODO: verify if doesn't work with AutoScenarioDbTable
"""
num_caught_exceptions = 0
dfs = {**inputs, **outputs} # Combine all dfs in one dict
@@ -666,13 +712,19 @@ def insert_tables_in_db(self, inputs: Inputs = {}, outputs: Outputs = {},
############################################################################################
# Read scenario
############################################################################################
- def get_scenarios_df(self):
+ def get_scenarios_df(self) -> pd.DataFrame:
"""Return all scenarios in df. Result is indexed by `scenario_name`.
Main API to get all scenarios.
The API called by a cached procedure in the dse_do_dashboard.DoDashApp.
"""
- sql = f"SELECT * FROM SCENARIO"
- df = pd.read_sql(sql, con=self.engine).set_index(['scenario_name'])
+ # sql = f"SELECT * FROM SCENARIO"
+ sa_scenario_table = list(self.input_db_tables.values())[0].table_metadata
+ sql = sa_scenario_table.select()
+ if self.enable_transactions:
+ with self.engine.begin() as connection:
+ df = pd.read_sql(sql, con=connection).set_index(['scenario_name'])
+ else:
+ df = pd.read_sql(sql, con=self.engine).set_index(['scenario_name'])
return df
def read_scenario_table_from_db(self, scenario_name: str, scenario_table_name: str) -> pd.DataFrame:
@@ -691,36 +743,460 @@ def read_scenario_table_from_db(self, scenario_name: str, scenario_table_name: s
# error!
raise ValueError(f"Scenario table name '{scenario_table_name}' unknown. Cannot load data from DB.")
- df = self._read_scenario_db_table_from_db(scenario_name, db_table)
+ if self.enable_transactions:
+ with self.engine.begin() as connection:
+ df = self._read_scenario_db_table_from_db(scenario_name, db_table, connection)
+ else:
+ df = self._read_scenario_db_table_from_db(scenario_name, db_table, self.engine)
return df
- def read_scenario_from_db(self, scenario_name: str) -> (Inputs, Outputs):
+ # def read_scenario_from_db(self, scenario_name: str) -> (Inputs, Outputs):
+ # """Single scenario load.
+ # Main API to read a complete scenario.
+ # Reads all tables for a single scenario.
+ # Returns all tables in one dict"""
+ # inputs = {}
+ # for scenario_table_name, db_table in self.input_db_tables.items():
+ # inputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table)
+ #
+ # outputs = {}
+ # for scenario_table_name, db_table in self.output_db_tables.items():
+ # outputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table)
+ #
+ # return inputs, outputs
+
+
+ # def _read_scenario_from_db(self, scenario_name: str, connection) -> (Inputs, Outputs):
+ # """Single scenario load.
+ # Main API to read a complete scenario.
+ # Reads all tables for a single scenario.
+ # Returns all tables in one dict
+ # """
+ # inputs = {}
+ # for scenario_table_name, db_table in self.input_db_tables.items():
+ # # print(f"scenario_table_name = {scenario_table_name}")
+ # if scenario_table_name != 'Scenario': # Skip the Scenario table as an input
+ # inputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table, connection=connection)
+ #
+ # outputs = {}
+ # for scenario_table_name, db_table in self.output_db_tables.items():
+ # outputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table, connection=connection)
+ # # if scenario_table_name == 'kpis':
+ # # # print(f"kpis table columns = {outputs[scenario_table_name].columns}")
+ # # outputs[scenario_table_name] = outputs[scenario_table_name].rename(columns={'name': 'NAME'}) #HACK!!!!!
+ # return inputs, outputs
+ def read_scenario_from_db(self, scenario_name: str, multi_threaded: bool = False) -> (Inputs, Outputs):
"""Single scenario load.
Main API to read a complete scenario.
Reads all tables for a single scenario.
- Returns all tables in one dict"""
+ Returns all tables in one dict
+
+ Note: multi_threaded doesn't seem to lead to performance improvement.
+ Fixed: omit reading scenario table as an input.
+ """
+ # print(f"read_scenario_from_db.multi_threaded = {multi_threaded}")
+ if multi_threaded:
+ inputs, outputs = self._read_scenario_from_db_multi_threaded(scenario_name)
+ else:
+ if self.enable_transactions:
+ with self.engine.begin() as connection:
+ inputs, outputs = self._read_scenario_from_db(scenario_name, connection)
+ else:
+ inputs, outputs = self._read_scenario_from_db(scenario_name, self.engine)
+ return inputs, outputs
+
+ def _read_scenario_from_db(self, scenario_name: str, connection) -> (Inputs, Outputs):
+ """Single scenario load.
+ Main API to read a complete scenario.
+ Reads all tables for a single scenario.
+ Returns all tables in one dict
+ """
inputs = {}
for scenario_table_name, db_table in self.input_db_tables.items():
- inputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table)
+ # print(f"scenario_table_name = {scenario_table_name}")
+ if scenario_table_name != 'Scenario': # Skip the Scenario table as an input
+ inputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table, connection=connection)
outputs = {}
for scenario_table_name, db_table in self.output_db_tables.items():
- outputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table)
+ outputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table, connection=connection)
return inputs, outputs
- def _read_scenario_db_table_from_db(self, scenario_name: str, db_table: ScenarioDbTable) -> pd.DataFrame:
+ def _read_scenario_from_db_multi_threaded(self, scenario_name) -> (Inputs, Outputs):
+ """Reads all tables from a scenario using multi-threading.
+ Does NOT seem to result in performance improvement!"""
+ class ReadTableFunction(object):
+ def __init__(self, dbm):
+ self.dbm = dbm
+ def __call__(self, scenario_table_name, db_table):
+ return self._read_scenario_db_table_from_db_thread(scenario_table_name, db_table)
+ def _read_scenario_db_table_from_db_thread(self, scenario_table_name, db_table):
+ with self.dbm.engine.begin() as connection:
+ df = self.dbm._read_scenario_db_table_from_db(scenario_name, db_table, connection)
+ dict = {scenario_table_name: df}
+ return dict
+
+ thread_number = 8
+ pool = ThreadPool(thread_number)
+ thread_worker = ReadTableFunction(self)
+ # print("ThreadPool created")
+ all_tables = [(scenario_table_name, db_table) for scenario_table_name, db_table in self.db_tables.items() if scenario_table_name != 'Scenario']
+ # print(all_tables)
+ all_results = pool.starmap(thread_worker, all_tables)
+ inputs = {k:v for element in all_results for k,v in element.items() if k in self.input_db_tables.keys()}
+ outputs = {k:v for element in all_results for k,v in element.items() if k in self.output_db_tables.keys()}
+ # print("All tables loaded")
+
+ return inputs, outputs
+
+ def read_scenario_input_tables_from_db(self, scenario_name: str):
+ """Convenience method to load all input tables.
+ Typically used at start if optimization model."""
+ return self.read_scenario_tables_from_db(scenario_name, input_table_names=['*'])
+
+ def read_scenario_tables_from_db(self, scenario_name: str,
+ input_table_names: Optional[List[str]] = None,
+ output_table_names: Optional[List[str]] = None) -> (Inputs, Outputs):
+ """Read selected set input and output tables from scenario.
+ If input_table_names/output_table_names contains a '*', then all input/output tables will be read.
+ If empty list or None, then no tables will be read.
+ """
+ if self.enable_transactions:
+ with self.engine.begin() as connection:
+ inputs, outputs = self._read_scenario_tables_from_db(connection, scenario_name, input_table_names, output_table_names)
+ else:
+ inputs, outputs = self._read_scenario_tables_from_db(self.engine, scenario_name, input_table_names, output_table_names)
+ return inputs, outputs
+
+ def _read_scenario_tables_from_db(self, connection, scenario_name: str,
+ input_table_names: List[str] = None,
+ output_table_names: List[str] = None) -> (Inputs, Outputs):
+ """Loads data for selected input and output tables.
+ If either list is names is ['*'], will load all tables as defined in db_tables configuration.
+ """
+ if input_table_names is None: # load no tables by default
+ input_table_names = []
+ elif '*' in input_table_names:
+ input_table_names = list(self.input_db_tables.keys())
+ if 'Scenario' in input_table_names: input_table_names.remove('Scenario') # Remove the scenario table
+
+ if output_table_names is None: # load no tables by default
+ output_table_names = []
+ elif '*' in output_table_names:
+ output_table_names = self.output_db_tables.keys()
+
+ inputs = {}
+ for scenario_table_name, db_table in self.input_db_tables.items():
+ if scenario_table_name in input_table_names:
+ inputs[scenario_table_name] = self._read_scenario_table_from_db(scenario_name, db_table, connection=connection)
+ outputs = {}
+ for scenario_table_name, db_table in self.output_db_tables.items():
+ if scenario_table_name in output_table_names:
+ outputs[scenario_table_name] = self._read_scenario_db_table_from_db(scenario_name, db_table, connection=connection)
+ return inputs, outputs
+
+ # def _read_scenario_db_table_from_db(self, scenario_name: str, db_table: ScenarioDbTable) -> pd.DataFrame:
+ # """Read one table from the DB.
+ # Removes the `scenario_name` column."""
+ # db_table_name = db_table.db_table_name
+ # sql = f"SELECT * FROM {db_table_name} WHERE scenario_name = '{scenario_name}'"
+ # df = pd.read_sql(sql, con=self.engine)
+ # if db_table_name != 'scenario':
+ # df = df.drop(columns=['scenario_name'])
+ #
+ # return df
+ def _read_scenario_db_table_from_db(self, scenario_name: str, db_table: ScenarioDbTable, connection) -> pd.DataFrame:
"""Read one table from the DB.
- Removes the `scenario_name` column."""
+ Removes the `scenario_name` column.
+
+ Modification: based on SQLAlchemy syntax. If doing the plain text SQL, then some column names not properly extracted
+ """
db_table_name = db_table.db_table_name
- sql = f"SELECT * FROM {db_table_name} WHERE scenario_name = '{scenario_name}'"
- df = pd.read_sql(sql, con=self.engine)
+ # sql = f"SELECT * FROM {db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
+ # db_table.table_metadata is a Table()
+ t: sqlalchemy.Table = db_table.get_sa_table() #table_metadata
+ sql = t.select().where(t.c.scenario_name == scenario_name) # This is NOT a simple string!
+ df = pd.read_sql(sql, con=connection)
if db_table_name != 'scenario':
df = df.drop(columns=['scenario_name'])
return df
+ ############################################################################################
+ # Update scenario
+ ############################################################################################
+ def update_cell_changes_in_db(self, db_cell_updates: List[DbCellUpdate]):
+ """Update a set of cells in the DB.
+
+ :param db_cell_updates:
+ :return:
+ """
+ if self.enable_transactions:
+ print("Update cells with transaction")
+ with self.engine.begin() as connection:
+ self._update_cell_changes_in_db(db_cell_updates, connection=connection)
+ else:
+ self._update_cell_changes_in_db(db_cell_updates)
+
+ def _update_cell_changes_in_db(self, db_cell_updates: List[DbCellUpdate], connection=None):
+ """Update an ordered list of single value changes (cell) in the DB."""
+ for db_cell_change in db_cell_updates:
+ self._update_cell_change_in_db(db_cell_change, connection)
+
+ def _update_cell_change_in_db(self, db_cell_update: DbCellUpdate, connection):
+ """Update a single value (cell) change in the DB."""
+ # db_table_name = self.db_tables[db_cell_update.table_name].db_table_name
+ # column_change = f"{db_cell_update.column_name} = '{db_cell_update.current_value}'"
+ # scenario_condition = f"scenario_name = '{db_cell_update.scenario_name}'"
+ # pk_conditions = ' AND '.join([f"{pk['column']} = '{pk['value']}'" for pk in db_cell_update.row_index])
+ # old_sql = f"UPDATE {db_table_name} SET {column_change} WHERE {pk_conditions} AND {scenario_condition};"
+
+ db_table: ScenarioDbTable = self.db_tables[db_cell_update.table_name]
+ t: sqlalchemy.Table = db_table.get_sa_table()
+ pk_conditions = [(db_table.get_sa_column(pk['column']) == pk['value']) for pk in db_cell_update.row_index]
+ target_col: sqlalchemy.Column = db_table.get_sa_column(db_cell_update.column_name)
+ sql = t.update().where(sqlalchemy.and_((t.c.scenario_name == db_cell_update.scenario_name), *pk_conditions)).values({target_col:db_cell_update.current_value})
+ # print(f"_update_cell_change_in_db = {sql}")
+
+ connection.execute(sql)
+
+ ############################################################################################
+ # Update/Replace tables in scenario
+ ############################################################################################
+ def update_scenario_output_tables_in_db(self, scenario_name, outputs: Outputs):
+ """Main API to update output from a DO solve in the scenario.
+ Deletes ALL output tables. Then inserts the given set of tables.
+ Since this only touches the output tables, more efficient than replacing the whole scenario."""
+ if self.enable_transactions:
+ with self.engine.begin() as connection:
+ self._update_scenario_output_tables_in_db(scenario_name, outputs, connection)
+ else:
+ self._update_scenario_output_tables_in_db(scenario_name, outputs, self.engine)
+
+ def _update_scenario_output_tables_in_db(self, scenario_name, outputs: Outputs, connection):
+ """Deletes ALL output tables. Then inserts the given set of tables.
+ Note that if a defined output table is not included in the outputs, it will still be deleted from the scenario data."""
+ # 1. Add scenario name to dfs:
+ outputs = ScenarioDbManager.add_scenario_name_to_dfs(scenario_name, outputs)
+ # 2. Delete all output tables
+ for scenario_table_name, db_table in reversed(self.output_db_tables.items()): # Note this INCLUDES the SCENARIO table!
+ if (scenario_table_name != 'Scenario'):
+ db_table._delete_scenario_table_from_db(scenario_name, connection)
+ # 3. Insert new data
+ for scenario_table_name, db_table in self.output_db_tables.items(): # Note this INCLUDES the SCENARIO table!
+ if (scenario_table_name != 'Scenario') and db_table.db_table_name in outputs.keys(): # If in given set of tables to replace
+ df = outputs[scenario_table_name]
+ db_table.insert_table_in_db_bulk(df=df, mgr=self, connection=connection) # The scenario_name is a column in the df
+
+ def replace_scenario_tables_in_db(self, scenario_name, inputs={}, outputs={}):
+ """Untested"""
+ if self.enable_transactions:
+ with self.engine.begin() as connection:
+ self._replace_scenario_tables_in_db(connection, scenario_name, inputs, outputs)
+ else:
+ self._replace_scenario_tables_in_db(self.engine, scenario_name, inputs, outputs)
+
+ def _replace_scenario_tables_in_db(self, connection, scenario_name, inputs={}, outputs={}):
+ """Untested
+ Replace only the tables listed in the inputs and outputs. But leave all other tables untouched.
+ Will first delete all given tables (in reverse cascading order), then insert the new ones (in cascading order)"""
+
+ # Add scenario name to dfs:
+ inputs = ScenarioDbManager.add_scenario_name_to_dfs(scenario_name, inputs)
+ outputs = ScenarioDbManager.add_scenario_name_to_dfs(scenario_name, outputs)
+ dfs = {**inputs, **outputs}
+ # 1. Delete tables
+ for scenario_table_name, db_table in reversed(self.db_tables.items()): # Note this INCLUDES the SCENARIO table!
+ if (scenario_table_name != 'Scenario') and db_table.db_table_name in dfs.keys(): # If in given set of tables to replace
+ db_table._delete_scenario_table_from_db()
+ # 2. Insert new data
+ for scenario_table_name, db_table in self.db_tables.items(): # Note this INCLUDES the SCENARIO table!
+ if (scenario_table_name != 'Scenario') and db_table.db_table_name in dfs.keys(): # If in given set of tables to replace
+ df = dfs[scenario_table_name]
+ db_table.insert_table_in_db_bulk(df=df, mgr=self, connection=connection) # The scenario_name is a column in the df
+
+ ############################################################################################
+ # CRUD operations on scenarios in DB:
+ # - Delete scenario
+ # - Duplicate scenario
+ # - Rename scenario
+ ############################################################################################
+ def delete_scenario_from_db(self, scenario_name: str):
+ """Delete a scenario. Uses a transaction (when enabled)."""
+ if self.enable_transactions:
+ print("Delete scenario within a transaction")
+ with self.engine.begin() as connection:
+ self._delete_scenario_from_db(scenario_name=scenario_name, connection=connection)
+ else:
+ self._delete_scenario_from_db(scenario_name=scenario_name, connection=self.engine)
+
+ ##########################################################
+ def duplicate_scenario_in_db(self, source_scenario_name: str, target_scenario_name: str):
+ """Duplicate a scenario. Uses a transaction (when enabled)."""
+ if self.enable_transactions:
+ print("Duplicate scenario within a transaction")
+ with self.engine.begin() as connection:
+ self._duplicate_scenario_in_db(connection, source_scenario_name, target_scenario_name)
+ else:
+ self._duplicate_scenario_in_db(self.engine, source_scenario_name, target_scenario_name)
+
+ def _duplicate_scenario_in_db(self, connection, source_scenario_name: str, target_scenario_name: str = None):
+ """Is fully done in DB using SQL in one SQL execute statement
+ :param source_scenario_name:
+ :param target_scenario_name:
+ :param connection:
+ :return:
+ """
+ if target_scenario_name is None:
+ new_scenario_name = self._find_free_duplicate_scenario_name(source_scenario_name)
+ elif self._check_free_scenario_name(target_scenario_name):
+ new_scenario_name = target_scenario_name
+ else:
+ raise ValueError(f"Target name for duplicate scenario '{target_scenario_name}' already exists.")
+
+ # inputs, outputs = self.read_scenario_from_db(source_scenario_name)
+ # self._replace_scenario_in_db_transaction(scenario_name=new_scenario_name, inputs=inputs, outputs=outputs,
+ # bulk=True, connection=connection)
+ self._duplicate_scenario_in_db_sql(connection, source_scenario_name, new_scenario_name)
+
+ def _duplicate_scenario_in_db_sql(self, connection, source_scenario_name: str, target_scenario_name: str = None):
+ """
+ :param source_scenario_name:
+ :param target_scenario_name:
+ :param connection:
+ :return:
+
+ See https://stackoverflow.com/questions/9879830/select-modify-and-insert-into-the-same-table
+
+ Problem: the table Parameter/parameters has a column 'value' (lower-case).
+ Almost all of the column names in the DFs are lower-case, as are the column names in the ScenarioDbTable.
+ Typically, the DB schema converts that the upper-case column names in the DB.
+ But probably because 'VALUE' is a reserved word, it does NOT do this for 'value'. But that means in order to refer to this column in SQL,
+ one needs to put "value" between double quotes.
+ Problem is that you CANNOT do that for other columns, since these are in upper-case in the DB.
+ Note that the kpis table uses upper case 'VALUE' and that seems to work fine
+
+ Resolution: use SQLAlchemy to construct the SQL. Do NOT create SQL expressions by text manipulation.
+ SQLAlchemy has the smarts to properly deal with these complex names.
+ """
+ if target_scenario_name is None:
+ new_scenario_name = self._find_free_duplicate_scenario_name(source_scenario_name)
+ elif self._check_free_scenario_name(target_scenario_name):
+ new_scenario_name = target_scenario_name
+ else:
+ raise ValueError(f"Target name for duplicate scenario '{target_scenario_name}' already exists.")
+
+ batch_sql=False # BEWARE: batch = True does NOT work!
+ sql_statements = []
+
+ # 1. Insert scenario in scenario table
+ # sql_insert = f"INSERT INTO SCENARIO (scenario_name) VALUES ('{new_scenario_name}')" # Old SQL
+ # sa_scenario_table = list(self.input_db_tables.values())[0].get_sa_table() # Scenario table must be the first
+ sa_scenario_table = self.get_scenario_db_table().get_sa_table()
+ sql_insert = sa_scenario_table.insert().values(scenario_name = new_scenario_name)
+ # print(f"_duplicate_scenario_in_db_sql - Insert SQL = {sql_insert}")
+ if batch_sql:
+ sql_statements.append(sql_insert)
+ else:
+ connection.execute(sql_insert)
+
+ # 2. Do 'insert into select' to duplicate rows in each table
+ for scenario_table_name, db_table in self.db_tables.items():
+ if scenario_table_name == 'Scenario':
+ continue
+
+ t: sqlalchemy.table = db_table.table_metadata # The table at hand
+ s: sqlalchemy.table = sa_scenario_table # The scenario table
+ # print("+++++++++++SQLAlchemy insert-select")
+ select_columns = [s.c.scenario_name if c.name == 'scenario_name' else c for c in t.columns] # Replace the t.c.scenario_name with s.c.scenario_name, so we get the new value
+ # print(f"select columns = {select_columns}")
+ select_sql = (sqlalchemy.select(select_columns)
+ .where(sqlalchemy.and_(t.c.scenario_name == source_scenario_name, s.c.scenario_name == target_scenario_name)))
+ target_columns = [c for c in t.columns]
+ sql_insert = t.insert().from_select(target_columns, select_sql)
+ # print(f"sql_insert = {sql_insert}")
+
+ # sql_insert = f"INSERT INTO {db_table.db_table_name} ({target_columns_txt}) SELECT '{target_scenario_name}',{other_source_columns_txt} FROM {db_table.db_table_name} WHERE scenario_name = '{source_scenario_name}'"
+ if batch_sql:
+ sql_statements.append(sql_insert)
+ else:
+ connection.execute(sql_insert)
+ if batch_sql:
+ batch_sql = ";\n".join(sql_statements)
+ print(batch_sql)
+ connection.execute(batch_sql)
+
+ def _find_free_duplicate_scenario_name(self, scenario_name: str, scenarios_df=None) -> Optional[str]:
+ """Finds next free scenario name based on pattern '{scenario_name}_copy_n'.
+ Will try at maximum 20 attempts.
+ """
+ max_num_attempts = 20
+ for i in range(1, max_num_attempts + 1):
+ new_name = f"{scenario_name}({i})"
+ free = self._check_free_scenario_name(new_name, scenarios_df)
+ if free:
+ return new_name
+ raise ValueError(f"Cannot find free name for duplicate scenario. Tried {max_num_attempts}. Last attempt = {new_name}. Rename scenarios.")
+ return None
+
+ def _check_free_scenario_name(self, scenario_name, scenarios_df=None) -> bool:
+ if scenarios_df is None:
+ scenarios_df = self.get_scenarios_df()
+ free = (False if scenario_name in scenarios_df.index else True)
+ return free
+
+ ##############################################
+ def rename_scenario_in_db(self, source_scenario_name: str, target_scenario_name: str):
+ """Rename a scenario. Uses a transaction (when enabled)."""
+ if self.enable_transactions:
+ print("Rename scenario within a transaction")
+ with self.engine.begin() as connection:
+ # self._rename_scenario_in_db(source_scenario_name, target_scenario_name, connection=connection)
+ self._rename_scenario_in_db_sql(connection, source_scenario_name, target_scenario_name)
+ else:
+ # self._rename_scenario_in_db(source_scenario_name, target_scenario_name)
+ self._rename_scenario_in_db_sql(self.engine, source_scenario_name, target_scenario_name)
+
+ def _rename_scenario_in_db_sql(self, connection, source_scenario_name: str, target_scenario_name: str = None):
+ """Rename scenario.
+ Uses 2 steps:
+ 1. Duplicate scenario
+ 2. Delete source scenario.
+
+ Problem is that we use scenario_name as a primary key. You should not change the value of primary keys in a DB.
+ Instead, first copy the data using a new scenario_name, i.e. duplicate a scenario. Next, delete the original scenario.
+
+ Long-term solution: use a scenario_seq sequence key as the PK. With scenario_name as a ordinary column in the scenario table.
+
+ Use of 'insert into select': https://stackoverflow.com/questions/9879830/select-modify-and-insert-into-the-same-table
+ """
+ # 1. Duplicate scenario
+ self._duplicate_scenario_in_db_sql(connection, source_scenario_name, target_scenario_name)
+ # 2. Delete scenario
+ self._delete_scenario_from_db(source_scenario_name, connection=connection)
+
+ def _delete_scenario_from_db(self, scenario_name: str, connection):
+ """Deletes all rows associated with a given scenario.
+ Note that it only deletes rows from tables defined in the self.db_tables, i.e. will NOT delete rows in 'auto-inserted' tables!
+ Must do a 'cascading' delete to ensure not violating FK constraints. In reverse order of how they are inserted.
+ Also deletes entry in scenario table
+ Uses SQLAlchemy syntax to generate SQL
+ TODO: check with 'auto-inserted' tables
+ TODO: batch all sql statements in single execute. Faster? And will that do the defer integrity checks?
+ """
+ # batch_sql=False # Batch=True does NOT work!
+ insp = sqlalchemy.inspect(connection)
+ tables_in_db = insp.get_table_names(schema=self.schema)
+ # sql_statements = []
+ for scenario_table_name, db_table in reversed(self.db_tables.items()): # Note this INCLUDES the SCENARIO table!
+ if db_table.db_table_name in tables_in_db:
+ # # sql = f"DELETE FROM {db_table.db_table_name} WHERE scenario_name = '{scenario_name}'" # Old
+ # t = db_table.table_metadata # A Table()
+ # sql = t.delete().where(t.c.scenario_name == scenario_name)
+ # connection.execute(sql)
+ db_table._delete_scenario_table_from_db(scenario_name, connection)
############################################################################################
# Old Read scenario APIs
@@ -866,29 +1342,13 @@ def read_scenario_tables_from_db_cached(self, scenario_name: str,
#######################################################################################################
# Review
#######################################################################################################
- def read_scenario_tables_from_db(self, scenario_name: str,
- input_table_names: List[str] = None,
- output_table_names: List[str] = None) -> (Inputs, Outputs):
- """Loads data for selected input and output tables.
- If either list is names is None, will load all tables as defined in db_tables configuration.
- """
- if input_table_names is None: # load all tables by default
- input_table_names = list(self.input_db_tables.keys())
- if 'Scenario' in input_table_names: input_table_names.remove('Scenario') # Remove the scenario table
- if output_table_names is None: # load all tables by default
- output_table_names = self.output_db_tables.keys()
- inputs = {}
- for scenario_table_name in input_table_names:
- inputs[scenario_table_name] = self.read_scenario_table_from_db(scenario_name, scenario_table_name)
- outputs = {}
- for scenario_table_name in output_table_names:
- outputs[scenario_table_name] = self.read_scenario_table_from_db(scenario_name, scenario_table_name)
- return inputs, outputs
def read_scenarios_from_db(self, scenario_names: List[str] = []) -> (Inputs, Outputs):
"""Multi scenario load.
- Reads all tables from set of scenarios"""
+ Reads all tables from set of scenarios
+ TODO: avoid use of text SQL. Use SQLAlchemy sql generation.
+ """
where_scenarios = ','.join([f"'{n}'" for n in scenario_names])
inputs = {}
diff --git a/dse_do_utils/version.py b/dse_do_utils/version.py
index c1a7924..cedca3c 100644
--- a/dse_do_utils/version.py
+++ b/dse_do_utils/version.py
@@ -9,4 +9,4 @@
See https://stackoverflow.com/questions/458550/standard-way-to-embed-version-into-python-package
"""
-__version__ = "0.5.3.1"
+__version__ = "0.5.4.0"