Skip to content

Commit 144cc0c

Browse files
tkirschketkirschke
authored
Merge pull request #47 from ScalefreeCOM/Consecutive-runs
Fixed generating models multiple times on a single run
2 parents 519946e + 20eff9a commit 144cc0c

File tree

10 files changed

+65
-39
lines changed

10 files changed

+65
-39
lines changed

backend/db.py renamed to backend/Database.py

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
image_path = os.path.join(os.path.dirname(__file__),"images")
1010
log = Logger('log')
1111

12-
class DB:
12+
class Database:
1313
def __init__(self, **kwargs):
1414
self.todo = []
1515
self.config = kwargs.get('turboVaultconfigs')
@@ -24,7 +24,7 @@ def __init__(self, **kwargs):
2424
'console_outputs': True,
2525
'cursor': None,
2626
'source': None,
27-
'generated_timestamp': datetime.now().strftime("%Y%m%d%H%M%S"),
27+
'generated_timestamp': None,
2828
'rdv_default_schema': self.config.get("rdv_schema"),
2929
'model_path': self.model_path,
3030
'hashdiff_naming': self.config.get('hashdiff_naming'),
@@ -48,6 +48,7 @@ def __initializeInMemoryDatabase(self):
4848
return db.cursor()
4949

5050
def read(self):
51+
self.data_structure['generated_timestamp'] = datetime.now().strftime("%Y%m%d%H%M%S")
5152
self.data_structure['cursor'] = self.__initializeInMemoryDatabase()
5253
self.data_structure['cursor'].execute("SELECT DISTINCT SOURCE_SYSTEM || '_*-*_' || SOURCE_OBJECT FROM source_data")
5354
results = self.data_structure['cursor'].fetchall()
@@ -60,17 +61,19 @@ def run(self):
6061
self.read()
6162
if self.SourceYML:
6263
sources.gen_sources(self.data_structure)
63-
try:
64-
for self.data_structure['source'] in self.selectedSources:
65-
seperatedNameAsList = self.data_structure['source'].split('_*-*_')
66-
self.data_structure['source_name'] = seperatedNameAsList[0]
67-
self.data_structure['source_object'] = ''.join(seperatedNameAsList[1:])
68-
generate_selected_entities.generate_selected_entities(self.todo, self.data_structure)
69-
if self.Properties:
64+
65+
for self.data_structure['source'] in self.selectedSources:
66+
seperatedNameAsList = self.data_structure['source'].split('_*-*_')
67+
self.data_structure['source_name'] = seperatedNameAsList[0]
68+
self.data_structure['source_object'] = ''.join(seperatedNameAsList[1:])
69+
generate_selected_entities.generate_selected_entities(self.todo, self.data_structure)
70+
if self.Properties:
71+
try:
7072
properties.gen_properties(self.data_structure)
71-
self.data_structure['print2FeedbackConsole'](message= 'Process successfully executed and models are ready to be used in Datavault 4dbt.')
72-
except Exception as e:
73-
self.data_structure['print2FeedbackConsole'](message= 'No sources selected!')
73+
except:
74+
self.data_structure['print2FeedbackConsole'](message= 'Failed to generate properties for {0} : {1}.'.format(self.data_structure['source_name'], self.data_structure['source_object']))
75+
self.data_structure['print2FeedbackConsole'](message= 'Process successfully executed and models are ready to be used in Datavault 4dbt.')
76+
7477

7578
if self.DBDocs:
7679
generate_erd.generate_erd(self.data_structure['cursor'], self.selectedSources, self.data_structure['generated_timestamp'],self.data_structure['model_path'],self.data_structure['hashdiff_naming'])

‎backend/bigquery.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ def __init__(self, **kwargs):
2424
'console_outputs': True,
2525
'cursor': None,
2626
'source': None,
27-
'generated_timestamp': datetime.now().strftime("%Y%m%d%H%M%S"),
27+
'generated_timestamp': None,
2828
'rdv_default_schema': self.config.get("rdv_schema"),
2929
'model_path': self.model_path,
3030
'hashdiff_naming': self.config.get('hashdiff_naming'),
@@ -103,6 +103,7 @@ def __initializeInMemoryDatabase(self):
103103
return db.cursor()
104104

105105
def read(self):
106+
self.data_structure['generated_timestamp'] = datetime.now().strftime("%Y%m%d%H%M%S")
106107
self.data_structure['cursor']= self.__initializeInMemoryDatabase()
107108
self.data_structure['cursor'].execute("SELECT DISTINCT SOURCE_SYSTEM || '_' || SOURCE_OBJECT FROM source_data")
108109
results = self.data_structure['cursor'].fetchall()

‎backend/config/config.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1-
## TODO
21
import os
32
from configparser import ConfigParser
43

54
class MetadataInputConfig:
65
def __init__(self)->None:
7-
self.supportedPlatforms: list = ['Excel','Google Sheets','Snowflake','BigQuery','db']
6+
self.supportedPlatforms: list = ['Excel','Google Sheets','Snowflake','BigQuery','Database']
87
self.configExpectedFields: dict ={
98
'Snowflake':[
109
'stage_schema',
@@ -43,7 +42,7 @@ def __init__(self)->None:
4342
'model_path',
4443
'excel_path',
4544
],
46-
'db': [
45+
'Database': [
4746
'stage_schema',
4847
'rdv_schema',
4948
'hashdiff_naming',

‎backend/excel.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ def __init__(self, **kwargs):
1919
'console_outputs': True,
2020
'cursor': None,
2121
'source': None,
22-
'generated_timestamp': datetime.now().strftime("%Y%m%d%H%M%S"),
22+
'generated_timestamp': None,
2323
'rdv_default_schema': self.config.get("rdv_schema"),
2424
'model_path': self.model_path,
2525
'hashdiff_naming': self.config.get('hashdiff_naming'),
@@ -47,6 +47,7 @@ def __initializeInMemoryDatabase(self):
4747
return db.cursor()
4848

4949
def read(self):
50+
self.data_structure['generated_timestamp'] = datetime.now().strftime("%Y%m%d%H%M%S")
5051
self.data_structure['cursor'] = self.__initializeInMemoryDatabase()
5152
self.data_structure['cursor'].execute("SELECT DISTINCT SOURCE_SYSTEM || '_' || SOURCE_OBJECT FROM source_data")
5253
results = self.data_structure['cursor'].fetchall()
@@ -55,7 +56,7 @@ def read(self):
5556
source_list.append(row[0])
5657
self.data_structure['source_list'] = source_list
5758
self.catchDatabase()
58-
59+
5960
def catchDatabase(self):
6061
if os.path.exists('dump.db'):
6162
os.remove('dump.db')
@@ -89,4 +90,4 @@ def run(self):
8990

9091
if self.DBDocs:
9192
generate_erd.generate_erd(self.data_structure['cursor'], self.selectedSources,self.data_structure['generated_timestamp'],self.data_structure['model_path'],self.data_structure['hashdiff_naming'])
92-
self.data_structure['cursor'].close()
93+
self.data_structure['cursor'].close()

‎backend/googleSheets.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ def __init__(self, **kwargs):
2525
'console_outputs': True,
2626
'cursor': None,
2727
'source': None,
28-
'generated_timestamp': datetime.now().strftime("%Y%m%d%H%M%S"),
28+
'generated_timestamp': None,
2929
'rdv_default_schema': self.config.get("rdv_schema"),
3030
'model_path': self.model_path,
3131
'hashdiff_naming': self.config.get('hashdiff_naming'),
@@ -88,6 +88,7 @@ def __initializeInMemoryDatabase(self):
8888
return db.cursor()
8989

9090
def read(self):
91+
self.data_structure['generated_timestamp'] = datetime.now().strftime("%Y%m%d%H%M%S")
9192
self.data_structure['cursor'] = self.__initializeInMemoryDatabase()
9293
self.data_structure['cursor'].execute("SELECT DISTINCT SOURCE_SYSTEM || '_' || SOURCE_OBJECT FROM source_data")
9394
results = self.data_structure['cursor'].fetchall()

‎backend/procs/sqlite3/nh_link.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from backend.procs.sqlite3.hub import generate_source_models
44

55
def get_groupname(cursor,object_id):
6-
query = f"""SELECT DISTINCT GROUP_NAME from non_historized_link where NH_Link_Identifier = '{object_id}' ORDER BY Target_Column_Sort_Order LIMIT 1"""
6+
query = f"""SELECT DISTINCT Group_Name from non_historized_link where NH_Link_Identifier = '{object_id}' ORDER BY Target_Column_Sort_Order LIMIT 1"""
77
cursor.execute(query)
88
return cursor.fetchone()[0]
99

‎backend/snowflake.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ def __initializeInMemoryDatabase(self):
161161
return db.cursor()
162162

163163
def read(self):
164+
self.data_structure['generated_timestamp'] = datetime.now().strftime("%Y%m%d%H%M%S")
164165
self.configParser()
165166
self.data_structure['cursor'] = self.__initializeInMemoryDatabase()
166167
self.data_structure['cursor'].execute("SELECT DISTINCT SOURCE_SYSTEM || '_' || SOURCE_OBJECT FROM source_data")

‎backend/turbovault.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
from backend.googleSheets import GoogleSheets
33
from backend.snowflake import Snowflake
44
from backend.bigquery import BigQuery
5-
from backend.db import DB
5+
from backend.Database import Database
66
class TurboVault():
77
def __init__(self, **kwargs):
88
self.turboVaultconfigs = kwargs.get('turboVaultconfigs')
@@ -16,8 +16,8 @@ def __init__(self, **kwargs):
1616
self.snowflake = Snowflake(turboVaultconfigs = self.turboVaultconfigs['Snowflake'], print2FeedbackConsole = self.print2FeedbackConsole)
1717
if 'BigQuery' in validSourcePlatforms:
1818
self.bigquery = BigQuery(turboVaultconfigs = self.turboVaultconfigs['BigQuery'], print2FeedbackConsole = self.print2FeedbackConsole)
19-
if 'db' in validSourcePlatforms:
20-
self.db = DB(turboVaultconfigs = self.turboVaultconfigs['db'], print2FeedbackConsole = self.print2FeedbackConsole)
19+
if 'Database' in validSourcePlatforms:
20+
self.Database = Database(turboVaultconfigs = self.turboVaultconfigs['Database'], print2FeedbackConsole = self.print2FeedbackConsole)
2121

2222
def doRunForExcel(self):
2323
self.excel.run()
@@ -31,5 +31,5 @@ def doRunForGoogleSheets(self):
3131
def doRunForSnowflake(self):
3232
self.snowflake.run()
3333

34-
def doRunForDb(self):
35-
self.db.run()
34+
def doRunForDatabase(self):
35+
self.Database.run()

‎frontend/events.py‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ def onDropdownSelection(self, selection: str) -> list:
2323
elif selection == 'BigQuery':
2424
self.TurboVault.bigquery.read()
2525
newSources : list = self.TurboVault.bigquery.data_structure['source_list']
26-
elif selection == 'db':
27-
self.TurboVault.db.read()
28-
newSources : list = self.TurboVault.db.data_structure['source_list']
26+
elif selection == 'Database':
27+
self.TurboVault.Database.read()
28+
newSources : list = self.TurboVault.Database.data_structure['source_list']
2929
return newSources
3030

3131
def onPressStart(self, selections) -> None:
@@ -38,7 +38,7 @@ def onPressStart(self, selections) -> None:
3838
Properties= selections['Properties'],
3939
)
4040
self.TurboVault.doRunForExcel()
41-
41+
self.TurboVault.excel.read()
4242
elif selections['SourcePlatform'] == 'Google Sheets':
4343
self.TurboVault.googleSheets.setTODO(
4444
SourceYML = selections['SourceYML'],
@@ -69,12 +69,12 @@ def onPressStart(self, selections) -> None:
6969
)
7070
self.TurboVault.doRunForBigQuery()
7171

72-
elif selections['SourcePlatform'] == 'db':
73-
self.TurboVault.db.setTODO(
72+
elif selections['SourcePlatform'] == 'Database':
73+
self.TurboVault.Database.setTODO(
7474
SourceYML = selections['SourceYML'],
7575
Tasks = selections['Tasks'] ,
7676
Sources= selections['Sources'],
7777
DBDocs= selections['DBDocs'],
7878
Properties= selections['Properties'],
7979
)
80-
self.TurboVault.doRunForDb()
80+
self.TurboVault.doRunForDatabase()

‎frontend/pyqt5.py‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
import ctypes
44
from datetime import datetime
55
from threading import Thread, Lock
6-
from PyQt5.QtCore import Qt
6+
from queue import Queue
7+
from PyQt5.QtCore import Qt, QTimer
78
from PyQt5.QtGui import (
89
QColor, QIcon, QMovie, QPixmap, QStandardItem, QStandardItemModel
910
)
@@ -15,13 +16,19 @@
1516
from frontend.PyQt5CustomClasses import QPushButton
1617
from frontend.events import Events
1718
from frontend.styles import styles
19+
1820
class MainApp(QWidget):
1921
def __init__(self, **kwargs) -> None:
2022
super().__init__()
2123
ConfigData: dict = kwargs.pop('configData') # Config data gettin poped
2224
self.validSourcePlatforms : list = ConfigData['validSourcePlatforms']
2325
self.invalidSourcePlatforms: list = ConfigData['invalidSourcePlatforms']
2426
self.config: object = ConfigData['config']
27+
28+
# --- Start of Changes ---
29+
self.message_queue = Queue()
30+
# --- End of Changes ---
31+
2532
self.events: object = Events(
2633
config = self.config,
2734
print2FeedbackConsole= self.print2FeedbackConsole,
@@ -46,6 +53,12 @@ def __init__(self, **kwargs) -> None:
4653
ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID("TurboVault4dbt")
4754
self.setGeometry(100, 100, 800, 1280)
4855
self.setupUI()
56+
57+
# --- Start of Changes ---
58+
self.timer = QTimer(self)
59+
self.timer.timeout.connect(self.process_queue)
60+
self.timer.start(100) # Check the queue every 100ms
61+
# --- End of Changes ---
4962

5063
def setupUI(self) -> None:
5164
mainLayout: QVBoxLayout = QVBoxLayout()
@@ -118,7 +131,7 @@ def __createDropdownLayout(self) -> QVBoxLayout:
118131
# Add invalid platforms as unselectable with custom color
119132
for platform in self.invalidSourcePlatforms:
120133
item = QStandardItem(platform)
121-
item.setForeground(QColor("#eb5a50")) # Custom red color
134+
item.setForeground(QColor("#eb5a50"))
122135
item.setEnabled(False)
123136
model.appendRow(item)
124137

@@ -403,16 +416,23 @@ def onStart(self):
403416
).start()
404417

405418
def onCancel(self):
406-
self.feedbackConsole.append("Process canceled.")
419+
self.print2FeedbackConsole("Process canceled.")
407420

421+
# --- Start of Changes ---
408422
def print2FeedbackConsole(self, message=None) -> None:
409-
self.feedbackConsole.append(message)
423+
if message:
424+
self.message_queue.put(message)
425+
426+
def process_queue(self):
427+
while not self.message_queue.empty():
428+
message = self.message_queue.get_nowait()
429+
self.feedbackConsole.append(message)
430+
# --- End of Changes ---
410431

411432
def enableWidgets(self, state) -> None:
412433
self.sourcePlatformCombo.setEnabled(state)
413434
self.selectAllSourcesBtn.setEnabled(state)
414435
self.sourcesList.setEnabled(state)
415436
self.deselectAllTasksBtn.setEnabled(state)
416437
self.tasksList.setEnabled(state)
417-
self.startButton.setEnabled(state)
418-
438+
self.startButton.setEnabled(state)

0 commit comments

Comments
 (0)