Skip to content

Commit 720a6f2

Browse files
cindy-mcindy-m
committed
Merge pull request #11 from topicus-ini/story-89263798
Story 89263798-adding no-switch and switch-only flags
2 parents d8f62ea + 2b77969 commit 720a6f2

File tree

3 files changed

+161
-103
lines changed

3 files changed

+161
-103
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
<groupId>nl.topicus.mssql2monetdb</groupId>
55
<artifactId>mssql2monetdb</artifactId>
66
<packaging>jar</packaging>
7-
<version>0.6</version>
7+
<version>0.7</version>
88
<name>mssql2monetdb</name>
99
<url>http://maven.apache.org</url>
1010

‎src/main/java/nl/topicus/mssql2monetdb/CopyTool.java‎

Lines changed: 119 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,9 @@ private void doCopy () throws CopyToolException
146146
{
147147
HashMap<String, CopyTable> tablesToCopy = config.getTablesToCopy();
148148

149+
boolean switchOnly = config.isSwitchOnly();
150+
boolean noSwitch = config.hasNoSwitch();
151+
149152
// no tables to copy?
150153
if (tablesToCopy.size() == 0)
151154
{
@@ -157,126 +160,140 @@ private void doCopy () throws CopyToolException
157160
// check if trigger is enabled and if so, if there is any new data
158161
boolean anyErrors = false;
159162

160-
if (config.isTriggerEnabled() && !checkForNewData())
161-
{
162-
LOG.info("No indication of new data from trigger source '" + config.getTriggerTable() + "." + config.getTriggerColumn() + "'");
163-
CopyToolConnectionManager.getInstance().closeConnections();
164-
return;
165-
}
166-
167-
// verify all MSSQL tables have data
168-
if (!MssqlUtil.allMSSQLTablesHaveData(tablesToCopy))
169-
{
170-
LOG.warn("Not all tables have data");
171-
CopyToolConnectionManager.getInstance().closeConnections();
172-
return;
173-
}
174-
175-
// verify MonetDB database is working by opening connection
176-
try {
177-
CopyToolConnectionManager.getInstance().openMonetDbConnection();
178-
} catch (SQLException e) {
179-
LOG.error("Unable to open connection to target MonetDB database: " + e.getMessage(), e);
180-
CopyToolConnectionManager.getInstance().closeConnections();
181-
return;
182-
}
183-
184-
LOG.info("STARTING PHASE 1: copying data from MS SQL source databases to local disk");
185-
186-
// phase 1: copy data from MS SQL sources to local disk
187-
try {
188-
for(CopyTable table : tablesToCopy.values())
163+
if(!switchOnly){
164+
165+
if (config.isTriggerEnabled() && !checkForNewData())
189166
{
190-
copyData(table);
167+
LOG.info("No indication of new data from trigger source '" + config.getTriggerTable() + "." + config.getTriggerColumn() + "'");
168+
CopyToolConnectionManager.getInstance().closeConnections();
169+
return;
191170
}
192-
} catch (Exception e) {
193-
anyErrors = true;
194-
LOG.error("Unable to copy data to disk", e);
195-
EmailUtil.sendMail("Unable to copy data with the following error: "+ e.getStackTrace(), "Unable to copy data from table in monetdb", config.getDatabaseProperties());
196-
}
197-
198-
LOG.info("PHASE 1 FINISHED: all data copied from MS SQL source databases to local disk");
199-
200-
LOG.info("STARTING PHASE 2: loading data into target MonetDB database");
201-
202-
// get a SQL-friendly representation of the current date/time of the load
203-
// used for the fast view switching tables
204-
DateFormat dateFormat = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss");
205-
Calendar cal = Calendar.getInstance();
206-
String loadDateStr = dateFormat.format(cal.getTime());
207-
208-
// phase 2: load data from disk into MonetDB
209-
try {
210-
for (CopyTable table : tablesToCopy.values())
171+
172+
// verify all MSSQL tables have data
173+
if (!MssqlUtil.allMSSQLTablesHaveData(tablesToCopy))
211174
{
212-
// pass load date to table
213-
table.setLoadDate(loadDateStr);
214-
215-
// load new data into MonetDB
216-
loadData(table);
175+
LOG.warn("Not all tables have data");
176+
CopyToolConnectionManager.getInstance().closeConnections();
177+
return;
217178
}
218-
} catch (Exception e) {
219-
anyErrors = true;
220-
LOG.error("Unable to load data into MonetDB", e);
221-
EmailUtil.sendMail("Unable to load data with the following error: "+ e.getStackTrace(), "Unable to load data table into monetdb", config.getDatabaseProperties());
222-
}
223-
224-
LOG.info("PAHSE 2 FINISHED: all data loaded into target MonetDB database");
225-
226-
LOG.info("STARTING PHASE 3: switching all view-based tables to new data");
227-
228-
// we need another loop through the tables for temp table copying and view
229-
// switching. We do this after the copy actions to reduce down-time
230-
231-
// phase 3: switch views (for view-based tables)
232-
for (CopyTable copyTable : tablesToCopy.values())
233-
{
234-
// if there are any temp table copies configured, then copy the
235-
// temp tables to result tables. We do this after the rest is done to
236-
// reduce down-time
237-
if (copyTable.isCopyViaTempTable())
238-
{
239-
copyTempTableToCurrentTable(copyTable);
179+
180+
// verify MonetDB database is working by opening connection
181+
try {
182+
CopyToolConnectionManager.getInstance().openMonetDbConnection();
183+
} catch (SQLException e) {
184+
LOG.error("Unable to open connection to target MonetDB database: " + e.getMessage(), e);
185+
CopyToolConnectionManager.getInstance().closeConnections();
186+
return;
240187
}
241188

242-
try
243-
{
244-
// set view to current table because it contains the new data now
245-
if (copyTable.isUseFastViewSwitching())
189+
LOG.info("STARTING PHASE 1: copying data from MS SQL source databases to local disk");
190+
191+
// phase 1: copy data from MS SQL sources to local disk
192+
try {
193+
for(CopyTable table : tablesToCopy.values())
246194
{
247-
MonetDBUtil.dropAndRecreateViewForTable(copyTable.getSchema(),
248-
copyTable.getToName(), copyTable.getCurrentTable());
195+
copyData(table);
249196
}
197+
} catch (Exception e) {
198+
anyErrors = true;
199+
LOG.error("Unable to copy data to disk", e);
200+
EmailUtil.sendMail("Unable to copy data with the following error: "+ e.getStackTrace(), "Unable to copy data from table in monetdb", config.getDatabaseProperties());
250201
}
251-
catch (SQLException e)
252-
{
202+
203+
LOG.info("PHASE 1 FINISHED: all data copied from MS SQL source databases to local disk");
204+
205+
LOG.info("STARTING PHASE 2: loading data into target MonetDB database");
206+
207+
// get a SQL-friendly representation of the current date/time of the load
208+
// used for the fast view switching tables
209+
DateFormat dateFormat = new SimpleDateFormat("yyyy_MM_dd_HH_mm_ss");
210+
Calendar cal = Calendar.getInstance();
211+
String loadDateStr = dateFormat.format(cal.getTime());
212+
213+
// phase 2: load data from disk into MonetDB
214+
try {
215+
for (CopyTable table : tablesToCopy.values())
216+
{
217+
// pass load date to table
218+
table.setLoadDate(loadDateStr);
219+
220+
// load new data into MonetDB
221+
loadData(table);
222+
}
223+
} catch (Exception e) {
253224
anyErrors = true;
254-
LOG.error("Unable to create view '" + copyTable.getToViewSql() + "'", e);
255-
EmailUtil.sendMail("Unable to create view" + copyTable.getToViewSql() + " with the following error: "+ e.toString(), "Unable to create view in monetdb", config.getDatabaseProperties());
225+
LOG.error("Unable to load data into MonetDB", e);
226+
EmailUtil.sendMail("Unable to load data with the following error: "+ e.getStackTrace(), "Unable to load data table into monetdb", config.getDatabaseProperties());
256227
}
228+
229+
LOG.info("PHASE 2 FINISHED: all data loaded into target MonetDB database");
230+
}
231+
else {
232+
LOG.info("Switch-Only requested with flag. Therefore, PHASE 1 and 2 skipped");
257233
}
258234

259-
LOG.info("PHASE 3 FINISHED: all views have been switched");
235+
//if switch-ONly flag set or when the no-switch-Flag is NOT set, than switching
236+
if(switchOnly || !noSwitch){
237+
238+
LOG.info("STARTING PHASE 3: switching all view-based tables to new data");
239+
240+
// we need another loop through the tables for temp table copying and view
241+
// switching. We do this after the copy actions to reduce down-time
242+
243+
// phase 3: switch views (for view-based tables)
244+
for (CopyTable copyTable : tablesToCopy.values())
245+
{
246+
// if there are any temp table copies configured, then copy the
247+
// temp tables to result tables. We do this after the rest is done to
248+
// reduce down-time
249+
if (copyTable.isCopyViaTempTable())
250+
{
251+
copyTempTableToCurrentTable(copyTable);
252+
}
253+
254+
try
255+
{
256+
// set view to current table because it contains the new data now
257+
if (copyTable.isUseFastViewSwitching())
258+
{
259+
MonetDBUtil.dropAndRecreateViewForTable(copyTable.getSchema(),
260+
copyTable.getToName(), copyTable.getCurrentTable());
261+
}
262+
}
263+
catch (SQLException e)
264+
{
265+
anyErrors = true;
266+
LOG.error("Unable to create view '" + copyTable.getToViewSql() + "'", e);
267+
EmailUtil.sendMail("Unable to create view" + copyTable.getToViewSql() + " with the following error: "+ e.toString(), "Unable to create view in monetdb", config.getDatabaseProperties());
268+
}
269+
}
270+
271+
LOG.info("PHASE 3 FINISHED: all views have been switched");
260272

261-
LOG.info("STARTING PHASE 4: cleanup of data from disk and database");
262273

263-
// phase 4: remove temp data from disk and target database
264-
for (CopyTable copyTable : tablesToCopy.values())
265-
{
266-
// remove temp data from disk
267-
removeTempData(copyTable);
274+
LOG.info("STARTING PHASE 4: cleanup of data from disk and database");
268275

269-
// remove old versions of view-based tables
270-
// that are no longer needed
271-
try {
272-
dropOldTables(copyTable);
273-
} catch (SQLException e) {
274-
LOG.warn("Got SQLException when trying to drop older versions of table '" + copyTable.getToName() + "': " + e.getMessage(), e);
276+
// phase 4: remove temp data from disk and target database
277+
for (CopyTable copyTable : tablesToCopy.values())
278+
{
279+
// remove temp data from disk
280+
removeTempData(copyTable);
281+
282+
// remove old versions of view-based tables
283+
// that are no longer needed
284+
try {
285+
dropOldTables(copyTable);
286+
} catch (SQLException e) {
287+
LOG.warn("Got SQLException when trying to drop older versions of table '" + copyTable.getToName() + "': " + e.getMessage(), e);
288+
}
275289
}
276-
}
277-
278-
LOG.info("PHASE 4 FINISHED: all data removed from disk and database");
290+
291+
LOG.info("PHASE 4 FINISHED: all data removed from disk and database");
279292

293+
}
294+
else{
295+
LOG.info("PHASE 3 (switching) and PHASE 4 (cleanup) skipped because no-switch-flag setting");
296+
}
280297
// write out info for trigger
281298
if (config.isTriggerEnabled() && !anyErrors)
282299
{

‎src/main/java/nl/topicus/mssql2monetdb/CopyToolConfig.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ public class CopyToolConfig
5454

5555
private String tempDirectory;
5656

57+
private boolean noSwitch;
58+
59+
private boolean switchOnly;
60+
5761
private HashMap<String, SourceDatabase> sourceDatabases = new HashMap<String, SourceDatabase>();
5862

5963
private HashMap<String, CopyTable> tablesToCopy = new HashMap<String, CopyTable>();
@@ -106,6 +110,19 @@ public CopyToolConfig(String args[]) throws ConfigException
106110
OptionBuilder.withDescription("Specify the configuration properties file");
107111
OptionBuilder.withLongOpt("config");
108112
options.addOption(OptionBuilder.create("c"));
113+
114+
OptionBuilder.hasArg(false);
115+
OptionBuilder.isRequired(false);
116+
OptionBuilder.withDescription("Specify if views will be switched or not");
117+
OptionBuilder.withLongOpt("no-switch");
118+
options.addOption(OptionBuilder.create("ns"));
119+
120+
OptionBuilder.hasArg(false);
121+
OptionBuilder.isRequired(false);
122+
OptionBuilder.withDescription("Specify if views will be switched or not");
123+
OptionBuilder.withLongOpt("switch-only");
124+
options.addOption(OptionBuilder.create("so"));
125+
109126

110127
CommandLineParser parser = new BasicParser();
111128
CommandLine cmd = null;
@@ -129,6 +146,7 @@ public CopyToolConfig(String args[]) throws ConfigException
129146
}
130147

131148
configFile = new File(cmd.getOptionValue("config"));
149+
132150
LOG.info("Using config file: " + configFile.getAbsolutePath());
133151

134152
Properties config = new Properties();
@@ -155,6 +173,12 @@ public CopyToolConfig(String args[]) throws ConfigException
155173
findTriggerProperties(config);
156174

157175
this.tempDirectory = findTempDirectory(config);
176+
177+
this.noSwitch = cmd.hasOption("no-switch");
178+
LOG.info("No-Switch-flag set to: " + noSwitch);
179+
180+
this.switchOnly = cmd.hasOption("switch-only");
181+
LOG.info("Switch-Only-flag set to: " + switchOnly);
158182

159183
// verify scheduling source
160184
//checkSchedulingSource();
@@ -835,5 +859,22 @@ public String getTempDirectory ()
835859
{
836860
return tempDirectory;
837861
}
862+
863+
864+
public boolean hasNoSwitch() {
865+
return noSwitch;
866+
}
867+
868+
public void setNoSwitch(boolean noSwitch) {
869+
this.noSwitch = noSwitch;
870+
}
871+
872+
public boolean isSwitchOnly() {
873+
return switchOnly;
874+
}
875+
876+
public void setSwitchOnly(boolean switchOnly) {
877+
this.switchOnly = switchOnly;
878+
}
838879

839880
}

0 commit comments

Comments
 (0)