Merge pull request #7 from DennisPallett/master

Several new features + improved speed and stability

Author: cindy-m

config.properties.example

@@ -1,41 +1,26 @@
-# indicates how many records should be insert at a time into MonetDB
-batch.size = 10000
-
-# Scheduling options
-# With this the tool will check this table/column in SQL server and determine
-# if there have been any changes since the last run
-#
-# scheduler.table = updated
-# scheduler.column = last_updatetime
-
-mssql.user = dbuser
-mssql.password = dbpassword
-mssql.server = localhost
-mssql.database = my_db
-# optional instance name
-# mssql.instance = instance_name
-
+# target MonetDB Database:
 monetdb.user = monetdb
-monetdb.password = monetdb
+monetdb.password = ENV:MONETDB_PASSWORD:monetdb
 monetdb.server = localhost
 monetdb.database = my_monetdb
 
+# source MS SQL Server database(s):
+# multiple databases can be specified, each under their own sub-key
+mssql.db1.user = dbuser
+mssql.db1.password = dbpassword
+mssql.db1.server = localhost
+mssql.db1.database = my_db
+# optional instance name
+# mssql.db1.instance = instance_name
+
 # copy a table
+table.example.source = db1
 table.example.from = example_table
 table.example.to = example_table_new
 table.example.schema = sys
 table.example.create = true
 table.example.truncate = true
 
-# Mail settings
-monetdb.mail.sendmail=true
-monetdb.mail.to=my.name@something.com
-monetdb.mail.from=from@someone.com
-monetdb.mail.server=1.1.1.1
-monetdb.mail.port=8888
-monetdb.mail.username=
-monetdb.mail.password=
-
 # use tmp_tables to copy the data to so we can recreate the 
 # to-table for faster data transfer to the end table which 
 # reduces down-time
@@ -49,5 +34,36 @@ table.example.tempTablePrefix = tmp_
 # resulting in almost no down-time of your database table
 # prefixes can be defined if you want custom naming
 table.example.useFastViewSwitching = false
-table.example.currentTablePrefix = current_
-table.example.backupTablePrefix = backup_
+
+# Mail settings
+monetdb.mail.sendmail=true
+monetdb.mail.to=my.name@something.com
+monetdb.mail.from=from@someone.com
+monetdb.mail.server=1.1.1.1
+monetdb.mail.port=8888
+monetdb.mail.username=
+monetdb.mail.password=
+
+# specifies a custom temp directory instead of the default system temp directory
+# temp.directory = /home/me/temp
+
+# indicates how many records should be insert at a time into MonetDB
+# only used when using INSERT method to insert new data. 
+batch.size = 10000
+
+# Scheduling options
+# Enabling the scheduler will run the tool indefinitely and run the copy job
+# every X minutes/hours/days. Very useful in combination with the trigger options
+#
+# scheduler.enabled = true
+# scheduler.interval = every 5 minutes
+
+# Trigger options
+# With this the tool will check this table/column in SQL server and determine
+# if there have been any changes since the last run. The copy job will only be executed
+# if there is new data in the trigger column.
+#
+# trigger.enabled = true
+# trigger.source = db1
+# trigger.table = updated
+# trigger.column = last_updatetime

pom.xml

@@ -4,7 +4,7 @@
 	<groupId>nl.topicus.mssql2monetdb</groupId>
 	<artifactId>mssql2monetdb</artifactId>
 	<packaging>jar</packaging>
-	<version>0.4-SNAPSHOT</version>
+	<version>0.5</version>
 	<name>mssql2monetdb</name>
 	<url>http://maven.apache.org</url>
 
@@ -43,7 +43,13 @@
 			<groupId>commons-lang</groupId>
 			<artifactId>commons-lang</artifactId>
 			<version>2.6</version>
-		</dependency>            
+		</dependency>   
+		
+		<dependency>
+			<groupId>net.sf.opencsv</groupId>
+			<artifactId>opencsv</artifactId>
+			<version>2.0</version>
+		</dependency>         
 
 		<!-- MonetDB JDBC driver -->
 		<dependency>

src/main/java/nl/topicus/mssql2monetdb/CONFIG_KEYS.java

@@ -3,11 +3,6 @@
 public enum CONFIG_KEYS
 {
 	JOB_ID("job.id"),
-	MSSQL_USER("mssql.user"),
-	MSSQL_PASSWORD("mssql.password"),
-	MSSQL_SERVER("mssql.server"),
-	MSSQL_DATABASE("mssql.database"),
-	MSSQL_INSTANCE("mssql.instance"),
 	MONETDB_USER("monetdb.user"),
 	MONETDB_PASSWORD("monetdb.password"),
 	MONETDB_SERVER("monetdb.server"),
@@ -19,10 +14,14 @@ public enum CONFIG_KEYS
 	MONETDB_MAIL_PORT("monetdb.mail.port"),
 	MONETDB_MAIL_USERNAME("monetdb.mail.username"),
 	MONETDB_MAIL_PASSWORD("monetdb.mail.password"),
-	ALLOW_MULTIPLE_INSTANCES("allow.multiple.instances", false),
-	SCHEDULER_TABLE("scheduler.table", false),
-	SCHEDULER_COLUMN("scheduler.column", false),
-	BATCH_SIZE("batch.size", false);
+	SCHEDULER_ENABLED("scheduler.enabled", false),
+	SCHEDULER_INTERVAL("scheduler.interval", false),
+	TRIGGER_ENABLED("trigger.enabled", false),
+	TRIGGER_SOURCE("trigger.source", false),
+	TRIGGER_TABLE("trigger.table", false),
+	TRIGGER_COLUMN("trigger.column", false),
+	BATCH_SIZE("batch.size", false),
+	TEMP_DIR("temp.directory", false);
 
 	private final String key;
 

src/main/java/nl/topicus/mssql2monetdb/ConfigException.java

@@ -0,0 +1,10 @@
+package nl.topicus.mssql2monetdb;
+
+public class ConfigException extends Exception {
+	
+	public ConfigException (String msg)
+	{
+		super(msg);
+	}
+
+}

src/main/java/nl/topicus/mssql2monetdb/CopyTable.java

@@ -22,8 +22,6 @@ public class CopyTable
 	public static final int COPY_METHOD_INSERT = 0;
 
 	public static final int COPY_METHOD_COPYINTO = 1;
-	
-	public static final int COPY_METHOD_COPYINTO_VIA_TEMP_FILE = 2;
 
 	// contains the actual result table and possible a temp table
 	private List<MonetDBTable> monetDBTables = new ArrayList<MonetDBTable>();
@@ -33,13 +31,15 @@ public class CopyTable
 	private boolean create = true;
 
 	private boolean drop = false;
+	
+	private String source;
 
 	// secret view name
 	private String toName;
 
 	private String fromName;
 
-	private String schema;
+	private String schema = "sys";
 
 	private int copyMethod = COPY_METHOD_NOTSET;
 
@@ -62,6 +62,18 @@ public class CopyTable
 
 	// this will do COPY INTO with LOCKED MODE
 	private boolean useLockedMode = false;
+	
+	private String loadDate;
+	
+	public void setLoadDate(String loadDateStr)
+	{
+		this.loadDate = loadDateStr;
+	}
+	
+	public String getLoadDate()
+	{
+		return this.loadDate;
+	}
 
 	public void setCopyMethod(int copyMethod)
 	{
@@ -102,6 +114,16 @@ public boolean create()
 	{
 		return this.create;
 	}
+	
+	public String getSource ()
+	{
+		return source;
+	}
+	
+	public void setSource(String source)
+	{
+		this.source = source;
+	}
 
 	public String getFromName()
 	{
@@ -132,6 +154,11 @@ public void setSchema(String schema)
 	{
 		this.schema = schema;
 	}
+	
+	public String getTempFilePrefix ()
+	{
+		return "table_" + this.source + "_" + this.fromName;
+	}
 
 	public List<MonetDBTable> getMonetDBTables()
 	{