Fix IF NOT EXISTS statement handling (#62)

Author: bakwc

mysql_ch_replicator/converter.py

@@ -49,6 +49,7 @@
     'utf32': 'utf_32',
     'utf8mb3': 'utf_8',        # Both utf8mb3 and utf8mb4 can be mapped to UTF-8
     'utf8mb4': 'utf_8',
+    'utf8': 'utf_8',
 }
 
 
@@ -558,6 +559,16 @@ def parse_mysql_table_structure(self, create_statement, required_table_name=None
         tokens = sqlparse.parse(create_statement.replace('\n', ' ').strip())[0].tokens
         tokens = [t for t in tokens if not t.is_whitespace and not t.is_newline]
 
+        # remove "IF NOT EXISTS"
+        if (len(tokens) > 5 and
+                tokens[0].normalized.upper() == 'CREATE' and
+                tokens[1].normalized.upper() == 'TABLE' and
+                tokens[2].normalized.upper() == 'IF' and
+                tokens[3].normalized.upper() == 'NOT' and
+                tokens[4].normalized.upper() == 'EXISTS'):
+            del tokens[2:5]  # Remove the 'IF', 'NOT', 'EXISTS' tokens
+            structure.if_not_exists = True
+
         if tokens[0].ttype != sqlparse.tokens.DDL:
             raise Exception('wrong create statement', create_statement)
         if tokens[0].normalized.lower() != 'create':

mysql_ch_replicator/table_structure.py

@@ -14,6 +14,7 @@ class TableStructure:
     table_name: str = ''
     charset: str = ''
     charset_python: str = ''
+    if_not_exists: bool = False
 
     def preprocess(self):
         field_names = [f.name for f in self.fields]

test_mysql_ch_replicator.py

@@ -10,6 +10,7 @@
 from mysql_ch_replicator import clickhouse_api
 from mysql_ch_replicator.binlog_replicator import State as BinlogState
 from mysql_ch_replicator.db_replicator import State as DbReplicatorState, DbReplicator
+from mysql_ch_replicator.converter import MysqlToClickhouseConverter
 
 from mysql_ch_replicator.runner import ProcessRunner
 
@@ -993,3 +994,13 @@ def test_string_primary_key(monkeypatch):
         commit=True,
     )
     assert_wait(lambda: len(ch.select(TEST_TABLE_NAME)) == 3)
+
+
+def test_parse_mysql_table_structure():
+    query = "CREATE TABLE IF NOT EXISTS user_preferences_portal (\n\t\t\tid char(36) NOT NULL,\n\t\t\tcategory varchar(50) DEFAULT NULL,\n\t\t\tdeleted tinyint(1) DEFAULT 0,\n\t\t\tdate_entered datetime DEFAULT NULL,\n\t\t\tdate_modified datetime DEFAULT NULL,\n\t\t\tassigned_user_id char(36) DEFAULT NULL,\n\t\t\tcontents longtext DEFAULT NULL\n\t\t ) ENGINE=InnoDB DEFAULT CHARSET=utf8"
+
+    converter = MysqlToClickhouseConverter()
+
+    structure = converter.parse_mysql_table_structure(query)
+
+    assert structure.table_name == 'user_preferences_portal'