chore: Staging -> master v0.3.11 (#726)

Signed-off-by: badalprasadsingh <badal@datazip.io>
Co-authored-by: Ishani10 <ishanigupta1002@gmail.com>
Co-authored-by: Badal Prasad Singh <badal@datazip.io>
Co-authored-by: vishal-datazip <vishal@datazip.io>
Co-authored-by: Vaibhav <vaibhav@datazip.io>
Co-authored-by: Shubham Baldava <shubham@datazip.io>
Co-authored-by: Ankit Sharma <111491139+hash-data@users.noreply.github.com>

Author: 7 people

.github/workflows/documentation.yaml

@@ -18,10 +18,10 @@ jobs:
           const { body } = context.payload.pull_request;
 
           // If N/A is checked, we're done
-          if (/-\s*\[\s*x\s*\]\s*N\/A/.test(body)) return;
+          if (/-\s*\[\s*[xX]\s*\]\s*N\/A/.test(body)) return;
 
           // Check for valid documentation link in the Documentation Link line
-          const docLine = body.match(/-\s*\[\s*x\s*\]\s*Documentation Link:[^\n]*/i);
+          const docLine = body.match(/-\s*\[\s*[xX]\s*\]\s*Documentation Link:[^\n]*/i);
           if (!docLine) {
             core.setFailed(
               'Documentation Required: Please either check "Documentation Link" and provide a valid link, or check "N/A" for bug fixes, refactors, or test changes.'

drivers/abstract/abstract.go

@@ -110,6 +110,12 @@ func (a *AbstractDriver) Discover(ctx context.Context) ([]*types.Stream, error)
 			convStream.Schema.Properties.Delete(constants.CdcTimestamp)
 		}
 
+		// add default stream properties
+		convStream.DefaultStreamProperties = &types.DefaultStreamProperties{
+			Normalization: types.IsDriverRelational(a.driver.Type()),
+			AppendMode:    a.driver.Type() == string(constants.Kafka),
+		}
+
 		finalStreams = append(finalStreams, convStream)
 		return true
 	})

drivers/mongodb/internal/testdata/test_streams.json

@@ -1 +1 @@
-{"selected_streams":{"olake_mongodb_test":[{"partition_regex":"","stream_name":"test_collection","normalization":false},{"partition_regex":"","stream_name":"mongodb_test_table_olake","normalization":false}]},"streams":[{"stream":{"name":"test_collection","namespace":"olake_mongodb_test","type_schema":{"properties":{"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id"},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp"},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type"}}},"supported_sync_modes":["full_refresh","incremental","cdc","strict_cdc"],"source_defined_primary_key":["_id"],"available_cursor_fields":[],"sync_mode":"cdc","destination_database":"mongodb:olake_mongodb_test","destination_table":"test_collection"}},{"stream":{"name":"mongodb_test_table_olake","namespace":"olake_mongodb_test","type_schema":{"properties":{"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp"},"_id":{"type":["string"],"destination_column_name":"_id"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id"},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp"},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type"},"created_timestamp":{"type":["integer_small"],"destination_column_name":"created_timestamp"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_bigint":{"type":["integer"],"destination_column_name":"id_bigint"},"id_bool":{"type":["boolean"],"destination_column_name":"id_bool"},"id_double":{"type":["number"],"destination_column_name":"id_double"},"id_cursor":{"type":["integer_small"],"destination_column_name":"id_cursor"},"id_int":{"type":["integer_small"],"destination_column_name":"id_int"},"id_maxkey":{"type":["unknown"],"destination_column_name":"id_maxkey"},"id_minkey":{"type":["unknown"],"destination_column_name":"id_minkey"},"id_nested":{"type":["object"],"destination_column_name":"id_nested"},"id_nil":{"type":["null"],"destination_column_name":"id_nil"},"id_regex":{"type":["unknown"],"destination_column_name":"id_regex"},"id_timestamp":{"type":["timestamp"],"destination_column_name":"id_timestamp"},"name_varchar":{"type":["string"],"destination_column_name":"name_varchar"}}},"supported_sync_modes":["full_refresh","incremental","cdc","strict_cdc"],"source_defined_primary_key":["_id"],"available_cursor_fields":["id_bigint","id_timestamp","id_nil","id_bool","name_varchar","id_minkey","id_regex","id_double","id","id_nested","_id","id_int","id_maxkey","created_timestamp","id_cursor"],"sync_mode":"cdc","destination_database":"mongodb:olake_mongodb_test","destination_table":"mongodb_test_table_olake"}}]}
+{"selected_streams":{"olake_mongodb_test":[{"stream_name":"test_collection","normalization":false, "partition_regex":""},{"stream_name":"mongodb_test_table_olake","normalization":false,"partition_regex":""}]},"streams":[{"stream":{"name":"test_collection","namespace":"olake_mongodb_test","type_schema":{"properties":{"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp"},"_olake_id":{"type":["null","string"],"destination_column_name":"_olake_id"},"_olake_timestamp":{"type":["null","timestamp_micro"],"destination_column_name":"_olake_timestamp"},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type"}}},"supported_sync_modes":["cdc","strict_cdc","full_refresh","incremental"],"source_defined_primary_key":["_id"],"available_cursor_fields":[],"sync_mode":"cdc","destination_database":"mongodb:olake_mongodb_test","destination_table":"test_collection","default_stream_properties":{"normalization":false,"append_mode":false}}},{"stream":{"name":"mongodb_test_table_olake","namespace":"olake_mongodb_test","type_schema":{"properties":{"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp"},"_id":{"type":["string"],"destination_column_name":"_id"},"_olake_id":{"type":["string","null"],"destination_column_name":"_olake_id"},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp"},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type"},"created_timestamp":{"type":["integer_small"],"destination_column_name":"created_timestamp"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_bigint":{"type":["integer"],"destination_column_name":"id_bigint"},"id_bool":{"type":["boolean"],"destination_column_name":"id_bool"},"id_cursor":{"type":["integer_small"],"destination_column_name":"id_cursor"},"id_double":{"type":["number"],"destination_column_name":"id_double"},"id_int":{"type":["integer_small"],"destination_column_name":"id_int"},"id_maxkey":{"type":["unknown"],"destination_column_name":"id_maxkey"},"id_minkey":{"type":["unknown"],"destination_column_name":"id_minkey"},"id_nested":{"type":["object"],"destination_column_name":"id_nested"},"id_nil":{"type":["null"],"destination_column_name":"id_nil"},"id_regex":{"type":["unknown"],"destination_column_name":"id_regex"},"id_timestamp":{"type":["timestamp"],"destination_column_name":"id_timestamp"},"name_varchar":{"type":["string"],"destination_column_name":"name_varchar"}}},"supported_sync_modes":["incremental","cdc","strict_cdc","full_refresh"],"source_defined_primary_key":["_id"],"available_cursor_fields":["created_timestamp","id_bool","id_cursor","name_varchar","id_timestamp","id_int","id_maxkey","id_bigint","id_regex","id_nested","id_nil","_id","id","id_double","id_minkey"],"sync_mode":"cdc","destination_database":"mongodb:olake_mongodb_test","destination_table":"mongodb_test_table_olake","default_stream_properties":{"normalization":false,"append_mode":false}}}]}

drivers/mysql/internal/testdata/test_streams.json

@@ -1 +1 @@
-{"selected_streams":{"olake_mysql_test":[{"partition_regex":"","stream_name":"mysql_test_table_olake","normalization":true}]},"streams":[{"stream":{"name":"mysql_test_table_olake","namespace":"olake_mysql_test","type_schema":{"properties":{"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp"},"_olake_id":{"type":["null","string"],"destination_column_name":"_olake_id"},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp"},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type"},"amount_decimal_9_2":{"type":["number","null"],"destination_column_name":"amount_decimal_9_2"},"created_date":{"type":["timestamp","null"],"destination_column_name":"created_date"},"created_timestamp":{"type":["timestamp","null"],"destination_column_name":"created_timestamp"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_bigint":{"type":["null","integer"],"destination_column_name":"id_bigint"},"id_cursor":{"type":["null","integer_small"],"destination_column_name":"id_cursor"},"id_int":{"type":["null","integer_small"],"destination_column_name":"id_int"},"id_int_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_int_unsigned"},"id_integer":{"type":["integer_small","null"],"destination_column_name":"id_integer"},"id_integer_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_integer_unsigned"},"id_mediumint":{"type":["null","integer_small"],"destination_column_name":"id_mediumint"},"id_mediumint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_mediumint_unsigned"},"id_smallint":{"type":["integer_small","null"],"destination_column_name":"id_smallint"},"id_smallint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_smallint_unsigned"},"id_tinyint":{"type":["integer_small","null"],"destination_column_name":"id_tinyint"},"id_tinyint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_tinyint_unsigned"},"is_active":{"type":["integer_small","null"],"destination_column_name":"is_active"},"long_varchar":{"type":["string","null"],"destination_column_name":"long_varchar"},"name_bool":{"type":["integer_small","null"],"destination_column_name":"name_bool"},"name_char":{"type":["string","null"],"destination_column_name":"name_char"},"name_longtext":{"type":["string","null"],"destination_column_name":"name_longtext"},"name_mediumtext":{"type":["string","null"],"destination_column_name":"name_mediumtext"},"name_text":{"type":["string","null"],"destination_column_name":"name_text"},"name_tinytext":{"type":["null","string"],"destination_column_name":"name_tinytext"},"name_varchar":{"type":["string","null"],"destination_column_name":"name_varchar"},"price_decimal":{"type":["number","null"],"destination_column_name":"price_decimal"},"price_double":{"type":["number","null"],"destination_column_name":"price_double"},"price_double_precision":{"type":["null","number"],"destination_column_name":"price_double_precision"},"price_float":{"type":["number_small","null"],"destination_column_name":"price_float"},"price_numeric":{"type":["number","null"],"destination_column_name":"price_numeric"},"price_real":{"type":["null","number"],"destination_column_name":"price_real"},"priority":{"type":["null","string"],"destination_column_name":"priority"},"status":{"type":["string","null"],"destination_column_name":"status"}}},"supported_sync_modes":["strict_cdc","full_refresh","incremental","cdc"],"source_defined_primary_key":["id"],"available_cursor_fields":["price_numeric","name_longtext","long_varchar","name_text","id_bigint","id_mediumint_unsigned","price_real","name_varchar","id_integer","id_mediumint","name_mediumtext","id_integer_unsigned","id_smallint","id_tinyint_unsigned","price_double","created_timestamp","name_tinytext","created_date","id_int","id_smallint_unsigned","id_tinyint","price_double_precision","id","id_cursor","price_decimal","amount_decimal_9_2","name_char","is_active","name_bool","id_int_unsigned","price_float","priority","status"],"sync_mode":"cdc","destination_database":"mysql:olake_mysql_test","destination_table":"mysql_test_table_olake"}}]}
+{"selected_streams":{"olake_mysql_test":[{"partition_regex":"","stream_name":"mysql_test_table_olake","normalization":true}]},"streams":[{"stream":{"name":"mysql_test_table_olake","namespace":"olake_mysql_test","type_schema":{"properties":{"_cdc_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_cdc_timestamp"},"_olake_id":{"type":["null","string"],"destination_column_name":"_olake_id"},"_olake_timestamp":{"type":["timestamp_micro","null"],"destination_column_name":"_olake_timestamp"},"_op_type":{"type":["string","null"],"destination_column_name":"_op_type"},"amount_decimal_9_2":{"type":["number","null"],"destination_column_name":"amount_decimal_9_2"},"created_date":{"type":["timestamp","null"],"destination_column_name":"created_date"},"created_timestamp":{"type":["timestamp","null"],"destination_column_name":"created_timestamp"},"id":{"type":["integer_small"],"destination_column_name":"id"},"id_bigint":{"type":["null","integer"],"destination_column_name":"id_bigint"},"id_cursor":{"type":["null","integer_small"],"destination_column_name":"id_cursor"},"id_int":{"type":["null","integer_small"],"destination_column_name":"id_int"},"id_int_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_int_unsigned"},"id_integer":{"type":["integer_small","null"],"destination_column_name":"id_integer"},"id_integer_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_integer_unsigned"},"id_mediumint":{"type":["null","integer_small"],"destination_column_name":"id_mediumint"},"id_mediumint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_mediumint_unsigned"},"id_smallint":{"type":["integer_small","null"],"destination_column_name":"id_smallint"},"id_smallint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_smallint_unsigned"},"id_tinyint":{"type":["integer_small","null"],"destination_column_name":"id_tinyint"},"id_tinyint_unsigned":{"type":["integer_small","null"],"destination_column_name":"id_tinyint_unsigned"},"is_active":{"type":["integer_small","null"],"destination_column_name":"is_active"},"long_varchar":{"type":["string","null"],"destination_column_name":"long_varchar"},"name_bool":{"type":["integer_small","null"],"destination_column_name":"name_bool"},"name_char":{"type":["string","null"],"destination_column_name":"name_char"},"name_longtext":{"type":["string","null"],"destination_column_name":"name_longtext"},"name_mediumtext":{"type":["string","null"],"destination_column_name":"name_mediumtext"},"name_text":{"type":["string","null"],"destination_column_name":"name_text"},"name_tinytext":{"type":["null","string"],"destination_column_name":"name_tinytext"},"name_varchar":{"type":["string","null"],"destination_column_name":"name_varchar"},"price_decimal":{"type":["number","null"],"destination_column_name":"price_decimal"},"price_double":{"type":["number","null"],"destination_column_name":"price_double"},"price_double_precision":{"type":["null","number"],"destination_column_name":"price_double_precision"},"price_float":{"type":["number_small","null"],"destination_column_name":"price_float"},"price_numeric":{"type":["number","null"],"destination_column_name":"price_numeric"},"price_real":{"type":["null","number"],"destination_column_name":"price_real"},"priority":{"type":["null","string"],"destination_column_name":"priority"},"status":{"type":["string","null"],"destination_column_name":"status"}}},"supported_sync_modes":["strict_cdc","full_refresh","incremental","cdc"],"source_defined_primary_key":["id"],"available_cursor_fields":["price_numeric","name_longtext","long_varchar","name_text","id_bigint","id_mediumint_unsigned","price_real","name_varchar","id_integer","id_mediumint","name_mediumtext","id_integer_unsigned","id_smallint","id_tinyint_unsigned","price_double","created_timestamp","name_tinytext","created_date","id_int","id_smallint_unsigned","id_tinyint","price_double_precision","id","id_cursor","price_decimal","amount_decimal_9_2","name_char","is_active","name_bool","id_int_unsigned","price_float","priority","status"],"sync_mode":"cdc","destination_database":"mysql:olake_mysql_test","destination_table":"mysql_test_table_olake","default_stream_properties":{"normalization":true,"append_mode":false}}}]}