diff --git a/airbyte-integrations/connectors/source-mysql/metadata.yaml b/airbyte-integrations/connectors/source-mysql/metadata.yaml index 83517f64152f..9e69f36d0c80 100644 --- a/airbyte-integrations/connectors/source-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.50.9 + dockerImageTag: 3.50.10 dockerRepository: airbyte/source-mysql documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql diff --git a/airbyte-integrations/connectors/source-mysql/src/main/kotlin/io/airbyte/integrations/source/mysql/MySqlSourceCdcTemporalConverter.kt b/airbyte-integrations/connectors/source-mysql/src/main/kotlin/io/airbyte/integrations/source/mysql/MySqlSourceCdcTemporalConverter.kt index ded4475e2c2e..b2597602cc34 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/kotlin/io/airbyte/integrations/source/mysql/MySqlSourceCdcTemporalConverter.kt +++ b/airbyte-integrations/connectors/source-mysql/src/main/kotlin/io/airbyte/integrations/source/mysql/MySqlSourceCdcTemporalConverter.kt @@ -37,7 +37,12 @@ class MySqlSourceCdcTemporalConverter : RelationalColumnCustomConverter { TimeHandler, TimestampHandler ) - + /** + * Handles zero-dates (e.g., '0000-00-00 00:00:00') which MySQL allows in NON-NULLABLE columns + * but are invalid dates that JDBC drivers return as NULL. We are now converting those NULL + * values to epoch (1970-01-01...) to match default value behavior and satisfy Debezium's + * non-nullable schema constraints. + */ data object DatetimeMillisHandler : RelationalColumnCustomConverter.Handler { override fun matches(column: RelationalColumn): Boolean = @@ -48,7 +53,14 @@ class MySqlSourceCdcTemporalConverter : RelationalColumnCustomConverter { override val partialConverters: List = listOf( - NullFallThrough, + PartialConverter { + if (it == null) { + val epoch = LocalDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC) + Converted(epoch.format(LocalDateTimeCodec.formatter)) + } else { + NoConversion + } + }, PartialConverter { if (it is LocalDateTime) { Converted(it.format(LocalDateTimeCodec.formatter)) @@ -80,7 +92,14 @@ class MySqlSourceCdcTemporalConverter : RelationalColumnCustomConverter { override val partialConverters: List = listOf( - NullFallThrough, + PartialConverter { + if (it == null) { + val epoch = LocalDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC) + Converted(epoch.format(LocalDateTimeCodec.formatter)) + } else { + NoConversion + } + }, PartialConverter { if (it is LocalDateTime) { Converted(it.format(LocalDateTimeCodec.formatter)) @@ -112,7 +131,14 @@ class MySqlSourceCdcTemporalConverter : RelationalColumnCustomConverter { override val partialConverters: List = listOf( - NullFallThrough, + PartialConverter { + if (it == null) { + val epoch = LocalDate.ofEpochDay(0) + Converted(epoch.format(LocalDateCodec.formatter)) + } else { + NoConversion + } + }, PartialConverter { if (it is LocalDate) { Converted(it.format(LocalDateCodec.formatter)) @@ -131,7 +157,11 @@ class MySqlSourceCdcTemporalConverter : RelationalColumnCustomConverter { } ) } - + /** + * TIME supports '00:00:00' and it is considered valid, see + * https://dev.mysql.com/doc/refman/8.0/en/time.html. If we get a null value from the server, it + * means the TIME is invalid/corrupt and in that case we should return null. + */ data object TimeHandler : RelationalColumnCustomConverter.Handler { override fun matches(column: RelationalColumn): Boolean = @@ -164,6 +194,7 @@ class MySqlSourceCdcTemporalConverter : RelationalColumnCustomConverter { } data object TimestampHandler : RelationalColumnCustomConverter.Handler { + override fun matches(column: RelationalColumn): Boolean = column.typeName().equals("TIMESTAMP", ignoreCase = true) @@ -171,7 +202,14 @@ class MySqlSourceCdcTemporalConverter : RelationalColumnCustomConverter { override val partialConverters: List = listOf( - NullFallThrough, + PartialConverter { + if (it == null) { + val epoch = OffsetDateTime.ofInstant(Instant.EPOCH, ZoneOffset.UTC) + Converted(epoch.format(OffsetDateTimeCodec.formatter)) + } else { + NoConversion + } + }, PartialConverter { if (it is ZonedDateTime) { val offsetDateTime: OffsetDateTime = it.toOffsetDateTime() diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 95540492579e..dfe8530d3738 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -184,42 +184,42 @@ Any database or table encoding combination of charset and collation is supported
MySQL Data Type Mapping -| MySQL Type | Resulting Type | Notes | -| :---------------------------------------- | :--------------------- | :------------------------------------------------------------------------------------------------------------- | -| `bit(1)` | boolean | | -| `bit(>1)` | base64 binary string | | -| `boolean` | boolean | | -| `tinyint(1)` | boolean | | -| `tinyint(>1)` | number | | -| `tinyint(>=1) unsigned` | number | | -| `smallint` | number | | -| `mediumint` | number | | -| `int` | number | | -| `bigint` | number | | -| `float` | number | | -| `double` | number | | -| `decimal` | number | | -| `binary` | string | | -| `blob` | string | | -| `date` | string | ISO 8601 date string. ZERO-DATE value will be converted to NULL. If column is mandatory, convert to EPOCH. | -| `datetime`, `timestamp` | string | ISO 8601 datetime string. ZERO-DATE value will be converted to NULL. If column is mandatory, convert to EPOCH. | -| `time` | string | ISO 8601 time string. Values are in range between 00:00:00 and 23:59:59. | -| `year` | year string | [Doc](https://dev.mysql.com/doc/refman/8.0/en/year.html) | -| `char`, `varchar` with non-binary charset | string | | -| `tinyblob` | base64 binary string | | -| `blob` | base64 binary string | | -| `mediumblob` | base64 binary string | | -| `longblob` | base64 binary string | | -| `binary` | base64 binary string | | -| `varbinary` | base64 binary string | | -| `tinytext` | string | | -| `text` | string | | -| `mediumtext` | string | | -| `longtext` | string | | -| `json` | serialized json string | E.g. `{"a": 10, "b": 15}` | -| `enum` | string | | -| `set` | string | E.g. `blue,green,yellow` | -| `geometry` | base64 binary string | | +| MySQL Type | Resulting Type | Notes | +| :---------------------------------------- | :--------------------- |:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `bit(1)` | boolean | | +| `bit(>1)` | base64 binary string | | +| `boolean` | boolean | | +| `tinyint(1)` | boolean | | +| `tinyint(>1)` | number | | +| `tinyint(>=1) unsigned` | number | | +| `smallint` | number | | +| `mediumint` | number | | +| `int` | number | | +| `bigint` | number | | +| `float` | number | | +| `double` | number | | +| `decimal` | number | | +| `binary` | string | | +| `blob` | string | | +| `date` | string | ISO-8601 datetime string. Zero-date values will be converted to Unix epoch (`1970-01-01`) when using CDC mode to prevent sync failures. In non-CDC syncs, zero-dates will be NULL. If using a zero-date column as a cursor or if the column is NOT NULL, convert to EPOCH. | +| `datetime`, `timestamp` | string | ISO-8601 datetime string. Zero-date values will be converted to Unix epoch (`1970-01-01`) when using CDC mode to prevent sync failures. In non-CDC syncs, zero-dates will be NULL. If using a zero-date column as a cursor or if the column is NOT NULL, convert to EPOCH. | +| `time` | string | ISO 8601 time string. Values are in range between 00:00:00 and 23:59:59. | +| `year` | year string | [Doc](https://dev.mysql.com/doc/refman/8.0/en/year.html) | +| `char`, `varchar` with non-binary charset | string | | +| `tinyblob` | base64 binary string | | +| `blob` | base64 binary string | | +| `mediumblob` | base64 binary string | | +| `longblob` | base64 binary string | | +| `binary` | base64 binary string | | +| `varbinary` | base64 binary string | | +| `tinytext` | string | | +| `text` | string | | +| `mediumtext` | string | | +| `longtext` | string | | +| `json` | serialized json string | E.g. `{"a": 10, "b": 15}` | +| `enum` | string | | +| `set` | string | E.g. `blue,green,yellow` | +| `geometry` | base64 binary string | |
@@ -230,7 +230,8 @@ Any database or table encoding combination of charset and collation is supported | Version | Date | Pull Request | Subject | |:------------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| -| 3.50.9 | 2025-10-06 | [67151](https://github.com/airbytehq/airbyte/pull/66515) | Fix CDC decorating fields encoding to Protobuf | +| 3.50.10 | 2025-10-10 | [67623](https://github.com/airbytehq/airbyte/pull/67623) | Fix CDC sync failures caused by zero-dates in non-nullable columns. | +| 3.50.9 | 2025-10-06 | [67151](https://github.com/airbytehq/airbyte/pull/67151) | Fix CDC decorating fields encoding to Protobuf | | 3.50.8 | 2025-09-18 | [66515](https://github.com/airbytehq/airbyte/pull/66515) | Fix division by zero in partition creation when sampling produces no split boundaries. | | 3.50.7 | 2025-09-10 | [66179](https://github.com/airbytehq/airbyte/pull/66179) | Bump to the latest CDK fixing protobuf encoding of certain column types | | 3.50.6 | 2025-08-08 | [64569](https://github.com/airbytehq/airbyte/pull/64569) | Moved db version logging from connector to new CDK version |