Skip to content

Commit 59f8c11

Browse files
dtunikovclaude
andauthored
mysql: convert BIT to UInt64 in binary type path for new mirrors (#4423)
## What `qkindFromMysqlType` (the binary binlog / QRep type-conversion path) mapped MySQL `BIT` to `Int64`, while `QkindFromMysqlColumnType` (the `information_schema` / DDL text path) already mapped it to `UInt64`. This made `BIT` columns that flow through the binary path inconsistent with the rest of the system. The binary path is reached for: - columns discovered via `TABLE_MAP_EVENT` without a captured DDL event (e.g. gh-ost / online-schema-change migrations) — `processTableMapEventSchema` - the QRep watermark column type - the `QRecordSchemaFromMysqlFields` fallback for columns absent from the table schema A plain `ALTER TABLE ... ADD COLUMN bit(...)` is **not** affected — it's captured as a DDL query event and already routes through the `UInt64` text path. ## Change - Map `BIT` to `UInt64` in `qkindFromMysqlType`, gated behind a new `InternalVersion_MySQLConvertBitToUInt64`. - Thread the mirror version through `qkindFromMysqlType` / `QRecordSchemaFromMysqlFields` and their callers (CDC, QRep, e2e helper). - Add a unit test covering the version gate. ## Why gate it The destination column type is fixed when a column is first created. An existing mirror that already materialized a `BIT` column as `Int64` via the binary path would break if the code started emitting `UInt64` values into that `Int64` column on upgrade (schema deltas add columns, they don't retype existing ones). Gating on `InternalVersion` keeps existing mirrors on their previous behavior and gives only new mirrors the consistent `UInt64` mapping. ## Testing - New unit test `TestQkindFromMysqlType_Bit` asserts `Int64` below the gate version and `UInt64` at/above it. - `go build ./...` and the `connectors/mysql` unit tests pass. > Note: a normal full-table BIT e2e test does **not** exercise this path, since in-schema BIT columns resolve their type via the ungated text path. The meaningful behavioral difference is limited to the gh-ost/`TABLE_MAP_EVENT`-detected-column and QRep cases described above. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 842a37b commit 59f8c11

6 files changed

Lines changed: 43 additions & 7 deletions

File tree

flow/connectors/mysql/cdc.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -914,7 +914,7 @@ func (c *MySqlConnector) processTableMapEventSchema(
914914
charset = uint16(collation)
915915
}
916916
mytype := tableMap.ColumnType[idx]
917-
qkind, err := qkindFromMysqlType(mytype, unsignedMap[idx], charset)
917+
qkind, err := qkindFromMysqlType(mytype, unsignedMap[idx], charset, req.InternalVersion)
918918
if err != nil {
919919
c.logger.Warn("Unknown MySQL type for new column, skipping",
920920
slog.String("table", sourceTableName),

flow/connectors/mysql/qrep.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ func (c *MySqlConnector) GetQRepPartitions(
141141
watermarkField := rs.Fields[1]
142142
watermarkMyType := watermarkField.Type
143143
watermarkUnsigned := (watermarkField.Flag & mysql.UNSIGNED_FLAG) != 0
144-
watermarkQKind, err := qkindFromMysqlType(watermarkField.Type, watermarkUnsigned, watermarkField.Charset)
144+
watermarkQKind, err := qkindFromMysqlType(watermarkField.Type, watermarkUnsigned, watermarkField.Charset, config.Version)
145145
if err != nil {
146146
return nil, fmt.Errorf("failed to convert mysql type to qvaluekind: %w", err)
147147
}
@@ -231,7 +231,7 @@ func (c *MySqlConnector) PullQRepRecords(
231231
c.deltaBytesRead.Store(0)
232232
totalRecords := int64(0)
233233
onResult := func(rs *mysql.Result) error {
234-
schema, err := QRecordSchemaFromMysqlFields(tableSchema, rs.Fields)
234+
schema, err := QRecordSchemaFromMysqlFields(tableSchema, rs.Fields, config.Version)
235235
if err != nil {
236236
return err
237237
}

flow/connectors/mysql/qvalue_convert.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ import (
2424
"github.com/PeerDB-io/peerdb/flow/shared/types"
2525
)
2626

27-
func qkindFromMysqlType(mytype byte, unsigned bool, charset uint16) (types.QValueKind, error) {
27+
func qkindFromMysqlType(mytype byte, unsigned bool, charset uint16, version uint32) (types.QValueKind, error) {
2828
switch mytype {
2929
case mysql.MYSQL_TYPE_TINY:
3030
if unsigned {
@@ -66,6 +66,9 @@ func qkindFromMysqlType(mytype byte, unsigned bool, charset uint16) (types.QValu
6666
case mysql.MYSQL_TYPE_YEAR:
6767
return types.QValueKindInt16, nil
6868
case mysql.MYSQL_TYPE_BIT:
69+
if version >= shared.InternalVersion_MySQLConvertBitToUInt64 {
70+
return types.QValueKindUInt64, nil
71+
}
6972
return types.QValueKindInt64, nil
7073
case mysql.MYSQL_TYPE_JSON:
7174
return types.QValueKindJSON, nil
@@ -92,7 +95,7 @@ func qkindFromMysqlType(mytype byte, unsigned bool, charset uint16) (types.QValu
9295
}
9396
}
9497

95-
func QRecordSchemaFromMysqlFields(tableSchema *protos.TableSchema, fields []*mysql.Field) (types.QRecordSchema, error) {
98+
func QRecordSchemaFromMysqlFields(tableSchema *protos.TableSchema, fields []*mysql.Field, version uint32) (types.QRecordSchema, error) {
9699
tableColumns := make(map[string]*protos.FieldDescription, len(tableSchema.Columns))
97100
for _, col := range tableSchema.Columns {
98101
tableColumns[col.Name] = col
@@ -112,7 +115,7 @@ func QRecordSchemaFromMysqlFields(tableSchema *protos.TableSchema, fields []*mys
112115
} else {
113116
var err error
114117
unsigned := (field.Flag & mysql.UNSIGNED_FLAG) != 0
115-
qkind, err = qkindFromMysqlType(field.Type, unsigned, field.Charset)
118+
qkind, err = qkindFromMysqlType(field.Type, unsigned, field.Charset, version)
116119
if err != nil {
117120
return types.QRecordSchema{}, err
118121
}

flow/connectors/mysql/qvalue_convert_test.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,40 @@ import (
44
"testing"
55
"time"
66

7+
"github.com/go-mysql-org/go-mysql/mysql"
78
"github.com/stretchr/testify/require"
9+
10+
"github.com/PeerDB-io/peerdb/flow/shared"
11+
"github.com/PeerDB-io/peerdb/flow/shared/types"
812
)
913

14+
func TestQkindFromMysqlType_Bit(t *testing.T) {
15+
for _, tc := range []struct {
16+
name string
17+
version uint32
18+
want types.QValueKind
19+
}{
20+
{"first version maps BIT to Int64", shared.InternalVersion_First, types.QValueKindInt64},
21+
{
22+
"version just before the gate maps BIT to Int64",
23+
shared.InternalVersion_MySQLConvertBitToUInt64 - 1,
24+
types.QValueKindInt64,
25+
},
26+
{
27+
"gating version maps BIT to UInt64",
28+
shared.InternalVersion_MySQLConvertBitToUInt64,
29+
types.QValueKindUInt64,
30+
},
31+
{"latest version maps BIT to UInt64", shared.InternalVersion_Latest, types.QValueKindUInt64},
32+
} {
33+
t.Run(tc.name, func(t *testing.T) {
34+
qkind, err := qkindFromMysqlType(mysql.MYSQL_TYPE_BIT, false, 0, tc.version)
35+
require.NoError(t, err)
36+
require.Equal(t, tc.want, qkind)
37+
})
38+
}
39+
}
40+
1041
func TestProcessTime(t *testing.T) {
1142
epoch := time.Unix(0, 0).UTC()
1243
for _, ts := range []struct {

flow/e2e/mysql.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ func (s *MySqlSource) GetRows(ctx context.Context, suffix string, table string,
193193
return nil, err
194194
}
195195

196-
schema, err := connmysql.QRecordSchemaFromMysqlFields(tableSchemas[tableName], rs.Fields)
196+
schema, err := connmysql.QRecordSchemaFromMysqlFields(tableSchemas[tableName], rs.Fields, shared.InternalVersion_Latest)
197197
if err != nil {
198198
return nil, err
199199
}

flow/shared/constants.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ const (
3737
InternalVersion_MongoDBIdWithoutRedundantQuotes
3838
// MySQL: convert enums to integers for older versions without binlog row metadata support
3939
InternalVersion_MySQL5ConvertEnumsToInts
40+
// MySQL: convert BIT to UInt64
41+
InternalVersion_MySQLConvertBitToUInt64
4042

4143
TotalNumberOfInternalVersions
4244
InternalVersion_Latest = TotalNumberOfInternalVersions - 1

0 commit comments

Comments
 (0)