Skip to content

Commit 9cd01df

Browse files
notfelineitnotfelineit
authored
Merge pull request #124 from planetscale/use-oneOf
Nullable columns: output in `oneOf` JSON schema format
2 parents d800f4c + 26600e1 commit 9cd01df

File tree

4 files changed

+83
-41
lines changed

4 files changed

+83
-41
lines changed

‎cmd/airbyte-source/read_test.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ func TestRead_StartingGtidsAndState(t *testing.T) {
2020
Password: "pscale_password",
2121
StartingGtids: "{\"sharded\": {\"-80\": \"MySQL56/MyGTID:1-3\"}}",
2222
}
23+
24+
numberType := "number"
25+
2326
streams := []internal.ConfiguredStream{
2427
{
2528
Stream: internal.Stream{
@@ -28,7 +31,7 @@ func TestRead_StartingGtidsAndState(t *testing.T) {
2831
Type: "object",
2932
Properties: map[string]internal.PropertyType{
3033
"id": {
31-
Type: []string{"number"},
34+
Type: &numberType,
3235
AirbyteType: "integer",
3336
},
3437
},
@@ -55,7 +58,7 @@ func TestRead_StartingGtidsAndState(t *testing.T) {
5558
Type: "object",
5659
Properties: map[string]internal.PropertyType{
5760
"id": {
58-
Type: []string{"number"},
61+
Type: &numberType,
5962
AirbyteType: "integer",
6063
},
6164
},

‎cmd/internal/planetscale_edge_database.go

+35-10
Original file line numberDiff line numberDiff line change
@@ -135,31 +135,56 @@ func (p PlanetScaleEdgeDatabase) getStreamForTable(ctx context.Context, psc Plan
135135
func getJsonSchemaType(mysqlType string, treatTinyIntAsBoolean bool, nullable string) PropertyType {
136136
// Support custom airbyte types documented here :
137137
// https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types
138-
var propertyType PropertyType
138+
var (
139+
jsonSchemaType string
140+
customFormat string
141+
airbyteType string
142+
oneOf []OneOfType
143+
)
139144

140145
switch {
141146
case strings.HasPrefix(mysqlType, "tinyint(1)"):
142147
if treatTinyIntAsBoolean {
143-
propertyType = PropertyType{Type: []string{"boolean"}}
148+
jsonSchemaType = "boolean"
144149
} else {
145-
propertyType = PropertyType{Type: []string{"number"}, AirbyteType: "integer"}
150+
jsonSchemaType = "number"
151+
airbyteType = "integer"
146152
}
147153
case strings.HasPrefix(mysqlType, "int"), strings.HasPrefix(mysqlType, "smallint"), strings.HasPrefix(mysqlType, "mediumint"), strings.HasPrefix(mysqlType, "bigint"), strings.HasPrefix(mysqlType, "tinyint"):
148-
propertyType = PropertyType{Type: []string{"number"}, AirbyteType: "integer"}
154+
jsonSchemaType = "number"
155+
airbyteType = "integer"
149156
case strings.HasPrefix(mysqlType, "decimal"), strings.HasPrefix(mysqlType, "double"), strings.HasPrefix(mysqlType, "float"):
150-
propertyType = PropertyType{Type: []string{"number"}}
157+
jsonSchemaType = "number"
151158
case strings.HasPrefix(mysqlType, "datetime"), strings.HasPrefix(mysqlType, "timestamp"):
152-
propertyType = PropertyType{Type: []string{"string"}, CustomFormat: "date-time", AirbyteType: "timestamp_without_timezone"}
159+
jsonSchemaType = "string"
160+
customFormat = "date-time"
161+
airbyteType = "timestamp_without_timezone"
153162
case strings.HasPrefix(mysqlType, "date"):
154-
propertyType = PropertyType{Type: []string{"string"}, CustomFormat: "date", AirbyteType: "date"}
163+
jsonSchemaType = "string"
164+
customFormat = "date"
165+
airbyteType = "date"
155166
case strings.HasPrefix(mysqlType, "time"):
156-
propertyType = PropertyType{Type: []string{"string"}, CustomFormat: "time", AirbyteType: "time_without_timezone"}
167+
jsonSchemaType = "string"
168+
customFormat = "time"
169+
airbyteType = "time_without_timezone"
157170
default:
158-
propertyType = PropertyType{Type: []string{"string"}}
171+
jsonSchemaType = "string"
172+
}
173+
174+
propertyType := PropertyType{
175+
Type: &jsonSchemaType,
176+
CustomFormat: customFormat,
177+
AirbyteType: airbyteType,
159178
}
160179

161180
if strings.ToLower(nullable) == "yes" {
162-
propertyType.Type = append(propertyType.Type, "null")
181+
oneOf = []OneOfType{
182+
{Type: jsonSchemaType},
183+
{Type: "null"},
184+
}
185+
186+
propertyType.Type = nil
187+
propertyType.OneOf = oneOf
163188
}
164189

165190
return propertyType

‎cmd/internal/planetscale_edge_database_test.go

+35-26
Original file line numberDiff line numberDiff line change
@@ -357,127 +357,135 @@ func TestRead_CanPickRdonlyForShardedKeyspaces(t *testing.T) {
357357
}
358358

359359
func TestDiscover_CanPickRightAirbyteType(t *testing.T) {
360+
numberType := "number"
361+
booleanType := "boolean"
362+
stringType := "string"
363+
360364
var tests = []struct {
361365
MysqlType string
362-
JSONSchemaType []string
366+
JSONSchemaType *string
367+
OneOf []OneOfType
363368
AirbyteType string
364369
TreatTinyIntAsBoolean bool
365370
IsNullable string
366371
}{
367372
{
368373
MysqlType: "int(11)",
369-
JSONSchemaType: []string{"number"},
374+
JSONSchemaType: &numberType,
370375
AirbyteType: "integer",
371376
},
372377
{
373378
MysqlType: "smallint(4)",
374-
JSONSchemaType: []string{"number"},
379+
JSONSchemaType: &numberType,
375380
AirbyteType: "integer",
376381
},
377382
{
378383
MysqlType: "mediumint(8)",
379-
JSONSchemaType: []string{"number"},
384+
JSONSchemaType: &numberType,
380385
AirbyteType: "integer",
381386
},
382387
{
383388
MysqlType: "tinyint",
384-
JSONSchemaType: []string{"number"},
389+
JSONSchemaType: &numberType,
385390
AirbyteType: "integer",
386391
TreatTinyIntAsBoolean: true,
387392
},
388393
{
389394
MysqlType: "tinyint(1)",
390-
JSONSchemaType: []string{"boolean"},
395+
JSONSchemaType: &booleanType,
391396
AirbyteType: "",
392397
TreatTinyIntAsBoolean: true,
393398
},
394399
{
395400
MysqlType: "tinyint(1) unsigned",
396-
JSONSchemaType: []string{"boolean"},
401+
JSONSchemaType: &booleanType,
397402
AirbyteType: "",
398403
TreatTinyIntAsBoolean: true,
399404
},
400405
{
401406
MysqlType: "tinyint(1)",
402-
JSONSchemaType: []string{"number"},
407+
JSONSchemaType: &numberType,
403408
AirbyteType: "integer",
404409
TreatTinyIntAsBoolean: false,
405410
},
406411
{
407412
MysqlType: "tinyint(1) unsigned",
408-
JSONSchemaType: []string{"number"},
413+
JSONSchemaType: &numberType,
409414
AirbyteType: "integer",
410415
TreatTinyIntAsBoolean: false,
411416
},
412417
{
413418
MysqlType: "bigint(16)",
414-
JSONSchemaType: []string{"number"},
419+
JSONSchemaType: &numberType,
415420
AirbyteType: "integer",
416421
},
417422
{
418423
MysqlType: "bigint unsigned",
419-
JSONSchemaType: []string{"number"},
424+
JSONSchemaType: &numberType,
420425
AirbyteType: "integer",
421426
},
422427
{
423428
MysqlType: "bigint zerofill",
424-
JSONSchemaType: []string{"number"},
429+
JSONSchemaType: &numberType,
425430
AirbyteType: "integer",
426431
},
427432
{
428433
MysqlType: "datetime",
429-
JSONSchemaType: []string{"string"},
434+
JSONSchemaType: &stringType,
430435
AirbyteType: "timestamp_without_timezone",
431436
},
432437
{
433438
MysqlType: "datetime(6)",
434-
JSONSchemaType: []string{"string"},
439+
JSONSchemaType: &stringType,
435440
AirbyteType: "timestamp_without_timezone",
436441
},
437442
{
438443
MysqlType: "time",
439-
JSONSchemaType: []string{"string"},
444+
JSONSchemaType: &stringType,
440445
AirbyteType: "time_without_timezone",
441446
},
442447
{
443448
MysqlType: "time(6)",
444-
JSONSchemaType: []string{"string"},
449+
JSONSchemaType: &stringType,
445450
AirbyteType: "time_without_timezone",
446451
},
447452
{
448453
MysqlType: "date",
449-
JSONSchemaType: []string{"string"},
454+
JSONSchemaType: &stringType,
450455
AirbyteType: "date",
451456
},
452457
{
453458
MysqlType: "text",
454-
JSONSchemaType: []string{"string"},
459+
JSONSchemaType: &stringType,
455460
AirbyteType: "",
456461
},
457462
{
458463
MysqlType: "varchar(256)",
459-
JSONSchemaType: []string{"string"},
464+
JSONSchemaType: &stringType,
460465
AirbyteType: "",
461466
},
462467
{
463-
MysqlType: "varchar(256)",
464-
JSONSchemaType: []string{"string", "null"},
465-
AirbyteType: "",
466-
IsNullable: "YES",
468+
MysqlType: "varchar(256)",
469+
AirbyteType: "",
470+
IsNullable: "YES",
471+
OneOf: []OneOfType{
472+
{Type: "string"},
473+
{Type: "null"},
474+
},
467475
},
468476
{
469477
MysqlType: "decimal(12,5)",
470-
JSONSchemaType: []string{"number"},
478+
JSONSchemaType: &numberType,
471479
AirbyteType: "",
472480
},
473481
{
474482
MysqlType: "double",
475-
JSONSchemaType: []string{"number"},
483+
JSONSchemaType: &numberType,
476484
AirbyteType: "",
477485
},
478486
{
479487
MysqlType: "float(30)",
480-
JSONSchemaType: []string{"number"},
488+
JSONSchemaType: &numberType,
481489
AirbyteType: "",
482490
},
483491
}
@@ -488,6 +496,7 @@ func TestDiscover_CanPickRightAirbyteType(t *testing.T) {
488496
p := getJsonSchemaType(typeTest.MysqlType, typeTest.TreatTinyIntAsBoolean, typeTest.IsNullable)
489497
assert.Equal(t, typeTest.AirbyteType, p.AirbyteType)
490498
assert.Equal(t, typeTest.JSONSchemaType, p.Type)
499+
assert.Equal(t, typeTest.OneOf, p.OneOf)
491500
})
492501
}
493502
}

‎cmd/internal/types.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,14 @@ const (
5454
)
5555

5656
type PropertyType struct {
57-
Type []string `json:"type"`
58-
CustomFormat string `json:"format,omitempty"`
59-
AirbyteType string `json:"airbyte_type,omitempty"`
57+
Type *string `json:"type,omitempty"`
58+
CustomFormat string `json:"format,omitempty"`
59+
AirbyteType string `json:"airbyte_type,omitempty"`
60+
OneOf []OneOfType `json:"oneOf,omitempty"`
61+
}
62+
63+
type OneOfType struct {
64+
Type string `json:"type"`
6065
}
6166

6267
type StreamSchema struct {

0 commit comments

Comments
 (0)