Skip to content

Commit b0cb727

Browse files
scgkiranscgkiran
authored
feat: add is_null support in testfile parser (#120)
Add nullable parameter to literal creation utils Literal creation helpers now include an additional boolean parameter for nullability.
1 parent bd507d3 commit b0cb727

File tree

12 files changed

+2119
-1338
lines changed

12 files changed

+2119
-1338
lines changed

‎expr/literals.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,19 @@ func (t *PrimitiveLiteral[T]) ToProtoFuncArg() *proto.FunctionArgument {
239239
func (t *PrimitiveLiteral[T]) Visit(VisitFunc) Expression { return t }
240240
func (*PrimitiveLiteral[T]) IsScalar() bool { return true }
241241

242+
func (t *PrimitiveLiteral[T]) WithType(newType types.Type) (Literal, error) {
243+
switch newType.(type) {
244+
case *types.FixedLenType[types.FixedChar]:
245+
if val, ok := any(t.Value).(types.FixedChar); ok {
246+
return &PrimitiveLiteral[types.FixedChar]{
247+
Value: val,
248+
Type: newType,
249+
}, nil
250+
}
251+
}
252+
return nil, fmt.Errorf("invalid type %T for primitive literal", newType)
253+
}
254+
242255
// NestedLiteral is either a Struct or List literal, both of which are
243256
// represented as a slice of other literals.
244257
type NestedLiteral[T nestedLiteral] struct {
@@ -680,7 +693,7 @@ func newVarCharWithType(literal *ProtoLiteral, vcharType *types.VarCharType) (Li
680693
return nil, fmt.Errorf("literal type is not varchar")
681694
}
682695
if len(literal.Value.(string)) > int(vcharType.GetLength()) {
683-
return nil, fmt.Errorf("varchar litearl value length is greater than type length")
696+
return nil, fmt.Errorf("varchar literal value length is greater than type length")
684697
}
685698
return &ProtoLiteral{Value: literal.Value, Type: vcharType}, nil
686699
}

‎expr/literals_test.go‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func TestNewDecimalWithType(t *testing.T) {
2828
}
2929
for _, tt := range tests {
3030
t.Run(tt.name, func(t *testing.T) {
31-
lit, err := literal.NewDecimalFromString(tt.name)
31+
lit, err := literal.NewDecimalFromString(tt.name, false)
3232
require.NoError(t, err)
3333
got, err := lit.(*expr.ProtoLiteral).WithType(tt.decType)
3434
if tt.expectedToFail {
@@ -79,7 +79,7 @@ func TestNewFixedLenWithType(t *testing.T) {
7979
}
8080
for _, tt := range tests {
8181
t.Run(tt.name, func(t *testing.T) {
82-
input, _ := literal.NewVarChar(tt.name)
82+
input, _ := literal.NewVarChar(tt.name, false)
8383
got, err := input.(*expr.ProtoLiteral).WithType(tt.inputType.(*types.VarCharType))
8484
if tt.wantErr {
8585
require.Error(t, err)
@@ -104,7 +104,7 @@ func TestNewPrecisionTimestampWithType(t *testing.T) {
104104
}
105105
for _, tt := range tests {
106106
t.Run(tt.name, func(t *testing.T) {
107-
lit, err := literal.NewPrecisionTimestampFromString(tt.inputPrecision, tt.name)
107+
lit, err := literal.NewPrecisionTimestampFromString(tt.inputPrecision, tt.name, false)
108108
require.NoError(t, err)
109109
got, err := lit.(*expr.ProtoLiteral).WithType(tt.inputType)
110110
if tt.wantErr {
@@ -131,7 +131,7 @@ func TestNewPrecisionTimestampTzWithType(t *testing.T) {
131131
}
132132
for _, tt := range tests {
133133
t.Run(tt.name, func(t *testing.T) {
134-
lit, err := literal.NewPrecisionTimestampTzFromString(tt.inputPrecision, tt.name)
134+
lit, err := literal.NewPrecisionTimestampTzFromString(tt.inputPrecision, tt.name, false)
135135
require.NoError(t, err)
136136
inputType := &types.PrecisionTimestampTzType{PrecisionTimestampType: tt.inputType}
137137
got, err := lit.(*expr.ProtoLiteral).WithType(inputType)
@@ -161,7 +161,7 @@ func TestNewIntervalDayWithType(t *testing.T) {
161161
}
162162
for _, tt := range tests {
163163
t.Run(tt.name, func(t *testing.T) {
164-
lit, err := literal.NewIntervalDaysToSecondFromString(tt.name)
164+
lit, err := literal.NewIntervalDaysToSecondFromString(tt.name, false)
165165
require.NoError(t, err)
166166
got, err := lit.(*expr.ProtoLiteral).WithType(tt.inputType)
167167
if tt.wantErr {
@@ -179,11 +179,11 @@ func TestNewIntervalDayWithType(t *testing.T) {
179179
}
180180

181181
func TestProtoLiteral_WithType(t1 *testing.T) {
182-
dec123, _ := literal.NewDecimalFromString("123.45")
183-
iday, _ := literal.NewIntervalDaysToSecondFromString("PT23H59M59.999S")
184-
pts, _ := literal.NewPrecisionTimestampFromString(3, "1991-01-01T01:02:03.456")
185-
ptstz, _ := literal.NewPrecisionTimestampTzFromString(3, "1991-01-01T01:02:03.456")
186-
vchar, _ := literal.NewVarChar("sun")
182+
dec123, _ := literal.NewDecimalFromString("123.45", false)
183+
iday, _ := literal.NewIntervalDaysToSecondFromString("PT23H59M59.999S", false)
184+
pts, _ := literal.NewPrecisionTimestampFromString(3, "1991-01-01T01:02:03.456", false)
185+
ptstz, _ := literal.NewPrecisionTimestampTzFromString(3, "1991-01-01T01:02:03.456", false)
186+
vchar, _ := literal.NewVarChar("sun", false)
187187
tests := []struct {
188188
name string
189189
protoLiteral *expr.ProtoLiteral
@@ -213,8 +213,10 @@ func TestByteSliceLiteral_WithType(t1 *testing.T) {
213213
uuid := expr.NewByteSliceLiteral[types.UUID]([]byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10}, false)
214214

215215
list := expr.NewNestedLiteral(expr.ListLiteralValue{
216-
literal.NewString("sun"), literal.NewString("moon"), literal.NewString("mars"),
216+
literal.NewString("sun", false), literal.NewString("moon", false), literal.NewString("mars", false),
217217
}, false)
218+
219+
fchar := expr.NewFixedCharLiteral("moon", false)
218220
type testCase struct {
219221
name string
220222
t expr.WithTypeLiteral
@@ -224,6 +226,7 @@ func TestByteSliceLiteral_WithType(t1 *testing.T) {
224226
}
225227
tests := []testCase{
226228
{"FixedBinary", fbin, &types.FixedBinaryType{Length: 3, Nullability: types.NullabilityNullable}, nil, assert.NoError},
229+
{"FixedChar", fchar, &types.FixedCharType{Length: 3, Nullability: types.NullabilityNullable}, nil, assert.NoError},
227230
{"UUID", uuid, &types.UUIDType{Nullability: types.NullabilityNullable}, nil, assert.NoError},
228231
{"List", list.(expr.WithTypeLiteral), &types.ListType{Type: &types.StringType{Nullability: types.NullabilityNullable}, Nullability: types.NullabilityNullable}, nil, assert.NoError},
229232
}

‎expr/string_test.go‎

Lines changed: 23 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ func TestLiteralToString(t *testing.T) {
4747
{expr.NewPrecisionTimestampTzLiteral(123456, types.PrecisionMilliSeconds, types.NullabilityNullable), "precision_timestamp_tz?<3>(1970-01-01T00:02:03.456Z)"},
4848
{expr.NewPrecisionTimestampTzLiteral(123456, types.PrecisionMicroSeconds, types.NullabilityNullable), "precision_timestamp_tz?<6>(1970-01-01T00:00:00.123456Z)"},
4949
{expr.NewPrecisionTimestampTzLiteral(123456, types.PrecisionNanoSeconds, types.NullabilityNullable), "precision_timestamp_tz?<9>(1970-01-01T00:00:00.000123456Z)"},
50-
{MustLiteral(literal.NewDecimalFromString("12.345")), "decimal<5,3>(12.345)"},
51-
{MustLiteral(literal.NewDecimalFromString("-12.345")), "decimal<5,3>(-12.345)"},
50+
{MustLiteral(literal.NewDecimalFromString("12.345", false)), "decimal<5,3>(12.345)"},
51+
{MustLiteral(literal.NewDecimalFromString("-12.345", false)), "decimal<5,3>(-12.345)"},
5252
}
5353

5454
for _, tt := range tests {
@@ -64,33 +64,32 @@ func TestLiteralToValueString(t *testing.T) {
6464
exp string
6565
}{
6666
{expr.NewNullLiteral(&types.Float32Type{}), "null"},
67-
{literal.NewBool(true), "true"},
68-
{literal.NewInt8(12), "12"},
67+
{literal.NewBool(true, false), "true"},
68+
{literal.NewInt8(12, false), "12"},
6969
{expr.NewPrimitiveLiteral[int8](0, true), "0"},
70-
{literal.NewInt16(0), "0"},
71-
{literal.NewInt32(99), "99"},
72-
{literal.NewFloat32(99.10), "99.1"},
73-
{literal.NewFloat64(99.20), "99.2"},
74-
{literal.NewString("99.30"), "99.30"},
75-
{MustLiteral(literal.NewDate(365)), "1971-01-01"},
76-
{MustLiteral(literal.NewTimeFromString("12:34:56")), "12:34:56"},
77-
{MustLiteral(literal.NewTimestampFromString("2021-03-05T12:34:56")), "2021-03-05 12:34:56"},
78-
{MustLiteral(literal.NewTimestampTZFromString("2021-03-05T12:34:56")), "2021-03-05T12:34:56Z"},
70+
{literal.NewInt16(0, false), "0"},
71+
{literal.NewInt32(99, false), "99"},
72+
{literal.NewFloat32(99.10, false), "99.1"},
73+
{literal.NewFloat64(99.20, false), "99.2"},
74+
{literal.NewString("99.30", false), "99.30"},
75+
{MustLiteral(literal.NewDate(365, false)), "1971-01-01"},
76+
{MustLiteral(literal.NewTimeFromString("12:34:56", false)), "12:34:56"},
77+
{MustLiteral(literal.NewTimestampFromString("2021-03-05T12:34:56", false)), "2021-03-05 12:34:56"},
78+
{MustLiteral(literal.NewTimestampTZFromString("2021-03-05T12:34:56", false)), "2021-03-05T12:34:56Z"},
7979
// Test the first implementation.
80-
{MustLiteral(literal.NewIntervalYearsToMonth(5, 4)), "5 years, 4 months"},
80+
{MustLiteral(literal.NewIntervalYearsToMonth(5, 4, false)), "5 years, 4 months"},
8181
// Test the other implementation.
8282
{&expr.IntervalYearToMonthLiteral{Years: 7, Months: 6}, "7 years, 6 months"},
83-
{MustLiteral(literal.NewIntervalDaysToSecond(5, 4, 3)), "5 days, 4 seconds, 3 subseconds"},
83+
{MustLiteral(literal.NewIntervalDaysToSecond(5, 4, 3, false)), "5 days, 4 seconds, 3 subseconds"},
8484
{t: &expr.IntervalCompoundLiteral{
8585
Years: 5, Months: 4, Days: 3,
8686
Seconds: 2, SubSeconds: 1, SubSecondPrecision: types.PrecisionMicroSeconds,
8787
Nullability: types.NullabilityRequired}, exp: "5 years, 4 months, 3 days, 2 seconds, 1 subseconds"},
88-
{MustLiteral(literal.NewUUIDFromBytes(
89-
[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16})),
88+
{MustLiteral(literal.NewUUIDFromBytes([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, false)),
9089
"01020304-0506-0708-090a-0b0c0d0e0f10"},
91-
{MustLiteral(literal.NewFixedChar("text")), "text"},
92-
{MustLiteral(literal.NewFixedBinary([]byte{1, 2, 3})), "0x010203"},
93-
{MustLiteral(literal.NewVarChar("vartext")), "vartext"},
90+
{MustLiteral(literal.NewFixedChar("text", false)), "text"},
91+
{MustLiteral(literal.NewFixedBinary([]byte{1, 2, 3}, false)), "0x010203"},
92+
{MustLiteral(literal.NewVarChar("vartext", false)), "vartext"},
9493
{expr.NewNestedLiteral(expr.ListLiteralValue{
9594
expr.NewNestedLiteral(expr.MapLiteralValue{
9695
{
@@ -135,9 +134,9 @@ func TestLiteralToValueString(t *testing.T) {
135134
{expr.NewPrecisionTimestampTzLiteral(123456, types.PrecisionEMinus7Seconds, types.NullabilityNullable), "1970-01-01T00:00:00.0123456Z"},
136135
{expr.NewPrecisionTimestampTzLiteral(123456, types.PrecisionEMinus8Seconds, types.NullabilityNullable), "1970-01-01T00:00:00.00123456Z"},
137136
{expr.NewPrecisionTimestampTzLiteral(123456, types.PrecisionNanoSeconds, types.NullabilityNullable), "1970-01-01T00:00:00.000123456Z"},
138-
{MustLiteral(literal.NewDecimalFromString("12.345")), "12.345"},
139-
{MustLiteral(literal.NewDecimalFromString("-12.345")), "-12.345"},
140-
{MustLiteral(literal.NewList([]expr.Literal{literal.NewInt8(2), literal.NewInt8(4), literal.NewInt8(6)})), "[2, 4, 6]"},
137+
{MustLiteral(literal.NewDecimalFromString("12.345", false)), "12.345"},
138+
{MustLiteral(literal.NewDecimalFromString("-12.345", false)), "-12.345"},
139+
{MustLiteral(literal.NewList([]expr.Literal{literal.NewInt8(2, false), literal.NewInt8(4, false), literal.NewInt8(6, false)}, false)), "[2, 4, 6]"},
141140
}
142141

143142
for _, tt := range tests {
@@ -148,7 +147,7 @@ func TestLiteralToValueString(t *testing.T) {
148147
}
149148

150149
func TestLiteralToStringBrokenDecimal(t *testing.T) {
151-
brokenDecimalLit, _ := literal.NewDecimalFromString("1234.56")
150+
brokenDecimalLit, _ := literal.NewDecimalFromString("1234.56", false)
152151
brokenDecimalLitAsProtoLit := brokenDecimalLit.(*expr.ProtoLiteral)
153152
brokenDecimalLitAsProtoLit.Value = []byte{1, 2, 3}
154153

‎grammar/generate.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
package grammar
22

33
//go:generate wget -nc https://www.antlr.org/download/antlr-4.13.2-complete.jar
4-
//go:generate wget https://raw.githubusercontent.com/substrait-io/substrait/3d2ff77575a7177f82a4d5b53408a059e9818922/grammar/SubstraitLexer.g4
5-
//go:generate wget https://raw.githubusercontent.com/substrait-io/substrait/3d2ff77575a7177f82a4d5b53408a059e9818922/grammar/SubstraitType.g4
6-
//go:generate wget https://raw.githubusercontent.com/substrait-io/substrait/3d2ff77575a7177f82a4d5b53408a059e9818922/grammar/FuncTestCaseLexer.g4
4+
//go:generate wget https://raw.githubusercontent.com/substrait-io/substrait/a30b3e2d7ec667a6da8fee083d7823b11768bd2c/grammar/SubstraitLexer.g4
5+
//go:generate wget https://raw.githubusercontent.com/substrait-io/substrait/a30b3e2d7ec667a6da8fee083d7823b11768bd2c/grammar/SubstraitType.g4
6+
//go:generate wget https://raw.githubusercontent.com/substrait-io/substrait/a30b3e2d7ec667a6da8fee083d7823b11768bd2c/grammar/FuncTestCaseLexer.g4
77
//go:generate wget https://raw.githubusercontent.com/substrait-io/substrait/3d2ff77575a7177f82a4d5b53408a059e9818922/grammar/FuncTestCaseParser.g4
88
//go:generate -command antlr java -Xmx500M -cp "./antlr-4.13.2-complete.jar:$CLASSPATH" org.antlr.v4.Tool
99
//go:generate antlr -Dlanguage=Go -visitor -Dlanguage=Go -package baseparser -o "../types/parser/baseparser" SubstraitLexer.g4 SubstraitType.g4

0 commit comments

Comments
 (0)