Skip to content
This repository was archived by the owner on Feb 21, 2024. It is now read-only.

Commit 9386fc7

Browse files
paddyjokpaddyjok
authored
implement select from time quantum columns (fb-1654) (#2282)
* first time quantum queries working * implement select from timequantum columns * skip a dax test * make linter happy * addressed review feedback * reverted over eager test elimination
1 parent b35c240 commit 9386fc7

26 files changed

+530
-282
lines changed

‎dax/table.go

+13-9
Original file line numberDiff line numberDiff line change
@@ -73,14 +73,16 @@ const PrefixTable = "tbl"
7373

7474
// Base types.
7575
const (
76-
BaseTypeBool = "bool" //
77-
BaseTypeDecimal = "decimal" //
78-
BaseTypeID = "id" // non-keyed mutex
79-
BaseTypeIDSet = "idset" // non-keyed set
80-
BaseTypeInt = "int" //
81-
BaseTypeString = "string" // keyed mutex
82-
BaseTypeStringSet = "stringset" // keyed set
83-
BaseTypeTimestamp = "timestamp" //
76+
BaseTypeBool = "bool" //
77+
BaseTypeDecimal = "decimal" //
78+
BaseTypeID = "id" // non-keyed mutex
79+
BaseTypeIDSet = "idset" // non-keyed set
80+
BaseTypeIDSetQ = "idsetq" // non-keyed set timequantum
81+
BaseTypeInt = "int" //
82+
BaseTypeString = "string" // keyed mutex
83+
BaseTypeStringSet = "stringset" // keyed set
84+
BaseTypeStringSetQ = "stringsetq" // keyed set timequantum
85+
BaseTypeTimestamp = "timestamp" //
8486

8587
DefaultPartitionN = 256
8688

@@ -679,9 +681,11 @@ func BaseTypeFromString(s string) (BaseType, error) {
679681
BaseTypeDecimal,
680682
BaseTypeID,
681683
BaseTypeIDSet,
684+
BaseTypeIDSetQ,
682685
BaseTypeInt,
683686
BaseTypeString,
684687
BaseTypeStringSet,
688+
BaseTypeStringSetQ,
685689
BaseTypeTimestamp:
686690
return BaseType(lowered), nil
687691
default:
@@ -706,7 +710,7 @@ func (f *Field) String() string {
706710
// StringKeys returns true if the field uses string keys.
707711
func (f *Field) StringKeys() bool {
708712
switch f.Type {
709-
case BaseTypeString, BaseTypeStringSet:
713+
case BaseTypeString, BaseTypeStringSet, BaseTypeStringSetQ:
710714
return true
711715
}
712716
return false

‎dax/test/dax/dax_test.go

+1
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ func TestDAXIntegration(t *testing.T) {
143143
"viewtests/drop-view", // drop view does a delete
144144
"viewtests/drop-view-if-exists-after-drop",
145145
"viewtests/select-view-after-drop",
146+
"time_quantum_insert/test-12", // orchestrator currently does not support to,from args on Rows()
146147
}
147148

148149
doSkip := func(name string) bool {

‎schema.go

+23-25
Original file line numberDiff line numberDiff line change
@@ -243,9 +243,9 @@ func FieldInfoToField(fi *FieldInfo) *dax.Field {
243243
fieldType = dax.BaseTypeBool
244244
case FieldTypeTime:
245245
if fo.Keys {
246-
fieldType = dax.BaseTypeStringSet
246+
fieldType = dax.BaseTypeStringSetQ
247247
} else {
248-
fieldType = dax.BaseTypeIDSet
248+
fieldType = dax.BaseTypeIDSetQ
249249
}
250250
timeQuantum = dax.TimeQuantum(fo.TimeQuantum)
251251
default:
@@ -397,11 +397,13 @@ func fieldToFieldType(f *dax.Field) string {
397397
return string(f.Type)
398398
}
399399
return "mutex"
400+
400401
case dax.BaseTypeIDSet, dax.BaseTypeStringSet:
401-
if f.Options.TimeQuantum != "" {
402-
return "time"
403-
}
404402
return "set"
403+
404+
case dax.BaseTypeIDSetQ, dax.BaseTypeStringSetQ:
405+
return "time"
406+
405407
default:
406408
return string(f.Type)
407409
}
@@ -449,15 +451,13 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) {
449451
OptFieldTypeMutex(cacheType, cacheSize),
450452
)
451453
case dax.BaseTypeIDSet:
452-
if fld.Options.TimeQuantum != "" {
453-
opts = append(opts,
454-
OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()),
455-
)
456-
} else {
457-
opts = append(opts,
458-
OptFieldTypeSet(cacheType, cacheSize),
459-
)
460-
}
454+
opts = append(opts,
455+
OptFieldTypeSet(cacheType, cacheSize),
456+
)
457+
case dax.BaseTypeIDSetQ:
458+
opts = append(opts,
459+
OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()),
460+
)
461461
case dax.BaseTypeInt:
462462
opts = append(opts,
463463
OptFieldTypeInt(fld.Options.Min.ToInt64(0), fld.Options.Max.ToInt64(0)),
@@ -468,17 +468,15 @@ func FieldOptionsFromField(fld *dax.Field) ([]FieldOption, error) {
468468
OptFieldKeys(),
469469
)
470470
case dax.BaseTypeStringSet:
471-
if fld.Options.TimeQuantum != "" {
472-
opts = append(opts,
473-
OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()),
474-
OptFieldKeys(),
475-
)
476-
} else {
477-
opts = append(opts,
478-
OptFieldTypeSet(cacheType, cacheSize),
479-
OptFieldKeys(),
480-
)
481-
}
471+
opts = append(opts,
472+
OptFieldTypeSet(cacheType, cacheSize),
473+
OptFieldKeys(),
474+
)
475+
case dax.BaseTypeStringSetQ:
476+
opts = append(opts,
477+
OptFieldTypeTime(TimeQuantum(fld.Options.TimeQuantum), fld.Options.TTL.String()),
478+
OptFieldKeys(),
479+
)
482480
case dax.BaseTypeTimestamp:
483481
opts = append(opts,
484482
OptFieldTypeTimestamp(fld.Options.Epoch, fld.Options.TimeUnit),

‎sql3/errors.go

+37-1
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ const (
3737
ErrIntOrDecimalOrTimestampOrStringExpressionExpected errors.Code = "ErrIntOrDecimalOrTimestampOrStringExpressionExpected"
3838
ErrStringExpressionExpected errors.Code = "ErrStringExpressionExpected"
3939
ErrSetExpressionExpected errors.Code = "ErrSetExpressionExpected"
40+
ErrTimeQuantumExpressionExpected errors.Code = "ErrTimeQuantumExpressionExpected"
4041
ErrSingleRowExpected errors.Code = "ErrSingleRowExpected"
4142

4243
// type related errors
@@ -58,7 +59,8 @@ const (
5859
ErrInvalidColumnInFilterExpression errors.Code = "ErrInvalidColumnInFilterExpression"
5960
ErrInvalidTypeInFilterExpression errors.Code = "ErrInvalidTypeInFilterExpression"
6061

61-
ErrTypeAssignmentIncompatible errors.Code = "ErrTypeAssignmentIncompatible"
62+
ErrTypeAssignmentIncompatible errors.Code = "ErrTypeAssignmentIncompatible"
63+
ErrTypeAssignmentToTimeQuantumIncompatible errors.Code = "ErrTypeAssignmentToTimeQuantumIncompatible"
6264

6365
ErrInvalidUngroupedColumnReference errors.Code = "ErrInvalidUngroupedColumnReference"
6466
ErrInvalidUngroupedColumnReferenceInHaving errors.Code = "ErrInvalidUngroupedColumnReferenceInHaving"
@@ -132,6 +134,10 @@ const (
132134
ErrValueOutOfRange errors.Code = "ErrValueOutOfRange"
133135
ErrStringLengthMismatch errors.Code = "ErrStringLengthMismatch"
134136
ErrUnexpectedTypeConversion errors.Code = "ErrUnexpectedTypeConversion"
137+
138+
// time quantum function eval
139+
ErrQRangeFromAndToTimeCannotBeBothNull errors.Code = "ErrQRangeFromAndToTimeCannotBeBothNull"
140+
ErrQRangeInvalidUse errors.Code = "ErrQRangeInvalidUse"
135141
)
136142

137143
func NewErrDuplicateColumn(line int, col int, column string) error {
@@ -235,6 +241,13 @@ func NewErrInvalidTypeCoercion(line, col int, from, to string) error {
235241
)
236242
}
237243

244+
func NewErrTypeAssignmentToTimeQuantumIncompatible(line, col int, type1 string) error {
245+
return errors.New(
246+
ErrTypeAssignmentToTimeQuantumIncompatible,
247+
fmt.Sprintf("[%d:%d] an expression of type '%s' cannot be assigned to a timequantum", line, col, type1),
248+
)
249+
}
250+
238251
func NewErrLiteralExpected(line, col int) error {
239252
return errors.New(
240253
ErrLiteralExpected,
@@ -445,6 +458,13 @@ func NewErrSetExpressionExpected(line, col int) error {
445458
)
446459
}
447460

461+
func NewErrTimeQuantumExpressionExpected(line, col int) error {
462+
return errors.New(
463+
ErrTimeQuantumExpressionExpected,
464+
fmt.Sprintf("[%d:%d] time quantum expression expected", line, col),
465+
)
466+
}
467+
448468
func NewErrSingleRowExpected(line, col int) error {
449469
return errors.New(
450470
ErrSingleRowExpected,
@@ -816,3 +836,19 @@ func NewErrUnexpectedTypeConversion(line, col int, val interface{}) error {
816836
NewErrInternalf("unexpected type conversion %T", val).Error(),
817837
)
818838
}
839+
840+
// time quantum function evaluation
841+
842+
func NewErrQRangeFromAndToTimeCannotBeBothNull(line, col int) error {
843+
return errors.New(
844+
ErrQRangeFromAndToTimeCannotBeBothNull,
845+
fmt.Sprintf("[%d:%d] calling ranqeq() 'from' and 'to' parameters cannot both be null", line, col),
846+
)
847+
}
848+
849+
func NewErrQRangeInvalidUse(line, col int) error {
850+
return errors.New(
851+
ErrQRangeInvalidUse,
852+
fmt.Sprintf("[%d:%d] calling ranqeq() usage invalid", line, col),
853+
)
854+
}

‎sql3/parser/astdatatype.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@ func IsValidTypeName(typeName string) bool {
1313
dax.BaseTypeDecimal,
1414
dax.BaseTypeID,
1515
dax.BaseTypeIDSet,
16+
dax.BaseTypeIDSetQ,
1617
dax.BaseTypeInt,
1718
dax.BaseTypeString,
1819
dax.BaseTypeStringSet,
20+
dax.BaseTypeStringSetQ,
1921
dax.BaseTypeTimestamp:
2022
return true
2123
default:
@@ -238,7 +240,6 @@ func (*DataTypeIDSet) TypeInfo() map[string]interface{} {
238240
return nil
239241
}
240242

241-
// TODO (pok) should time quantum be it's own type and not a constraint?
242243
type DataTypeIDSetQuantum struct {
243244
}
244245

@@ -247,7 +248,7 @@ func NewDataTypeIDSetQuantum() *DataTypeIDSetQuantum {
247248
}
248249

249250
func (*DataTypeIDSetQuantum) BaseTypeName() string {
250-
return dax.BaseTypeIDSet
251+
return dax.BaseTypeIDSetQ
251252
}
252253

253254
func (dt *DataTypeIDSetQuantum) TypeDescription() string {
@@ -323,7 +324,7 @@ func NewDataTypeStringSetQuantum() *DataTypeStringSetQuantum {
323324
}
324325

325326
func (*DataTypeStringSetQuantum) BaseTypeName() string {
326-
return dax.BaseTypeStringSet
327+
return dax.BaseTypeStringSetQ
327328
}
328329

329330
func (dt *DataTypeStringSetQuantum) TypeDescription() string {

‎sql3/planner/compilecreatetable.go

+19-16
Original file line numberDiff line numberDiff line change
@@ -196,19 +196,17 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column
196196
switch strings.ToLower(typeName) {
197197
case dax.BaseTypeBool:
198198
column.fos = append(column.fos, pilosa.OptFieldTypeBool())
199-
case dax.BaseTypeDecimal:
200199

200+
case dax.BaseTypeDecimal:
201201
// if we don't have a scale, it's an error
202202
if col.Type.Scale == nil {
203203
return nil, sql3.NewErrDecimalScaleExpected(col.Type.Name.NamePos.Line, col.Type.Name.NamePos.Column)
204204
}
205-
206205
// get the scale value
207206
scale, err = strconv.ParseInt(col.Type.Scale.Value, 10, 64)
208207
if err != nil {
209208
return nil, err
210209
}
211-
212210
// Adjust min/max to fit within the scaled min/max.
213211
scaledMin, scaledMax := pql.MinMax(scale)
214212
if scaledMax.LessThan(max) {
@@ -217,30 +215,35 @@ func (p *ExecutionPlanner) compileColumn(ctx context.Context, col *parser.Column
217215
if scaledMin.GreaterThan(min) {
218216
min = scaledMin
219217
}
220-
221218
column.fos = append(column.fos, pilosa.OptFieldTypeDecimal(scale, min, max))
219+
222220
case dax.BaseTypeID:
223221
column.fos = append(column.fos, pilosa.OptFieldTypeMutex(cacheType, cacheSize))
222+
224223
case dax.BaseTypeIDSet:
225-
if timeQuantum != "" {
226-
column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl))
227-
} else {
228-
column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize))
229-
}
224+
column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize))
225+
226+
case dax.BaseTypeIDSetQ:
227+
column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl))
228+
230229
case dax.BaseTypeInt:
231230
column.fos = append(column.fos, pilosa.OptFieldTypeInt(min.ToInt64(0), max.ToInt64(0)))
231+
232232
case dax.BaseTypeString:
233233
column.fos = append(column.fos, pilosa.OptFieldTypeMutex(cacheType, cacheSize))
234234
column.fos = append(column.fos, pilosa.OptFieldKeys())
235+
235236
case dax.BaseTypeStringSet:
236-
if timeQuantum != "" {
237-
column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl))
238-
} else {
239-
column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize))
240-
}
237+
column.fos = append(column.fos, pilosa.OptFieldTypeSet(cacheType, cacheSize))
241238
column.fos = append(column.fos, pilosa.OptFieldKeys())
239+
240+
case dax.BaseTypeStringSetQ:
241+
column.fos = append(column.fos, pilosa.OptFieldTypeTime(timeQuantum, ttl))
242+
column.fos = append(column.fos, pilosa.OptFieldKeys())
243+
242244
case dax.BaseTypeTimestamp:
243245
column.fos = append(column.fos, pilosa.OptFieldTypeTimestamp(epoch, timeUnit))
246+
244247
}
245248
return column, nil
246249
}
@@ -392,8 +395,8 @@ func (p *ExecutionPlanner) analyzeColumn(typeName string, col *parser.ColumnDefi
392395
handledConstraints[parser.TIMEUNIT] = struct{}{}
393396

394397
case *parser.TimeQuantumConstraint:
395-
//make sure we have a set type
396-
if !(strings.EqualFold(typeName, dax.BaseTypeStringSet) || strings.EqualFold(typeName, dax.BaseTypeIDSet)) {
398+
//make sure we have one of the time quantum types
399+
if !(strings.EqualFold(typeName, dax.BaseTypeStringSetQ) || strings.EqualFold(typeName, dax.BaseTypeIDSetQ)) {
397400
return sql3.NewErrBadColumnConstraint(col.Name.NamePos.Line, col.Name.NamePos.Column, "TIMEQUANTUM", typeName)
398401
}
399402
//check the type of the expression

‎sql3/planner/compileselect.go

+1-29
Original file line numberDiff line numberDiff line change
@@ -573,35 +573,7 @@ func (p *ExecutionPlanner) analyzeSource(ctx context.Context, source parser.Sour
573573
return source, nil
574574

575575
case *parser.TableValuedFunction:
576-
// check it actually is a table valued function - we only support one right now; subtable()
577-
switch strings.ToUpper(source.Name.Name) {
578-
case "SUBTABLE":
579-
_, err := p.analyzeCallExpression(ctx, source.Call, scope)
580-
if err != nil {
581-
return nil, err
582-
}
583-
584-
tvfResultType, ok := source.Call.ResultDataType.(*parser.DataTypeSubtable)
585-
if !ok {
586-
return nil, sql3.NewErrInternalf("unexepected tvf return type")
587-
}
588-
589-
// populate the output columns from the source
590-
for idx, member := range tvfResultType.Columns {
591-
soc := &parser.SourceOutputColumn{
592-
TableName: "", // TODO (pok) use the tq column actually referenced as the table name
593-
ColumnName: member.Name,
594-
ColumnIndex: idx,
595-
Datatype: member.DataType,
596-
}
597-
source.OutputColumns = append(source.OutputColumns, soc)
598-
}
599-
600-
default:
601-
return nil, sql3.NewErrInternalf("table valued function expected")
602-
}
603-
604-
return source, nil
576+
return nil, sql3.NewErrInternalf("table valued function expected")
605577

606578
case *parser.SelectStatement:
607579
expr, err := p.analyzeSelectStatement(ctx, source)

‎sql3/planner/expression.go

+16
Original file line numberDiff line numberDiff line change
@@ -1576,6 +1576,9 @@ func (n *callPlanExpression) Evaluate(currentRow []interface{}) (interface{}, er
15761576
return n.EvaluateToTimestamp(currentRow)
15771577
case "STR":
15781578
return n.EvaluateStr(currentRow)
1579+
// time quantum functions
1580+
case "RANGEQ":
1581+
return n.EvaluateRangeQ(currentRow)
15791582
default:
15801583
return nil, sql3.NewErrInternalf("unhandled function name '%s'", n.name)
15811584
}
@@ -2063,6 +2066,19 @@ func (n *stringLiteralPlanExpression) WithChildren(children ...types.PlanExpress
20632066
return n, nil
20642067
}
20652068

2069+
func (expr *stringLiteralPlanExpression) ConvertToTimestamp() *time.Time {
2070+
//try to coerce to a date
2071+
if tm, err := time.ParseInLocation(time.RFC3339Nano, expr.value, time.UTC); err == nil {
2072+
return &tm
2073+
} else if tm, err := time.ParseInLocation(time.RFC3339, expr.value, time.UTC); err == nil {
2074+
return &tm
2075+
} else if tm, err := time.ParseInLocation("2006-01-02", expr.value, time.UTC); err == nil {
2076+
return &tm
2077+
} else {
2078+
return nil
2079+
}
2080+
}
2081+
20662082
// castPlanExpressionis a cast op
20672083
type castPlanExpression struct {
20682084
lhs types.PlanExpression

0 commit comments

Comments
 (0)