Skip to content
This repository was archived by the owner on Sep 28, 2022. It is now read-only.

Commit b811c3b

Browse files
committed
start adding time quantum support to batch import
adds QuantizedTime type to support efficiently tracking a time quantum with each Row which will be applied to all the values of time-capable fields in that row.
1 parent c2b019c commit b811c3b

File tree

2 files changed

+178
-0
lines changed

2 files changed

+178
-0
lines changed

gpexp/importbatch.go

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
package gpexp
22

33
import (
4+
"time"
5+
46
"github.com/pilosa/go-pilosa"
57
"github.com/pilosa/pilosa/roaring"
68
"github.com/pkg/errors"
@@ -153,6 +155,72 @@ func NewBatch(client *pilosa.Client, size int, index *pilosa.Index, fields []*pi
153155
type Row struct {
154156
ID interface{}
155157
Values []interface{}
158+
Time *QuantizedTime
159+
}
160+
161+
// QuantizedTime represents a moment in time down to some granularity
162+
// (year, month, day, or hour).
163+
type QuantizedTime struct {
164+
ymdh [10]byte
165+
}
166+
167+
// Set sets the Quantized time to the given timestamp (down to hour
168+
// granularity).
169+
func (qt *QuantizedTime) Set(t time.Time) {
170+
copy(qt.ymdh[:], t.Format("2006010215"))
171+
}
172+
173+
// SetYear sets the quantized time's year, but leaves month, day, and
174+
// hour untouched.
175+
func (qt *QuantizedTime) SetYear(year string) {
176+
copy(qt.ymdh[:4], year)
177+
}
178+
179+
// SetMonth sets the QuantizedTime's month, but leaves year, day, and
180+
// hour untouched.
181+
func (qt *QuantizedTime) SetMonth(month string) {
182+
copy(qt.ymdh[4:6], month)
183+
}
184+
185+
// SetDay sets the QuantizedTime's day, but leaves year, month, and
186+
// hour untouched.
187+
func (qt *QuantizedTime) SetDay(day string) {
188+
copy(qt.ymdh[6:8], day)
189+
}
190+
191+
// SetHour sets the QuantizedTime's hour, but leaves year, month, and
192+
// day untouched.
193+
func (qt *QuantizedTime) SetHour(hour string) {
194+
copy(qt.ymdh[8:10], hour)
195+
}
196+
197+
func (qt *QuantizedTime) views(q pilosa.TimeQuantum) ([]string, error) {
198+
views := make([]string, 0, len(q))
199+
for _, unit := range q {
200+
switch unit {
201+
case 'Y':
202+
if qt.ymdh[0] == 0 {
203+
return nil, errors.New("no data set for year")
204+
}
205+
views = append(views, string(qt.ymdh[:4]))
206+
case 'M':
207+
if qt.ymdh[4] == 0 {
208+
return nil, errors.New("no data set for month")
209+
}
210+
views = append(views, string(qt.ymdh[:6]))
211+
case 'D':
212+
if qt.ymdh[6] == 0 {
213+
return nil, errors.New("no data set for day")
214+
}
215+
views = append(views, string(qt.ymdh[:8]))
216+
case 'H':
217+
if qt.ymdh[8] == 0 {
218+
return nil, errors.New("no data set for hour")
219+
}
220+
views = append(views, string(qt.ymdh[:10]))
221+
}
222+
}
223+
return views, nil
156224
}
157225

158226
// Add adds a record to the batch. Performance will be best if record

gpexp/importbatch_test.go

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"reflect"
55
"strconv"
66
"testing"
7+
"time"
78

89
"github.com/pilosa/go-pilosa"
910
"github.com/pkg/errors"
@@ -483,3 +484,112 @@ outer:
483484
}
484485
return nil
485486
}
487+
488+
func TestQuantizedTime(t *testing.T) {
489+
cases := []struct {
490+
name string
491+
time time.Time
492+
year string
493+
month string
494+
day string
495+
hour string
496+
quantum pilosa.TimeQuantum
497+
exp []string
498+
expErr string
499+
}{
500+
{
501+
name: "no time quantum",
502+
exp: []string{},
503+
expErr: "",
504+
},
505+
{
506+
name: "no data",
507+
quantum: pilosa.TimeQuantumYear,
508+
expErr: "no data set for year",
509+
},
510+
{
511+
name: "timestamp",
512+
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
513+
quantum: "YMDH",
514+
exp: []string{"2013", "201310", "20131016", "2013101617"},
515+
},
516+
{
517+
name: "timestamp-less-granular",
518+
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
519+
quantum: "YM",
520+
exp: []string{"2013", "201310"},
521+
},
522+
{
523+
name: "timestamp-mid-granular",
524+
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
525+
quantum: "MD",
526+
exp: []string{"201310", "20131016"},
527+
},
528+
{
529+
name: "justyear",
530+
year: "2013",
531+
quantum: "Y",
532+
exp: []string{"2013"},
533+
},
534+
{
535+
name: "justyear-wantmonth",
536+
year: "2013",
537+
quantum: "YM",
538+
expErr: "no data set for month",
539+
},
540+
{
541+
name: "timestamp-changeyear",
542+
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
543+
year: "2019",
544+
quantum: "YMDH",
545+
exp: []string{"2019", "201910", "20191016", "2019101617"},
546+
},
547+
{
548+
name: "yearmonthdayhour",
549+
year: "2013",
550+
month: "10",
551+
day: "16",
552+
hour: "17",
553+
quantum: "YMDH",
554+
exp: []string{"2013", "201310", "20131016", "2013101617"},
555+
},
556+
{
557+
name: "timestamp-changehour",
558+
time: time.Date(2013, time.October, 16, 17, 34, 43, 0, time.FixedZone("UTC-5", -5*60*60)),
559+
hour: "05",
560+
quantum: "MDH",
561+
exp: []string{"201310", "20131016", "2013101605"},
562+
},
563+
}
564+
565+
for i, test := range cases {
566+
t.Run(test.name+strconv.Itoa(i), func(t *testing.T) {
567+
tq := &QuantizedTime{}
568+
var zt time.Time
569+
if zt != test.time {
570+
tq.Set(test.time)
571+
}
572+
if test.year != "" {
573+
tq.SetYear(test.year)
574+
}
575+
if test.month != "" {
576+
tq.SetMonth(test.month)
577+
}
578+
if test.day != "" {
579+
tq.SetDay(test.day)
580+
}
581+
if test.hour != "" {
582+
tq.SetHour(test.hour)
583+
}
584+
585+
views, err := tq.views(test.quantum)
586+
if !reflect.DeepEqual(views, test.exp) {
587+
t.Errorf("unexpected views, got/want:\n%v\n%v\n", views, test.exp)
588+
}
589+
if (err != nil && err.Error() != test.expErr) || (err == nil && test.expErr != "") {
590+
t.Errorf("unexpected error, got/want:\n%v\n%s\n", err, test.expErr)
591+
}
592+
})
593+
}
594+
595+
}

0 commit comments

Comments
 (0)