Skip to content
This repository was archived by the owner on Feb 21, 2024. It is now read-only.

Commit 713dbb6

Browse files
jaffeejaffee
authored
Merge pull request #1921 from jaffee/shardwidth22
add support to modify shard width at build time
2 parents 9805179 + 7ede65b commit 713dbb6

27 files changed

+130
-32
lines changed

‎.circleci/config.yml

+6
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,12 @@ jobs:
4444
- *fast-checkout
4545
- run: sudo apt-get install lsof
4646
- run: make test
47+
test-golang-1.12-shard22: &base-test
48+
<<: *defaults
49+
steps:
50+
- *fast-checkout
51+
- run: sudo apt-get install lsof
52+
- run: make test SHARD_WIDTH=22
4753
test-golang-1.12-race:
4854
<<: *defaults
4955
steps:

‎Makefile

+3-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ VERSION_ID = $(if $(ENTERPRISE_ENABLED),enterprise-)$(VERSION)-$(GOOS)-$(GOARCH)
66
BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD)))
77
BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH)
88
BUILD_TIME := $(shell date -u +%FT%T%z)
9+
SHARD_WIDTH = 20
910
LDFLAGS="-X github.com/pilosa/pilosa.Version=$(VERSION) -X github.com/pilosa/pilosa.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa.Enterprise=$(if $(ENTERPRISE_ENABLED),1)"
1011
GO_VERSION=latest
1112
ENTERPRISE ?= 0
@@ -14,6 +15,7 @@ RELEASE ?= 0
1415
RELEASE_ENABLED = $(subst 0,,$(RELEASE))
1516
BUILD_TAGS += $(if $(ENTERPRISE_ENABLED),enterprise)
1617
BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
18+
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
1719
export GO111MODULE=on
1820

1921
# Run tests and compile Pilosa
@@ -29,7 +31,7 @@ vendor: go.mod
2931

3032
# Run test suite
3133
test:
32-
go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS)
34+
go test ./... -tags='$(BUILD_TAGS)' $(TESTFLAGS)
3335

3436
bench:
3537
go test ./... -bench=. -run=NoneZ -timeout=127m $(TESTFLAGS)

‎cluster_internal_test.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -157,15 +157,15 @@ func TestFragSources(t *testing.T) {
157157
if err != nil {
158158
t.Fatal(err)
159159
}
160-
_, err = field.SetBit(1, 1300000, nil)
160+
_, err = field.SetBit(1, ShardWidth+1, nil)
161161
if err != nil {
162162
t.Fatal(err)
163163
}
164-
_, err = field.SetBit(1, 2600000, nil)
164+
_, err = field.SetBit(1, ShardWidth*2+1, nil)
165165
if err != nil {
166166
t.Fatal(err)
167167
}
168-
_, err = field.SetBit(1, 3900000, nil)
168+
_, err = field.SetBit(1, ShardWidth*3+1, nil)
169169
if err != nil {
170170
t.Fatal(err)
171171
}
@@ -755,7 +755,7 @@ func TestCluster_ResizeStates(t *testing.T) {
755755
t.Fatal(err)
756756
}
757757
tc.SetBit("i", "f", 1, 101, nil)
758-
tc.SetBit("i", "f", 1, 1300000, nil)
758+
tc.SetBit("i", "f", 1, ShardWidth+1, nil)
759759

760760
// Before starting the resize, get the CheckSum to use for
761761
// comparison later.

‎executor.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"time"
2323

2424
"github.com/pilosa/pilosa/pql"
25+
"github.com/pilosa/pilosa/shardwidth"
2526
"github.com/pilosa/pilosa/tracing"
2627
"github.com/pkg/errors"
2728
)
@@ -1234,7 +1235,7 @@ func (e *executor) executeRowsShard(_ context.Context, index string, fieldName s
12341235
if columnID, ok, err := c.UintArg("column"); err != nil {
12351236
return nil, err
12361237
} else if ok {
1237-
colShard := columnID >> shardWidthExponent
1238+
colShard := columnID >> shardwidth.Exponent
12381239
if colShard != shard {
12391240
return rowIDs, nil
12401241
}

‎executor_test.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -2237,11 +2237,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
22372237
})
22382238

22392239
t.Run("Remote SetBit", func(t *testing.T) {
2240-
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, f=7)`}); err != nil {
2241-
t.Fatalf("quuerying remote: %v", err)
2240+
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, f=7)`, pilosa.ShardWidth+1)}); err != nil {
2241+
t.Fatalf("querying remote: %v", err)
22422242
}
22432243

2244-
if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{1500000}) {
2244+
if !reflect.DeepEqual(hldr1.Row("i", "f", 7).Columns(), []uint64{pilosa.ShardWidth + 1}) {
22452245
t.Fatalf("unexpected cols from row 7: %v", hldr1.Row("i", "f", 7).Columns())
22462246
}
22472247
})
@@ -2252,11 +2252,11 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
22522252
t.Fatalf("creating field: %v", err)
22532253
}
22542254

2255-
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: `Set(1500000, z=5, 2010-07-08T00:00)`}); err != nil {
2255+
if _, err := c[0].API.Query(context.Background(), &pilosa.QueryRequest{Index: "i", Query: fmt.Sprintf(`Set(%d, z=5, 2010-07-08T00:00)`, pilosa.ShardWidth+1)}); err != nil {
22562256
t.Fatalf("quuerying remote: %v", err)
22572257
}
22582258

2259-
if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{1500000}) {
2259+
if !reflect.DeepEqual(hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns(), []uint64{pilosa.ShardWidth + 1}) {
22602260
t.Fatalf("unexpected cols from row 7: %v", hldr1.RowTime("i", "z", 5, time.Date(2010, time.January, 1, 0, 0, 0, 0, time.UTC), "Y").Columns())
22612261
}
22622262
})

‎fragment.go

+5-3
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ import (
4040
"github.com/pilosa/pilosa/logger"
4141
"github.com/pilosa/pilosa/pql"
4242
"github.com/pilosa/pilosa/roaring"
43+
"github.com/pilosa/pilosa/shardwidth"
4344
"github.com/pilosa/pilosa/stats"
4445
"github.com/pilosa/pilosa/syswrap"
4546
"github.com/pilosa/pilosa/tracing"
@@ -48,8 +49,9 @@ import (
4849

4950
const (
5051
// ShardWidth is the number of column IDs in a shard. It must be a power of 2 greater than or equal to 16.
51-
shardWidthExponent = 20
52-
ShardWidth = 1 << shardWidthExponent
52+
// shardWidthExponent = 20 // set in shardwidthNN.go files
53+
54+
ShardWidth = 1 << shardwidth.Exponent
5355

5456
// shardVsContainerExponent is the power of 2 of ShardWith minus the power
5557
// of two of roaring container width (which is 16).
@@ -59,7 +61,7 @@ const (
5961
// which a given container is in means dividing by the number of rows per
6062
// container which is performantly expressed as a right shift by this
6163
// exponent.
62-
shardVsContainerExponent = shardWidthExponent - 16
64+
shardVsContainerExponent = shardwidth.Exponent - 16
6365

6466
// width of roaring containers is 2^16
6567
containerWidth = 1 << 16

‎fragment_internal_test.go

+7-7
Original file line numberDiff line numberDiff line change
@@ -137,14 +137,14 @@ func TestFragment_SetRow(t *testing.T) {
137137
rowID := uint64(1000)
138138

139139
// Set bits on the fragment.
140-
if _, err := f.setBit(rowID, 8000001); err != nil {
140+
if _, err := f.setBit(rowID, 7*ShardWidth+1); err != nil {
141141
t.Fatal(err)
142-
} else if _, err := f.setBit(rowID, 8065536); err != nil {
142+
} else if _, err := f.setBit(rowID, 7*ShardWidth+65536); err != nil {
143143
t.Fatal(err)
144144
}
145145

146146
// Verify data on row.
147-
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000001, 8065536}) {
147+
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65536}) {
148148
t.Fatalf("unexpected columns: %+v", cols)
149149
}
150150
// Verify count on row.
@@ -153,15 +153,15 @@ func TestFragment_SetRow(t *testing.T) {
153153
}
154154

155155
// Set row (overwrite existing data).
156-
row := NewRow(8000002, 8065537, 8131074)
156+
row := NewRow(7*ShardWidth+1, 7*ShardWidth+65537, 7*ShardWidth+140000)
157157
if changed, err := f.unprotectedSetRow(row, rowID); err != nil {
158158
t.Fatal(err)
159159
} else if !changed {
160160
t.Fatalf("expected changed value: %v", changed)
161161
}
162162

163163
// Verify data on row.
164-
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{8000002, 8065537, 8131074}) {
164+
if cols := f.row(rowID).Columns(); !reflect.DeepEqual(cols, []uint64{7*ShardWidth + 1, 7*ShardWidth + 65537, 7*ShardWidth + 140000}) {
165165
t.Fatalf("unexpected columns after set row: %+v", cols)
166166
}
167167
// Verify count on row.
@@ -1914,7 +1914,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
19141914
f := mustOpenFragment("i", "f", viewStandard, 0, "")
19151915
defer f.Clean(b)
19161916
// Generate some intersecting data.
1917-
maxX := 1048576 / 2
1917+
maxX := ShardWidth / 2
19181918
sz := maxX
19191919
rows := make([]uint64, sz)
19201920
cols := make([]uint64, sz)
@@ -1950,7 +1950,7 @@ func BenchmarkFragment_FullSnapshot(b *testing.B) {
19501950

19511951
func BenchmarkFragment_Import(b *testing.B) {
19521952
b.StopTimer()
1953-
maxX := 1048576 * 5 * 2
1953+
maxX := ShardWidth * 5 * 2
19541954
sz := maxX
19551955
rows := make([]uint64, sz)
19561956
cols := make([]uint64, sz)

‎roaring/roaring_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -299,10 +299,10 @@ func TestBitmap_Max(t *testing.T) {
299299

300300
// Ensure CountRange is correct even if rangekey is prior to initial container.
301301
func TestBitmap_BitmapCountRangeEdgeCase(t *testing.T) {
302-
s := uint64(2009 * 1048576)
303-
e := uint64(2010 * 1048576)
302+
s := uint64(2009 * pilosa.ShardWidth)
303+
e := uint64(2010 * pilosa.ShardWidth)
304304

305-
start := s + (39314024 % 1048576)
305+
start := s + (39314024 % pilosa.ShardWidth)
306306
bm0 := roaring.NewFileBitmap()
307307
for i := uint64(0); i < 65536; i++ {
308308
if (i+1)%4096 == 0 {

‎server/cluster_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,10 @@ func TestMain_SendReceiveMessage(t *testing.T) {
8585
}
8686

8787
// Write data on first node.
88-
if _, err := m0.Query("i", "", `
88+
if _, err := m0.Query("i", "", fmt.Sprintf(`
8989
Set(1, f=1)
90-
Set(2400000, f=1)
91-
`); err != nil {
90+
Set(%d, f=1)
91+
`, 2*pilosa.ShardWidth+1)); err != nil {
9292
t.Fatal(err)
9393
}
9494

‎server/handler_test.go

+7-5
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"context"
2020
"encoding/hex"
2121
"encoding/json"
22+
"fmt"
2223
"io"
2324
"io/ioutil"
2425
gohttp "net/http"
@@ -109,8 +110,8 @@ func TestHandler_Endpoints(t *testing.T) {
109110
t.Fatalf("unexpected status code: %d", w.Code)
110111
}
111112
body := w.Body.String()
112-
target := `{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":1048576}]}
113-
`
113+
target := fmt.Sprintf(`{"indexes":[{"name":"i0","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}},{"name":"f1","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%d},{"name":"i1","options":{"keys":false,"trackExistence":false},"fields":[{"name":"f0","options":{"type":"set","cacheType":"ranked","cacheSize":50000,"keys":false}}],"shardWidth":%[1]d}]}
114+
`, pilosa.ShardWidth)
114115
if body != target {
115116
t.Fatalf("%s != %s", target, body)
116117
}
@@ -298,7 +299,7 @@ func TestHandler_Endpoints(t *testing.T) {
298299
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)")))
299300
if w.Code != gohttp.StatusOK {
300301
t.Fatalf("unexpected status code: %d", w.Code)
301-
} else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" {
302+
} else if body := w.Body.String(); body != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[%d,%d,%d]}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4)+"\n" {
302303
t.Fatalf("unexpected body: %s", body)
303304
}
304305
})
@@ -315,10 +316,11 @@ func TestHandler_Endpoints(t *testing.T) {
315316
t.Run("ColumnAttrs_JSON", func(t *testing.T) {
316317
w := httptest.NewRecorder()
317318
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)")))
319+
exp := fmt.Sprintf(`{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[%[1]d,%[2]d,%[3]d]}],"columnAttrs":[{"id":%[1]d,"attrs":{"x":"y"}},{"id":%[2]d,"attrs":{"y":123,"z":false}}]}`, pilosa.ShardWidth+1, pilosa.ShardWidth+2, 3*pilosa.ShardWidth+4) + "\n"
318320
if w.Code != gohttp.StatusOK {
319321
t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String())
320-
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" {
321-
t.Fatalf("unexpected body: %s", body)
322+
} else if body := w.Body.String(); body != exp {
323+
t.Fatalf("unexpected body: \n%s\ngot:\n%s", body, exp)
322324
}
323325
})
324326

‎shardwidth/16.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth16
2+
3+
package shardwidth
4+
5+
const Exponent = 16

‎shardwidth/17.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth17
2+
3+
package shardwidth
4+
5+
const Exponent = 17

‎shardwidth/18.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth18
2+
3+
package shardwidth
4+
5+
const Exponent = 18

‎shardwidth/19.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth19
2+
3+
package shardwidth
4+
5+
const Exponent = 19

‎shardwidth/20.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build !shardwidth16,!shardwidth17,!shardwidth18,!shardwidth19,!shardwidth21,!shardwidth22,!shardwidth23,!shardwidth24,!shardwidth25,!shardwidth26,!shardwidth27,!shardwidth28,!shardwidth29,!shardwidth30,!shardwidth31,!shardwidth32
2+
3+
package shardwidth
4+
5+
const Exponent = 20

‎shardwidth/21.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth21
2+
3+
package shardwidth
4+
5+
const Exponent = 21

‎shardwidth/22.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth22
2+
3+
package shardwidth
4+
5+
const Exponent = 22

‎shardwidth/23.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth23
2+
3+
package shardwidth
4+
5+
const Exponent = 23

‎shardwidth/24.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth24
2+
3+
package shardwidth
4+
5+
const Exponent = 24

‎shardwidth/25.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth25
2+
3+
package shardwidth
4+
5+
const Exponent = 25

‎shardwidth/26.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth26
2+
3+
package shardwidth
4+
5+
const Exponent = 26

‎shardwidth/27.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth27
2+
3+
package shardwidth
4+
5+
const Exponent = 27

‎shardwidth/28.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth28
2+
3+
package shardwidth
4+
5+
const Exponent = 28

‎shardwidth/29.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth29
2+
3+
package shardwidth
4+
5+
const Exponent = 29

‎shardwidth/30.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth30
2+
3+
package shardwidth
4+
5+
const Exponent = 30

‎shardwidth/31.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth31
2+
3+
package shardwidth
4+
5+
const Exponent = 31

‎shardwidth/32.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
// +build shardwidth32
2+
3+
package shardwidth
4+
5+
const Exponent = 32

0 commit comments

Comments
 (0)