Skip to content

Commit 5054979

Browse files
gkirokdinaltalIguaz
authored
Development (#452)
* Break up lint target for usability. * Avoid redownloading and rebuilding golangci-lint unnecessarily * Run impi before downloading and building golangci-lint (in case impi fails) * Separate prints about building impi and building golangci-lint * IG 14948 (#440) * avoid panic when ingesting data with pre label aggregations * go fmt * new v3io-go (#450) Co-authored-by: Dina Nimrodi <[email protected]> Co-authored-by: Tal Neiman <[email protected]>
1 parent 042795b commit 5054979

File tree

7 files changed

+40
-18
lines changed

7 files changed

+40
-18
lines changed

Diff for: ‎Makefile

+14-7
Original file line numberDiff line numberDiff line change
@@ -69,31 +69,38 @@ build:
6969
bin:
7070
${TSDB_BUILD_COMMAND}
7171

72-
.PHONY: lint
73-
lint:
72+
PHONY: gofmt
73+
gofmt:
7474
ifeq ($(shell gofmt -l .),)
7575
# gofmt OK
7676
else
7777
$(error Please run `go fmt ./...` to format the code)
7878
endif
79-
@echo Installing linters...
80-
go get -u github.com/pavius/impi/cmd/impi
81-
curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.10.2
82-
cp ./bin/golangci-lint $(GOPATH)/bin/
8379

80+
.PHONY: impi
81+
impi:
82+
@echo Installing impi...
83+
go get -u github.com/pavius/impi/cmd/impi
8484
@echo Verifying imports...
8585
$(GOPATH)/bin/impi \
8686
--local github.com/iguazio/provazio \
8787
--skip pkg/controller/apis \
8888
--skip pkg/controller/client \
89+
--ignore-generated \
8990
--scheme stdLocalThirdParty \
9091
./...
9192

93+
$(GOPATH)/bin/golangci-lint:
94+
@echo Installing golangci-lint...
95+
curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.10.2
96+
cp ./bin/golangci-lint $(GOPATH)/bin/
97+
98+
.PHONY: lint
99+
lint: gofmt impi $(GOPATH)/bin/golangci-lint
92100
@echo Linting...
93101
@$(GOPATH)/bin/golangci-lint run \
94102
--disable-all --enable=deadcode --enable=goconst --enable=golint --enable=ineffassign \
95103
--enable=interfacer --enable=unconvert --enable=varcheck --enable=errcheck --enable=gofmt --enable=misspell \
96104
--enable=staticcheck --enable=gosimple --enable=govet --enable=goconst \
97105
cmd/... pkg/... internal/...
98106
@echo done linting
99-

Diff for: ‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ require (
2323

2424
replace (
2525
github.com/v3io/frames => github.com/v3io/frames v0.6.9-v0.9.12.0.20200219120609-981ffb872c73
26+
github.com/v3io/v3io-go => github.com/v3io/v3io-go v0.1.5-0.20200224125003-964a745e51aa
2627
github.com/xwb1989/sqlparser => github.com/v3io/sqlparser v0.0.0-20190306105200-4d7273501871
2728
labix.org/v2/mgo => github.com/go-mgo/mgo v0.0.0-20180705113738-7446a0344b7872c067b3d6e1b7642571eafbae17
2829
launchpad.net/gocheck => github.com/go-check/check v0.0.0-20180628173108-788fd78401277ebd861206a03c884797c6ec5541

Diff for: ‎go.sum

+5-2
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,11 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
7676
github.com/prometheus/prometheus v2.5.0+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s=
7777
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
7878
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
79-
github.com/russross/blackfriday v1.5.2+incompatible h1:/YIL6L1Deczl4O/cQ7ZVdrdKwuB6y7EWpw9LkD8xofE=
80-
github.com/russross/blackfriday v1.5.2+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
79+
github.com/rs/xid v1.1.0/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
8180
github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
8281
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
82+
github.com/russross/blackfriday v1.5.2+incompatible h1:/YIL6L1Deczl4O/cQ7ZVdrdKwuB6y7EWpw9LkD8xofE=
83+
github.com/russross/blackfriday v1.5.2+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
8384
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
8485
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
8586
github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
@@ -109,6 +110,8 @@ github.com/v3io/v3io-go v0.0.0-20191024084247-042df6b5ee40eb60996ab7f4e74ec9aa07
109110
github.com/v3io/v3io-go v0.0.0-20191120130819-9003ae83f0b673afb88b862d8f46dcc818684450/go.mod h1:IFb6dJiyvJnOjXUoCoPJ5UViaYjgVYmqJb4fD1qDeLk=
110111
github.com/v3io/v3io-go v0.0.5-0.20191205125653-9003ae83f0b6 h1:+52DyMCjcWg6uXAlTe0KgbOsiQqUKrtL9tBPSERhyFg=
111112
github.com/v3io/v3io-go v0.0.5-0.20191205125653-9003ae83f0b6/go.mod h1:IFb6dJiyvJnOjXUoCoPJ5UViaYjgVYmqJb4fD1qDeLk=
113+
github.com/v3io/v3io-go v0.1.5-0.20200224125003-964a745e51aa h1:nsgWtNfdTdhaHaNjb5E6+MWJlsCB/vyPrVZenI4BrOA=
114+
github.com/v3io/v3io-go v0.1.5-0.20200224125003-964a745e51aa/go.mod h1:D0W1tjsVgcp4xk3ZI2fjKTKaOpYJLewN1BPN0x2osO4=
112115
github.com/v3io/v3io-go-http v0.0.0-20190221115935-53e2b487c9a2/go.mod h1:GXYcR9MxgfbE3BJdkXki5EclvtS8Nxu2RQNLA8hMMog=
113116
github.com/v3io/v3io-go-http v0.0.0-20190415143924-cc2fbcde6663 h1:WZcM/GRBAastacksmv5pODbtr8fJ/0/9EsPDpPfXkRk=
114117
github.com/v3io/v3io-go-http v0.0.0-20190415143924-cc2fbcde6663/go.mod h1:GXYcR9MxgfbE3BJdkXki5EclvtS8Nxu2RQNLA8hMMog=

Diff for: ‎pkg/appender/store.go

+9-4
Original file line numberDiff line numberDiff line change
@@ -424,10 +424,15 @@ func (cs *chunkStore) writeChunks(mc *MetricsCache, metric *MetricState) (hasPen
424424
expr = lblexpr + encodingExpr + lsetExpr + expr
425425
}
426426

427-
// Call the V3IO async UpdateItem method
428-
conditionExpr := fmt.Sprintf("NOT exists(%s) OR (exists(%s) AND %s == '%d')",
429-
config.EncodingAttrName, config.EncodingAttrName,
430-
config.EncodingAttrName, activeChunk.appender.Encoding())
427+
conditionExpr := ""
428+
429+
// Only add the condition when adding to a data chunk, not when writing data to label pre-aggregated
430+
if activeChunk != nil {
431+
// Call the V3IO async UpdateItem method
432+
conditionExpr = fmt.Sprintf("NOT exists(%s) OR (exists(%s) AND %s == '%d')",
433+
config.EncodingAttrName, config.EncodingAttrName,
434+
config.EncodingAttrName, activeChunk.appender.Encoding())
435+
}
431436
expr += fmt.Sprintf("%v=%d;", config.MaxTimeAttrName, cs.maxTime) // TODO: use max() expr
432437
path := partition.GetMetricPath(metric.name, metric.hash, cs.labelNames, cs.isAggr())
433438
request, err := mc.container.UpdateItem(

Diff for: ‎pkg/config/config.go

+1
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ type V3ioConfig struct {
171171
// use server aggregations if ` <requested step> / <rollup interval> > UseServerAggregateCoefficient`
172172
UseServerAggregateCoefficient int `json:"useServerAggregateCoefficient,omitempty"`
173173
LoadPartitionsFromSchemaAttr bool `json:"loadPartitionsFromSchemaAttr,omitempty"`
174+
RequestChanLength int `json:"RequestChanLength,omitempty"`
174175
}
175176

176177
type MetricsReporterConfig struct {

Diff for: ‎pkg/tsdb/v3iotsdb.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,10 @@ func NewV3ioAdapter(cfg *config.V3ioConfig, container v3io.Container, logger log
129129
}
130130

131131
func NewContainer(v3ioURL string, numWorkers int, accessKey string, username string, password string, containerName string, logger logger.Logger) (v3io.Container, error) {
132-
ctx, err := v3iohttp.NewContext(logger, v3iohttp.NewDefaultClient(), &v3io.NewContextInput{NumWorkers: numWorkers})
132+
newContextInput := &v3iohttp.NewContextInput{
133+
NumWorkers: numWorkers,
134+
}
135+
ctx, err := v3iohttp.NewContext(logger, newContextInput)
133136
if err != nil {
134137
return nil, err
135138
}

Diff for: ‎pkg/utils/container.go

+6-4
Original file line numberDiff line numberDiff line change
@@ -63,11 +63,13 @@ func CreateContainer(logger logger.Logger, cfg *config.V3ioConfig, httpTimeout t
6363
return nil, err
6464
}
6565

66-
newClient := v3iohttp.NewClient(nil, httpTimeout)
67-
newContextInput := &v3io.NewContextInput{
68-
NumWorkers: cfg.Workers,
66+
newClient := v3iohttp.NewClient(&v3iohttp.NewClientInput{DialTimeout: httpTimeout})
67+
newContextInput := &v3iohttp.NewContextInput{
68+
HTTPClient: newClient,
69+
NumWorkers: cfg.Workers,
70+
RequestChanLen: cfg.RequestChanLength,
6971
}
70-
context, err := v3iohttp.NewContext(logger, newClient, newContextInput)
72+
context, err := v3iohttp.NewContext(logger, newContextInput)
7173
if err != nil {
7274
return nil, errors.Wrap(err, "Failed to create a V3IO TSDB client.")
7375
}

0 commit comments

Comments
 (0)