Merge pull request #16 from innogames/issue15

Fix client timeout and some side improvements

Author: Felixoid

Makefile

@@ -9,7 +9,11 @@ endef
 GO_FILES = $(shell find -name '*.go')
 PKG_FILES = build/$(NAME)_$(VERSION)_amd64.deb build/$(NAME)-$(VERSION)-1.x86_64.rpm
 SUM_FILES = build/sha256sum build/md5sum
+MODULE = github.com/innogames/$(NAME)
 
+GO ?= go
+export GOFLAGS +=  -mod=vendor
+export GO111MODULE := on
 
 .PHONY: all clean docker test version
 
@@ -27,8 +31,8 @@ rebuild: clean all
 
 # Run tests
 test:
-	go vet $(GO_FILES)
-	go test $(GO_FILES)
+	$(GO) vet $(MODULE)
+	$(GO) test $(MODULE)
 
 build: | $(NAME)
 	mkdir build
@@ -38,7 +42,7 @@ docker:
 	docker build -t innogames/$(NAME):latest -f docker/$(NAME)/Dockerfile .
 
 $(NAME): $(NAME).go
-	go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
+	$(GO) build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
 
 #########################################################
 # Prepare artifact directory and set outputs for upload #
@@ -73,7 +77,7 @@ build/pkg: build/$(NAME) build/config.toml.example
 	cp -l config.toml.example pkg/etc/$(NAME)
 
 build/$(NAME): $(NAME).go
-	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
+	GOOS=linux GOARCH=amd64 $(GO) build -ldflags "-X 'main.version=$(VERSION)'" -o $@ .
 
 build/config.toml.example: build/$(NAME)
 	./build/$(NAME) --print-defaults > $@

README.md

@@ -33,7 +33,9 @@ To launch the container run the following command on the host with a running Cli
 * Daemon mode is preferable over one-shot script for the normal work
 * It's safe to run it on the cluster hosts
 * You could either run it on the one of replicated host or just over the all hosts
-* If you have big partitions (month or something like this) and will get exceptions about timeout, then you need to adjust `receive_timeout` parameter in DSN
+* There are two different parameters in DSN, that should be adjusted together to fix timeouts for big partitions (month or something like this) optimizing:
+  * `read_timeout` - `clickhouse-go` parameter. Controls the timeout between the `graphite-ch-optimizer` and ClickHouse server.
+  * `receive_timeout` - ClickHouse parameter, used when the OPTIMIZE query is applied in the cluster. See [comment](https://github.com/ClickHouse/ClickHouse/issues/4831#issuecomment-708721042) in the issue.
 * `optimize_throw_if_noop=1` is not mandatory, but good to have.
 * The next picture demonstrates the result of running the daemon for the first time on ~3 years old GraphiteMergeTree table:  
 <img src="./docs/result.jpg" alt="example"/>
@@ -160,7 +162,7 @@ Default config:
 ```toml
 [clickhouse]
   optimize-interval = "72h0m0s"
-  server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true"
+  server-dsn = "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true&read_timeout=3600"
 
 [daemon]
   dry-run = false
@@ -180,7 +182,7 @@ Usage of graphite-ch-optimizer:
       --print-defaults               Print default config values and exit
   -v, --version                      Print version and exit
       --optimize-interval duration   The partition will be merged after having no writes for more than the given duration (default 72h0m0s)
-  -s, --server-dsn string            DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true")
+  -s, --server-dsn string            DSN to connect to ClickHouse server (default "tcp://localhost:9000?&optimize_throw_if_noop=1&receive_timeout=3600&debug=true&read_timeout=3600")
   -n, --dry-run                      Will print how many partitions would be merged without actions
       --loop-interval duration       Daemon will check if there partitions to merge once per this interval (default 1h0m0s)
       --one-shot                     Program will make only one optimization instead of working in the loop (true if dry-run)

go.mod

@@ -3,17 +3,18 @@ module github.com/innogames/graphite-ch-optimizer
 go 1.13
 
 require (
-	github.com/ClickHouse/clickhouse-go v1.3.13
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/pelletier/go-toml v1.6.0
-	github.com/sirupsen/logrus v1.4.2
-	github.com/spf13/afero v1.2.2 // indirect
+	github.com/ClickHouse/clickhouse-go v1.4.3
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/magiconair/properties v1.8.4 // indirect
+	github.com/mitchellh/mapstructure v1.3.3 // indirect
+	github.com/pelletier/go-toml v1.8.1
+	github.com/sirupsen/logrus v1.7.0
+	github.com/spf13/afero v1.4.1 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.6.2
-	golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 // indirect
-	golang.org/x/text v0.3.2 // indirect
-	gopkg.in/ini.v1 v1.51.1 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
+	github.com/spf13/viper v1.7.1
+	golang.org/x/sys v0.0.0-20201027140754-0fcbb8f4928c // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 )