Skip to content

Commit 08bfe10

Browse files
lomiklomik
committed
Merge branch 'server-rollup'
2 parents c9023d4 + 708c5a3 commit 08bfe10

File tree

22 files changed

+964
-366
lines changed

22 files changed

+964
-366
lines changed

‎Gopkg.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/config.go‎

Lines changed: 60 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -39,77 +39,77 @@ func (d *Duration) Value() time.Duration {
3939
}
4040

4141
type Common struct {
42-
Listen string `toml:"listen"`
42+
Listen string `toml:"listen" json:"listen"`
4343
// MetricPrefix string `toml:"metric-prefix"`
4444
// MetricInterval *Duration `toml:"metric-interval"`
4545
// MetricEndpoint string `toml:"metric-endpoint"`
46-
MaxCPU int `toml:"max-cpu"`
47-
MaxMetricsInFindAnswer int `toml:"max-metrics-in-find-answer"` //zero means infinite
48-
TargetBlacklist []string `toml:"target-blacklist"`
49-
Blacklist []*regexp.Regexp `toml:"-"` // compiled TargetBlacklist
46+
MaxCPU int `toml:"max-cpu" json:"max-cpu"`
47+
MaxMetricsInFindAnswer int `toml:"max-metrics-in-find-answer" json:"max-metrics-in-find-answer"` //zero means infinite
48+
TargetBlacklist []string `toml:"target-blacklist" json:"target-blacklist"`
49+
Blacklist []*regexp.Regexp `toml:"-" json:"-"` // compiled TargetBlacklist
5050
}
5151

5252
type ClickHouse struct {
53-
Url string `toml:"url"`
54-
DataTable string `toml:"data-table"`
55-
DataTimeout *Duration `toml:"data-timeout"`
56-
TreeTable string `toml:"tree-table"`
57-
DateTreeTable string `toml:"date-tree-table"`
58-
DateTreeTableVersion int `toml:"date-tree-table-version"`
59-
IndexTable string `toml:"index-table"`
60-
IndexUseDaily bool `toml:"index-use-daily"`
61-
IndexTimeout *Duration `toml:"index-timeout"`
62-
TaggedTable string `toml:"tagged-table"`
63-
TaggedAutocompleDays int `toml:"tagged-autocomplete-days"`
64-
ReverseTreeTable string `toml:"reverse-tree-table"`
65-
TreeTimeout *Duration `toml:"tree-timeout"`
66-
TagTable string `toml:"tag-table"`
67-
RollupConf string `toml:"rollup-conf"`
68-
ExtraPrefix string `toml:"extra-prefix"`
69-
ConnectTimeout *Duration `toml:"connect-timeout"`
53+
Url string `toml:"url" json:"url"`
54+
DataTable string `toml:"data-table" json:"data-table"`
55+
DataTimeout *Duration `toml:"data-timeout" json:"data-timeout"`
56+
TreeTable string `toml:"tree-table" json:"tree-table"`
57+
DateTreeTable string `toml:"date-tree-table" json:"date-tree-table"`
58+
DateTreeTableVersion int `toml:"date-tree-table-version" json:"date-tree-table-version"`
59+
IndexTable string `toml:"index-table" json:"index-table"`
60+
IndexUseDaily bool `toml:"index-use-daily" json:"index-use-daily"`
61+
IndexTimeout *Duration `toml:"index-timeout" json:"index-timeout"`
62+
TaggedTable string `toml:"tagged-table" json:"tagged-table"`
63+
TaggedAutocompleDays int `toml:"tagged-autocomplete-days" json:"tagged-autocomplete-days"`
64+
ReverseTreeTable string `toml:"reverse-tree-table" json:"reverse-tree-table"`
65+
TreeTimeout *Duration `toml:"tree-timeout" json:"tree-timeout"`
66+
TagTable string `toml:"tag-table" json:"tag-table"`
67+
RollupConf string `toml:"rollup-conf" json:"-"`
68+
ExtraPrefix string `toml:"extra-prefix" json:"extra-prefix"`
69+
ConnectTimeout *Duration `toml:"connect-timeout" json:"connect-timeout"`
70+
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
7071
}
7172

7273
type Tags struct {
73-
Rules string `toml:"rules"`
74-
Date string `toml:"date"`
75-
ExtraWhere string `toml:"extra-where"`
76-
InputFile string `toml:"input-file"`
77-
OutputFile string `toml:"output-file"`
74+
Rules string `toml:"rules" json:"rules"`
75+
Date string `toml:"date" json:"date"`
76+
ExtraWhere string `toml:"extra-where" json:"extra-where"`
77+
InputFile string `toml:"input-file" json:"input-file"`
78+
OutputFile string `toml:"output-file" json:"output-file"`
7879
}
7980

8081
type Carbonlink struct {
81-
Server string `toml:"server"`
82-
Threads int `toml:"threads-per-request"`
83-
Retries int `toml:"-"`
84-
ConnectTimeout *Duration `toml:"connect-timeout"`
85-
QueryTimeout *Duration `toml:"query-timeout"`
86-
TotalTimeout *Duration `toml:"total-timeout"`
82+
Server string `toml:"server" json:"server"`
83+
Threads int `toml:"threads-per-request" json:"threads-per-request"`
84+
Retries int `toml:"-" json:"-"`
85+
ConnectTimeout *Duration `toml:"connect-timeout" json:"connect-timeout"`
86+
QueryTimeout *Duration `toml:"query-timeout" json:"query-timeout"`
87+
TotalTimeout *Duration `toml:"total-timeout" json:"total-timeout"`
8788
}
8889

8990
type DataTable struct {
90-
Table string `toml:"table"`
91-
Reverse bool `toml:"reverse"`
92-
MaxAge *Duration `toml:"max-age"`
93-
MinAge *Duration `toml:"min-age"`
94-
MaxInterval *Duration `toml:"max-interval"`
95-
MinInterval *Duration `toml:"min-interval"`
96-
TargetMatchAny string `toml:"target-match-any"`
97-
TargetMatchAll string `toml:"target-match-all"`
98-
TargetMatchAnyRegexp *regexp.Regexp `toml:"-"`
99-
TargetMatchAllRegexp *regexp.Regexp `toml:"-"`
100-
RollupConf string `toml:"rollup-conf"`
101-
Rollup *rollup.Rollup `toml:"-"`
91+
Table string `toml:"table" json:"table"`
92+
Reverse bool `toml:"reverse" json:"reverse"`
93+
MaxAge *Duration `toml:"max-age" json:"max-age"`
94+
MinAge *Duration `toml:"min-age" json:"min-age"`
95+
MaxInterval *Duration `toml:"max-interval" json:"max-interval"`
96+
MinInterval *Duration `toml:"min-interval" json:"min-interval"`
97+
TargetMatchAny string `toml:"target-match-any" json:"target-match-any"`
98+
TargetMatchAll string `toml:"target-match-all" json:"target-match-all"`
99+
TargetMatchAnyRegexp *regexp.Regexp `toml:"-" json:"-"`
100+
TargetMatchAllRegexp *regexp.Regexp `toml:"-" json:"-"`
101+
RollupConf string `toml:"rollup-conf" json:"-"`
102+
Rollup *rollup.Rollup `toml:"-" json:"rollup-conf"`
102103
}
103104

104105
// Config ...
105106
type Config struct {
106-
Common Common `toml:"common"`
107-
ClickHouse ClickHouse `toml:"clickhouse"`
108-
DataTable []DataTable `toml:"data-table"`
109-
Tags Tags `toml:"tags"`
110-
Carbonlink Carbonlink `toml:"carbonlink"`
111-
Logging []zapwriter.Config `toml:"logging"`
112-
Rollup *rollup.Rollup `toml:"-"`
107+
Common Common `toml:"common" json:"common"`
108+
ClickHouse ClickHouse `toml:"clickhouse" json:"clickhouse"`
109+
DataTable []DataTable `toml:"data-table" json:"data-table"`
110+
Tags Tags `toml:"tags" json:"tags"`
111+
Carbonlink Carbonlink `toml:"carbonlink" json:"carbonlink"`
112+
Logging []zapwriter.Config `toml:"logging" json:"logging"`
113113
}
114114

115115
// NewConfig ...
@@ -225,18 +225,15 @@ func ReadConfig(filename string) (*Config, error) {
225225
return nil, err
226226
}
227227

228-
rollupConfBody, err := ioutil.ReadFile(cfg.ClickHouse.RollupConf)
229-
if err != nil {
230-
return nil, err
228+
if cfg.ClickHouse.RollupConf == "auto" {
229+
cfg.ClickHouse.Rollup, err = rollup.Auto(cfg.ClickHouse.Url, cfg.ClickHouse.DataTable, time.Minute)
230+
} else {
231+
cfg.ClickHouse.Rollup, err = rollup.ReadFromXMLFile(cfg.ClickHouse.RollupConf)
231232
}
232-
233-
r, err := rollup.ParseXML(rollupConfBody)
234233
if err != nil {
235234
return nil, err
236235
}
237236

238-
cfg.Rollup = r
239-
240237
l := len(cfg.Common.TargetBlacklist)
241238
if l > 0 {
242239
cfg.Common.Blacklist = make([]*regexp.Regexp, l)
@@ -266,17 +263,14 @@ func ReadConfig(filename string) (*Config, error) {
266263
}
267264

268265
if cfg.DataTable[i].RollupConf != "" {
269-
rollupConfBody, err := ioutil.ReadFile(cfg.DataTable[i].RollupConf)
270-
if err != nil {
271-
return nil, err
266+
if cfg.DataTable[i].RollupConf == "auto" {
267+
cfg.DataTable[i].Rollup, err = rollup.Auto(cfg.ClickHouse.Url, cfg.DataTable[i].Table, time.Minute)
268+
} else {
269+
cfg.DataTable[i].Rollup, err = rollup.ReadFromXMLFile(cfg.DataTable[i].RollupConf)
272270
}
273-
274-
r, err := rollup.ParseXML(rollupConfBody)
275271
if err != nil {
276272
return nil, err
277273
}
278-
279-
cfg.DataTable[i].Rollup = r
280274
}
281275
}
282276

‎config/json.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package config
2+
3+
import (
4+
"encoding/json"
5+
"net/url"
6+
)
7+
8+
func (c *ClickHouse) MarshalJSON() ([]byte, error) {
9+
type ClickHouseRaw ClickHouse
10+
11+
// make copy
12+
a := *c
13+
14+
u, err := url.Parse(a.Url)
15+
if err != nil {
16+
a.Url = "<parse error>"
17+
} else {
18+
if _, isSet := u.User.Password(); isSet {
19+
u.User = url.UserPassword(u.User.Username(), "xxxxxx")
20+
}
21+
a.Url = u.String()
22+
}
23+
24+
return json.Marshal((*ClickHouseRaw)(&a))
25+
}

‎config/json_test.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package config
2+
3+
import (
4+
"encoding/json"
5+
"testing"
6+
7+
"github.com/stretchr/testify/assert"
8+
)
9+
10+
func TestClickhouseUrlPassword(t *testing.T) {
11+
assert := assert.New(t)
12+
13+
result := make(map[string]interface{})
14+
15+
c := &ClickHouse{Url: "http://user:qwerty@localhost:8123/?param=value"}
16+
b, err := json.Marshal(c)
17+
assert.NoError(err)
18+
19+
assert.NoError(json.Unmarshal(b, &result))
20+
assert.Equal("http://user:xxxxxx@localhost:8123/?param=value", result["url"].(string))
21+
assert.Equal("http://user:qwerty@localhost:8123/?param=value", c.Url)
22+
}

‎graphite-clickhouse.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package main
33
import (
44
"context"
55
"encoding/binary"
6+
"encoding/json"
67
"flag"
78
"fmt"
89
"log"
@@ -165,6 +166,14 @@ func main() {
165166
http.Handle("/read", Handler(zapwriter.Default(), prometheus.NewHandler(cfg)))
166167
http.Handle("/tags/autoComplete/tags", Handler(zapwriter.Default(), autocomplete.NewTags(cfg)))
167168
http.Handle("/tags/autoComplete/values", Handler(zapwriter.Default(), autocomplete.NewValues(cfg)))
169+
http.HandleFunc("/debug/config", func(w http.ResponseWriter, r *http.Request) {
170+
b, err := json.MarshalIndent(cfg, "", " ")
171+
if err != nil {
172+
http.Error(w, err.Error(), http.StatusInternalServerError)
173+
return
174+
}
175+
w.Write(b)
176+
})
168177

169178
http.Handle("/", Handler(zapwriter.Default(), http.HandlerFunc(http.NotFound)))
170179

0 commit comments

Comments
 (0)