Skip to content

Commit 4363669

Browse files
williamchanricowilliamchanrico
authored
Merge pull request #11 from williamchanrico/support_custom_task_inventory_format
Add Custom Inventory Format Support
2 parents 9a7773f + 270a864 commit 4363669

File tree

8 files changed

+235
-16
lines changed

8 files changed

+235
-16
lines changed

‎README.md

+27-3
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,18 @@ Usage of planet-exporter:
7676
Darkstat target address
7777
-task-darkstat-enabled
7878
Enable darkstat collector task
79+
-task-ebpf-addr string
80+
Ebpf target address (default "http://localhost:9435/metrics")
81+
-task-ebpf-enabled
82+
Enable Ebpf collector task
7983
-task-interval string
8084
Interval between collection of expensive data into memory (default "7s")
8185
-task-inventory-addr string
82-
Darkstat target address
86+
HTTP endpoint that returns the inventory data
8387
-task-inventory-enabled
8488
Enable inventory collector task
89+
-task-inventory-format string
90+
Inventory format to parse the returned inventory data (default "arrayjson")
8591
-task-socketstat-enabled
8692
Enable socketstat collector task (default true)
8793
-version
@@ -94,7 +100,7 @@ Running with minimum collector tasks (just the socketstat)
94100
# planet-exporter
95101
```
96102

97-
Running with inventory and darkstat (installed separately rev >= [e7e6652](https://www.unix4lyfe.org/gitweb/darkstat/commit/e7e6652113099e33930ab0f39630bf280e38f769))
103+
Running with inventory and darkstat (darkstat has to be installed separately rev >= [e7e6652](https://www.unix4lyfe.org/gitweb/darkstat/commit/e7e6652113099e33930ab0f39630bf280e38f769))
98104

99105
```
100106
# planet-exporter \
@@ -104,6 +110,15 @@ Running with inventory and darkstat (installed separately rev >= [e7e6652](https
104110
-task-darkstat-addr http://localhost:51666/metrics
105111
```
106112

113+
Running with another inventory format
114+
115+
```
116+
# planet-exporter \
117+
-task-inventory-enabled \
118+
-task-inventory-format "ndjson" \
119+
-task-inventory-addr http://link-to-your.net/inventory_hosts.json
120+
```
121+
107122
### Collector Tasks
108123

109124
#### Inventory
@@ -112,7 +127,9 @@ Query inventory data to map IP into `hostgroup` (an identifier based on ansible
112127

113128
Without this task enabled, those hostgroup and domain fields will be left empty.
114129

115-
The flag `--task-inventory-addr` should contain an http url to an array of json objects:
130+
The flag `--task-inventory-addr` should contain an HTTP endpoint that returns inventory data in the supported format:
131+
132+
##### --task-inventory-format=arrayjson
116133

117134
```json
118135
[
@@ -129,6 +146,13 @@ The flag `--task-inventory-addr` should contain an http url to an array of json
129146
]
130147
```
131148

149+
##### --task-inventory-format=ndjson
150+
151+
```json
152+
{"ip_address":"10.0.1.2","domain":"xyz.service.consul","hostgroup":"xyz"}
153+
{"ip_address":"172.16.1.2","domain":"abc.service.consul","hostgroup":"abc"}
154+
```
155+
132156
#### Socketstat
133157

134158
Query local connections socket similar to `ss` or `netstat` to build upstream and downstream dependency metrics.

‎cmd/planet-exporter/internal/internal.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ type Config struct {
5555

5656
TaskInventoryEnabled bool
5757
TaskInventoryAddr string // InventoryAddr url for inventory hostgroup mapping table data
58+
TaskInventoryFormat string // InventoryFormat returned by inventory address [jsonarray,ndjson]
5859

5960
TaskEbpfEnabled bool
6061
TaskEbpfAddr string // TaskEbpfAddr url for scraping the ebpf data
@@ -163,7 +164,7 @@ func (s Service) collect(ctx context.Context, interval time.Duration) {
163164
taskebpf.InitTask(ctx, s.Config.TaskEbpfEnabled, s.Config.TaskEbpfAddr)
164165

165166
log.Infof("Task Inventory: %v", s.Config.TaskInventoryEnabled)
166-
taskinventory.InitTask(ctx, s.Config.TaskInventoryEnabled, s.Config.TaskInventoryAddr)
167+
taskinventory.InitTask(ctx, s.Config.TaskInventoryEnabled, s.Config.TaskInventoryAddr, s.Config.TaskInventoryFormat)
167168

168169
log.Infof("Task Socketstat: %v", s.Config.TaskSocketstatEnabled)
169170
tasksocketstat.InitTask(ctx, s.Config.TaskSocketstatEnabled)

‎cmd/planet-exporter/main.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,8 @@ func main() {
5353
flag.StringVar(&config.TaskEbpfAddr, "task-ebpf-addr", "http://localhost:9435/metrics", "Ebpf target address")
5454

5555
flag.BoolVar(&config.TaskInventoryEnabled, "task-inventory-enabled", false, "Enable inventory collector task")
56-
flag.StringVar(&config.TaskInventoryAddr, "task-inventory-addr", "", "Darkstat target address")
56+
flag.StringVar(&config.TaskInventoryAddr, "task-inventory-addr", "", "HTTP endpoint that returns the inventory data")
57+
flag.StringVar(&config.TaskInventoryFormat, "task-inventory-format", "arrayjson", "Inventory format to parse the returned inventory data")
5758

5859
flag.Parse()
5960

‎collector/task/inventory/inventory.go

+53-11
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@ import (
3030

3131
// task that queries inventory data and aggregates them into usable mapping table
3232
type task struct {
33-
enabled bool
34-
inventoryAddr string
33+
enabled bool
34+
inventoryAddr string
35+
inventoryFormat string
3536

3637
mu sync.Mutex
3738
values map[string]Host
@@ -41,6 +42,11 @@ type task struct {
4142
var (
4243
once sync.Once
4344
singleton task
45+
46+
supportedInventoryFormats = map[string]bool{
47+
"arrayjson": true,
48+
"ndjson": true,
49+
}
4450
)
4551

4652
const (
@@ -59,10 +65,18 @@ func init() {
5965
}
6066

6167
// InitTask initial states
62-
func InitTask(ctx context.Context, enabled bool, inventoryAddr string) {
68+
func InitTask(ctx context.Context, enabled bool, inventoryAddr string, inventoryFormat string) {
69+
// Validate inventory format
70+
if _, ok := supportedInventoryFormats[inventoryFormat]; !ok {
71+
log.Warningf("Unsupported inventory format '%v', fallback to the default format", inventoryFormat)
72+
inventoryFormat = "arrayjson"
73+
}
74+
log.Infof("Using inventory format '%v'", inventoryFormat)
75+
6376
once.Do(func() {
6477
singleton.enabled = enabled
6578
singleton.inventoryAddr = inventoryAddr
79+
singleton.inventoryFormat = inventoryFormat
6680
})
6781
}
6882

@@ -107,17 +121,10 @@ func Collect(ctx context.Context) error {
107121
}
108122
defer response.Body.Close()
109123

110-
var metrics []Host
111-
decoder := json.NewDecoder(response.Body)
112-
decoder.DisallowUnknownFields()
113-
err = decoder.Decode(&metrics)
124+
metrics, err := parseInventory(singleton.inventoryFormat, response.Body)
114125
if err != nil {
115126
return err
116127
}
117-
if decoder.More() {
118-
bytesCopied, _ := io.Copy(ioutil.Discard, response.Body)
119-
log.Warnf("Unexpected remaining data (%v Bytes) in inventory response: %v", bytesCopied, singleton.inventoryAddr)
120-
}
121128

122129
hosts := make(map[string]Host)
123130
for _, v := range metrics {
@@ -153,3 +160,38 @@ func GetLocalInventory() Host {
153160

154161
return inv
155162
}
163+
164+
func parseInventory(format string, inventoryData io.ReadCloser) ([]Host, error) {
165+
var result []Host
166+
167+
decoder := json.NewDecoder(inventoryData)
168+
decoder.DisallowUnknownFields()
169+
170+
switch format {
171+
case "ndjson":
172+
inventoryEntry := Host{}
173+
for decoder.More() {
174+
err := decoder.Decode(&inventoryEntry)
175+
if err != nil {
176+
log.Errorf("Skip an inventory entry due to parser error: %v", err)
177+
continue
178+
}
179+
result = append(result, inventoryEntry)
180+
}
181+
182+
case "arrayjson":
183+
err := decoder.Decode(&result)
184+
if err != nil {
185+
return nil, err
186+
}
187+
188+
// We expect a single JSON array object here. Clear unexpected data that's left in the io.ReadCloser
189+
if decoder.More() {
190+
bytesCopied, _ := io.Copy(ioutil.Discard, inventoryData)
191+
log.Warnf("Unexpected remaining data (%v Bytes) in inventory response: %v", bytesCopied, singleton.inventoryAddr)
192+
}
193+
}
194+
log.Debugf("Parsed %v inventory data", len(result))
195+
196+
return result, nil
197+
}

‎collector/task/inventory/inventory_test.go

+113
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// Copyright 2021 - [email protected]
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package inventory
16+
17+
import (
18+
"io"
19+
"reflect"
20+
"strings"
21+
"testing"
22+
)
23+
24+
// mockInventoryResponse returns an io.ReadCloser simulating data coming from an inventory endpoint
25+
func mockInventoryResponse(raw string) io.ReadCloser {
26+
return io.NopCloser(strings.NewReader(raw))
27+
}
28+
29+
func Test_parseInventory(t *testing.T) {
30+
type args struct {
31+
inventoryFormat string
32+
inventoryData io.ReadCloser
33+
}
34+
35+
tests := []struct {
36+
name string
37+
args args
38+
want []Host
39+
wantErr bool
40+
}{
41+
// Format: 'ndjson'
42+
{
43+
name: "Test single ndjson inventory entry",
44+
args: args{
45+
inventoryFormat: "ndjson",
46+
inventoryData: mockInventoryResponse(`
47+
{"ip_address":"10.0.1.2","domain":"xyz.service.consul","hostgroup":"xyz"}
48+
`),
49+
},
50+
want: []Host{
51+
{IPAddress: "10.0.1.2", Domain: "xyz.service.consul", Hostgroup: "xyz"},
52+
},
53+
},
54+
{
55+
name: "Test multiple ndjson inventory entries",
56+
args: args{
57+
inventoryFormat: "ndjson",
58+
inventoryData: mockInventoryResponse(`
59+
{"ip_address":"10.0.1.2","domain":"xyz.service.consul","hostgroup":"xyz"}
60+
{"ip_address":"172.16.1.2","domain":"abc.service.consul","hostgroup":"abc"}
61+
`),
62+
},
63+
want: []Host{
64+
{IPAddress: "10.0.1.2", Domain: "xyz.service.consul", Hostgroup: "xyz"},
65+
{IPAddress: "172.16.1.2", Domain: "abc.service.consul", Hostgroup: "abc"},
66+
},
67+
},
68+
69+
// Format: 'arrayjson'
70+
{
71+
name: "Test single arrayjson inventory entry",
72+
args: args{
73+
inventoryFormat: "arrayjson",
74+
inventoryData: mockInventoryResponse(`
75+
[
76+
{"ip_address":"10.0.1.2","domain":"xyz.service.consul","hostgroup":"xyz"}
77+
]
78+
`),
79+
},
80+
want: []Host{
81+
{IPAddress: "10.0.1.2", Domain: "xyz.service.consul", Hostgroup: "xyz"},
82+
},
83+
},
84+
{
85+
name: "Test multiple arrayjson inventory entries",
86+
args: args{
87+
inventoryFormat: "arrayjson",
88+
inventoryData: mockInventoryResponse(`
89+
[
90+
{"ip_address":"10.0.1.2","domain":"xyz.service.consul","hostgroup":"xyz"},
91+
{"ip_address":"172.16.1.2","domain":"abc.service.consul","hostgroup":"abc"}
92+
]
93+
`),
94+
},
95+
want: []Host{
96+
{IPAddress: "10.0.1.2", Domain: "xyz.service.consul", Hostgroup: "xyz"},
97+
{IPAddress: "172.16.1.2", Domain: "abc.service.consul", Hostgroup: "abc"},
98+
},
99+
},
100+
}
101+
for _, tt := range tests {
102+
t.Run(tt.name, func(t *testing.T) {
103+
got, err := parseInventory(tt.args.inventoryFormat, tt.args.inventoryData)
104+
if (err != nil) != tt.wantErr {
105+
t.Errorf("parseInventory() error = %v, wantErr %v", err, tt.wantErr)
106+
return
107+
}
108+
if !reflect.DeepEqual(got, tt.want) {
109+
t.Errorf("parseInventory() = %v, want %v", got, tt.want)
110+
}
111+
})
112+
}
113+
}

‎setup/inventories-examples/task-inventory-arrayjson.json

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[
2+
{
3+
"ip_address": "10.0.1.2",
4+
"domain": "xyz.service.consul",
5+
"hostgroup": "xyz"
6+
},
7+
{
8+
"ip_address": "172.16.1.2",
9+
"domain": "abc.service.consul",
10+
"hostgroup": "abc"
11+
}
12+
]

‎setup/inventories-examples/task-inventory-ndjson.json

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"ip_address":"10.0.1.2","domain":"xyz.service.consul","hostgroup":"xyz"}
2+
{"ip_address":"172.16.1.2","domain":"abc.service.consul","hostgroup":"abc"}

‎setup/systemd/planet-exporter.service

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[Unit]
2+
Description=Planet Exporter
3+
After=network-online.target
4+
5+
[Service]
6+
User=root
7+
ExecStart=/usr/bin/planet-exporter \
8+
-listen-address 0.0.0.0:11910 \
9+
-log-level info \
10+
-log-disable-colors \
11+
-log-disable-timestamp \
12+
-task-darkstat-enabled=true \
13+
-task-darkstat-addr http://127.0.0.1:11560/metrics \
14+
-task-inventory-enabled=true \
15+
-task-inventory-addr https://s3-ap-southeast-1.amazonaws.com/example/inventory.json
16+
LimitNOFILE=8192
17+
CPUQuota=20%
18+
MemoryHigh=256M
19+
MemoryMax=512M
20+
21+
Restart=always
22+
23+
[Install]
24+
WantedBy=multi-user.target

0 commit comments

Comments
 (0)