Skip to content

Commit b5002f4

Browse files
TrolldemortedTrolldemorted
authored
Cleanup routerarkime and elk setup (#85)
Fixes enowars/EnoELK#2 Fixes enowars/EnoELK#9 Fixes #65
1 parent 828e9a4 commit b5002f4

File tree

19 files changed

+450
-97
lines changed

19 files changed

+450
-97
lines changed

‎ansible/roles/enoelk/files/EnoELK/config-dir/beats.conf‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
input {
2+
beats {
3+
port => 5044
4+
}
5+
}
6+
7+
output {
8+
elasticsearch {
9+
hosts => ["http://elasticsearch:9200"]
10+
index => "%{[@metadata][beat]}-%{[@metadata][version]}"
11+
"action" => "create"
12+
}
13+
}

‎ansible/roles/enoelk/files/EnoELK/config-dir/enologmessage.conf‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
input {
2+
beats {
3+
port => 5045
4+
}
5+
}
6+
7+
filter {
8+
grok {
9+
match => { "message" => "##ENOLOGMESSAGE %{GREEDYDATA:message}" }
10+
overwrite => ["message"]
11+
add_tag => [ "enologmessage" ]
12+
}
13+
grok {
14+
match => { "message" => "##ENOSTATISTICSMESSAGE %{GREEDYDATA:message}" }
15+
overwrite => ["message"]
16+
add_tag => [ "enostatisticsmessage" ]
17+
}
18+
if "enologmessage" in [tags] {
19+
json {
20+
source => "message"
21+
target => "enologs"
22+
skip_on_invalid_json => true
23+
}
24+
mutate {
25+
replace => {
26+
"[@metadata][index_prefix]" => "enologmessage"
27+
}
28+
}
29+
}
30+
if "enostatisticsmessage" in [tags] {
31+
json {
32+
source => "message"
33+
target => "enostatistics"
34+
skip_on_invalid_json => true
35+
}
36+
mutate {
37+
replace => {
38+
"[@metadata][index_prefix]" => "enostatisticsmessage"
39+
}
40+
}
41+
}
42+
if "enologmessage" not in [tags] and "enostatisticsmessage" not in [tags] {
43+
drop { }
44+
}
45+
}
46+
47+
output {
48+
elasticsearch {
49+
hosts => ["http://elasticsearch:9200"]
50+
index => "%{[@metadata][index_prefix]}"
51+
}
52+
}

‎ansible/roles/enoelk/files/EnoELK/config-dir/pipelines.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
- pipeline.id: enologmessage
2+
path.config: "/config-dir/enologmessage.conf"
3+
- pipeline.id: beats
4+
path.config: "/config-dir/beats.conf"

‎ansible/roles/enoelk/files/EnoELK/docker-compose.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
version: "3"
2+
services:
3+
elasticsearch:
4+
image: elasticsearch:8.4.3
5+
restart: unless-stopped
6+
environment:
7+
- discovery.type=single-node
8+
- xpack.security.enabled=false
9+
ports:
10+
- "127.0.0.1:9200:9200"
11+
volumes:
12+
- ./data/elasticsearch:/usr/share/elasticsearch/data
13+
kibana:
14+
image: kibana:8.4.3
15+
restart: unless-stopped
16+
ports:
17+
- "5601:5601"
18+
environment:
19+
- ELASTICSEARCH_HOSTS=http://elasticsearch:9200
20+
- xpack.security.enabled=false
21+
logstash:
22+
image: logstash:8.4.3
23+
restart: unless-stopped
24+
links:
25+
- elasticsearch
26+
volumes:
27+
- ./config-dir:/config-dir:ro
28+
command: logstash --path.settings=/config-dir
29+
ports:
30+
- "5044:5044"
31+
- "5045:5045"
32+
init:
33+
image: ubuntu
34+
restart: "no"
35+
entrypoint: /bin/sh
36+
volumes:
37+
- /usr/share/metricbeat/:/usr/share/metricbeat/:ro
38+
- /usr/bin/metricbeat:/usr/bin/metricbeat:ro
39+
- /etc/metricbeat/metricbeat.yml:/etc/metricbeat/metricbeat.yml:ro
40+
- .:/EnoELK:ro
41+
command:
42+
- /EnoELK/setup.sh

‎ansible/roles/enoelk/files/EnoELK/setup.sh‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
#!/bin/sh
2+
set -e
3+
cd "$(dirname "$0")"
4+
5+
apt-get update && apt-get install -y --no-install-recommends curl
6+
7+
while ! curl -sq http://elasticsearch:9200; do
8+
echo "Waiting for elasticsearch to start...";
9+
sleep 3;
10+
done
11+
while ! curl -sq --fail http://kibana:5601/api/status; do
12+
echo "Waiting for kibana to start..."
13+
sleep 3;
14+
done
15+
16+
echo -e "\nCreating enologmessage index...";
17+
curl -sq -X PUT http://elasticsearch:9200/enologmessage -H 'Content-Type: application/json' -d '
18+
{
19+
"mappings": {
20+
"dynamic": false,
21+
"properties": {
22+
"@timestamp": {
23+
"type": "date_nanos"
24+
},
25+
"@version": {
26+
"type": "text",
27+
"fields": {
28+
"keyword": {
29+
"type": "keyword",
30+
"ignore_above": 256
31+
}
32+
}
33+
},
34+
"agent": {
35+
"properties": {
36+
"ephemeral_id": {
37+
"type": "text",
38+
"fields": {
39+
"keyword": {
40+
"type": "keyword",
41+
"ignore_above": 256
42+
}
43+
}
44+
},
45+
"hostname": {
46+
"type": "text",
47+
"fields": {
48+
"keyword": {
49+
"type": "keyword",
50+
"ignore_above": 256
51+
}
52+
}
53+
},
54+
"id": {
55+
"type": "text",
56+
"fields": {
57+
"keyword": {
58+
"type": "keyword",
59+
"ignore_above": 256
60+
}
61+
}
62+
},
63+
"type": {
64+
"type": "text",
65+
"fields": {
66+
"keyword": {
67+
"type": "keyword",
68+
"ignore_above": 256
69+
}
70+
}
71+
},
72+
"version": {
73+
"type": "text",
74+
"fields": {
75+
"keyword": {
76+
"type": "keyword",
77+
"ignore_above": 256
78+
}
79+
}
80+
}
81+
}
82+
},
83+
"ecs": {
84+
"properties": {
85+
"version": {
86+
"type": "text",
87+
"fields": {
88+
"keyword": {
89+
"type": "keyword",
90+
"ignore_above": 256
91+
}
92+
}
93+
}
94+
}
95+
},
96+
"enologs": {
97+
"properties": {
98+
"currentRoundId": {
99+
"type": "long"
100+
},
101+
"flag": {
102+
"type": "text",
103+
"fields": {
104+
"keyword": {
105+
"type": "keyword",
106+
"ignore_above": 256
107+
}
108+
}
109+
},
110+
"function": {
111+
"type": "text",
112+
"fields": {
113+
"keyword": {
114+
"type": "keyword",
115+
"ignore_above": 256
116+
}
117+
}
118+
},
119+
"message": {
120+
"type": "text",
121+
"fields": {
122+
"keyword": {
123+
"type": "keyword",
124+
"ignore_above": 256
125+
}
126+
}
127+
},
128+
"method": {
129+
"type": "text",
130+
"fields": {
131+
"keyword": {
132+
"type": "keyword",
133+
"ignore_above": 256
134+
}
135+
}
136+
},
137+
"module": {
138+
"type": "text",
139+
"fields": {
140+
"keyword": {
141+
"type": "keyword",
142+
"ignore_above": 256
143+
}
144+
}
145+
},
146+
"relatedRoundId": {
147+
"type": "long"
148+
},
149+
"serviceName": {
150+
"type": "text",
151+
"fields": {
152+
"keyword": {
153+
"type": "keyword",
154+
"ignore_above": 256
155+
}
156+
}
157+
},
158+
"severity": {
159+
"type": "text",
160+
"fields": {
161+
"keyword": {
162+
"type": "keyword",
163+
"ignore_above": 256
164+
}
165+
}
166+
},
167+
"severityLevel": {
168+
"type": "long"
169+
},
170+
"taskChainId": {
171+
"type": "text",
172+
"fields": {
173+
"keyword": {
174+
"type": "keyword",
175+
"ignore_above": 256
176+
}
177+
}
178+
},
179+
"taskId": {
180+
"type": "long"
181+
},
182+
"teamId": {
183+
"type": "long"
184+
},
185+
"teamName": {
186+
"type": "text",
187+
"fields": {
188+
"keyword": {
189+
"type": "keyword",
190+
"ignore_above": 256
191+
}
192+
}
193+
},
194+
"timestamp": {
195+
"type": "date_nanos"
196+
},
197+
"tool": {
198+
"type": "text",
199+
"fields": {
200+
"keyword": {
201+
"type": "keyword",
202+
"ignore_above": 256
203+
}
204+
}
205+
},
206+
"type": {
207+
"type": "text",
208+
"fields": {
209+
"keyword": {
210+
"type": "keyword",
211+
"ignore_above": 256
212+
}
213+
}
214+
},
215+
"variantId": {
216+
"type": "long"
217+
}
218+
}
219+
},
220+
"log": {
221+
"properties": {
222+
"file": {
223+
"properties": {
224+
"path": {
225+
"type": "text",
226+
"fields": {
227+
"keyword": {
228+
"type": "keyword",
229+
"ignore_above": 256
230+
}
231+
}
232+
}
233+
}
234+
},
235+
"offset": {
236+
"type": "long"
237+
}
238+
}
239+
},
240+
"tags": {
241+
"type": "text",
242+
"fields": {
243+
"keyword": {
244+
"type": "keyword",
245+
"ignore_above": 256
246+
}
247+
}
248+
}
249+
}
250+
}
251+
}
252+
'
253+
254+
echo -e "Creating enologmessage data view...";
255+
curl -sq -X POST http://kibana:5601/api/data_views/data_view -H 'Content-Type: application/json' -H 'kbn-xsrf: reporting' -d '
256+
{
257+
"data_view": {
258+
"title": "enologmessage",
259+
"name": "enologmessage",
260+
"timeFieldName": "enologs.timestamp"
261+
}
262+
}'
263+
264+
echo -e "\nCreating enostatisticsmessage data view...";
265+
curl -sq -X POST http://kibana:5601/api/data_views/data_view -H 'Content-Type: application/json' -H 'kbn-xsrf: reporting' -d '
266+
{
267+
"data_view": {
268+
"title": "enostatisticsmessage",
269+
"name": "enostatisticsmessage",
270+
"timeFieldName": "enostatistics.timestamp"
271+
}
272+
}'
273+
274+
echo -e "\nCreating visualizations";
275+
curl -vvv -X POST -F "file=@./visualizations/saved-obj.ndjson" -H 'kbn-xsrf: true' http://kibana:5601/api/saved_objects/_import?overwrite=True
276+
277+
#echo -e "\nRunning metricbeat setup...";
278+
#/usr/bin/metricbeat setup -e \
279+
# -E output.logstash.enabled=false \
280+
# -E output.elasticsearch.hosts=['elasticsearch:9200'] \
281+
# -E setup.kibana.host=kibana:5601
282+
283+
echo "done!"

0 commit comments

Comments
 (0)