Skip to content

Commit 312340f

Browse files
ChristopherGSChristopherGS
committed
Monitoring Logs with the Elastic Stack - Basic ELK Setup
1 parent 21639fe commit 312340f

File tree

11 files changed

+243
-0
lines changed

11 files changed

+243
-0
lines changed

‎exercise_notebooks/elk_exercise/Dockerfile

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
FROM python:3.7-alpine
2+
WORKDIR /application
3+
4+
COPY ./requirements.txt requirements.txt
5+
RUN apk add --no-cache \
6+
gcc \
7+
libc-dev \
8+
linux-headers \
9+
bash; \
10+
pip install -r requirements.txt;
11+
12+
COPY . /application
13+
14+
15+
EXPOSE 5000
16+
VOLUME /application
17+
CMD gunicorn --bind 0.0.0.0:5000 \
18+
--workers=1 \
19+
--log-config gunicorn_logging.conf \
20+
--log-level=DEBUG \
21+
--access-logfile=- \
22+
--error-logfile=- \
23+
application:application

‎exercise_notebooks/elk_exercise/app/__init__.py

Whitespace-only changes.

‎exercise_notebooks/elk_exercise/app/flask_app.py

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import logging
2+
3+
from flask import Flask
4+
5+
gunicorn_error_logger = logging.getLogger('gunicorn.error')
6+
gunicorn_error_logger.setLevel(logging.DEBUG)
7+
8+
9+
def index():
10+
gunicorn_error_logger.info('hello')
11+
return 'home'
12+
13+
14+
def create_app():
15+
main_app = Flask(__name__)
16+
main_app.add_url_rule('/', 'index', index)
17+
18+
return main_app

‎exercise_notebooks/elk_exercise/application.py

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from app.flask_app import create_app
2+
3+
4+
application = create_app()
5+
6+
if __name__ == '__main__':
7+
application.run()

‎exercise_notebooks/elk_exercise/docker-compose.yml

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
version: '3.2'
2+
3+
services:
4+
# The environment variable "ELK_VERSION" is used throughout this file to
5+
# specify the version of the images to run. The default is set in the
6+
# '.env' file in this folder. It can be overridden with any normal
7+
# technique for setting environment variables, for example:
8+
#
9+
# ELK_VERSION=6.0.0-beta1 docker-compose up
10+
#
11+
# REF: https://docs.docker.com/compose/compose-file/#variable-substitution
12+
webapp:
13+
build: .
14+
container_name: webapp
15+
expose:
16+
- 5000
17+
ports:
18+
- 5000:5000
19+
links:
20+
- logstash
21+
networks:
22+
- elk
23+
depends_on:
24+
- logstash
25+
- kibana
26+
- elasticsearch
27+
volumes:
28+
- ./:/application
29+
elasticsearch:
30+
image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
31+
volumes:
32+
- type: bind
33+
source: ./elasticsearch/config/elasticsearch.yml
34+
target: /usr/share/elasticsearch/config/elasticsearch.yml
35+
read_only: true
36+
- type: volume
37+
source: elasticsearch
38+
target: /usr/share/elasticsearch/data
39+
ports:
40+
- "9200:9200"
41+
- "9300:9300"
42+
environment:
43+
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
44+
ELASTIC_PASSWORD: changeme
45+
# Use single node discovery in order to disable production mode and avoid bootstrap checks
46+
# see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
47+
discovery.type: single-node
48+
networks:
49+
- elk
50+
51+
logstash:
52+
image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
53+
volumes:
54+
- type: bind
55+
source: ./logstash/config/logstash.yml
56+
target: /usr/share/logstash/config/logstash.yml
57+
read_only: true
58+
- type: bind
59+
source: ./logstash/pipeline
60+
target: /usr/share/logstash/pipeline
61+
read_only: true
62+
ports:
63+
- "5001:5001"
64+
- "9600:9600"
65+
environment:
66+
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
67+
networks:
68+
- elk
69+
depends_on:
70+
- elasticsearch
71+
72+
kibana:
73+
image: docker.elastic.co/kibana/kibana:${ELK_VERSION}
74+
volumes:
75+
- type: bind
76+
source: ./kibana/config/kibana.yml
77+
target: /usr/share/kibana/config/kibana.yml
78+
read_only: true
79+
ports:
80+
- "5601:5601"
81+
networks:
82+
- elk
83+
depends_on:
84+
- elasticsearch
85+
86+
networks:
87+
elk:
88+
driver: bridge
89+
90+
volumes:
91+
elasticsearch:

‎exercise_notebooks/elk_exercise/elasticsearch/config/elasticsearch.yml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
## Default Elasticsearch configuration from Elasticsearch base image.
3+
## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
4+
cluster.name: "docker-cluster"
5+
network.host: 0.0.0.0
6+
7+
## X-Pack settings
8+
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
9+
xpack.license.self_generated.type: basic
10+
xpack.security.enabled: true
11+
xpack.monitoring.collection.enabled: true

‎exercise_notebooks/elk_exercise/gunicorn_logging.conf

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
[loggers]
2+
keys=root, logstash.error, logstash.access
3+
4+
[handlers]
5+
keys=console, logstash
6+
7+
[formatters]
8+
keys=generic, access, json
9+
10+
[logger_root]
11+
level=INFO
12+
handlers=console
13+
14+
[logger_logstash.error]
15+
level=INFO
16+
handlers=logstash
17+
propagate=1
18+
qualname=gunicorn.error
19+
20+
[logger_logstash.access]
21+
level=INFO
22+
handlers=logstash
23+
propagate=0
24+
qualname=gunicorn.access
25+
26+
[handler_console]
27+
class=StreamHandler
28+
formatter=generic
29+
args=(sys.stdout, )
30+
31+
[handler_logstash]
32+
class=logstash.TCPLogstashHandler
33+
formatter=json
34+
args=('logstash', 5001)
35+
36+
[formatter_generic]
37+
format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s
38+
datefmt=%Y-%m-%d %H:%M:%S
39+
class=logging.Formatter
40+
41+
[formatter_access]
42+
format=%(message)s
43+
class=logging.Formatter
44+
45+
[formatter_json]
46+
class=pythonjsonlogger.jsonlogger.JsonFormatter

‎exercise_notebooks/elk_exercise/kibana/config/kibana.yml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
## Default Kibana configuration from Kibana base image.
3+
## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js
4+
#
5+
server.name: kibana
6+
server.host: "0"
7+
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
8+
xpack.monitoring.ui.container.elasticsearch.enabled: true
9+
10+
## X-Pack security credentials
11+
#
12+
elasticsearch.username: elastic
13+
elasticsearch.password: changeme

‎exercise_notebooks/elk_exercise/logstash/config/logstash.yml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
## Default Logstash configuration from Logstash base image.
3+
## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
4+
#
5+
http.host: "0.0.0.0"
6+
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
7+
8+
## X-Pack security credentials
9+
#
10+
xpack.monitoring.enabled: true
11+
xpack.monitoring.elasticsearch.username: elastic
12+
xpack.monitoring.elasticsearch.password: changeme

‎exercise_notebooks/elk_exercise/logstash/pipeline/logstash.conf

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
input {
2+
tcp {
3+
port => 5001
4+
tags => ["webapp_logs"]
5+
type => "webapp_logs"
6+
codec => json
7+
}
8+
}
9+
10+
output {
11+
elasticsearch {
12+
hosts => "elasticsearch:9200"
13+
user => "elastic"
14+
password => "changeme"
15+
index => "webapp_logs-%{+YYYY.MM.dd}"
16+
}
17+
}

‎exercise_notebooks/elk_exercise/requirements.txt

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
Flask>=1.1.1,<1.2.0
2+
python3-logstash>=0.4.80,<0.5.0
3+
python-json-logger>=0.1.11,<0.2.0
4+
gunicorn>=20.0.4,<20.1.0
5+

0 commit comments

Comments
 (0)