Skip to content

Commit fd88d66

Browse files
ChristopherGSChristopherGS
authored
Monitoring Logs with the Elastic Stack - Integrate with API
1 parent 36d6804 commit fd88d66

File tree

15 files changed

+235
-21
lines changed

15 files changed

+235
-21
lines changed

‎packages/ml_api/Makefile

+7-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,13 @@ run-service-development:
2727

2828
run-service-wsgi:
2929
@echo "+ $@"
30-
gunicorn --workers=1 --bind 0.0.0.0:5000 run:application
30+
gunicorn --bind 0.0.0.0:5000 \
31+
--workers=1 \
32+
--log-config gunicorn_logging.conf \
33+
--log-level=DEBUG \
34+
--access-logfile=- \
35+
--error-logfile=- \
36+
run:application
3137

3238
db-migrations:
3339
@echo "+ $@"

‎packages/ml_api/api/app.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
from api.monitoring.middleware import setup_metrics
88
from api.persistence.core import init_database
99

10-
_logger = logging.getLogger(__name__)
10+
_logger = logging.getLogger('mlapi')
1111

1212

1313
def create_app(

‎packages/ml_api/api/config.py

+4-5
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,10 @@
22
import os
33
import pathlib
44
import sys
5+
from logging.config import fileConfig
56

67
import api
78

8-
99
# logging format
1010
FORMATTER = logging.Formatter(
1111
"%(asctime)s — %(name)s — %(levelname)s —" "%(funcName)s:%(lineno)d — %(message)s"
@@ -82,10 +82,9 @@ def get_console_handler():
8282
def setup_app_logging(config: Config) -> None:
8383
"""Prepare custom logging for our application."""
8484
_disable_irrelevant_loggers()
85-
root = logging.getLogger()
86-
root.setLevel(config.LOGGING_LEVEL)
87-
root.addHandler(get_console_handler())
88-
root.propagate = False
85+
fileConfig(ROOT / 'gunicorn_logging.conf')
86+
logger = logging.getLogger('mlapi')
87+
logger.setLevel(config.LOGGING_LEVEL)
8988

9089

9190
def _disable_irrelevant_loggers() -> None:

‎packages/ml_api/api/controller.py

+15-6
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,16 @@
33
import threading
44

55
from flask import request, jsonify, Response, current_app
6+
from prometheus_client import Histogram, Gauge, Info
7+
from regression_model import __version__ as live_version
68

9+
from api.config import APP_NAME
10+
from api.persistence.data_access import PredictionPersistence, ModelType
711
from gradient_boosting_model import __version__ as shadow_version
8-
from regression_model import __version__ as live_version
9-
from prometheus_client import Histogram, Gauge, Info
1012
from gradient_boosting_model.predict import make_prediction
11-
from api.persistence.data_access import PredictionPersistence, ModelType
12-
from api.config import APP_NAME
1313

14+
_logger = logging.getLogger('mlapi')
1415

15-
_logger = logging.getLogger(__name__)
1616

1717
PREDICTION_TRACKER = Histogram(
1818
name='house_price_prediction_dollars',
@@ -45,13 +45,18 @@
4545

4646
def health():
4747
if request.method == "GET":
48-
return jsonify({"status": "ok"})
48+
status = {"status": "ok"}
49+
_logger.debug(status)
50+
return jsonify(status)
4951

5052

5153
def predict():
5254
if request.method == "POST":
5355
# Step 1: Extract POST data from request body as JSON
5456
json_data = request.get_json()
57+
_logger.info(
58+
f'Inputs for model: {ModelType.LASSO.name} '
59+
f'Input values: {json_data}')
5560

5661
# Step 2a: Get and save live model predictions
5762
persistence = PredictionPersistence(db_session=current_app.db_session)
@@ -89,6 +94,10 @@ def predict():
8994
app_name=APP_NAME,
9095
model_name=ModelType.LASSO.name,
9196
model_version=live_version).set(_prediction)
97+
_logger.info(
98+
f'Prediction results for model: {ModelType.LASSO.name} '
99+
f'version: {result.model_version} '
100+
f'Output values: {result.predictions}')
92101

93102
# Step 5: Prepare prediction response
94103
return jsonify(

‎packages/ml_api/api/persistence/core.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
from api.config import Config, ROOT
1313

14-
_logger = logging.getLogger(__name__)
14+
_logger = logging.getLogger('mlapi')
1515

1616
# Base class for SQLAlchemy models
1717
Base = declarative_base()

‎packages/ml_api/api/persistence/data_access.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -5,23 +5,24 @@
55

66
import numpy as np
77
import pandas as pd
8-
from gradient_boosting_model.predict import make_prediction as make_shadow_prediction
98
from regression_model.predict import make_prediction as make_live_prediction
109
from sqlalchemy.orm.session import Session
1110

1211
from api.persistence.models import (
1312
LassoModelPredictions,
1413
GradientBoostingModelPredictions,
1514
)
15+
from gradient_boosting_model.predict import make_prediction as make_shadow_prediction
16+
17+
_logger = logging.getLogger('mlapi')
18+
1619

1720
SECONDARY_VARIABLES_TO_RENAME = {
1821
"FirstFlrSF": "1stFlrSF",
1922
"SecondFlrSF": "2ndFlrSF",
2023
"ThreeSsnPortch": "3SsnPorch",
2124
}
2225

23-
_logger = logging.getLogger(__name__)
24-
2526

2627
class ModelType(enum.Enum):
2728
LASSO = "lasso"

‎packages/ml_api/docker/docker-compose-elk.yml

+99
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
version: '3.2'
2+
services:
3+
ml_api:
4+
build:
5+
context: ../
6+
dockerfile: docker/Dockerfile
7+
environment:
8+
DB_HOST: database
9+
DB_PORT: 5432
10+
DB_USER: user
11+
DB_PASSWORD: ${DB_PASSWORD:-password}
12+
DB_NAME: ml_api_dev
13+
networks:
14+
- elk
15+
depends_on:
16+
- database
17+
- logstash
18+
ports:
19+
- "5000:5000" # expose webserver to localhost host:container
20+
command: bash -c "make db-migrations && make run-service-wsgi"
21+
22+
database:
23+
image: postgres:latest
24+
environment:
25+
POSTGRES_USER: user
26+
POSTGRES_PASSWORD: password
27+
POSTGRES_DB: ml_api_dev
28+
ports:
29+
# expose postgres container on different host port to default (host:container)
30+
- "6609:5432"
31+
volumes:
32+
- my_dbdata:/var/lib/postgresql/data
33+
networks:
34+
- elk
35+
36+
elasticsearch:
37+
image: docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION}
38+
volumes:
39+
- type: bind
40+
source: ./elasticsearch/config/elasticsearch.yml
41+
target: /usr/share/elasticsearch/config/elasticsearch.yml
42+
read_only: true
43+
- type: volume
44+
source: elasticsearch
45+
target: /usr/share/elasticsearch/data
46+
ports:
47+
- "9200:9200"
48+
- "9300:9300"
49+
environment:
50+
ES_JAVA_OPTS: "-Xmx256m -Xms256m"
51+
ELASTIC_PASSWORD: changeme
52+
# Use single node discovery in order to disable production mode and avoid bootstrap checks
53+
# see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html
54+
discovery.type: single-node
55+
networks:
56+
- elk
57+
58+
logstash:
59+
image: docker.elastic.co/logstash/logstash:${ELK_VERSION}
60+
volumes:
61+
- type: bind
62+
source: ./logstash/config/logstash.yml
63+
target: /usr/share/logstash/config/logstash.yml
64+
read_only: true
65+
- type: bind
66+
source: ./logstash/pipeline
67+
target: /usr/share/logstash/pipeline
68+
read_only: true
69+
ports:
70+
- "5001:5001"
71+
- "9600:9600"
72+
environment:
73+
LS_JAVA_OPTS: "-Xmx256m -Xms256m"
74+
networks:
75+
- elk
76+
depends_on:
77+
- elasticsearch
78+
79+
kibana:
80+
image: docker.elastic.co/kibana/kibana:${ELK_VERSION}
81+
volumes:
82+
- type: bind
83+
source: ./kibana/config/kibana.yml
84+
target: /usr/share/kibana/config/kibana.yml
85+
read_only: true
86+
ports:
87+
- "5601:5601"
88+
networks:
89+
- elk
90+
depends_on:
91+
- elasticsearch
92+
93+
networks:
94+
elk:
95+
driver: bridge
96+
97+
volumes:
98+
my_dbdata:
99+
elasticsearch:

‎packages/ml_api/docker/docker-compose.yml

-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
version: '3'
22
services:
33
ml_api:
4-
image: christophergs/ml_api:master
54
build:
65
context: ../
76
dockerfile: docker/Dockerfile

‎packages/ml_api/docker/elasticsearch/config/elasticsearch.yml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
## Default Elasticsearch configuration from Elasticsearch base image.
2+
## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
3+
cluster.name: "docker-cluster"
4+
network.host: 0.0.0.0
5+
6+
## X-Pack settings
7+
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
8+
xpack.license.self_generated.type: basic
9+
xpack.security.enabled: true
10+
xpack.monitoring.collection.enabled: true

‎packages/ml_api/docker/kibana/config/kibana.yml

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
---
2+
## Default Kibana configuration from Kibana base image.
3+
## https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js
4+
5+
server.name: kibana
6+
server.host: "0"
7+
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
8+
xpack.monitoring.ui.container.elasticsearch.enabled: true
9+
10+
## X-Pack security credentials
11+
elasticsearch.username: elastic
12+
elasticsearch.password: changeme

‎packages/ml_api/docker/logstash/config/logstash.yml

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
---
2+
## Default Logstash configuration from Logstash base image.
3+
## https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
4+
#
5+
http.host: "0.0.0.0"
6+
xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ]
7+
8+
## X-Pack security credentials
9+
xpack.monitoring.enabled: true
10+
xpack.monitoring.elasticsearch.username: elastic
11+
xpack.monitoring.elasticsearch.password: changeme

‎packages/ml_api/docker/logstash/pipeline/logstash.conf

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
input {
2+
tcp {
3+
port => 5001
4+
tags => ["webapp_logs"]
5+
type => "webapp_logs"
6+
codec => json
7+
}
8+
}
9+
10+
output {
11+
elasticsearch {
12+
hosts => "elasticsearch:9200"
13+
user => "elastic"
14+
password => "changeme"
15+
index => "webapp_logs-%{+YYYY.MM.dd}"
16+
}
17+
}

‎packages/ml_api/gunicorn_logging.conf

+49
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
[loggers]
2+
keys=root, mlapi, logstash.error, logstash.access
3+
4+
[handlers]
5+
keys=console, logstash
6+
7+
[formatters]
8+
keys=generic, json
9+
10+
[logger_root]
11+
level=INFO
12+
handlers=console
13+
propagate=1
14+
15+
[logger_mlapi]
16+
level=INFO
17+
handlers=console,logstash
18+
propagate=0
19+
qualname=mlapi
20+
21+
[logger_logstash.error]
22+
level=INFO
23+
handlers=logstash
24+
propagate=1
25+
qualname=gunicorn.error
26+
27+
[logger_logstash.access]
28+
level=INFO
29+
handlers=logstash
30+
propagate=0
31+
qualname=gunicorn.access
32+
33+
[handler_console]
34+
class=StreamHandler
35+
formatter=generic
36+
args=(sys.stdout, )
37+
38+
[handler_logstash]
39+
class=logstash.TCPLogstashHandler
40+
formatter=json
41+
args=('logstash', 5001)
42+
43+
[formatter_generic]
44+
format=%(asctime)s [%(process)d] [%(levelname)s] %(message)s
45+
datefmt=%Y-%m-%d %H:%M:%S
46+
class=logging.Formatter
47+
48+
[formatter_json]
49+
class=pythonjsonlogger.jsonlogger.JsonFormatter

‎packages/ml_api/requirements/requirements.txt

+5-1
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,12 @@ psycopg2>=2.8.4,<2.9.0 # DB Driver
1919
alembic>=1.3.1,<1.4.0 # DB Migrations
2020
sqlalchemy_utils>=0.36.0,<0.37.0 # DB Utils
2121

22-
# Monitoring
22+
# Metrics
2323
prometheus_client>=0.7.1,<0.8.0
2424

25+
# Logging
26+
python3-logstash>=0.4.80,<0.5.0
27+
python-json-logger>=0.1.11,<0.2.0
28+
2529
# Deployment
2630
gunicorn>=20.0.4,<20.1.0

‎packages/ml_api/run.py

-2
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,10 @@
44
from api.app import create_app
55
from api.config import DevelopmentConfig, setup_app_logging
66

7-
87
_config = DevelopmentConfig()
98

109
# setup logging as early as possible
1110
setup_app_logging(config=_config)
12-
1311
main_app = create_app(config_object=_config).app
1412
application = DispatcherMiddleware(
1513
app=main_app.wsgi_app,

0 commit comments

Comments
 (0)