PB-2183: implement otel instrumentation

Author: hbollon

.env.test

@@ -0,0 +1,9 @@
+# OTEL
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-alti
+OTEL_PYTHON_EXCLUDED_URLS="checker"
+OTEL_PYTHON_LOG_CORRELATION=true

.pylintrc

@@ -36,10 +36,6 @@ load-plugins=pylint_flask
 # Pickle collected data for later comparisons.
 persistent=yes
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no

Makefile

@@ -94,6 +94,9 @@ ci:
 	# Create virtual env with all packages for development using the Pipfile.lock
 	pipenv sync --dev
 
+.PHONY: otelrequirements
+otelrequirements:
+	edot-bootstrap --action=requirements
 
 # linting target, calls upon yapf to make sure your code is easier to read and respects some conventions.
 

Pipfile

@@ -12,6 +12,12 @@ logging-utilities = "~=5.0"
 Flask = "~=3.1"
 PyYAML = "~=6.0"
 
+# OpenTelemetry packages
+opentelemetry-sdk = "*"
+opentelemetry-exporter-otlp = "*"
+opentelemetry-instrumentation-flask = "*"
+opentelemetry-instrumentation-logging = "*"
+
 [dev-packages]
 mock = "*"
 nose2 = "*"

Pipfile.lock

@@ -0,0 +1,41 @@
+from os import getenv
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import \
+    OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+from app.helpers.utils import strtobool
+
+
+def initialize() -> None:
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_LOGGING", "false")):
+            LoggingInstrumentor().instrument()
+
+
+def initialize_flask(app):
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_FLASK", "false")):
+            FlaskInstrumentor().instrument_app(app)
+
+
+def setup_trace_provider():
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        # Since we created a new tracer, the default span processor is gone. We need to
+        # create a new one using the default OTEL env variables and ad it to the tracer.
+        span_processor = BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
+                headers=getenv('OTEL_EXPORTER_OTLP_HEADERS'),
+                insecure=strtobool(getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+            )
+        )
+
+        provider = TracerProvider(resource=Resource.create())
+        provider.add_span_processor(span_processor)
+        trace.set_tracer_provider(provider)

app/helpers/otel.py

@@ -0,0 +1,16 @@
+version: "2"
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: --config otel-local-config.yaml
+    env_file:
+      - .env.test
+    volumes:
+      - ./otel-local-config.yaml:/otel-local-config.yaml
+    ports:
+      - "4317:4317"
+
+  zipkin:
+    image: openzipkin/zipkin:latest
+    ports:
+      - "9411:9411"

docker-compose-otel.yaml

@@ -82,6 +82,9 @@ formatters:
         duration: "%(duration)s"
         payload: "%(response.json).128s"
       message: message
+      otelSpanID: otelSpanID
+      otelTraceID: otelTraceID
+      otelTraceSampled: otelTraceSampled
 
 handlers:
   console:

logging-cfg-local.yml

@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+  zipkin:
+    endpoint: http://zipkin:9411/api/v2/spans
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, zipkin]

otel-local-config.yaml

@@ -1,3 +1,15 @@
+# Initialize OTEL.
+# Initialize should be called as early as possible, but at least before the app is imported
+# The order has a impact on how the libraries are instrumented. If called after app import,
+# e.g. the flask instrumentation has no effect. See:
+# https://github.com/open-telemetry/opentelemetry.io/blob/main/content/en/docs/zero-code/python/troubleshooting.md#use-programmatic-auto-instrumentation
+
+from app.helpers.otel import initialize
+from app.helpers.otel import initialize_flask
+from app.helpers.otel import setup_trace_provider
+
+initialize()
+
 import multiprocessing
 
 from gunicorn.app.base import BaseApplication
@@ -9,6 +21,13 @@
 from app.settings import GUNICORN_WORKER_TMP_DIR
 from app.settings import HTTP_PORT
 
+initialize_flask(application)
+
+def post_fork(server, worker):
+    server.log.info("Worker spawned (pid: %s)", worker.pid)
+
+    # Setup OTEL providers for this worker
+    setup_trace_provider()
 
 class StandaloneApplication(BaseApplication):  # pylint: disable=abstract-method
 
@@ -44,6 +63,7 @@ def load(self):
         'worker_tmp_dir': GUNICORN_WORKER_TMP_DIR,
         'timeout': 60,
         'keepalive': GUNICORN_KEEPALIVE,
-        'logconfig_dict': get_logging_cfg()
+        'logconfig_dict': get_logging_cfg(),
+        'post_fork': post_fork
     }
     StandaloneApplication(application, options).run()