Monitoring (#7)

* restructure

* replace node-exporter, promtail with alloy

* deploy tempo for tracing

* ci: update Helm values.yaml and Chart.yaml to ghcr.io/phuchoang2603/realtime-credit-card-fraud-detection:v1.0.0

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

Author: phuchoang2603

.github/workflows/release.yml

@@ -71,7 +71,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: deployments/docker/Dockerfile
+          file: ./Dockerfile
           push: true
           tags: |
             ${{ env.GHCR_IMAGE }}:${{ env.VERSION_TAG }}

deployments/docker/Dockerfile Dockerfiledeployments/docker/Dockerfile renamed to Dockerfile

@@ -14,4 +14,4 @@ EXPOSE 8000
 EXPOSE 8010
 
 
-CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-access-log"]

app/main.py

@@ -3,6 +3,7 @@
 import time
 import pandas as pd
 from fastapi import FastAPI, Request, HTTPException
+from opentelemetry import trace
 
 # Import configurations and schemas from separate modules
 from app.schema import TransactionFeatures, Prediction
@@ -88,17 +89,26 @@ async def health_check():
 
 
 @app.post("/predict", response_model=Prediction, tags=["Prediction"])
+@traceable
 async def predict_fraud(request: Request, transaction: TransactionFeatures):
     """
     Orchestrates the fraud detection process by calling traceable helper functions.
     """
     request_id = request.headers.get("X-Request-ID", "N/A")
+
     log.info(
         "Received prediction request",
         request_id=request_id,
         transaction_id=transaction.TRANSACTION_ID,
+        customer_id=transaction.CUSTOMER_ID,
+        terminal_id=transaction.TERMINAL_ID,
     )
 
+    span = trace.get_current_span()
+    span.set_attribute("transaction_id", transaction.TRANSACTION_ID)
+    span.set_attribute("customer_id", transaction.CUSTOMER_ID)
+    span.set_attribute("terminal_id", transaction.TERMINAL_ID)
+
     # 1. Run Pre-Prediction Checks
     await run_terminal_control_check(customer_id=transaction.CUSTOMER_ID)
     await run_transaction_blocking_rules(transaction=transaction)

app/utils/tracing_config.py

@@ -1,24 +1,28 @@
+from functools import wraps
+
 from fastapi import FastAPI
 from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
 from opentelemetry.sdk.resources import Resource
-from functools import wraps
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
 
 def setup_tracing(app: FastAPI, service_name: str):
     """
-    Sets up OpenTelemetry tracing and instruments the FastAPI application.
+    Sets up OpenTelemetry tracing to export traces to Grafana Alloy.
     """
     resource = Resource(attributes={"service.name": service_name})
-
-    # Set up the TracerProvider
     provider = TracerProvider(resource=resource)
-    provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
-    trace.set_tracer_provider(provider)
 
-    # Instrument the FastAPI app automatically
+    # Configure the exporter to send traces to Alloy's OTLP port
+    otlp_exporter = OTLPSpanExporter(
+        endpoint="http://alloy:4317/v1/traces", insecure=True
+    )
+    provider.add_span_processor(BatchSpanProcessor(otlp_exporter))
+
+    trace.set_tracer_provider(provider)
     FastAPIInstrumentor.instrument_app(app)
 
 
@@ -29,20 +33,17 @@ def get_tracer(name: str):
 
 def traceable(func):
     """
-    A decorator that adds an OpenTelemetry span to a function.
-    The span is automatically named after the function.
+    A decorator that adds an OpenTelemetry span to an async function.
     """
 
     @wraps(func)
     async def wrapper(*args, **kwargs):
         tracer = get_tracer(func.__module__)
         with tracer.start_as_current_span(func.__name__) as span:
             try:
-                # Execute the original async function
                 result = await func(*args, **kwargs)
                 return result
             except Exception as e:
-                # Record the exception in the span and re-raise it
                 span.record_exception(e)
                 raise
 

deployments/docker/client/Dockerfile client/Dockerfiledeployments/docker/client/Dockerfile renamed to client/Dockerfile

@@ -0,0 +1,131 @@
+// #################################################
+// # Define Endpoints
+// #################################################
+
+loki.write "default" {
+    endpoint {
+        url = "http://loki:3100/loki/api/v1/push"
+    }
+}
+
+prometheus.remote_write "default" {
+    endpoint {
+        url = "http://prometheus:9090/api/v1/write"
+    }
+}
+
+// #################################################
+// # Data Sources
+// #################################################
+
+// --- 1. Collect Logs from Docker Containers ---
+discovery.docker "linux" {
+  host = "unix:///var/run/docker.sock"
+}
+discovery.relabel "logs_integrations_docker" {
+      targets = []
+  
+      rule {
+          source_labels = ["__meta_docker_container_name"]
+          regex = "/(.*)"
+          target_label = "service_name"
+      }
+
+  }
+loki.source.docker "default" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.docker.linux.targets
+  labels     = {"platform" = "docker"}
+  relabel_rules = discovery.relabel.logs_integrations_docker.rules
+  forward_to = [loki.write.default.receiver]
+}
+
+// --- 2. Collect Metrics from Application ---
+// prometheus.scrape "fraud_detection_api" {
+//   targets = [{
+//     __address__ = "api:8010",
+//   }]
+//   forward_to     = [prometheus.remote_write.default.receiver]
+//   job_name = "fraud-detection-api"
+// }
+
+// --- 3. Collect Traces from Application ---
+otelcol.receiver.otlp "default" {
+  http {}
+  grpc {}
+
+  output {
+    traces = [otelcol.processor.batch.default.input]
+  }
+}
+otelcol.processor.batch "default" {
+  output {
+    traces = [otelcol.exporter.otlp.tempo.input]
+  }
+}
+otelcol.exporter.otlp "tempo" {
+  client {
+    endpoint = "tempo:4317"
+    tls {
+			insecure = true
+		}
+  }
+} 
+
+// --- 4. Collect Metrics from Docker Containers (cAdvisor) ---
+prometheus.exporter.cadvisor "container_metrics" {
+  docker_host = "unix:///var/run/docker.sock"
+
+  storage_duration = "5m"
+}
+prometheus.scrape "scrape_cadvisor" {
+  targets    = prometheus.exporter.cadvisor.container_metrics.targets
+  forward_to = [prometheus.remote_write.default.receiver]
+  scrape_interval = "10s"
+}
+
+// --- 5. Collect Metrics from the Host Machine (replaces node-exporter) ---
+discovery.relabel "integrations_node_exporter" {
+  targets = prometheus.exporter.unix.integrations_node_exporter.targets
+
+  rule {
+    // Set the instance label to the hostname of the machine
+    target_label = "instance"
+    replacement  = constants.hostname
+  }
+
+  rule {
+    // Set a standard job name for all node_exporter metrics
+    target_label = "job"
+    replacement = "integrations/node_exporter"
+  }
+}
+prometheus.exporter.unix "integrations_node_exporter" {
+  disable_collectors = ["ipvs", "btrfs", "infiniband", "xfs", "zfs"]
+  enable_collectors = ["meminfo"]
+
+  filesystem {
+    fs_types_exclude     = "^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|tmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$"
+    mount_points_exclude = "^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/)"
+    mount_timeout        = "5s"
+  }
+
+  netclass {
+    ignored_devices = "^(veth.*|cali.*|[a-f0-9]{15})$"
+  }
+
+  netdev {
+    device_exclude = "^(veth.*|cali.*|[a-f0-9]{15})$"
+  }
+}
+prometheus.scrape "integrations_node_exporter" {
+scrape_interval = "15s"
+  // Use the targets with labels from the discovery.relabel component
+  targets    = discovery.relabel.integrations_node_exporter.output
+  // Send the scraped metrics to the relabeling component
+  forward_to = [prometheus.remote_write.default.receiver]
+}
+
+livedebugging {
+  enabled = true
+}