feat: Varnish NCSA frontend/backend logs (#129)

algchoo · web-flow · commit 9f13fa735884 · 2024-07-15T14:30:54.000-04:00
diff --git a/charts/varnish/Chart.yaml b/charts/varnish/Chart.yaml
@@ -11,4 +11,4 @@ name: varnish
 sources:
   - https://varnish-cache.org/
   - https://artifacthub.io/packages/helm/varnish/varnish-cache
-version: 1.0.2
+version: 1.0.3
diff --git a/charts/varnish/templates/configmaps.yaml b/charts/varnish/templates/configmaps.yaml
@@ -47,17 +47,70 @@ metadata:
 data:
   default.vcl: |
     vcl 4.1;
-    
+
     probe health {
         .url = "/";
         .timeout = 2s;
         .interval = 5s;
         .window = 5;
         .threshold = 3;
-    } 
-    
-    backend server_nginx_0 {
+    }
+
+    backend default {
         .host = "localhost";
         .port = "8080";
         .probe = health;
     }
+
+    sub vcl_recv {
+        # Force a cache miss to ensure backend requests
+        return (pass);
+    }
+
+    sub vcl_backend_response {
+        set beresp.ttl = 10s;
+    }
+
+    sub vcl_deliver {
+        if (obj.hits > 0) {
+            set resp.http.X-Cache = "HIT";
+        } else {
+            set resp.http.X-Cache = "MISS";
+        }
+    }
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: alloy-varnish-log-config
+  labels:
+    alloy-config: varnish-logs
+data:
+  config.alloy: |
+    logging {
+      level  = "info"
+      format = "logfmt"
+    }
+
+    loki.write "logs_integrations" {
+      endpoint {
+        url = "http://loki.default.svc.cluster.local:3100/loki/api/v1/push"
+
+      }
+      external_labels = {"cluster" = "my-cluster"}
+    }
+
+    local.file_match "logs_integrations_integrations_varnish_cache" {
+      path_targets = [{
+        __address__ = "localhost",
+        __path__    = "/var/log/varnish/varnishncsa-*.log",
+        instance    = "custom-varnish-cache.sample-apps.svc.cluster.local:9131",
+        job         = "integrations/varnish-cache",
+      }]
+    }
+
+    loki.source.file "logs_integrations_integrations_apache_airflow" {
+      targets    = local.file_match.logs_integrations_integrations_varnish_cache.targets
+      forward_to = [loki.write.logs_integrations.receiver]
+    }
+---
diff --git a/charts/varnish/templates/deployment.yaml b/charts/varnish/templates/deployment.yaml
@@ -22,8 +22,18 @@ spec:
     spec:            
       serviceAccountName: custom-varnish-cache      
       securityContext:
-        fsGroup: 999
+        fsGroup: 101
       shareProcessNamespace: true
+      initContainers:
+      - name: init-script
+        image: alpine:latest
+        securityContext:
+          runAsNonRoot: true
+          runAsUser: 103
+        command: ['sh', '-c', 'touch /var/log/varnish/varnishncsa-frontend.log /var/log/varnish/varnishncsa-backend.log && chmod 666 /var/log/varnish/*.log']
+        volumeMounts:
+        - name: varnish-log
+          mountPath: /var/log/varnish
       containers:      
         - name: nginx
           image: {{ .Values.nginxImage }}
@@ -37,7 +47,7 @@ spec:
         - name: varnish-cache
           securityContext:
             runAsNonRoot: true
-            runAsUser: 1002
+            runAsUser: 103
           image: {{ .Values.varnishImage }}
           imagePullPolicy: IfNotPresent
           ports:
@@ -59,7 +69,9 @@ spec:
               mountPath: "/etc/varnish/default.vcl"
               subPath: default.vcl
             - name: custom-varnish-vsm
-              mountPath: /var/lib/varnish    
+              mountPath: /var/lib/varnish
+            - name: varnish-log
+              mountPath: /var/log/varnish
         - name: exporter
           command:
             - "/prometheus_varnish_exporter"
@@ -73,7 +85,49 @@ spec:
               subPath: secret
           ports:
           - containerPort: 9131
-            name: prometheus                
+            name: prometheus
+        - name: varnish-cache-ncsa-frontend
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 103
+          image: {{ .Values.varnishImage }}
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c"]
+          args: ["varnishncsa -n /var/lib/varnish -F '{\"Timestamp\": \"%t\", \"Varnish-Side\": \"%{Varnish:side}x\", \"Age\": %{age}o, \"Handling\": \"%{Varnish:handling}x\", \"Request\": \"%r\", \"Status\": \"%s\", \"Response-Reason\": \"%{VSL:RespReason}x\", \"Fetch-Error\": \"%{VSL:FetchError}x\", \"X-Forwarded-For\": \"%{x-forwarded-for}i\", \"Remote-User\": \"%u\", \"Bytes\": \"%b\", \"Time-To-Serve\": %D, \"User-Agent\": \"%{User-agent}i\", \"Referer\": \"%{Referer}i\", \"X-Varnish\": \"%{x-varnish}o\", \"X-Magento-Tags\": \"%{x-magento-tags}o\"}' -w /var/log/varnish/varnishncsa-frontend.log"]
+          volumeMounts:
+          - name: custom-varnish-vsm
+            mountPath: /var/lib/varnish
+            readOnly: true
+          - name: varnish-secret
+            mountPath: /etc/varnish/secret
+            subPath: secret
+          - name: varnish-log
+            mountPath: /var/log/varnish
+        - name: varnish-cache-ncsa-backend
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 103
+          image: {{ .Values.varnishImage }}
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c"]
+          args: ["varnishncsa -n /var/lib/varnish -b -F '{\"Timestamp\": \"%t\", \"Varnish-Side\": \"%{Varnish:side}x\", \"Handling\": \"%{Varnish:handling}x\", \"Request\": \"%r\", \"Status\": \"%s\", \"Response-Reason\": \"%{VSL:RespReason}x\", \"Fetch-Error\": \"%{VSL:FetchError}x\", \"Bytes\": \"%b\", \"Time-To-Serve\": %D}' -w /var/log/varnish/varnishncsa-backend.log"]
+          volumeMounts:
+          - name: custom-varnish-vsm
+            mountPath: /var/lib/varnish
+            readOnly: true
+          - name: varnish-secret
+            mountPath: /etc/varnish/secret
+            subPath: secret
+          - name: varnish-log
+            mountPath: /var/log/varnish
+        - name: alloy-varnish-logs
+          image: grafana/alloy:latest
+          volumeMounts:
+          - name: varnish-log
+            mountPath: /var/log/varnish
+          - name: alloy-varnish-log-config
+            mountPath: /etc/alloy/config.alloy
+            subPath: config.alloy
       volumes:
       - name: custom-config
         emptyDir:
@@ -90,6 +144,14 @@ spec:
       - name: nginx-config
         configMap:
           name: nginx-config
+      - name: varnish-log
+        emptyDir: {}
+      - name: alloy-varnish-log-config
+        configMap:
+          name: alloy-varnish-log-config
+          items:
+          - key: config.alloy
+            path: config.alloy
       affinity:
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
diff --git a/charts/varnish/templates/service.yaml b/charts/varnish/templates/service.yaml
@@ -13,7 +13,7 @@ spec:
   ports:
     - name: http
       port: 80
-      targetPort: 6081
+      targetPort: 80
     - name: varnish-exporter
       port: 9131
       targetPort: 9131
