Push agent v2 logs to QE grafana

Author: john-david3

.github/actions/start-promtail/action.yml

@@ -0,0 +1,25 @@
+name: Start Promtail
+description: Start promtail in a Docker container to send test results to loki
+runs:
+    using: "composite"
+    steps:
+        - name: Start Promtail container
+          shell: bash
+          run: |
+              docker run -d \
+               --name=promtail \
+               -v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \
+               -v "${{ github.workspace }}/test/dashboard/logs:/var/log" \
+               -e TEST_OUTDIR=test/dashboard/logs \
+               -e GITHUB_RUN_ID=${{ github.run_id }} \
+               -e GITHUB_JOB=${{ github.job }} \
+               -e GITHUB_WORKFLOW=${{ github.workflow }} \
+               -e GITHUB_EVENT_NAME=${{ github.event_name }} \
+               -e GITHUB_SERVER_URL=${{ github.server_url }} \
+               -e GITHUB_REPOSITORY=${{ github.repository }} \
+               -e GITHUB_HEAD_REF=${{ github.head_ref }} \
+               -e GITHUB_SHA=${{ github.sha }} \
+               -e GITHUB_ACTOR=${{ github.actor }} \
+               grafana/promtail:3.4.4 \
+               -config.file=/etc/promtail/config.yaml \
+               -config.expand-env=true

.github/workflows/ci.yml

@@ -161,11 +161,20 @@ jobs:
             with:
                 name: nginx-agent-unsigned-snapshots
                 path: build
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory for Results
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Run Integration Tests
             run: |
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-                  make integration-test
+                  make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
           - name: Container Output Logs
             if: failure()
             run: |
@@ -209,6 +218,12 @@ jobs:
       - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
           go-version-file: 'go.mod'
+      -   name: Set Start Time
+          run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+      -   name: Create Directory for Results
+          run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+      -   name: Start Promtail
+          uses: ./.github/actions/start-promtail
       - name: Download Packages
         uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
@@ -218,7 +233,10 @@ jobs:
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
-            make official-image-integration-test
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+      - name: Generate Test Results
+        if: always()
+        run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
       - name: Container Output Logs 
         if: failure()
         run: |
@@ -275,6 +293,12 @@ jobs:
             with:
                 name: nginx-agent-unsigned-snapshots
                 path: build
+          - name: Set Start Time
+            run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
+          - name: Create Directory for Results
+            run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
+          - name: Start Promtail
+            uses: ./.github/actions/start-promtail
           - name: Login to Docker Registry
             uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
             with:
@@ -286,7 +310,10 @@ jobs:
                 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
                 CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
                 OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
-                  make official-image-integration-test
+                  make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}-${{matrix.container.image}}-${{matrix-container-version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
+          - name: Generate Test Results
+            if: always()
+            run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}
           - name: Container Output Logs
             if: failure()
             run: |

scripts/workflow/generate_results.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+# TODO: `start_time` and `end_time`, separate test name from `$line`
+
+JOB_RESULT="$1"
+START_TIME="$2"
+TEST_TYPE="$3"
+
+INPUT_FILE="./test/dashboard/logs/$TEST_TYPE/raw_logs.log"
+JOB_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/result.json"
+LOG_OUTPUT_FILE="./test/dashboard/logs/$TEST_TYPE/test.log"
+OUTPUT_PATH="./test/dashboard/logs/$TEST_TYPE/"
+
+END_TIME="`date "+%Y-%m-%dT%H:%M:%S.%NZ"`"
+START_SECONDS=$(date -d "$START_TIME" +%s.%N)
+END_SECONDS=$(date -d "$END_TIME" +%s.%N)
+DURATION=$(echo "$END_SECONDS - $START_SECONDS" | bc)
+
+MSG=""        # individual test msg
+FAIL_MSG=""   # msg for entire job run
+RESULT=""
+HAS_FAILED=false
+IS_RUNNING=false
+
+load_job_status(){
+    if [ $JOB_RESULT == "success" ]; then
+        RESULT="pass"
+    elif [ $JOB_RESULT == "failure" ]; then
+        RESULT="fail"
+    else
+        RESULT="skip"
+    fi
+}
+
+format_logs_to_json(){
+    local LINE="$1"
+    local JSON="{"
+    local key value
+    
+    while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"[^\"]*\")|([:space:]]+)) ]]; do
+        key="${BASH_REMATCH[1]}"
+        value="${BASH_REMATCH[2]}"
+        LINE="${line#*${BASH_REMATCH[0]}}"
+        json+="\"$key\": \"$value\", "
+    done
+    
+    json="${json%, }"
+    json+="}"
+    
+    echo "$json"
+}
+
+format_results(){
+    while IFS= read -r line; do
+        
+        if [[ "$line" =~ ^===\ RUN[[:space:]]+(.+) ]]; then
+            TEST_NAME="${BASH_REMATCH[1]}"
+            IS_RUNNING=true
+            MSG=""
+            TEST_START=""
+            TEST_END=""
+            OUTPUT_FILE="${OUTPUT_PATH}${TEST_NAME}/result.json"
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then
+            TEST_START="${BASH_REMATCH[1]}"
+        elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished testing ]]; then
+            TEST_END="${BASH_REMATCH[1]}"
+        elif [[ "$line" == "FAIL" ]]; then
+            HAS_FAILED=false
+            MSG="$MSG_STR"
+            FAIL_MSG+="$MSG"
+            HAS_FAILED=false
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $OUTPUT_FILE
+        elif [[ "$line" == "--- PASS"* ]]; then
+            TEST_RES="pass"
+            IS_RUNNING=false
+            echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $OUTPUT_FILE
+        elif [[ "$line" == "--- FAIL"* ]]; then
+            TEST_RES="fail"
+            HAS_FAILED=true
+            IS_RUNNING=false
+        elif [[ "$line" == time=* && "$line" == *level=* ]]; then
+            LOG_LINE=$(format_logs_to_json "$line")
+            echo "$LOG_LINE" >> "$LOG_OUTPUT_FILE"
+        fi
+        
+        if [ $HAS_FAILED == true ]; then
+            MSG_STR+="$line"
+        fi
+
+    done < "$INPUT_FILE"
+            
+    # Store the result of the whole job
+    echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$RESULT\", \"msg\": \"$FAIL_MSG\"}" > $JOB_OUTPUT_FILE
+}
+
+# Main body of the script
+{
+    load_job_status
+    format_results
+}

test/dashboard/prep/promtail.yaml

@@ -0,0 +1,135 @@
+server:
+    http_listen_port: 9080
+    grpc_listen_port: 0
+    log_level: info
+    
+positions:
+    filename: /tmp/positions.yaml
+    sync_period: "10s"
+
+clients:
+    - url: ${{ secrets.LOKI_DASHBOARD_URL }}
+      external_labels:
+          systest_project: agent_v2
+          systest_type: ${GITHUB_JOB}
+      timeout: 30s
+      backoff_config:
+          min_period: 500ms
+          max_period: 5s
+          max_retries: 5
+
+scrape_configs:
+    - job_name: test-results
+      static_configs:
+          - targets:
+                - localhost
+            labels:
+                systest_job: test-results
+                __path__: /var/log/**/result.json
+      pipeline_stages:
+          - json:
+                expressions:
+                    time:
+          - timestamp:
+                source: time
+                format: RFC3339Nano
+          - template:
+                source: ci_pipeline_id
+                template: "${GITHUB_RUN_ID}"
+          - template:
+                source: ci_pipeline_url
+                template: "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          - template:
+                source: ci_pipeline_name
+                template: "${WORKFLOW}"
+          - template:
+                source: ci_pipeline_source
+                template: "${GITHUB_EVENT_NAME}"
+          - template:
+                source: ci_job_name
+                template: "${GITHUB_JOB}"
+          - template:
+                source: ci_commit_ref
+                template: "${GITHUB_HEAD_REF}"
+          - template:
+                source: ci_commit_sha
+                template: "${GITHUB_SHA}"
+          - template:
+                source: ci_commit_author
+                template: "${GITHUB_ACTOR}"
+          - template:
+                source: systest_path
+                template: '{{ trimPrefix "${TEST_OUTDIR}/" .filename | dir | replace "." "/" }}'
+
+          - structured_metadata:
+                ci_pipeline_id:
+                ci_pipeline_url:
+                ci_pipeline_name:
+                ci_pipeline_source:
+                ci_job_name:
+                ci_commit_ref:
+                ci_commit_sha:
+                ci_commit_author:
+                filename:
+                systest_path:
+
+          - labeldrop:
+                - filename
+
+
+    - job_name: test-logs
+      static_configs:
+        - targets:
+            - localhost
+          labels:
+            systest_job: test-logs
+            __path__: /var/log/**/test.log
+      pipeline_stages:
+        - json:
+            expressions:
+              time:
+        - timestamp:
+            source: time
+            format: RFC3339Nano
+        - template:
+            source: ci_pipeline_id
+            template: "${GITHUB_RUN_ID}"
+        - template:
+            source: ci_pipeline_url
+            template: "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        - template:
+            source: ci_pipeline_name
+            template: "${WORKFLOW}"
+        - template:
+            source: ci_pipeline_source
+            template: "${GITHUB_EVENT_NAME}"
+        - template:
+            source: ci_job_name
+            template: "${GITHUB_JOB}"
+        - template:
+            source: ci_commit_ref
+            template: "${GITHUB_HEAD_REF}"
+        - template:
+            source: ci_commit_sha
+            template: "${GITHUB_SHA}"
+        - template:
+            source: ci_commit_author
+            template: "${GITHUB_ACTOR}"
+        - template:
+            source: systest_path
+            template: '{{ trimPrefix "${TEST_OUTDIR}/" .filename | dir | replace "." "/" }}'
+
+        - structured_metadata:
+            ci_pipeline_id:
+            ci_pipeline_url:
+            ci_pipeline_name:
+            ci_pipeline_source:
+            ci_job_name:
+            ci_commit_ref:
+            ci_commit_sha:
+            ci_commit_author:
+            filename:
+            systest_path:
+
+        - labeldrop:
+            - filename

test/integration/api/api_test.go

@@ -25,6 +25,7 @@ import (
 var delay = time.Duration(5 * time.Second)
 
 func TestAPI_Nginx(t *testing.T) {
+	t.Log("testing nginx api")
 	ctx := context.Background()
 	containerNetwork := utils.CreateContainerNetwork(ctx, t)
 
@@ -80,9 +81,11 @@ func TestAPI_Nginx(t *testing.T) {
 	assert.Equal(t, "/etc/nginx/nginx.conf", nginxDetailsResponse[0].ConfPath)
 
 	utils.TestAgentHasNoErrorLogs(t, testContainer)
+	t.Log("finished testing nginx api")
 }
 
 func TestAPI_Metrics(t *testing.T) {
+	t.Log("testing metrics api")
 	ctx := context.Background()
 	containerNetwork := utils.CreateContainerNetwork(ctx, t)
 
@@ -157,4 +160,5 @@ func TestAPI_Metrics(t *testing.T) {
 	}
 
 	utils.TestAgentHasNoErrorLogs(t, testContainer)
+	t.Log("finished testing metrics api")
 }

test/integration/features/features_test.go

@@ -11,6 +11,7 @@ import (
 )
 
 func TestFeatures_NginxCountingEnabled(t *testing.T) {
+	t.Log("testing nginx counting enabled")
 	enabledFeatureLogs := []string{
 		"level=info msg=\"NGINX Counter initializing", "level=info msg=\"MetricsThrottle initializing\"", "level=info msg=\"DataPlaneStatus initializing\"",
 		"level=info msg=\"OneTimeRegistration initializing\"", "level=info msg=\"Metrics initializing\"",
@@ -56,9 +57,11 @@ func TestFeatures_NginxCountingEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
+	t.Log("finished testing nginx counting enabled")
 }
 
 func TestFeatures_MetricsEnabled(t *testing.T) {
+	t.Log("testing metrics enabled")
 	enabledFeatureLogs := []string{"level=info msg=\"Metrics initializing\"", "level=info msg=\"MetricsThrottle initializing\"", "level=info msg=\"DataPlaneStatus initializing\""}
 	disabledFeatureLogs := []string{"level=info msg=\"OneTimeRegistration initializing\"", "level=info msg=\"Events initializing\"", "level=info msg=\"Agent API initializing\""}
 
@@ -101,9 +104,11 @@ func TestFeatures_MetricsEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
+	t.Log("finished testing metrics enabled")
 }
 
 func TestFeatures_ConfigEnabled(t *testing.T) {
+	t.Log("testing config enabled")
 	enabledFeatureLogs := []string{"level=info msg=\"DataPlaneStatus initializing\""}
 	disabledFeatureLogs := []string{"level=info msg=\"Events initializing\"", "level=info msg=\"Agent API initializing\"", "level=info msg=\"Metrics initializing\"", "level=info msg=\"MetricsThrottle initializing\""}
 
@@ -146,4 +151,5 @@ func TestFeatures_ConfigEnabled(t *testing.T) {
 	for _, logLine := range disabledFeatureLogs {
 		assert.NotContains(t, string(agentLogContent), logLine, "agent log file contains disabled feature log")
 	}
+	t.Log("finished testing config enabled")
 }