adding tests for checking fluentbit version

.github/workflows/pull_request.yml

@@ -5,6 +5,7 @@ permissions:
   packages: write
   id-token: write
   checks: write
+  pull-requests: write
 
 on: [pull_request]
 
@@ -51,13 +52,10 @@ jobs:
           pre_release_title="Temporary release to build and test artifacts from PR#${{ github.event.pull_request.number }}"
           pre_release_notes="Created by PR: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pull/${{ github.event.pull_request.number }}"
 
-          # Releases created from a runner are always DRAFT
+          # Releases created from a runner are always DRAFT, so we create then edit
           echo "Creating release: $pre_release_tag"
-          gh release create "$pre_release_tag" --title "$pre_release_title" --notes "$pre_release_notes"
-          # We need the pre-release to NOT be a draft, otherwise it won't be visible to download packages from the Ansible-managed hosts
-          echo "Updating release to be a pre-release"
-          gh release edit "$pre_release_tag" --draft=false --prerelease
-
+          gh release create "$pre_release_tag" --title "$pre_release_title" --notes "$pre_release_notes" --prerelease
+
       - name: Install python
         # commit sha from v4 tag - verify with: git ls-remote https://github.com/actions/setup-python v4
         uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c

.github/workflows/run_e2e_tests.yml

@@ -4,6 +4,7 @@ permissions:
   contents: read
   id-token: write
   checks: write
+  pull-requests: write
 
 on:
   # To be able to launch it from pull_request.yml programmatically

.github/workflows/run_prerelease.yml

@@ -5,6 +5,7 @@ permissions:
   packages: write
   id-token: write
   checks: write
+  pull-requests: write
 
 on:
   # To be able to launch it from pull_request.yml programmatically

ansible/build-fb-suse/playbook.yml

@@ -122,6 +122,10 @@
         dest: "{{ home_path }}"
         remote_src: yes
         creates: "{{ bison_path }}"
+      register: bison_download
+      retries: 5
+      delay: 30
+      until: bison_download is succeeded
 
     - name: Configure Bison {{ bison_version }}
       command:
@@ -151,12 +155,20 @@
         dest: "{{ home_path }}"
         remote_src: yes
         creates: "{{ cmake_path }}"
+      register: cmake_download
+      retries: 5
+      delay: 30
+      until: cmake_download is succeeded
 
     - name: Checkout Fluent Bit {{ fluent_bit_version }}
       git:
         repo: 'https://github.com/fluent/fluent-bit.git'
         dest: "{{ fluent_bit_path }}"
         version: "v{{ fluent_bit_version }}"
+      register: git_checkout
+      retries: 5
+      delay: 30
+      until: git_checkout is succeeded
 
     - name: Configure Fluent Bit {{ fluent_bit_version }}
       command:

ansible/provision-and-execute-tests/Makefile

@@ -60,6 +60,11 @@ prerelease-linux-test-sles: dependencies prepare-inventory
 prerelease-linux-merge-all: dependencies
 	ansible-playbook $(ANSIBLE_FOLDER)/playbook-merge-partial-results.yml -e combined_test_report_name=$(TEST_REPORT_NAME) -e pre_release_name=$(PRE_RELEASE_NAME)
 
+# Aggregate installation reports into a summary table
+.PHONY: prerelease-aggregate-installation-reports
+prerelease-aggregate-installation-reports: dependencies
+	ansible-playbook $(ANSIBLE_FOLDER)/playbook-aggregate-installation-reports.yml
+
 .PHONY: prerelease-windows
 prerelease-windows: dependencies prepare-inventory
 	TEST_REPORT_NAME=$(TEST_REPORT_NAME) ansible-playbook $(ANSIBLE_FOLDER)/playbook-windows.yml -i $(ANSIBLE_INVENTORY) $(if $(NR_FB_OUTPUT_PLUGIN_VERSION),-e nr_fb_output_plugin_version=$(NR_FB_OUTPUT_PLUGIN_VERSION)) $(if $(NR_FB_OUTPUT_PLUGIN_TAG),-e plugin_tag=$(NR_FB_OUTPUT_PLUGIN_TAG))
@@ -107,6 +112,10 @@ staging-linux-test-sles: dependencies prepare-inventory
 staging-linux-merge-all: dependencies
 	ansible-playbook $(ANSIBLE_FOLDER)/playbook-merge-partial-results.yml -e combined_test_report_name=$(TEST_REPORT_NAME) -e pre_release_name=$(PRE_RELEASE_NAME)
 
+.PHONY: staging-aggregate-installation-reports
+staging-aggregate-installation-reports: dependencies
+	ansible-playbook $(ANSIBLE_FOLDER)/playbook-aggregate-installation-reports.yml
+
 .PHONY: staging-windows
 staging-windows: dependencies prepare-inventory
 	TEST_REPORT_NAME=$(TEST_REPORT_NAME) ansible-playbook $(ANSIBLE_FOLDER)/playbook-windows.yml -i $(ANSIBLE_INVENTORY) $(if $(NR_FB_OUTPUT_PLUGIN_VERSION),-e nr_fb_output_plugin_version=$(NR_FB_OUTPUT_PLUGIN_VERSION)) $(if $(NR_FB_OUTPUT_PLUGIN_TAG),-e plugin_tag=$(NR_FB_OUTPUT_PLUGIN_TAG))
@@ -154,6 +163,10 @@ production-linux-test-sles: dependencies prepare-inventory
 production-linux-merge-all: dependencies
 	ansible-playbook $(ANSIBLE_FOLDER)/playbook-merge-partial-results.yml -e combined_test_report_name=$(TEST_REPORT_NAME) -e pre_release_name=$(PRE_RELEASE_NAME)
 
+.PHONY: production-aggregate-installation-reports
+production-aggregate-installation-reports: dependencies
+	ansible-playbook $(ANSIBLE_FOLDER)/playbook-aggregate-installation-reports.yml
+
 .PHONY: production-windows
 production-windows: dependencies prepare-inventory
 	TEST_REPORT_NAME=$(TEST_REPORT_NAME) ansible-playbook $(ANSIBLE_FOLDER)/playbook-windows.yml -i $(ANSIBLE_INVENTORY) $(if $(NR_FB_OUTPUT_PLUGIN_VERSION),-e nr_fb_output_plugin_version=$(NR_FB_OUTPUT_PLUGIN_VERSION)) $(if $(NR_FB_OUTPUT_PLUGIN_TAG),-e plugin_tag=$(NR_FB_OUTPUT_PLUGIN_TAG))

ansible/provision-and-execute-tests/playbook-aggregate-installation-reports.yml

@@ -0,0 +1,99 @@
+- name: Aggregate Fluent Bit Installation Reports
+  hosts: localhost
+  gather_facts: no
+  vars:
+    test_reports_dir: /tmp/test-reports
+    pre_release_name: "{{ lookup('env', 'PRE_RELEASE_NAME') | default('unknown', true) }}"
+  tasks:
+    - name: Check if test reports directory exists
+      ansible.builtin.stat:
+        path: "{{ test_reports_dir }}"
+      register: reports_dir_stat
+
+    - name: Fail if reports directory doesn't exist
+      ansible.builtin.fail:
+        msg: "Test reports directory {{ test_reports_dir }} does not exist"
+      when: not reports_dir_stat.stat.exists
+
+    - name: Find installation report JSON files
+      ansible.builtin.find:
+        paths: "{{ test_reports_dir }}"
+        patterns: 'installation-report-*.json'
+      register: installation_reports
+
+    - name: Log number of reports found
+      ansible.builtin.debug:
+        msg: "Found {{ installation_reports.files | length }} installation reports"
+
+    - name: Check if aggregate script exists
+      ansible.builtin.stat:
+        path: "{{ playbook_dir }}/../../integration-tests/aggregate-reports.js"
+      register: aggregate_script_stat
+
+    - name: Ensure Node.js is available
+      ansible.builtin.command: which node
+      register: node_check
+      failed_when: false
+      changed_when: false
+
+    - name: Run aggregation script
+      ansible.builtin.command: node {{ playbook_dir }}/../../integration-tests/aggregate-reports.js {{ test_reports_dir }}
+      register: aggregation_result
+      changed_when: false
+      failed_when: false  # Don't fail the whole playbook if aggregation fails
+      when:
+        - installation_reports.files | length > 0
+        - aggregate_script_stat.stat.exists
+        - node_check.rc == 0
+
+    - name: Display aggregation output
+      ansible.builtin.debug:
+        msg: "{{ aggregation_result.stdout_lines }}"
+      when:
+        - aggregation_result is defined
+        - aggregation_result.stdout_lines is defined
+
+    - name: Check if aggregate markdown report was created
+      ansible.builtin.stat:
+        path: "{{ test_reports_dir }}/fluent-bit-aggregate-report.md"
+      register: aggregate_md_stat
+
+    - name: Display aggregate markdown report
+      ansible.builtin.command: cat {{ test_reports_dir }}/fluent-bit-aggregate-report.md
+      register: aggregate_report_content
+      changed_when: false
+      when: aggregate_md_stat.stat.exists
+
+    - name: Print aggregate report to console
+      ansible.builtin.debug:
+        msg: "{{ aggregate_report_content.stdout_lines }}"
+      when:
+        - aggregate_report_content is defined
+        - aggregate_report_content.stdout_lines is defined
+
+    - name: Upload aggregate report to GitHub release (if configured)
+      ansible.builtin.command: gh release upload {{ pre_release_name }} {{ test_reports_dir }}/fluent-bit-aggregate-report.md --clobber
+      register: gh_upload
+      changed_when: false
+      failed_when: false
+      when:
+        - aggregate_md_stat.stat.exists
+        - pre_release_name != 'unknown'
+        - pre_release_name is not regex('^local-.*')
+
+    - name: Upload aggregate JSON to GitHub release (if configured)
+      ansible.builtin.command: gh release upload {{ pre_release_name }} {{ test_reports_dir }}/fluent-bit-aggregate-report.json --clobber
+      register: gh_upload_json
+      changed_when: false
+      failed_when: false
+      when:
+        - aggregate_md_stat.stat.exists
+        - pre_release_name != 'unknown'
+        - pre_release_name is not regex('^local-.*')
+
+    - name: Log upload status
+      ansible.builtin.debug:
+        msg:
+          - "Markdown report upload: {{ 'success' if (gh_upload is defined and gh_upload.rc == 0) else 'skipped or failed' }}"
+          - "JSON report upload: {{ 'success' if (gh_upload_json is defined and gh_upload_json.rc == 0) else 'skipped or failed' }}"
+      when: gh_upload is defined or gh_upload_json is defined

ansible/provision-and-execute-tests/playbook-provision-prerelease.yml

@@ -50,15 +50,39 @@
 #        falcon_customer_id: "{{ lookup('env', 'CROWDSTRIKE_CUSTOMER_ID') }}"
 #        api_base_url: "https://api.laggar.gcw.crowdstrike.com"
 
-    - name: Install  newrelic-infra agent
-      ansible.builtin.include_role:
-        name: caos.ansible_roles.newrelic_cli
-      vars:
-        repo_endpoint: "https://nr-downloads-main.s3.amazonaws.com/"
-        recipe: "newrelic-infra"
-        region: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_REGION') }}" # US | EU | STAGING
-        nr_api_key: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_API_KEY') }}"
-        nr_account_id: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_ACCOUNT_ID') }}"
+    - name: Install newrelic-infra agent with retry logic
+      block:
+        - name: Install newrelic-infra agent
+          ansible.builtin.include_role:
+            name: caos.ansible_roles.newrelic_cli
+          vars:
+            repo_endpoint: "https://nr-downloads-main.s3.amazonaws.com/"
+            recipe: "newrelic-infra"
+            region: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_REGION') }}" # US | EU | STAGING
+            nr_api_key: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_API_KEY') }}"
+            nr_account_id: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_ACCOUNT_ID') }}"
+      rescue:
+        - name: Log installation failure for observability
+          ansible.builtin.debug:
+            msg: "WARNING: newrelic-infra installation failed on {{ inventory_hostname }}. Retrying in 30s..."
+
+        - name: Wait before retry
+          ansible.builtin.pause:
+            seconds: 30
+
+        - name: Retry newrelic-infra agent installation
+          ansible.builtin.include_role:
+            name: caos.ansible_roles.newrelic_cli
+          vars:
+            repo_endpoint: "https://nr-downloads-main.s3.amazonaws.com/"
+            recipe: "newrelic-infra"
+            region: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_REGION') }}" # US | EU | STAGING
+            nr_api_key: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_API_KEY') }}"
+            nr_account_id: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_ACCOUNT_ID') }}"
+
+        - name: Log successful retry
+          ansible.builtin.debug:
+            msg: "SUCCESS: newrelic-infra installation completed after retry on {{ inventory_hostname }}"
 
 - name: Install fluent-bit from GH prerelease
   hosts: linux

ansible/provision-and-execute-tests/playbook-run-tests.yml

@@ -257,6 +257,15 @@
       changed_when: false
       when: ansible_system == 'Linux'
 
+    - name: Validate fb_version variable is set and valid
+      ansible.builtin.assert:
+        that:
+          - fb_version is defined
+          - fb_version | length > 0
+          - fb_version is match('^[0-9]+\.[0-9]+\.[0-9]+(?:-[\w.-]+)?$')
+        fail_msg: "fb_version must be set to a valid semantic version (e.g., 4.2.2 or 4.2.2-beta). Current value: {{ fb_version | default('UNDEFINED') }}"
+        success_msg: "fb_version is valid: {{ fb_version }}"
+
     - name: Run test-suite
       ansible.builtin.shell: "source ~/.nvm/nvm.sh && npm run test"
       args:
@@ -275,6 +284,10 @@
         MONITORED_SYSLOG_RFC_5424_TCP_PORT: "{{ monitored_syslog_rfc_5424_tcp_port }}"
         MONITORED_SYSLOG_RFC_5424_UDP_PORT: "{{ monitored_syslog_rfc_5424_udp_port }}"
         MONITORED_SYSTEMD_UNIT: "{{ monitored_systemd_unit }}"
+        EXPECTED_FB_VERSION: "{{ fb_version }}"
+        OS_DISTRO: "{{ os_distro }}"
+        OS_VERSION: "{{ os_version }}"
+        OS_ARCH: "{{ arch }}"
       register: test_job
 
     # Robust async status check with connection recovery
@@ -435,3 +448,27 @@
       ansible.builtin.debug:
         msg: "Fetch completed for {{ inventory_hostname }}: {{ 'primary succeeded' if (fetch_result is defined and fetch_result is succeeded) else 'recovered via rescue' }}"
       when: fetch_result is defined or final_fetch_result is defined
+
+    # Fetch installation report JSON for aggregation
+    - name: Fetch installation report for aggregation
+      block:
+        - name: Check if installation report exists
+          ansible.builtin.stat:
+            path: /tmp/fluent-bit-installation-report.json
+          register: report_stat
+
+        - name: Fetch installation report
+          fetch:
+            flat: true
+            src: /tmp/fluent-bit-installation-report.json
+            dest: "{{ test_reports_dir }}/installation-report-{{ os_distro }}-{{ os_version }}-{{ arch }}.json"
+          when: report_stat.stat.exists
+          register: report_fetch_result
+          retries: 3
+          delay: 10
+          until: report_fetch_result is succeeded
+      rescue:
+        - name: Log installation report fetch failure
+          ansible.builtin.debug:
+            msg: "Could not fetch installation report for {{ inventory_hostname }}, continuing..."
+      ignore_errors: true

ansible/provision-and-execute-tests/playbook-windows.yml

@@ -190,6 +190,15 @@
         path: '{{ test_suite_folder }}\reports'
         state: absent
 
+    - name: Validate fb_version variable is set and valid
+      ansible.builtin.assert:
+        that:
+          - fb_version is defined
+          - fb_version | length > 0
+          - fb_version is match('^[0-9]+\.[0-9]+\.[0-9]+(?:-[\w.-]+)?$')
+        fail_msg: "fb_version must be set to a valid semantic version (e.g., 4.2.2 or 4.2.2-beta). Current value: {{ fb_version | default('UNDEFINED') }}"
+        success_msg: "fb_version is valid: {{ fb_version }}"
+
     - name: Run test-suite (async with timeout)
       ansible.windows.win_shell: 'npm run test'
       args:
@@ -207,6 +216,7 @@
         MONITORED_TCP_PORT: "{{ monitored_tcp_port }}"
         MONITORED_WINDOWS_LOG_NAME_USING_WINLOG: "{{ monitored_windows_log_name_using_winevtlog }}"
         MONITORED_WINDOWS_LOG_NAME_USING_WINEVTLOG: "{{ monitored_windows_log_name_using_winlog }}"
+        EXPECTED_FB_VERSION: "{{ fb_version }}"
       register: test_job
 
     # Robust async status check with connection recovery for Windows