diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index c6c18528..83ccb89e 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -32,7 +32,7 @@ jobs: - name: Install tox, tox-lsr run: | set -euxo pipefail - pip3 install "git+https://github.com/linux-system-roles/tox-lsr@3.5.1" + pip3 install "git+https://github.com/linux-system-roles/tox-lsr@3.6.0" - name: Convert role to collection format id: collection diff --git a/.github/workflows/ansible-managed-var-comment.yml b/.github/workflows/ansible-managed-var-comment.yml index 99bf5d52..eadd836b 100644 --- a/.github/workflows/ansible-managed-var-comment.yml +++ b/.github/workflows/ansible-managed-var-comment.yml @@ -30,7 +30,7 @@ jobs: - name: Install tox, tox-lsr run: | set -euxo pipefail - pip3 install "git+https://github.com/linux-system-roles/tox-lsr@3.5.1" + pip3 install "git+https://github.com/linux-system-roles/tox-lsr@3.6.0" - name: Run ansible-plugin-scan run: | diff --git a/.github/workflows/ansible-test.yml b/.github/workflows/ansible-test.yml index b98f69bf..f97384bb 100644 --- a/.github/workflows/ansible-test.yml +++ b/.github/workflows/ansible-test.yml @@ -33,7 +33,7 @@ jobs: - name: Install tox, tox-lsr run: | set -euxo pipefail - pip3 install "git+https://github.com/linux-system-roles/tox-lsr@3.5.1" + pip3 install "git+https://github.com/linux-system-roles/tox-lsr@3.6.0" - name: Convert role to collection format run: | diff --git a/.github/workflows/qemu-kvm-integration-tests.yml b/.github/workflows/qemu-kvm-integration-tests.yml index 2d65444c..ab0c69f4 100644 --- a/.github/workflows/qemu-kvm-integration-tests.yml +++ b/.github/workflows/qemu-kvm-integration-tests.yml @@ -74,7 +74,7 @@ jobs: python3 -m pip install --upgrade pip sudo apt update sudo apt install -y --no-install-recommends git ansible-core genisoimage qemu-system-x86 - pip3 install "git+https://github.com/linux-system-roles/tox-lsr@3.5.1" + pip3 install "git+https://github.com/linux-system-roles/tox-lsr@3.6.0" - name: Configure tox-lsr if: steps.check_platform.outputs.supported @@ -86,7 +86,7 @@ jobs: if: steps.check_platform.outputs.supported run: >- tox -e ${{ matrix.scenario.env }} -- --image-name ${{ matrix.scenario.image }} --make-batch - --log-level=debug --skip-tags tests::infiniband -- + --log-level=debug --skip-tags tests::infiniband,tests::nvme,tests::scsi -- - name: Test result summary if: steps.check_platform.outputs.supported && always() @@ -109,14 +109,29 @@ jobs: echo "$f" done < batch.report - - name: Show test logs on failure + - name: Upload test logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: "logs-${{ matrix.scenario.image }}-${{ matrix.scenario.env }}" + path: | + tests/*.log + artifacts/default_provisioners.log + artifacts/*.qcow2.*.log + batch.txt + batch.report + retention-days: 30 + + - name: Show test log failures if: steps.check_platform.outputs.supported && failure() run: | set -euo pipefail for f in tests/*.log; do - echo "::group::$(basename $f)" - cat "$f" - echo "::endgroup::" + if FAIL=$(grep -B100 -A30 "fatal:" "$f"); then + echo "::group::$(basename $f)" + echo "$FAIL" + echo "::endgroup::" + fi done - name: Set commit status as success with a description that platform is skipped diff --git a/.github/workflows/tft.yml b/.github/workflows/tft.yml index 0513589c..718a1269 100644 --- a/.github/workflows/tft.yml +++ b/.github/workflows/tft.yml @@ -154,7 +154,7 @@ jobs: targetUrl: "" - name: Run test in testing farm - uses: sclorg/testing-farm-as-github-action@v3 + uses: sclorg/testing-farm-as-github-action@v4 if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform) with: git_ref: main diff --git a/.ostree/packages-runtime-CentOS-10.txt b/.ostree/packages-runtime-CentOS-10.txt index 295a742f..b2140983 100644 --- a/.ostree/packages-runtime-CentOS-10.txt +++ b/.ostree/packages-runtime-CentOS-10.txt @@ -2,6 +2,7 @@ bpftrace grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-mssql pcp-system-tools diff --git a/.ostree/packages-runtime-CentOS-9.txt b/.ostree/packages-runtime-CentOS-9.txt index e87908b7..0c70f458 100644 --- a/.ostree/packages-runtime-CentOS-9.txt +++ b/.ostree/packages-runtime-CentOS-9.txt @@ -2,6 +2,7 @@ bpftrace grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-mssql pcp-system-tools diff --git a/.ostree/packages-runtime-Fedora.txt b/.ostree/packages-runtime-Fedora.txt index ef8418f0..94a93e49 100644 --- a/.ostree/packages-runtime-Fedora.txt +++ b/.ostree/packages-runtime-Fedora.txt @@ -2,6 +2,7 @@ bpftrace grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-mssql pcp-system-tools diff --git a/.ostree/packages-runtime-RedHat-10.txt b/.ostree/packages-runtime-RedHat-10.txt index 295a742f..b2140983 100644 --- a/.ostree/packages-runtime-RedHat-10.txt +++ b/.ostree/packages-runtime-RedHat-10.txt @@ -2,6 +2,7 @@ bpftrace grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-mssql pcp-system-tools diff --git a/.ostree/packages-runtime-RedHat-8.txt b/.ostree/packages-runtime-RedHat-8.txt index e87908b7..0c70f458 100644 --- a/.ostree/packages-runtime-RedHat-8.txt +++ b/.ostree/packages-runtime-RedHat-8.txt @@ -2,6 +2,7 @@ bpftrace grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-mssql pcp-system-tools diff --git a/.ostree/packages-runtime-RedHat-9.txt b/.ostree/packages-runtime-RedHat-9.txt index e87908b7..0c70f458 100644 --- a/.ostree/packages-runtime-RedHat-9.txt +++ b/.ostree/packages-runtime-RedHat-9.txt @@ -2,6 +2,7 @@ bpftrace grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-mssql pcp-system-tools diff --git a/.ostree/packages-testing-CentOS-10.txt b/.ostree/packages-testing-CentOS-10.txt index cacb9e19..851106a7 100644 --- a/.ostree/packages-testing-CentOS-10.txt +++ b/.ostree/packages-testing-CentOS-10.txt @@ -3,6 +3,7 @@ cyrus-sasl-scram grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-elasticsearch pcp-pmda-mssql diff --git a/.ostree/packages-testing-CentOS-9.txt b/.ostree/packages-testing-CentOS-9.txt index f6da445f..9d598d8c 100644 --- a/.ostree/packages-testing-CentOS-9.txt +++ b/.ostree/packages-testing-CentOS-9.txt @@ -3,6 +3,7 @@ cyrus-sasl-scram grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-elasticsearch pcp-pmda-mssql diff --git a/.ostree/packages-testing-Fedora.txt b/.ostree/packages-testing-Fedora.txt index cacb9e19..851106a7 100644 --- a/.ostree/packages-testing-Fedora.txt +++ b/.ostree/packages-testing-Fedora.txt @@ -3,6 +3,7 @@ cyrus-sasl-scram grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-elasticsearch pcp-pmda-mssql diff --git a/.ostree/packages-testing-RedHat-10.txt b/.ostree/packages-testing-RedHat-10.txt index cacb9e19..851106a7 100644 --- a/.ostree/packages-testing-RedHat-10.txt +++ b/.ostree/packages-testing-RedHat-10.txt @@ -3,6 +3,7 @@ cyrus-sasl-scram grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-elasticsearch pcp-pmda-mssql diff --git a/.ostree/packages-testing-RedHat-8.txt b/.ostree/packages-testing-RedHat-8.txt index f6da445f..9d598d8c 100644 --- a/.ostree/packages-testing-RedHat-8.txt +++ b/.ostree/packages-testing-RedHat-8.txt @@ -3,6 +3,7 @@ cyrus-sasl-scram grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-elasticsearch pcp-pmda-mssql diff --git a/.ostree/packages-testing-RedHat-9.txt b/.ostree/packages-testing-RedHat-9.txt index f6da445f..9d598d8c 100644 --- a/.ostree/packages-testing-RedHat-9.txt +++ b/.ostree/packages-testing-RedHat-9.txt @@ -3,6 +3,7 @@ cyrus-sasl-scram grafana grafana-pcp pcp-export-pcp2elasticsearch +pcp-export-pcp2spark pcp-pmda-bpftrace pcp-pmda-elasticsearch pcp-pmda-mssql diff --git a/README.md b/README.md index 80213ae1..69ab64da 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,14 @@ Boolean flag allowing metric values to be exported into Elasticsearch. Boolean flag allowing metrics from Elasticsearch to be made available. +### metrics_into_spark: false + +Boolean flag allowing metric values to be exported into Spark. + +### metrics_from_spark: false + +Boolean flag allowing metrics from Spark to be made available. + ### metrics_from_postfix: false Boolean flag allowing metrics from Postfix to be made available. diff --git a/defaults/main.yml b/defaults/main.yml index 8427d7ba..b6eca247 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -25,6 +25,12 @@ metrics_into_elasticsearch: false # Make metrics available from an Elasticsearch instance metrics_from_elasticsearch: false +# Export metrics to an Apache Spark data analysis setup +metrics_into_spark: false + +# Make metrics available from an Apache Spark instance +metrics_from_spark: false + # Make metrics available from a SQL Server database metrics_from_mssql: false diff --git a/examples/pcp_spark.yml b/examples/pcp_spark.yml new file mode 100644 index 00000000..5b3d7f3f --- /dev/null +++ b/examples/pcp_spark.yml @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: MIT +--- +- name: Set up Spark for use with PCP + hosts: monitoring + + roles: + - role: linux-system-roles.metrics + vars: + metrics_provider: pcp + metrics_from_spark: true + metrics_into_spark: true diff --git a/tasks/main.yml b/tasks/main.yml index d8609e59..3339a6de 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -49,6 +49,18 @@ metrics_into_elasticsearch | d(false) | bool # yamllint enable rule:line-length +- name: Configure Spark metrics + vars: + spark_metrics_agent: "{{ metrics_from_spark | d(false) | bool }}" + spark_export_metrics: "{{ metrics_into_spark | d(false) | bool }}" + spark_metrics_provider: "{{ metrics_provider }}" + include_role: + # noqa role-name[path] + name: "{{ role_path }}/roles/spark" + when: > + metrics_from_spark | d(false) | bool or + metrics_into_spark | d(false) | bool + - name: Configure SQL Server metrics. vars: mssql_metrics_provider: "{{ metrics_provider }}" diff --git a/tests/check_from_spark.yml b/tests/check_from_spark.yml new file mode 100644 index 00000000..a2ab79ff --- /dev/null +++ b/tests/check_from_spark.yml @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MIT +--- +- name: Check if OpenMetrics PMDA has Spark metrics registered + command: pmprobe -I openmetrics.control.status + register: status + until: status.stdout.find("spark") != -1 + retries: 10 + delay: 1 + changed_when: false + when: "'pcp-pmda-openmetrics' in __spark_packages_pcp" diff --git a/tests/check_into_spark.yml b/tests/check_into_spark.yml new file mode 100644 index 00000000..8221485d --- /dev/null +++ b/tests/check_into_spark.yml @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: MIT +--- +- name: Check if export to Spark is installed + command: test -x /usr/bin/pcp2spark + changed_when: false + +- name: Check the ansible_managed header in pcp2spark.service + vars: + __test_config_path: >- + {{ __spark_service_path }}/pcp2spark.service + include_tasks: check_header.yml diff --git a/tests/roles/spark b/tests/roles/spark new file mode 120000 index 00000000..7e4c0d7d --- /dev/null +++ b/tests/roles/spark @@ -0,0 +1 @@ +../../vendor/github.com/performancecopilot/ansible-pcp/roles/spark \ No newline at end of file diff --git a/tests/tests_verify_from_spark.yml b/tests/tests_verify_from_spark.yml new file mode 100644 index 00000000..7c78968d --- /dev/null +++ b/tests/tests_verify_from_spark.yml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: MIT +--- +- name: Test import from Spark + hosts: all + vars: + metrics_from_spark: true + + pre_tasks: + - name: Stop test + meta: end_host + when: (ansible_distribution in ['RedHat', 'CentOS'] and + ansible_distribution_major_version | int < 7) or + ansible_distribution not in ['Fedora', 'RedHat', 'CentOS'] + + - name: Save state of services + import_tasks: get_services_state.yml + + tasks: + - name: Run test + block: + - name: Run the metrics role to configure Spark + include_role: + name: linux-system-roles.metrics + public: true + + - name: Check if import from Spark works + include_tasks: check_from_spark.yml + + - name: Flush handlers + meta: flush_handlers + + rescue: + - name: Handle failure case + include_tasks: handle_test_failure.yml + + always: + - name: Restore state of services + import_tasks: restore_services_state.yml diff --git a/tests/tests_verify_into_spark.yml b/tests/tests_verify_into_spark.yml new file mode 100644 index 00000000..a371306d --- /dev/null +++ b/tests/tests_verify_into_spark.yml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: MIT +--- +- name: Test import to Spark + hosts: all + vars: + metrics_into_spark: true + + pre_tasks: + - name: Stop test + meta: end_host + when: (ansible_distribution in ['RedHat'] and + (ansible_facts['distribution_version'] is version('8.4', '<'))) or + ansible_distribution not in ['Fedora', 'RedHat'] + + - name: Save state of services + import_tasks: get_services_state.yml + + tasks: + - name: Run test + block: + - name: Run the role + include_role: + name: linux-system-roles.metrics + public: true + + - name: Check if import to Spark works + include_tasks: check_into_spark.yml + + - name: Flush handlers + meta: flush_handlers + + rescue: + - name: Handle failure case + include_tasks: handle_test_failure.yml + + always: + - name: Restore state of services + import_tasks: restore_services_state.yml diff --git a/vendor/github.com/performancecopilot/ansible-pcp/docs/pcp/setup.yml b/vendor/github.com/performancecopilot/ansible-pcp/docs/pcp/setup.yml index 4a5bcb83..9a7f7d5a 100644 --- a/vendor/github.com/performancecopilot/ansible-pcp/docs/pcp/setup.yml +++ b/vendor/github.com/performancecopilot/ansible-pcp/docs/pcp/setup.yml @@ -6,6 +6,7 @@ vars: pcp_pmie_endpoint: https://example.com/webhook pcp_pmlogger_interval: 10 + # NOTE: No openmetrics on EL7 pcp_optional_agents: [dm, nfsclient, openmetrics] pcp_explicit_labels: environment: production diff --git a/vendor/github.com/performancecopilot/ansible-pcp/roles/pcp/README.md b/vendor/github.com/performancecopilot/ansible-pcp/roles/pcp/README.md index 7a4c95fa..48cd08fd 100644 --- a/vendor/github.com/performancecopilot/ansible-pcp/roles/pcp/README.md +++ b/vendor/github.com/performancecopilot/ansible-pcp/roles/pcp/README.md @@ -89,6 +89,7 @@ Basic PCP setup with monitoring suited for a single host. - role: performancecopilot.metrics.pcp vars: pcp_pmlogger_interval: 10 + # NOTE: No openmetrics on EL7 pcp_optional_agents: [dm, nfsclient, openmetrics] pcp_explicit_labels: environment: production diff --git a/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/tasks/main.yml b/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/tasks/main.yml index a65dff49..5dc6fdbf 100644 --- a/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/tasks/main.yml +++ b/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/tasks/main.yml @@ -56,6 +56,7 @@ when: - spark_metrics_provider == 'pcp' - spark_metrics_agent | d(false) | bool + - "'pcp-pmda-openmetrics' in __spark_packages_pcp" - name: Ensure PCP OpenMetrics agent is enabled with Spark endpoint file: @@ -65,6 +66,7 @@ when: - spark_metrics_provider == 'pcp' - spark_metrics_agent | d(false) | bool + - "'pcp-pmda-openmetrics' in __spark_packages_pcp" - name: Ensure correct service path for ostree systems when: diff --git a/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/vars/CentOS_7.yml b/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/vars/CentOS_7.yml index 79c55328..98ed86af 100644 --- a/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/vars/CentOS_7.yml +++ b/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/vars/CentOS_7.yml @@ -5,3 +5,5 @@ __spark_packages_export_pcp: - pcp-export-pcp2spark - pcp-system-tools + +__spark_packages_pcp: [] diff --git a/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/vars/RedHat_7.yml b/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/vars/RedHat_7.yml index 8142ef30..c3e8bf87 100644 --- a/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/vars/RedHat_7.yml +++ b/vendor/github.com/performancecopilot/ansible-pcp/roles/spark/vars/RedHat_7.yml @@ -5,3 +5,5 @@ __spark_packages_export_pcp: - pcp-export-pcp2spark - pcp-system-tools + +__spark_packages_pcp: []