diff --git a/.github/actions/systemtests/generate-matrix/action.yml b/.github/actions/systemtests/generate-matrix/action.yml index 60148136b67..14f1b5df134 100644 --- a/.github/actions/systemtests/generate-matrix/action.yml +++ b/.github/actions/systemtests/generate-matrix/action.yml @@ -5,44 +5,36 @@ inputs: pipelines: description: "Comma-separated list of pipelines (if provided, takes precedence)" required: false - default: "" profiles: description: "Comma-separated list of profiles (used if pipelines is empty)" required: false - default: "" default_agent: description: "Default agent value" - required: true - default: "oracle-4cpu-16gb-x86-64" + required: false default_arch: description: "Default architecture value" - required: true - default: "amd64" + required: false default_strimzi_feature_gates: description: "Default value for strimzi_feature_gates" - required: true - default: "" + required: false default_strimzi_rbac_scope: description: "Default value for strimzi_rbac_scope" - required: true - default: "CLUSTER" + required: false default_cluster_operator_install_type: description: "Default value for cluster_operator_install_type" - required: true - default: "bundle" + required: false default_parallel: description: "Number of tests executed in parallel" - default: "1" + required: false default_groups: description: "Default JUnit5 test groups that will be executed" - default: "" + required: false default_tests: description: "Default JUnit5 tests that will be executed" - default: "" + required: false runnerArch: description: "Architecture of GitHub runner" required: false - default: "amd64" outputs: matrix: @@ -121,12 +113,12 @@ runs: env: INPUT_PIPELINES: "${{ inputs.pipelines }}" INPUT_PROFILES: "${{ inputs.profiles }}" - INPUT_DEFAULT_AGENT: "${{ inputs.default_agent }}" - INPUT_DEFAULT_ARCH: "${{ inputs.default_arch }}" + INPUT_DEFAULT_AGENT: "${{ inputs.default_agent != '' && inputs.default_agent || 'ubuntu-latest' }}" + INPUT_DEFAULT_ARCH: "${{ inputs.default_arch != '' && inputs.default_arch || 'amd64' }}" INPUT_DEFAULT_JDK_VERSION: "${{ inputs.default_jdk_version }}" INPUT_DEFAULT_STRIMZI_USE_FEATURE_GATES: "${{ inputs.default_strimzi_feature_gates }}" - INPUT_DEFAULT_STRIMZI_RBAC_SCOPE: "${{ inputs.default_strimzi_rbac_scope }}" - INPUT_DEFAULT_CLUSTER_OPERATOR_INSTALL_TYPE: "${{ inputs.default_cluster_operator_install_type }}" - INPUT_DEFAULT_PARALLEL: "${{ inputs.default_parallel }}" + INPUT_DEFAULT_STRIMZI_RBAC_SCOPE: "${{ inputs.default_strimzi_rbac_scope != '' && inputs.default_strimzi_rbac_scope || 'CLUSTER' }}" + INPUT_DEFAULT_CLUSTER_OPERATOR_INSTALL_TYPE: "${{ inputs.default_cluster_operator_install_type != '' && inputs.default_cluster_operator_install_type || 'bundle' }}" + INPUT_DEFAULT_PARALLEL: "${{ inputs.default_parallel != '' && inputs.default_parallel || '1' }}" INPUT_DEFAULT_GROUPS: "${{ inputs.default_groups }}" INPUT_DEFAULT_TESTS: "${{ inputs.default_tests }}" diff --git a/.github/actions/systemtests/parse-comment/action.yml b/.github/actions/systemtests/parse-comment/action.yml index d5d7cacbd78..db352f6e299 100644 --- a/.github/actions/systemtests/parse-comment/action.yml +++ b/.github/actions/systemtests/parse-comment/action.yml @@ -171,8 +171,8 @@ runs: strimzi_rbac_scope: getParameter('strimzi_rbac_scope', sanitized), // keep case cluster_operator_install_type: getParameter('cluster_operator_install_type', lower) || DEFAULTS.installType, parallel: getParameter('parallel', lower) || DEFAULTS.parallel, - groups: getParameter('groups', lower), - tests: getParameter('tests', lower), + groups: getParameter('groups', sanitized), + tests: getParameter('tests', sanitized), kafkaVersion: getParameter('kafkaversion', lower) || DEFAULTS.kafkaVersion, kubeVersion: getParameter('kubeversion', lower) || DEFAULTS.kubeVersion, releaseVersion: process.env.INPUT_RELEASE_VERSION || 'latest', diff --git a/.github/actions/systemtests/validate-matrix/action.yml b/.github/actions/systemtests/validate-matrix/action.yml new file mode 100644 index 00000000000..d2f3d1af490 --- /dev/null +++ b/.github/actions/systemtests/validate-matrix/action.yml @@ -0,0 +1,70 @@ +name: "Validate Matrix" +description: "Validates generated matrix and returns feedback message" + +inputs: + matrix: + description: "JSON matrix generated by generate-matrix action" + required: true + +outputs: + isValid: + description: "Whether the matrix is valid (not empty)" + value: ${{ steps.validate.outputs.isValid }} + message: + description: "Feedback message about the validation result" + value: ${{ steps.validate.outputs.message }} + +runs: + using: "composite" + steps: + - name: Validate Matrix + id: validate + uses: actions/github-script@v7 + with: + script: | + const matrix = JSON.parse(process.env.MATRIX || '[]'); + + // Check if matrix is empty + const isEmpty = !matrix || matrix.length === 0; + + if (isEmpty) { + core.setOutput('isValid', 'false'); + + let errorMessage = "❌ **System test validation failed**\n"; + errorMessage += "\n"; + errorMessage += "The provided pipeline/profile names did not match any configured tests.\n"; + errorMessage += "\n"; + errorMessage += "**Available pipelines:**\n"; + errorMessage += "`acceptance`, `acceptance-helm`, `regression`, `regression-fg`, `regression-rbac`, `smoke`, `upgrade`\n"; + errorMessage += "\n"; + errorMessage += "**Available profiles:**\n"; + errorMessage += "`acceptance`, `azp_connect_mirrormaker`, `azp_dynconfig_listeners_tracing_watcher`, `azp_kafka_oauth`, `azp_kafka_upgrade`, `azp_kraft_upgrade`, `azp_operators`, `azp_rbac_remaining`, `azp_rolling_update_bridge`, `azp_security`, `brokers-and-security`, `operands`, `operators`, `smoke`\n"; + errorMessage += "\n"; + errorMessage += "Please check your command for typos and try again."; + + core.setOutput('message', errorMessage); + return; + } + + // Matrix is valid - create summary of what will run + core.setOutput('isValid', 'true'); + core.info(`Matrix validation passed. ${matrix.length} job(s) will be executed.`); + + const jobs = matrix.map(job => { + const name = job.jobName || `${job.pipeline}-${job.profile || job.pipeline}-${job.arch}`; + const agent = job.agent || 'default'; + const timeout = job.timeout || 'default'; + return `- **${name}** (${agent}, ${timeout}min)`; + }).join('\n'); + + let successMessage = "✅ **System test validation passed**\n"; + successMessage += "\n"; + successMessage += `The following ${matrix.length} job(s) will be executed:\n`; + successMessage += "\n"; + successMessage += jobs.replace(/\n/g, '\n') + "\n"; + successMessage += "\n"; + successMessage += "Tests will start after successful build completion."; + + core.setOutput('message', successMessage); + env: + MATRIX: ${{ inputs.matrix }} \ No newline at end of file diff --git a/.github/docs/README.md b/.github/docs/README.md index c99f26a33d3..006e56e0248 100644 --- a/.github/docs/README.md +++ b/.github/docs/README.md @@ -62,14 +62,13 @@ Build process actions: - [build-strimzi-binaries](../actions/build/build-strimzi-binaries) - [containers-build](../actions/build/containers-build) - [containers-load](../actions/build/containers-load) -- [containers-push-manifest](../actions/build/containers-push-manifest) System tests execution actions: - [generate-matrix](../actions/systemtests/generate-matrix) - [set-defaults](../actions/utils/set-defaults) - [log-variables](../actions/utils/log-variables) - [parse-comment](../actions/systemtests/parse-comment) -- [determine-ref](../actions/determine-ref) +- [determine-ref](../actions/utils/determine-ref) Utils actions: - [check-permissions](../actions/utils/check-permissions) @@ -99,24 +98,24 @@ Comment for triggering the workflow has to starts with string `/gha run` and the The whole script that parse the trigger even is part of [parse-comment](../actions/systemtests/parse-comment) action. Currently, we have these parameters that can be passed through the comment: -| Name | Info | Default | -|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------| -| pipeline | Name of the pipeline from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that wil be executed | regression,upgrade | -| profile | Testing profile from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that will be executed, it has to be defined in pom file | smoke | -| agent | Agent that will be used for a specific pipeline (see list of runners in Strimzi org config for more info) | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) | -| strimzi_feature_gates | Which Strimzi Feature Gates will be used | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) | -| strimzi_rbac_scope | RBAC scope for Strimzi | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) | -| cluster_operator_install_type | How Strimzi will be installed during the tests | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) | -| parallel | Number of tests that will be executed in parallel | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) | -| architecture | Which architecture will be used (should match with agent arch) | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) | -| groups | Which Junit5 groups will be executed | all | -| tests | Which Junit5 tests will be executed | all | -| kubeVersion | Used Kubernetes version as part of Kind/Minikube setup | The one set as default in setup scripts | -| kafkaVersion | Which Kafka version will be used in the tests | Default one from STs config | +| Name | Info | Default | +|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| pipeline | Name of the pipeline from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that wil be executed | regression,upgrade | +| profile | Testing profile from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that will be executed, it has to be defined in pom file | smoke | +| agent | Agent that will be used for a specific pipeline (see list of runners in Strimzi org config for more info) | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) | +| strimzi_feature_gates | Which Strimzi Feature Gates will be used | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) | +| strimzi_rbac_scope | RBAC scope for Strimzi | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) | +| cluster_operator_install_type | How Strimzi will be installed during the tests | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) | +| parallel | Number of tests that will be executed in parallel | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) | +| architecture | Which architecture will be used (should match with agent arch) | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) | +| groups | Which Junit5 groups will be executed | all | +| tests | Which Junit5 tests will be executed | all | +| kubeVersion | Used Kubernetes version as part of Kind/Minikube setup | The one set as default in setup scripts | +| kafkaVersion | Which Kafka version will be used in the tests | Default one from STs config | The process of parameter usage is as follows: -- `pipeline` has the highest priority. If `pipeline` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that match specific _pipeline_. -- `profile` has the second-highest priority. If `profile` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that match specific _profile_. +- `pipeline` has the highest priority. If `pipeline` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that match specific _pipeline_. +- `profile` has the second-highest priority. If `profile` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that match specific _profile_. - `agent` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment. - `strimzi_feature_gates` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment. - `strimzi_rbac_scope` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment. diff --git a/.github/tests/inputs/validate-matrix/empty_matrix.json b/.github/tests/inputs/validate-matrix/empty_matrix.json new file mode 100644 index 00000000000..0637a088a01 --- /dev/null +++ b/.github/tests/inputs/validate-matrix/empty_matrix.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.github/tests/inputs/validate-matrix/null_matrix.json b/.github/tests/inputs/validate-matrix/null_matrix.json new file mode 100644 index 00000000000..ec747fa47dd --- /dev/null +++ b/.github/tests/inputs/validate-matrix/null_matrix.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/.github/tests/inputs/validate-matrix/valid_custom.json b/.github/tests/inputs/validate-matrix/valid_custom.json new file mode 100644 index 00000000000..6946550b3b4 --- /dev/null +++ b/.github/tests/inputs/validate-matrix/valid_custom.json @@ -0,0 +1,30 @@ +[ + { + "agent": "oracle-4cpu-16gb-x86-64", + "arch": "amd64", + "pipeline": "custom", + "profile": "all", + "timeout": 720, + "strimzi_feature_gates": "", + "strimzi_rbac_scope": "CLUSTER", + "cluster_operator_install_type": "bundle", + "parallel": "1", + "groups": "", + "tests": "", + "jobName": "custom-all-amd64" + }, + { + "agent": "oracle-4cpu-16gb-arm64", + "arch": "arm64", + "pipeline": "custom", + "profile": "all", + "timeout": 720, + "strimzi_feature_gates": "", + "strimzi_rbac_scope": "CLUSTER", + "cluster_operator_install_type": "bundle", + "parallel": "1", + "groups": "", + "tests": "", + "jobName": "custom-all-arm64" + } +] \ No newline at end of file diff --git a/.github/tests/inputs/validate-matrix/valid_multiple.json b/.github/tests/inputs/validate-matrix/valid_multiple.json new file mode 100644 index 00000000000..00128a6b367 --- /dev/null +++ b/.github/tests/inputs/validate-matrix/valid_multiple.json @@ -0,0 +1,38 @@ +[ + { + "agent": "oracle-vm-8cpu-32gb-x86-64", + "arch": "amd64", + "pipeline": "regression", + "profile": "brokers-and-security", + "timeout": 480, + "strimzi_feature_gates": "false", + "strimzi_rbac_scope": "CLUSTER", + "cluster_operator_install_type": "yaml", + "parallel": 4, + "jobName": "regression-brokers-and-security-amd64" + }, + { + "agent": "oracle-vm-8cpu-32gb-x86-64", + "arch": "amd64", + "pipeline": "regression", + "profile": "operators", + "timeout": 480, + "strimzi_feature_gates": "false", + "strimzi_rbac_scope": "CLUSTER", + "cluster_operator_install_type": "yaml", + "parallel": 4, + "jobName": "regression-operators-amd64" + }, + { + "agent": "oracle-vm-2cpu-8gb-x86-64", + "arch": "amd64", + "pipeline": "smoke", + "profile": "smoke", + "timeout": 60, + "strimzi_feature_gates": "false", + "strimzi_rbac_scope": "CLUSTER", + "cluster_operator_install_type": "yaml", + "parallel": 1, + "jobName": "smoke-amd64" + } +] \ No newline at end of file diff --git a/.github/tests/inputs/validate-matrix/valid_single.json b/.github/tests/inputs/validate-matrix/valid_single.json new file mode 100644 index 00000000000..59b8eea288c --- /dev/null +++ b/.github/tests/inputs/validate-matrix/valid_single.json @@ -0,0 +1,14 @@ +[ + { + "agent": "oracle-vm-2cpu-8gb-x86-64", + "arch": "amd64", + "pipeline": "smoke", + "profile": "smoke", + "timeout": 60, + "strimzi_feature_gates": "false", + "strimzi_rbac_scope": "CLUSTER", + "cluster_operator_install_type": "yaml", + "parallel": 1, + "jobName": "smoke-amd64" + } +] \ No newline at end of file diff --git a/.github/tests/scenarios/validate-matrix.yaml b/.github/tests/scenarios/validate-matrix.yaml new file mode 100644 index 00000000000..2c435e7ba30 --- /dev/null +++ b/.github/tests/scenarios/validate-matrix.yaml @@ -0,0 +1,40 @@ +scenarios: + - id: valid-multiple-jobs + description: "Validate matrix with multiple valid jobs" + event: push + matrix_file: .github/tests/inputs/validate-matrix/valid_multiple.json + expectations: + isValid: "true" + message_contains: "will be executed" + + - id: valid-single-job + description: "Validate matrix with single valid job" + event: push + matrix_file: .github/tests/inputs/validate-matrix/valid_single.json + expectations: + isValid: "true" + message_contains: "will be executed" + + - id: empty-matrix + description: "Validate empty matrix (should fail validation)" + event: push + matrix_file: .github/tests/inputs/validate-matrix/empty_matrix.json + expectations: + isValid: "false" + message_contains: "❌ **System test validation failed**" + + - id: null-matrix + description: "Validate null matrix input" + event: push + matrix_file: .github/tests/inputs/validate-matrix/null_matrix.json + expectations: + isValid: "false" + message_contains: "❌ **System test validation failed**" + + - id: valid-with-custom-jobs + description: "Validate matrix with custom job configurations" + event: push + matrix_file: .github/tests/inputs/validate-matrix/valid_custom.json + expectations: + isValid: "true" + message_contains: "will be executed" \ No newline at end of file diff --git a/.github/tests/workflows/validate-matrix-template.yaml b/.github/tests/workflows/validate-matrix-template.yaml new file mode 100644 index 00000000000..d76e03133c1 --- /dev/null +++ b/.github/tests/workflows/validate-matrix-template.yaml @@ -0,0 +1,64 @@ +name: Validate-matrix template + +on: + push: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + # Test the validate-matrix action + - id: validate-matrix + uses: ./.github/actions/systemtests/validate-matrix + with: + matrix: ${{ env.TEST_MATRIX }} + + # Validate the action outputs + - name: Validate action outputs + shell: bash + env: + EXPECT_ISVALID: ${{ env.EXPECT_ISVALID }} + EXPECT_MESSAGE_CONTAINS: ${{ env.EXPECT_MESSAGE_CONTAINS }} + EXPECT_ERROR_CONTAINS: ${{ env.EXPECT_ERROR_CONTAINS }} + EXPECT_JOB_COUNT: ${{ env.EXPECT_JOB_COUNT }} + run: | + set +e # Don't exit on first error + VALIDATION_ERRORS=0 + + IS_VALID='${{ steps.validate-matrix.outputs.isValid }}' + MESSAGE='${{ steps.validate-matrix.outputs.message }}' + + echo "::group::Action Outputs" + echo "isValid: $IS_VALID" + echo "message: $MESSAGE" + echo "::endgroup::" + + # Test expected values if provided + if [[ -n "$EXPECT_ISVALID" ]]; then + if [[ "$IS_VALID" == "$EXPECT_ISVALID" ]]; then + echo "✅ isValid matches expectation: $EXPECT_ISVALID" + else + echo "❌ isValid mismatch. Expected: $EXPECT_ISVALID, Got: $IS_VALID" + VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1)) + fi + fi + + if [[ -n "$EXPECT_MESSAGE_CONTAINS" ]]; then + if [[ "$MESSAGE" == *"$EXPECT_MESSAGE_CONTAINS"* ]]; then + echo "✅ Message contains expected text: $EXPECT_MESSAGE_CONTAINS" + else + echo "❌ Message does not contain expected text: $EXPECT_MESSAGE_CONTAINS" + VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1)) + fi + fi + + # Summary + if [[ $VALIDATION_ERRORS -eq 0 ]]; then + echo "✅ All validations passed!" + exit 0 + else + echo "❌ Validation failed with $VALIDATION_ERRORS error(s)" + exit 1 + fi \ No newline at end of file diff --git a/.github/workflows/actions-tests.yml b/.github/workflows/actions-tests.yml index e28ac5f0e75..e16c6a4ba7e 100644 --- a/.github/workflows/actions-tests.yml +++ b/.github/workflows/actions-tests.yml @@ -282,3 +282,98 @@ jobs: echo "───────────────────────────────────────" echo "🎉 All $TOTAL generate-matrix scenarios passed!" echo " Matrix generation logic validated ✅" + + test-validate-matrix: + needs: + - lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + # Install dependencies + - name: Install act + run: | + # Install act for workflow testing + curl -s https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash + sudo install -m 0755 ./bin/act /usr/local/bin/act + + - name: Install yq + uses: ./.github/actions/dependencies/install-yq + + # Test all validate-matrix scenarios + - name: Test validate-matrix action scenarios + run: | + set -e + + SCENARIOS_FILE=".github/tests/scenarios/validate-matrix.yaml" + WORKFLOW=".github/tests/workflows/validate-matrix-template.yaml" + + echo "✅ Running validate-matrix action tests..." + echo "📁 Loading scenarios from: $SCENARIOS_FILE" + + # Get total number of scenarios + TOTAL=$(yq eval '.scenarios | length' "$SCENARIOS_FILE") + echo "📊 Found $TOTAL test scenarios" + echo + + # Initialize overall result tracker + overall_result=true + + # Loop through each scenario + for i in $(seq 0 $((TOTAL - 1))); do + # Extract scenario details using yq + id=$(yq eval ".scenarios[$i].id" "$SCENARIOS_FILE") + description=$(yq eval ".scenarios[$i].description" "$SCENARIOS_FILE") + event=$(yq eval ".scenarios[$i].event" "$SCENARIOS_FILE") + matrix_file=$(yq eval ".scenarios[$i].matrix_file" "$SCENARIOS_FILE") + + echo "───────────────────────────────────────" + echo "🔍 Scenario $((i + 1))/$TOTAL: $id" + echo " Description: $description" + echo " Event: $event" + echo " Matrix file: $matrix_file" + + # Load matrix content from file and compact to single line + if [[ -f "$matrix_file" ]]; then + TEST_MATRIX=$(jq -c . "$matrix_file") + echo " ✅ Matrix loaded and compacted from file" + else + echo " ❌ Matrix file not found: $matrix_file" + overall_result=false + continue + fi + + # Build environment variables from expectations + env_args="--env TEST_MATRIX='$TEST_MATRIX'" + + # Get all expectation keys and build env vars + expectation_keys=$(yq eval ".scenarios[$i].expectations | keys | .[]" "$SCENARIOS_FILE") + for key in $expectation_keys; do + value=$(yq eval ".scenarios[$i].expectations.$key" "$SCENARIOS_FILE") + env_key="EXPECT_$(echo "$key" | tr '[:lower:]' '[:upper:]')" + env_args="$env_args --env $env_key=\"$value\"" + done + + # Run act with the scenario + echo "▶️ Running test..." + + token_args="-s GITHUB_TOKEN=\"${{ secrets.GITHUB_TOKEN }}\"" + + if eval "act '$event' -W '$WORKFLOW' -P ubuntu-latest=catthehacker/ubuntu:act-latest --pull=false $env_args $token_args"; then + echo "✅ $id completed" + else + echo "❌ $id failed" + overall_result=false + fi + # Print blank line for better readability + echo + done + + if [ "$overall_result" = "false" ]; then + echo "There are some test ❌. Please check the logs for more details." + exit 1 + fi + + echo "───────────────────────────────────────" + echo "🎉 All $TOTAL validate-matrix scenarios passed!" + echo " Matrix validation logic validated ✅" diff --git a/.github/workflows/run-system-tests.yml b/.github/workflows/run-system-tests.yml index 1b539e8bf90..876247b3aae 100644 --- a/.github/workflows/run-system-tests.yml +++ b/.github/workflows/run-system-tests.yml @@ -3,41 +3,6 @@ name: Run System Tests on: workflow_call: inputs: - profileList: - description: "Comma-separated or single profile name(s)" - required: false - default: "" - type: string - pipelineList: - description: "Comma-separated or single pipeline name(s)" - required: false - default: "" - type: string - excludedGroups: - description: "Excluded groups" - required: false - default: "" - type: string - cluster_operator_install_type: - description: "Install type method used for Strimzi operator" - required: false - default: "yaml" - type: string - strimzi_rbac_scope: - description: "RBAC configuration for operator access" - required: false - default: "CLUSTER" - type: string - parallel: - description: "How many tests should be executed in parallel" - required: false - default: "1" - type: string - strimzi_feature_gates: - description: "Which feature-gates will be used by operator" - required: false - default: "" - type: string releaseVersion: description: "Release version to test (i.e. 'latest', '0.46.0')" required: false @@ -48,26 +13,6 @@ on: required: false default: "latest" type: string - agent: - description: "On which agent the jobs will be executed" - required: true - type: string - architecture: - description: "Which architecture is used for the job" - required: true - type: string - groups: - description: "JUnit5 test groups that will be executed" - required: false - # Empty string means all groups - default: "" - type: string - tests: - description: "JUnit5 tests that will be executed" - required: false - # Empty string means all tests - default: "" - type: string kubeVersion: description: "Used Kubernetes version for Kind/Minikube. Note that for Kind you have to use whole image with sha digest!" required: false @@ -79,35 +24,16 @@ on: description: "Repository ref that will be checkout" required: true type: string + matrix: + description: "JSON matrix generated by validate-matrix action" + required: true + type: string jobs: - generate-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.gen.outputs.matrix }} - steps: - - uses: actions/checkout@v5 - with: - ref: ${{ inputs.ref }} - - name: Generate Matrix - id: gen - uses: ./.github/actions/systemtests/generate-matrix - with: - pipelines: ${{ inputs.pipelineList }} - profiles: ${{ inputs.profileList }} - default_agent: ${{ inputs.agent }} - default_arch: ${{ inputs.architecture }} - default_strimzi_feature_gates: ${{ inputs.strimzi_feature_gates }} - default_strimzi_rbac_scope: ${{ inputs.strimzi_rbac_scope }} - default_cluster_operator_install_type: ${{ inputs.cluster_operator_install_type }} - default_parallel: ${{ inputs.parallel }} - runnerArch: "amd64" - run-tests: - needs: generate-matrix strategy: matrix: - config: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} + config: ${{ fromJson(inputs.matrix) }} fail-fast: false # Max timeout ise set for 10h for the whole job, test execution has lower timeouts to not block the job for the whole time # but have some additional time to collect logs, etc. @@ -310,8 +236,8 @@ jobs: TEST_LOG_DIR: "systemtest-logs" CONNECT_IMAGE_WITH_FILE_SINK_PLUGIN: ${{ env.CONNECT_IMAGE_WITH_FILE_SINK_PLUGIN }} # Build the switch only when the input is non-empty - GROUPS_OPT: ${{ inputs.groups != '' && format('-Dgroups="{0}"', inputs.groups) || '' }} - TESTS_OPT: ${{ inputs.tests != '' && format('-Dit.test="{0}"', inputs.tests) || '' }} + GROUPS_OPT: ${{ matrix.config.groups != '' && format('-Dgroups={0}', matrix.config.groups) || '' }} + TESTS_OPT: ${{ matrix.config.tests != '' && format('-Dit.test={0}', matrix.config.tests) || '' }} EXCLUDED_GROUPS: ${{ matrix.config.excluded_groups || '' }} STRIMZI_TEST_LOG_LEVEL: INFO diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index 8d52bc33947..ed0c4f494ef 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -83,12 +83,64 @@ jobs: - uses: actions/checkout@v5 with: ref: ${{ needs.parse-params.outputs.ref }} - - name: Add comment + + generate-matrix: + needs: + - parse-params + - check-rights + if: |- + ${{ + always() && + needs.parse-params.outputs.shouldRun == 'true' && + (needs.check-rights.result == 'success' || github.event_name == 'pull_request') + }} + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + statuses: write + outputs: + isValid: ${{ steps.validate.outputs.isValid }} + message: ${{ steps.validate.outputs.message }} + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ needs.parse-params.outputs.ref }} + - name: Generate Matrix for Validation + id: gen + uses: ./.github/actions/systemtests/generate-matrix + with: + pipelines: ${{ needs.parse-params.outputs.pipelineList }} + profiles: ${{ needs.parse-params.outputs.profileList }} + default_agent: ${{ needs.parse-params.outputs.agent }} + default_arch: ${{ needs.parse-params.outputs.architecture }} + default_strimzi_feature_gates: ${{ needs.parse-params.outputs.strimzi_feature_gates }} + default_strimzi_rbac_scope: ${{ needs.parse-params.outputs.strimzi_rbac_scope }} + default_cluster_operator_install_type: ${{ needs.parse-params.outputs.cluster_operator_install_type }} + default_parallel: ${{ needs.parse-params.outputs.parallel }} + default_tests: ${{ needs.parse-params.outputs.tests }} + default_groups: ${{ needs.parse-params.outputs.groups }} + runnerArch: "amd64" + - name: Validate Matrix + id: validate + uses: ./.github/actions/systemtests/validate-matrix + with: + matrix: ${{ steps.gen.outputs.matrix }} + - name: Add validation error comment + if: ${{ github.event_name == 'issue_comment' && steps.validate.outputs.isValid == 'false' }} uses: ./.github/actions/utils/add-comment with: - commentMessage: ':hourglass_flowing_sand: System test verification started: [link](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})' + commentMessage: ${{ steps.validate.outputs.message }} + - name: Add trigger comment + if: ${{ github.event_name == 'issue_comment' && steps.validate.outputs.isValid == 'true' }} + uses: ./.github/actions/utils/add-comment + with: + commentMessage: ':hourglass_flowing_sand: System test verification started: [link](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) + + ${{ steps.validate.outputs.message }}' - name: Set check & commit status (pending) - if: ${{ needs.parse-params.outputs.shouldRun == 'true' }} + if: ${{ steps.validate.outputs.isValid == 'true' }} uses: ./.github/actions/utils/check-and-status with: checkState: pending @@ -100,8 +152,15 @@ jobs: needs: - parse-params - check-rights - # Run only if properly triggered, permissions check passed, and the run is not for release (already built images) - if: ${{ always() && needs.parse-params.outputs.shouldRun == 'true' && (needs.check-rights.result == 'success' || github.event_name == 'pull_request') && needs.parse-params.outputs.releaseVersion == 'latest' }} + - generate-matrix + # Run only if properly triggered, permissions check passed, matrix is valid, and the run is not for release (already built images) + if: |- + ${{ + always() &&needs.parse-params.outputs.shouldRun == 'true' && + (needs.check-rights.result == 'success' || github.event_name == 'pull_request') && + needs.generate-matrix.outputs.isValid == 'true' && + needs.parse-params.outputs.releaseVersion == 'latest' + }} runs-on: oracle-2cpu-8gb-arm64 steps: - uses: actions/checkout@v5 @@ -117,9 +176,17 @@ jobs: needs: - parse-params - check-rights + - generate-matrix - build-artifacts - # Run only if properly triggered, permissions check passed, and the run is not for release (already built images) - if: ${{ always() && needs.parse-params.outputs.shouldRun == 'true' && (needs.check-rights.result == 'success' || github.event_name == 'pull_request') && needs.parse-params.outputs.releaseVersion == 'latest' && needs.build-artifacts.result != 'failure'}} + # Run only if properly triggered, permissions check passed, matrix is valid, and the run is not for release (already built images) + if: |- + ${{ + always() &&needs.parse-params.outputs.shouldRun == 'true' && + (needs.check-rights.result == 'success' || github.event_name == 'pull_request')&& + needs.generate-matrix.outputs.isValid == 'true' && + needs.parse-params.outputs.releaseVersion == 'latest' && + needs.build-artifacts.result != 'failure' + }} strategy: matrix: architecture: [amd64, arm64] @@ -138,10 +205,19 @@ jobs: needs: - parse-params - check-rights + - generate-matrix - build-artifacts - build-images - # Run if properly triggered, permissions check passed, and build jobs succeeded or were skipped - if: ${{ always() && needs.parse-params.outputs.shouldRun == 'true' && (needs.check-rights.result == 'success' || github.event_name == 'pull_request') && needs.build-images.result != 'failure' && needs.build-artifacts.result != 'failure' }} + # Run if properly triggered, permissions check passed, matrix is valid, and build jobs succeeded or were skipped + if: |- + ${{ + always() && + needs.parse-params.outputs.shouldRun == 'true' && + (needs.check-rights.result == 'success' || github.event_name == 'pull_request')&& + needs.generate-matrix.outputs.isValid == 'true' && + needs.build-images.result != 'failure' && + needs.build-artifacts.result != 'failure' + }} # Job needs write rights for setting statuses on pull-requests permissions: contents: read @@ -149,26 +225,28 @@ jobs: statuses: write uses: ./.github/workflows/run-system-tests.yml with: - pipelineList: ${{ needs.parse-params.outputs.pipelineList }} - profileList: ${{ needs.parse-params.outputs.profileList }} - agent: ${{ needs.parse-params.outputs.agent }} - architecture: ${{ needs.parse-params.outputs.architecture }} - strimzi_feature_gates: ${{ needs.parse-params.outputs.strimzi_feature_gates }} - strimzi_rbac_scope: ${{ needs.parse-params.outputs.strimzi_rbac_scope }} - cluster_operator_install_type: ${{ needs.parse-params.outputs.cluster_operator_install_type }} - parallel: ${{ needs.parse-params.outputs.parallel }} releaseVersion: ${{ needs.parse-params.outputs.releaseVersion }} kafkaVersion: ${{ needs.parse-params.outputs.kafkaVersion }} kubeVersion: ${{ needs.parse-params.outputs.kubeVersion }} # Checkout ref params ref: ${{ needs.parse-params.outputs.ref }} + # Pass the validated matrix directly + matrix: ${{ needs.generate-matrix.outputs.matrix }} report: needs: - parse-params - check-rights + - generate-matrix - run-tests - if: ${{ always() && needs.parse-params.outputs.shouldRun == 'true' && needs.check-rights.result == 'success' && github.event_name != 'pull_request' }} + if: |- + ${{ + always() && + needs.parse-params.outputs.shouldRun == 'true' && + needs.check-rights.result == 'success' && + github.event_name != 'pull_request' && + needs.generate-matrix.outputs.isValid == 'true' + }} runs-on: ubuntu-latest permissions: contents: read