Add comment parameter validation passed to GHA and improve docs

.github/actions/systemtests/generate-matrix/action.yml

@@ -5,44 +5,36 @@ inputs:
   pipelines:
     description: "Comma-separated list of pipelines (if provided, takes precedence)"
     required: false
-    default: ""
   profiles:
     description: "Comma-separated list of profiles (used if pipelines is empty)"
     required: false
-    default: ""
   default_agent:
     description: "Default agent value"
-    required: true
-    default: "oracle-4cpu-16gb-x86-64"
+    required: false
   default_arch:
     description: "Default architecture value"
-    required: true
-    default: "amd64"
+    required: false
   default_strimzi_feature_gates:
     description: "Default value for strimzi_feature_gates"
-    required: true
-    default: ""
+    required: false
   default_strimzi_rbac_scope:
     description: "Default value for strimzi_rbac_scope"
-    required: true
-    default: "CLUSTER"
+    required: false
   default_cluster_operator_install_type:
     description: "Default value for cluster_operator_install_type"
-    required: true
-    default: "bundle"
+    required: false
   default_parallel:
     description: "Number of tests executed in parallel"
-    default: "1"
+    required: false
   default_groups:
     description: "Default JUnit5 test groups that will be executed"
-    default: ""
+    required: false
   default_tests:
     description: "Default JUnit5 tests that will be executed"
-    default: ""
+    required: false
   runnerArch:
     description: "Architecture of GitHub runner"
     required: false
-    default: "amd64"
 
 outputs:
   matrix:
@@ -121,12 +113,12 @@ runs:
       env:
         INPUT_PIPELINES: "${{ inputs.pipelines }}"
         INPUT_PROFILES: "${{ inputs.profiles }}"
-        INPUT_DEFAULT_AGENT: "${{ inputs.default_agent }}"
-        INPUT_DEFAULT_ARCH: "${{ inputs.default_arch }}"
+        INPUT_DEFAULT_AGENT: "${{ inputs.default_agent != '' && inputs.default_agent || 'ubuntu-latest' }}"
+        INPUT_DEFAULT_ARCH: "${{ inputs.default_arch != '' && inputs.default_arch || 'amd64' }}"
         INPUT_DEFAULT_JDK_VERSION: "${{ inputs.default_jdk_version }}"
         INPUT_DEFAULT_STRIMZI_USE_FEATURE_GATES: "${{ inputs.default_strimzi_feature_gates }}"
-        INPUT_DEFAULT_STRIMZI_RBAC_SCOPE: "${{ inputs.default_strimzi_rbac_scope }}"
-        INPUT_DEFAULT_CLUSTER_OPERATOR_INSTALL_TYPE: "${{ inputs.default_cluster_operator_install_type }}"
-        INPUT_DEFAULT_PARALLEL: "${{ inputs.default_parallel }}"
+        INPUT_DEFAULT_STRIMZI_RBAC_SCOPE: "${{ inputs.default_strimzi_rbac_scope != '' && inputs.default_strimzi_rbac_scope || 'CLUSTER' }}"
+        INPUT_DEFAULT_CLUSTER_OPERATOR_INSTALL_TYPE: "${{ inputs.default_cluster_operator_install_type != '' && inputs.default_cluster_operator_install_type || 'bundle' }}"
+        INPUT_DEFAULT_PARALLEL: "${{ inputs.default_parallel != '' && inputs.default_parallel || '1' }}"
         INPUT_DEFAULT_GROUPS: "${{ inputs.default_groups }}"
         INPUT_DEFAULT_TESTS: "${{ inputs.default_tests }}"

.github/actions/systemtests/parse-comment/action.yml

@@ -171,8 +171,8 @@ runs:
             strimzi_rbac_scope:            getParameter('strimzi_rbac_scope',  sanitized),  // keep case
             cluster_operator_install_type: getParameter('cluster_operator_install_type', lower) || DEFAULTS.installType,
             parallel:                      getParameter('parallel', lower) || DEFAULTS.parallel,
-            groups:                        getParameter('groups', lower),
-            tests:                         getParameter('tests', lower),
+            groups:                        getParameter('groups', sanitized),
+            tests:                         getParameter('tests', sanitized),
             kafkaVersion:                  getParameter('kafkaversion', lower) || DEFAULTS.kafkaVersion,
             kubeVersion:                   getParameter('kubeversion',  lower) || DEFAULTS.kubeVersion,
             releaseVersion:                process.env.INPUT_RELEASE_VERSION || 'latest',

.github/actions/systemtests/validate-matrix/action.yml

@@ -0,0 +1,70 @@
+name: "Validate Matrix"
+description: "Validates generated matrix and returns feedback message"
+
+inputs:
+  matrix:
+    description: "JSON matrix generated by generate-matrix action"
+    required: true
+
+outputs:
+  isValid:
+    description: "Whether the matrix is valid (not empty)"
+    value: ${{ steps.validate.outputs.isValid }}
+  message:
+    description: "Feedback message about the validation result"
+    value: ${{ steps.validate.outputs.message }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Validate Matrix
+      id: validate
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const matrix = JSON.parse(process.env.MATRIX || '[]');
+
+          // Check if matrix is empty
+          const isEmpty = !matrix || matrix.length === 0;
+
+          if (isEmpty) {
+            core.setOutput('isValid', 'false');
+
+            let errorMessage = "❌ **System test validation failed**\n";
+            errorMessage += "\n";
+            errorMessage += "The provided pipeline/profile names did not match any configured tests.\n";
+            errorMessage += "\n";
+            errorMessage += "**Available pipelines:**\n";
+            errorMessage += "`acceptance`, `acceptance-helm`, `regression`, `regression-fg`, `regression-rbac`, `smoke`, `upgrade`\n";
+            errorMessage += "\n";
+            errorMessage += "**Available profiles:**\n";
+            errorMessage += "`acceptance`, `azp_connect_mirrormaker`, `azp_dynconfig_listeners_tracing_watcher`, `azp_kafka_oauth`, `azp_kafka_upgrade`, `azp_kraft_upgrade`, `azp_operators`, `azp_rbac_remaining`, `azp_rolling_update_bridge`, `azp_security`, `brokers-and-security`, `operands`, `operators`, `smoke`\n";
+            errorMessage += "\n";
+            errorMessage += "Please check your command for typos and try again.";
+
+            core.setOutput('message', errorMessage);
+            return;
+          }
+
+          // Matrix is valid - create summary of what will run
+          core.setOutput('isValid', 'true');
+          core.info(`Matrix validation passed. ${matrix.length} job(s) will be executed.`);
+
+          const jobs = matrix.map(job => {
+            const name = job.jobName || `${job.pipeline}-${job.profile || job.pipeline}-${job.arch}`;
+            const agent = job.agent || 'default';
+            const timeout = job.timeout || 'default';
+            return `- **${name}** (${agent}, ${timeout}min)`;
+          }).join('\n');
+
+          let successMessage = "✅ **System test validation passed**\n";
+          successMessage += "\n";
+          successMessage += `The following ${matrix.length} job(s) will be executed:\n`;
+          successMessage += "\n";
+          successMessage += jobs.replace(/\n/g, '\n') + "\n";
+          successMessage += "\n";
+          successMessage += "Tests will start after successful build completion.";
+
+          core.setOutput('message', successMessage);
+      env:
+        MATRIX: ${{ inputs.matrix }}

.github/docs/README.md

@@ -62,14 +62,13 @@ Build process actions:
 - [build-strimzi-binaries](../actions/build/build-strimzi-binaries)
 - [containers-build](../actions/build/containers-build)
 - [containers-load](../actions/build/containers-load)
-- [containers-push-manifest](../actions/build/containers-push-manifest)
 
 System tests execution actions:
 - [generate-matrix](../actions/systemtests/generate-matrix)
 - [set-defaults](../actions/utils/set-defaults)
 - [log-variables](../actions/utils/log-variables)
 - [parse-comment](../actions/systemtests/parse-comment)
-- [determine-ref](../actions/determine-ref)
+- [determine-ref](../actions/utils/determine-ref)
 
 Utils actions:
 - [check-permissions](../actions/utils/check-permissions)
@@ -99,24 +98,24 @@ Comment for triggering the workflow has to starts with string `/gha run` and the
 The whole script that parse the trigger even is part of [parse-comment](../actions/systemtests/parse-comment) action.
 Currently, we have these parameters that can be passed through the comment:
 
-| Name                          | Info                                                                                                                                     | Default                                                                  |
-|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------|
-| pipeline                      | Name of the pipeline from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that wil be executed                               | regression,upgrade                                                       |
-| profile                       | Testing profile from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that will be executed, it has to be defined in pom file | smoke                                                                    |
-| agent                         | Agent that will be used for a specific pipeline (see list of runners in Strimzi org config for more info)                                | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
-| strimzi_feature_gates         | Which Strimzi Feature Gates will be used                                                                                                 | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
-| strimzi_rbac_scope            | RBAC scope for Strimzi                                                                                                                   | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
-| cluster_operator_install_type | How Strimzi will be installed during the tests                                                                                           | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
-| parallel                      | Number of tests that will be executed in parallel                                                                                        | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
-| architecture                  | Which architecture will be used (should match with agent arch)                                                                           | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
-| groups                        | Which Junit5 groups will be executed                                                                                                     | all                                                                      |
-| tests                         | Which Junit5 tests will be executed                                                                                                      | all                                                                      |
-| kubeVersion                   | Used Kubernetes version as part of Kind/Minikube setup                                                                                   | The one set as default in setup scripts                                  |
-| kafkaVersion                  | Which Kafka version will be used in the tests                                                                                            | Default one from STs config                                              |
+| Name                          | Info                                                                                                                                                 | Default                                                                              |
+|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|
+| pipeline                      | Name of the pipeline from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that wil be executed                               | regression,upgrade                                                                   |
+| profile                       | Testing profile from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that will be executed, it has to be defined in pom file | smoke                                                                                |
+| agent                         | Agent that will be used for a specific pipeline (see list of runners in Strimzi org config for more info)                                            | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
+| strimzi_feature_gates         | Which Strimzi Feature Gates will be used                                                                                                             | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
+| strimzi_rbac_scope            | RBAC scope for Strimzi                                                                                                                               | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
+| cluster_operator_install_type | How Strimzi will be installed during the tests                                                                                                       | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
+| parallel                      | Number of tests that will be executed in parallel                                                                                                    | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
+| architecture                  | Which architecture will be used (should match with agent arch)                                                                                       | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
+| groups                        | Which Junit5 groups will be executed                                                                                                                 | all                                                                                  |
+| tests                         | Which Junit5 tests will be executed                                                                                                                  | all                                                                                  |
+| kubeVersion                   | Used Kubernetes version as part of Kind/Minikube setup                                                                                               | The one set as default in setup scripts                                              |
+| kafkaVersion                  | Which Kafka version will be used in the tests                                                                                                        | Default one from STs config                                                          |
 
 The process of parameter usage is as follows:
-- `pipeline` has the highest priority. If `pipeline` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that match specific _pipeline_.
-- `profile` has the second-highest priority. If `profile` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that match specific _profile_.
+- `pipeline` has the highest priority. If `pipeline` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that match specific _pipeline_.
+- `profile` has the second-highest priority. If `profile` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that match specific _profile_.
 - `agent` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment.
 - `strimzi_feature_gates` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment.
 - `strimzi_rbac_scope` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment.

.github/tests/inputs/validate-matrix/empty_matrix.json

@@ -0,0 +1 @@
+[]

.github/tests/inputs/validate-matrix/null_matrix.json

@@ -0,0 +1 @@
+null

.github/tests/inputs/validate-matrix/valid_custom.json

@@ -0,0 +1,30 @@
+[
+  {
+    "agent": "oracle-4cpu-16gb-x86-64",
+    "arch": "amd64",
+    "pipeline": "custom",
+    "profile": "all",
+    "timeout": 720,
+    "strimzi_feature_gates": "",
+    "strimzi_rbac_scope": "CLUSTER",
+    "cluster_operator_install_type": "bundle",
+    "parallel": "1",
+    "groups": "",
+    "tests": "",
+    "jobName": "custom-all-amd64"
+  },
+  {
+    "agent": "oracle-4cpu-16gb-arm64",
+    "arch": "arm64",
+    "pipeline": "custom",
+    "profile": "all",
+    "timeout": 720,
+    "strimzi_feature_gates": "",
+    "strimzi_rbac_scope": "CLUSTER",
+    "cluster_operator_install_type": "bundle",
+    "parallel": "1",
+    "groups": "",
+    "tests": "",
+    "jobName": "custom-all-arm64"
+  }
+]

.github/tests/inputs/validate-matrix/valid_multiple.json

@@ -0,0 +1,38 @@
+[
+  {
+    "agent": "oracle-vm-8cpu-32gb-x86-64",
+    "arch": "amd64",
+    "pipeline": "regression",
+    "profile": "brokers-and-security",
+    "timeout": 480,
+    "strimzi_feature_gates": "false",
+    "strimzi_rbac_scope": "CLUSTER",
+    "cluster_operator_install_type": "yaml",
+    "parallel": 4,
+    "jobName": "regression-brokers-and-security-amd64"
+  },
+  {
+    "agent": "oracle-vm-8cpu-32gb-x86-64",
+    "arch": "amd64",
+    "pipeline": "regression",
+    "profile": "operators",
+    "timeout": 480,
+    "strimzi_feature_gates": "false",
+    "strimzi_rbac_scope": "CLUSTER",
+    "cluster_operator_install_type": "yaml",
+    "parallel": 4,
+    "jobName": "regression-operators-amd64"
+  },
+  {
+    "agent": "oracle-vm-2cpu-8gb-x86-64",
+    "arch": "amd64",
+    "pipeline": "smoke",
+    "profile": "smoke",
+    "timeout": 60,
+    "strimzi_feature_gates": "false",
+    "strimzi_rbac_scope": "CLUSTER",
+    "cluster_operator_install_type": "yaml",
+    "parallel": 1,
+    "jobName": "smoke-amd64"
+  }
+]

.github/tests/inputs/validate-matrix/valid_single.json

@@ -0,0 +1,14 @@
+[
+  {
+    "agent": "oracle-vm-2cpu-8gb-x86-64",
+    "arch": "amd64",
+    "pipeline": "smoke",
+    "profile": "smoke",
+    "timeout": 60,
+    "strimzi_feature_gates": "false",
+    "strimzi_rbac_scope": "CLUSTER",
+    "cluster_operator_install_type": "yaml",
+    "parallel": 1,
+    "jobName": "smoke-amd64"
+  }
+]

.github/tests/scenarios/validate-matrix.yaml

@@ -0,0 +1,40 @@
+scenarios:
+  - id: valid-multiple-jobs
+    description: "Validate matrix with multiple valid jobs"
+    event: push
+    matrix_file: .github/tests/inputs/validate-matrix/valid_multiple.json
+    expectations:
+      isValid: "true"
+      message_contains: "will be executed"
+
+  - id: valid-single-job
+    description: "Validate matrix with single valid job"
+    event: push
+    matrix_file: .github/tests/inputs/validate-matrix/valid_single.json
+    expectations:
+      isValid: "true"
+      message_contains: "will be executed"
+
+  - id: empty-matrix
+    description: "Validate empty matrix (should fail validation)"
+    event: push
+    matrix_file: .github/tests/inputs/validate-matrix/empty_matrix.json
+    expectations:
+      isValid: "false"
+      message_contains: "❌ **System test validation failed**"
+
+  - id: null-matrix
+    description: "Validate null matrix input"
+    event: push
+    matrix_file: .github/tests/inputs/validate-matrix/null_matrix.json
+    expectations:
+      isValid: "false"
+      message_contains: "❌ **System test validation failed**"
+
+  - id: valid-with-custom-jobs
+    description: "Validate matrix with custom job configurations"
+    event: push
+    matrix_file: .github/tests/inputs/validate-matrix/valid_custom.json
+    expectations:
+      isValid: "true"
+      message_contains: "will be executed"