Skip to content

Commit b57b480

Browse files
FrawlessFrawless
authored
Add comment parameter validation passed to GHA and improve docs (#11855)
Signed-off-by: Jakub Stejskal <xstejs24@gmail.com>
1 parent 4384943 commit b57b480

File tree

14 files changed

+488
-140
lines changed

14 files changed

+488
-140
lines changed

‎.github/actions/systemtests/generate-matrix/action.yml‎

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -5,44 +5,36 @@ inputs:
55
pipelines:
66
description: "Comma-separated list of pipelines (if provided, takes precedence)"
77
required: false
8-
default: ""
98
profiles:
109
description: "Comma-separated list of profiles (used if pipelines is empty)"
1110
required: false
12-
default: ""
1311
default_agent:
1412
description: "Default agent value"
15-
required: true
16-
default: "oracle-4cpu-16gb-x86-64"
13+
required: false
1714
default_arch:
1815
description: "Default architecture value"
19-
required: true
20-
default: "amd64"
16+
required: false
2117
default_strimzi_feature_gates:
2218
description: "Default value for strimzi_feature_gates"
23-
required: true
24-
default: ""
19+
required: false
2520
default_strimzi_rbac_scope:
2621
description: "Default value for strimzi_rbac_scope"
27-
required: true
28-
default: "CLUSTER"
22+
required: false
2923
default_cluster_operator_install_type:
3024
description: "Default value for cluster_operator_install_type"
31-
required: true
32-
default: "bundle"
25+
required: false
3326
default_parallel:
3427
description: "Number of tests executed in parallel"
35-
default: "1"
28+
required: false
3629
default_groups:
3730
description: "Default JUnit5 test groups that will be executed"
38-
default: ""
31+
required: false
3932
default_tests:
4033
description: "Default JUnit5 tests that will be executed"
41-
default: ""
34+
required: false
4235
runnerArch:
4336
description: "Architecture of GitHub runner"
4437
required: false
45-
default: "amd64"
4638

4739
outputs:
4840
matrix:
@@ -121,12 +113,12 @@ runs:
121113
env:
122114
INPUT_PIPELINES: "${{ inputs.pipelines }}"
123115
INPUT_PROFILES: "${{ inputs.profiles }}"
124-
INPUT_DEFAULT_AGENT: "${{ inputs.default_agent }}"
125-
INPUT_DEFAULT_ARCH: "${{ inputs.default_arch }}"
116+
INPUT_DEFAULT_AGENT: "${{ inputs.default_agent != '' && inputs.default_agent || 'ubuntu-latest' }}"
117+
INPUT_DEFAULT_ARCH: "${{ inputs.default_arch != '' && inputs.default_arch || 'amd64' }}"
126118
INPUT_DEFAULT_JDK_VERSION: "${{ inputs.default_jdk_version }}"
127119
INPUT_DEFAULT_STRIMZI_USE_FEATURE_GATES: "${{ inputs.default_strimzi_feature_gates }}"
128-
INPUT_DEFAULT_STRIMZI_RBAC_SCOPE: "${{ inputs.default_strimzi_rbac_scope }}"
129-
INPUT_DEFAULT_CLUSTER_OPERATOR_INSTALL_TYPE: "${{ inputs.default_cluster_operator_install_type }}"
130-
INPUT_DEFAULT_PARALLEL: "${{ inputs.default_parallel }}"
120+
INPUT_DEFAULT_STRIMZI_RBAC_SCOPE: "${{ inputs.default_strimzi_rbac_scope != '' && inputs.default_strimzi_rbac_scope || 'CLUSTER' }}"
121+
INPUT_DEFAULT_CLUSTER_OPERATOR_INSTALL_TYPE: "${{ inputs.default_cluster_operator_install_type != '' && inputs.default_cluster_operator_install_type || 'bundle' }}"
122+
INPUT_DEFAULT_PARALLEL: "${{ inputs.default_parallel != '' && inputs.default_parallel || '1' }}"
131123
INPUT_DEFAULT_GROUPS: "${{ inputs.default_groups }}"
132124
INPUT_DEFAULT_TESTS: "${{ inputs.default_tests }}"

‎.github/actions/systemtests/parse-comment/action.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -171,8 +171,8 @@ runs:
171171
strimzi_rbac_scope: getParameter('strimzi_rbac_scope', sanitized), // keep case
172172
cluster_operator_install_type: getParameter('cluster_operator_install_type', lower) || DEFAULTS.installType,
173173
parallel: getParameter('parallel', lower) || DEFAULTS.parallel,
174-
groups: getParameter('groups', lower),
175-
tests: getParameter('tests', lower),
174+
groups: getParameter('groups', sanitized),
175+
tests: getParameter('tests', sanitized),
176176
kafkaVersion: getParameter('kafkaversion', lower) || DEFAULTS.kafkaVersion,
177177
kubeVersion: getParameter('kubeversion', lower) || DEFAULTS.kubeVersion,
178178
releaseVersion: process.env.INPUT_RELEASE_VERSION || 'latest',

‎.github/actions/systemtests/validate-matrix/action.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: "Validate Matrix"
2+
description: "Validates generated matrix and returns feedback message"
3+
4+
inputs:
5+
matrix:
6+
description: "JSON matrix generated by generate-matrix action"
7+
required: true
8+
9+
outputs:
10+
isValid:
11+
description: "Whether the matrix is valid (not empty)"
12+
value: ${{ steps.validate.outputs.isValid }}
13+
message:
14+
description: "Feedback message about the validation result"
15+
value: ${{ steps.validate.outputs.message }}
16+
17+
runs:
18+
using: "composite"
19+
steps:
20+
- name: Validate Matrix
21+
id: validate
22+
uses: actions/github-script@v7
23+
with:
24+
script: |
25+
const matrix = JSON.parse(process.env.MATRIX || '[]');
26+
27+
// Check if matrix is empty
28+
const isEmpty = !matrix || matrix.length === 0;
29+
30+
if (isEmpty) {
31+
core.setOutput('isValid', 'false');
32+
33+
let errorMessage = "❌ **System test validation failed**\n";
34+
errorMessage += "\n";
35+
errorMessage += "The provided pipeline/profile names did not match any configured tests.\n";
36+
errorMessage += "\n";
37+
errorMessage += "**Available pipelines:**\n";
38+
errorMessage += "`acceptance`, `acceptance-helm`, `regression`, `regression-fg`, `regression-rbac`, `smoke`, `upgrade`\n";
39+
errorMessage += "\n";
40+
errorMessage += "**Available profiles:**\n";
41+
errorMessage += "`acceptance`, `azp_connect_mirrormaker`, `azp_dynconfig_listeners_tracing_watcher`, `azp_kafka_oauth`, `azp_kafka_upgrade`, `azp_kraft_upgrade`, `azp_operators`, `azp_rbac_remaining`, `azp_rolling_update_bridge`, `azp_security`, `brokers-and-security`, `operands`, `operators`, `smoke`\n";
42+
errorMessage += "\n";
43+
errorMessage += "Please check your command for typos and try again.";
44+
45+
core.setOutput('message', errorMessage);
46+
return;
47+
}
48+
49+
// Matrix is valid - create summary of what will run
50+
core.setOutput('isValid', 'true');
51+
core.info(`Matrix validation passed. ${matrix.length} job(s) will be executed.`);
52+
53+
const jobs = matrix.map(job => {
54+
const name = job.jobName || `${job.pipeline}-${job.profile || job.pipeline}-${job.arch}`;
55+
const agent = job.agent || 'default';
56+
const timeout = job.timeout || 'default';
57+
return `- **${name}** (${agent}, ${timeout}min)`;
58+
}).join('\n');
59+
60+
let successMessage = "✅ **System test validation passed**\n";
61+
successMessage += "\n";
62+
successMessage += `The following ${matrix.length} job(s) will be executed:\n`;
63+
successMessage += "\n";
64+
successMessage += jobs.replace(/\n/g, '\n') + "\n";
65+
successMessage += "\n";
66+
successMessage += "Tests will start after successful build completion.";
67+
68+
core.setOutput('message', successMessage);
69+
env:
70+
MATRIX: ${{ inputs.matrix }}

‎.github/docs/README.md‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,13 @@ Build process actions:
6262
- [build-strimzi-binaries](../actions/build/build-strimzi-binaries)
6363
- [containers-build](../actions/build/containers-build)
6464
- [containers-load](../actions/build/containers-load)
65-
- [containers-push-manifest](../actions/build/containers-push-manifest)
6665

6766
System tests execution actions:
6867
- [generate-matrix](../actions/systemtests/generate-matrix)
6968
- [set-defaults](../actions/utils/set-defaults)
7069
- [log-variables](../actions/utils/log-variables)
7170
- [parse-comment](../actions/systemtests/parse-comment)
72-
- [determine-ref](../actions/determine-ref)
71+
- [determine-ref](../actions/utils/determine-ref)
7372

7473
Utils actions:
7574
- [check-permissions](../actions/utils/check-permissions)
@@ -99,24 +98,24 @@ Comment for triggering the workflow has to starts with string `/gha run` and the
9998
The whole script that parse the trigger even is part of [parse-comment](../actions/systemtests/parse-comment) action.
10099
Currently, we have these parameters that can be passed through the comment:
101100

102-
| Name | Info | Default |
103-
|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------|
104-
| pipeline | Name of the pipeline from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that wil be executed | regression,upgrade |
105-
| profile | Testing profile from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that will be executed, it has to be defined in pom file | smoke |
106-
| agent | Agent that will be used for a specific pipeline (see list of runners in Strimzi org config for more info) | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
107-
| strimzi_feature_gates | Which Strimzi Feature Gates will be used | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
108-
| strimzi_rbac_scope | RBAC scope for Strimzi | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
109-
| cluster_operator_install_type | How Strimzi will be installed during the tests | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
110-
| parallel | Number of tests that will be executed in parallel | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
111-
| architecture | Which architecture will be used (should match with agent arch) | Value set in [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) |
112-
| groups | Which Junit5 groups will be executed | all |
113-
| tests | Which Junit5 tests will be executed | all |
114-
| kubeVersion | Used Kubernetes version as part of Kind/Minikube setup | The one set as default in setup scripts |
115-
| kafkaVersion | Which Kafka version will be used in the tests | Default one from STs config |
101+
| Name | Info | Default |
102+
|-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------|
103+
| pipeline | Name of the pipeline from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that wil be executed | regression,upgrade |
104+
| profile | Testing profile from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that will be executed, it has to be defined in pom file | smoke |
105+
| agent | Agent that will be used for a specific pipeline (see list of runners in Strimzi org config for more info) | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
106+
| strimzi_feature_gates | Which Strimzi Feature Gates will be used | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
107+
| strimzi_rbac_scope | RBAC scope for Strimzi | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
108+
| cluster_operator_install_type | How Strimzi will be installed during the tests | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
109+
| parallel | Number of tests that will be executed in parallel | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
110+
| architecture | Which architecture will be used (should match with agent arch) | Value set in [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) |
111+
| groups | Which Junit5 groups will be executed | all |
112+
| tests | Which Junit5 tests will be executed | all |
113+
| kubeVersion | Used Kubernetes version as part of Kind/Minikube setup | The one set as default in setup scripts |
114+
| kafkaVersion | Which Kafka version will be used in the tests | Default one from STs config |
116115

117116
The process of parameter usage is as follows:
118-
- `pipeline` has the highest priority. If `pipeline` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that match specific _pipeline_.
119-
- `profile` has the second-highest priority. If `profile` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/generate-matrix/pipelines.yaml) that match specific _profile_.
117+
- `pipeline` has the highest priority. If `pipeline` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that match specific _pipeline_.
118+
- `profile` has the second-highest priority. If `profile` is defined, the jobs will be loaded with data from [pipelines.yaml](../actions/systemtests/generate-matrix/pipelines.yaml) that match specific _profile_.
120119
- `agent` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment.
121120
- `strimzi_feature_gates` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment.
122121
- `strimzi_rbac_scope` is used only for a `custom` pipeline when either `pipeline` or `profile` are not specified via comment.

‎.github/tests/inputs/validate-matrix/empty_matrix.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎.github/tests/inputs/validate-matrix/null_matrix.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
null

‎.github/tests/inputs/validate-matrix/valid_custom.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
[
2+
{
3+
"agent": "oracle-4cpu-16gb-x86-64",
4+
"arch": "amd64",
5+
"pipeline": "custom",
6+
"profile": "all",
7+
"timeout": 720,
8+
"strimzi_feature_gates": "",
9+
"strimzi_rbac_scope": "CLUSTER",
10+
"cluster_operator_install_type": "bundle",
11+
"parallel": "1",
12+
"groups": "",
13+
"tests": "",
14+
"jobName": "custom-all-amd64"
15+
},
16+
{
17+
"agent": "oracle-4cpu-16gb-arm64",
18+
"arch": "arm64",
19+
"pipeline": "custom",
20+
"profile": "all",
21+
"timeout": 720,
22+
"strimzi_feature_gates": "",
23+
"strimzi_rbac_scope": "CLUSTER",
24+
"cluster_operator_install_type": "bundle",
25+
"parallel": "1",
26+
"groups": "",
27+
"tests": "",
28+
"jobName": "custom-all-arm64"
29+
}
30+
]

‎.github/tests/inputs/validate-matrix/valid_multiple.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
[
2+
{
3+
"agent": "oracle-vm-8cpu-32gb-x86-64",
4+
"arch": "amd64",
5+
"pipeline": "regression",
6+
"profile": "brokers-and-security",
7+
"timeout": 480,
8+
"strimzi_feature_gates": "false",
9+
"strimzi_rbac_scope": "CLUSTER",
10+
"cluster_operator_install_type": "yaml",
11+
"parallel": 4,
12+
"jobName": "regression-brokers-and-security-amd64"
13+
},
14+
{
15+
"agent": "oracle-vm-8cpu-32gb-x86-64",
16+
"arch": "amd64",
17+
"pipeline": "regression",
18+
"profile": "operators",
19+
"timeout": 480,
20+
"strimzi_feature_gates": "false",
21+
"strimzi_rbac_scope": "CLUSTER",
22+
"cluster_operator_install_type": "yaml",
23+
"parallel": 4,
24+
"jobName": "regression-operators-amd64"
25+
},
26+
{
27+
"agent": "oracle-vm-2cpu-8gb-x86-64",
28+
"arch": "amd64",
29+
"pipeline": "smoke",
30+
"profile": "smoke",
31+
"timeout": 60,
32+
"strimzi_feature_gates": "false",
33+
"strimzi_rbac_scope": "CLUSTER",
34+
"cluster_operator_install_type": "yaml",
35+
"parallel": 1,
36+
"jobName": "smoke-amd64"
37+
}
38+
]

‎.github/tests/inputs/validate-matrix/valid_single.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
[
2+
{
3+
"agent": "oracle-vm-2cpu-8gb-x86-64",
4+
"arch": "amd64",
5+
"pipeline": "smoke",
6+
"profile": "smoke",
7+
"timeout": 60,
8+
"strimzi_feature_gates": "false",
9+
"strimzi_rbac_scope": "CLUSTER",
10+
"cluster_operator_install_type": "yaml",
11+
"parallel": 1,
12+
"jobName": "smoke-amd64"
13+
}
14+
]

‎.github/tests/scenarios/validate-matrix.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
scenarios:
2+
- id: valid-multiple-jobs
3+
description: "Validate matrix with multiple valid jobs"
4+
event: push
5+
matrix_file: .github/tests/inputs/validate-matrix/valid_multiple.json
6+
expectations:
7+
isValid: "true"
8+
message_contains: "will be executed"
9+
10+
- id: valid-single-job
11+
description: "Validate matrix with single valid job"
12+
event: push
13+
matrix_file: .github/tests/inputs/validate-matrix/valid_single.json
14+
expectations:
15+
isValid: "true"
16+
message_contains: "will be executed"
17+
18+
- id: empty-matrix
19+
description: "Validate empty matrix (should fail validation)"
20+
event: push
21+
matrix_file: .github/tests/inputs/validate-matrix/empty_matrix.json
22+
expectations:
23+
isValid: "false"
24+
message_contains: "❌ **System test validation failed**"
25+
26+
- id: null-matrix
27+
description: "Validate null matrix input"
28+
event: push
29+
matrix_file: .github/tests/inputs/validate-matrix/null_matrix.json
30+
expectations:
31+
isValid: "false"
32+
message_contains: "❌ **System test validation failed**"
33+
34+
- id: valid-with-custom-jobs
35+
description: "Validate matrix with custom job configurations"
36+
event: push
37+
matrix_file: .github/tests/inputs/validate-matrix/valid_custom.json
38+
expectations:
39+
isValid: "true"
40+
message_contains: "will be executed"

0 commit comments

Comments
 (0)