From 0aed6ec4a03d188ed2ab35061fa9169584a3aed4 Mon Sep 17 00:00:00 2001 From: Jesse Simpson Date: Mon, 14 Apr 2025 14:57:47 -0400 Subject: [PATCH 1/7] test: test minor version upgrade --- .../actions/generate-chart-matrix/action.yml | 8 ++++-- .github/workflows/chart-release.yaml | 2 +- .../test-chart-version-template.yaml | 25 +++++++++++++++++++ .github/workflows/test-chart-version.yaml | 5 ++-- charts/camunda-platform-8.3/Chart.yaml | 1 + charts/camunda-platform-8.4/Chart.yaml | 1 + charts/camunda-platform-8.5/Chart.yaml | 1 + charts/camunda-platform-8.6/Chart.yaml | 1 + charts/camunda-platform-8.7/Chart.yaml | 1 + 9 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/actions/generate-chart-matrix/action.yml b/.github/actions/generate-chart-matrix/action.yml index 24c4687ea6..a42f9b54c3 100644 --- a/.github/actions/generate-chart-matrix/action.yml +++ b/.github/actions/generate-chart-matrix/action.yml @@ -38,9 +38,13 @@ runs: if [[ $(echo ${ALL_MODIFIED_FILES} | grep "${chart_dir}") ]]; then camunda_version="$(basename ${chart_dir} | sed 's/camunda-platform-//')"; echo "Camunda version: ${camunda_version}" - echo "${camunda_version}" >> matrix_versions.txt; + chart_version="$(cat ${chart_dir}/Chart.yaml | grep '^version:' | head -n 1 | sed 's/version: //')"; + chart_previous_camunda_version="$(echo ${camunda_version} | jq '. | (((.|tonumber) - .1)*10|round/10)')" + chart_previous_helm_version="$(cat charts/camunda-platform-${chart_previous_camunda_version}/Chart.yaml | grep '^version:' | head -n 1 | sed 's/version: //')" + echo "${camunda_version} ${chart_version} ${chart_previous_helm_version}" >> matrix_versions.txt; fi done - matrix="$(cat matrix_versions.txt | jq --slurp --compact-output --raw-input 'split("\n") | .[:-1]')" + matrix="$(cat matrix_versions.txt | jq --slurp --compact-output --raw-input '[ split("\n") | .[] | split(" ") | { version: .[0], chartVersion: .[1], previousHelmVersion: .[2] } | select(.version != null)] ')" + echo "matrix=${matrix}" | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/chart-release.yaml b/.github/workflows/chart-release.yaml index 9591a0da48..e1c97be523 100644 --- a/.github/workflows/chart-release.yaml +++ b/.github/workflows/chart-release.yaml @@ -52,7 +52,7 @@ jobs: # Create a tmp file and ensure it's empty. echo "" > "${matrix_file}" - echo '${{ steps.generate-chart-versions.outputs.matrix }}' | jq -cr '.[]' | while read -r dir_id; do + echo '${{ steps.generate-chart-versions.outputs.matrix }}' | jq -cr '.[].version' | while read -r dir_id; do chart_file="charts/camunda-platform-${dir_id}/Chart.yaml" # Extract version info. diff --git a/.github/workflows/test-chart-version-template.yaml b/.github/workflows/test-chart-version-template.yaml index b2191d6962..79d5cd8c9d 100644 --- a/.github/workflows/test-chart-version-template.yaml +++ b/.github/workflows/test-chart-version-template.yaml @@ -8,6 +8,10 @@ on: description: Chart version to test required: true type: string + previous-helm-version: + description: Previous helm version to upgrade from in minor upgrade test + required: false + type: string pr_number: description: Pull Request number. Required due to a github bug that is not giving the concurrency group the pull_request number from the github event required: true @@ -57,3 +61,24 @@ jobs: camunda-helm-dir: "camunda-platform-${{ inputs.chart-version }}" camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}" caller-git-ref: "${{ github.event.pull_request.head.sha }}" + + minor-upgrade-integration: + name: Camunda ${{ inputs.chart-version }} - Minor upgrade Integration Test + needs: [validation, unit] + permissions: + contents: read + id-token: write + deployments: write + secrets: inherit + uses: ./.github/workflows/test-integration-template.yaml + with: + identifier: "${{ github.event.pull_request.number }}-minr-${{ inputs.chart-version }}" + deployment-ttl: "${{ contains(github.event.pull_request.labels.*.name, 'test-persistent') && '1w' || '' }}" + platforms: "gke" + flows: "upgrade" + camunda-helm-dir: "camunda-platform-${{ inputs.chart-version }}" + camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}" + # TODO: find a way to auto bump + camunda-helm-upgrade-version: "${{ inputs.previous-helm-version }}" + caller-git-ref: "${{ github.event.pull_request.head.sha }}" + diff --git a/.github/workflows/test-chart-version.yaml b/.github/workflows/test-chart-version.yaml index b9d7c83537..b2b50964e2 100644 --- a/.github/workflows/test-chart-version.yaml +++ b/.github/workflows/test-chart-version.yaml @@ -42,10 +42,11 @@ jobs: needs: init strategy: fail-fast: false - matrix: - version: ${{ fromJson(needs.init.outputs.matrix) }} + matrix: + include: ${{ fromJson(needs.init.outputs.matrix) }} uses: ./.github/workflows/test-chart-version-template.yaml secrets: inherit with: chart-version: "${{ matrix.version }}" + previous-helm-version: "${{ matrix.previousHelmVersion }}" pr_number: "${{ github.event.pull_request.number }}" # Required as there is a github bug that prevent the concurrency group on the sub workflow getting the pull_request number from the github event diff --git a/charts/camunda-platform-8.3/Chart.yaml b/charts/camunda-platform-8.3/Chart.yaml index c84cd53ef9..e00bedc321 100644 --- a/charts/camunda-platform-8.3/Chart.yaml +++ b/charts/camunda-platform-8.3/Chart.yaml @@ -70,3 +70,4 @@ annotations: - kind: fixed description: "Changing helper function for identityURL" camunda.io/helmCLIVersion: 3.17.2 + test: test diff --git a/charts/camunda-platform-8.4/Chart.yaml b/charts/camunda-platform-8.4/Chart.yaml index 90a358ad39..d12372bf06 100644 --- a/charts/camunda-platform-8.4/Chart.yaml +++ b/charts/camunda-platform-8.4/Chart.yaml @@ -72,3 +72,4 @@ annotations: - kind: fixed description: "Changing helper function for identityURL" camunda.io/helmCLIVersion: 3.17.2 + test: test diff --git a/charts/camunda-platform-8.5/Chart.yaml b/charts/camunda-platform-8.5/Chart.yaml index a4215a9c7a..ff0fc75597 100644 --- a/charts/camunda-platform-8.5/Chart.yaml +++ b/charts/camunda-platform-8.5/Chart.yaml @@ -72,3 +72,4 @@ annotations: description: "Add global.extraManifests support for injecting arbitrary YAML" - kind: fixed description: "Changing helper function for identityURL" + test: test diff --git a/charts/camunda-platform-8.6/Chart.yaml b/charts/camunda-platform-8.6/Chart.yaml index d7ba6d1700..89ef4ae9e0 100644 --- a/charts/camunda-platform-8.6/Chart.yaml +++ b/charts/camunda-platform-8.6/Chart.yaml @@ -80,3 +80,4 @@ annotations: description: "Refactored unit tests for identity and operate in 8.6" - kind: fixed description: "Refactor unit tests in 8.6/test/unit/console to new style" + test: test diff --git a/charts/camunda-platform-8.7/Chart.yaml b/charts/camunda-platform-8.7/Chart.yaml index 21451a6d6b..f7803ba8ce 100644 --- a/charts/camunda-platform-8.7/Chart.yaml +++ b/charts/camunda-platform-8.7/Chart.yaml @@ -96,3 +96,4 @@ annotations: description: "Refactored zeebe gateway unit tests to new style" - kind: fixed description: "Refactor of connector unit tests to new style in 8.7" + test: test From 3be0a537a95146d7fc993c7fba33d7e067c2b502 Mon Sep 17 00:00:00 2001 From: Jesse Simpson Date: Fri, 18 Apr 2025 10:47:24 -0400 Subject: [PATCH 2/7] refactor: change jq expression to golang script --- .../actions/generate-chart-matrix/action.yml | 11 +--- scripts/generate-gha-matrix/Makefile | 11 ++++ scripts/generate-gha-matrix/errors.go | 25 ++++++++ .../generate-gha-matrix.go | 27 +++++++++ scripts/generate-gha-matrix/go.mod | 15 +++++ scripts/generate-gha-matrix/go.sum | 20 +++++++ scripts/generate-gha-matrix/inputs.go | 18 ++++++ scripts/generate-gha-matrix/process.go | 30 ++++++++++ scripts/generate-gha-matrix/process_test.go | 30 ++++++++++ scripts/generate-gha-matrix/types.go | 11 ++++ scripts/generate-gha-matrix/versions.go | 38 ++++++++++++ scripts/generate-gha-matrix/versions_test.go | 60 +++++++++++++++++++ 12 files changed, 288 insertions(+), 8 deletions(-) create mode 100644 scripts/generate-gha-matrix/Makefile create mode 100644 scripts/generate-gha-matrix/errors.go create mode 100644 scripts/generate-gha-matrix/generate-gha-matrix.go create mode 100644 scripts/generate-gha-matrix/go.mod create mode 100644 scripts/generate-gha-matrix/go.sum create mode 100644 scripts/generate-gha-matrix/inputs.go create mode 100644 scripts/generate-gha-matrix/process.go create mode 100644 scripts/generate-gha-matrix/process_test.go create mode 100644 scripts/generate-gha-matrix/types.go create mode 100644 scripts/generate-gha-matrix/versions.go create mode 100644 scripts/generate-gha-matrix/versions_test.go diff --git a/.github/actions/generate-chart-matrix/action.yml b/.github/actions/generate-chart-matrix/action.yml index a42f9b54c3..51ea697ea3 100644 --- a/.github/actions/generate-chart-matrix/action.yml +++ b/.github/actions/generate-chart-matrix/action.yml @@ -36,15 +36,10 @@ runs: ls -d ${{ inputs.charts-path }} | while read chart_dir; do if [[ $(echo ${ALL_MODIFIED_FILES} | grep "${chart_dir}") ]]; then - camunda_version="$(basename ${chart_dir} | sed 's/camunda-platform-//')"; - echo "Camunda version: ${camunda_version}" - chart_version="$(cat ${chart_dir}/Chart.yaml | grep '^version:' | head -n 1 | sed 's/version: //')"; - chart_previous_camunda_version="$(echo ${camunda_version} | jq '. | (((.|tonumber) - .1)*10|round/10)')" - chart_previous_helm_version="$(cat charts/camunda-platform-${chart_previous_camunda_version}/Chart.yaml | grep '^version:' | head -n 1 | sed 's/version: //')" - echo "${camunda_version} ${chart_version} ${chart_previous_helm_version}" >> matrix_versions.txt; + echo ${chart_dir} >> matrix_versions.txt fi done - matrix="$(cat matrix_versions.txt | jq --slurp --compact-output --raw-input '[ split("\n") | .[] | split(" ") | { version: .[0], chartVersion: .[1], previousHelmVersion: .[2] } | select(.version != null)] ')" - + cd scripts/generate-gha-matrix/ + matrix="$(cat ../../matrix_versions.txt | go run .)" echo "matrix=${matrix}" | tee -a $GITHUB_OUTPUT diff --git a/scripts/generate-gha-matrix/Makefile b/scripts/generate-gha-matrix/Makefile new file mode 100644 index 0000000000..5ab1007d57 --- /dev/null +++ b/scripts/generate-gha-matrix/Makefile @@ -0,0 +1,11 @@ + +PHONY: test + +build: + go build + +run: + go run . + +test: + go test *.go diff --git a/scripts/generate-gha-matrix/errors.go b/scripts/generate-gha-matrix/errors.go new file mode 100644 index 0000000000..bd94ec99e7 --- /dev/null +++ b/scripts/generate-gha-matrix/errors.go @@ -0,0 +1,25 @@ +package main + +import ( + "fmt" +) + +type ProcessError string + +func (e ProcessError) Error() string { + return "process: " + string(e) +} + +func ProcessErrorf(msg string, args ...interface{}) error { + return ProcessError(fmt.Sprintf(msg, args...)) +} + +type VersionParsingError string + +func (e VersionParsingError) Error() string { + return "versions: " + string(e) +} + +func VersionParsingErrorf(msg string, args ...interface{}) error { + return VersionParsingError(fmt.Sprintf(msg, args...)) +} diff --git a/scripts/generate-gha-matrix/generate-gha-matrix.go b/scripts/generate-gha-matrix/generate-gha-matrix.go new file mode 100644 index 0000000000..a4d6514f67 --- /dev/null +++ b/scripts/generate-gha-matrix/generate-gha-matrix.go @@ -0,0 +1,27 @@ +package main + +import ( + "encoding/json" + "fmt" +) + +/* +This script accepts input piped into it and outputs a github-actions readable +matrix of tests to run. + +ex: echo charts/camunda-platform-8.3 charts/camunda-platform-8.4 charts/camunda-platform-8.8 | go run . | jq +*/ +func main() { + input := parseInput() + + output, err := processInputs(input) + if err != nil { + panic(err) + } + + final, err := json.Marshal(output) + if err != nil { + panic(err) + } + fmt.Println(string(final)) +} diff --git a/scripts/generate-gha-matrix/go.mod b/scripts/generate-gha-matrix/go.mod new file mode 100644 index 0000000000..671476455b --- /dev/null +++ b/scripts/generate-gha-matrix/go.mod @@ -0,0 +1,15 @@ +module github.com/camunda/camunda-platform-helm/scripts/generate-gha-matrix + +go 1.23.7 + +require ( + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/stretchr/testify v1.10.0 // indirect + golang.org/x/mod v0.24.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + helm.sh/helm/v3 v3.17.3 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) diff --git a/scripts/generate-gha-matrix/go.sum b/scripts/generate-gha-matrix/go.sum new file mode 100644 index 0000000000..e2e61d7f5b --- /dev/null +++ b/scripts/generate-gha-matrix/go.sum @@ -0,0 +1,20 @@ +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +helm.sh/helm/v3 v3.17.3 h1:3n5rW3D0ArjFl0p4/oWO8IbY/HKaNNwJtOQFdH2AZHg= +helm.sh/helm/v3 v3.17.3/go.mod h1:+uJKMH/UiMzZQOALR3XUf3BLIoczI2RKKD6bMhPh4G8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/scripts/generate-gha-matrix/inputs.go b/scripts/generate-gha-matrix/inputs.go new file mode 100644 index 0000000000..d97aaa535d --- /dev/null +++ b/scripts/generate-gha-matrix/inputs.go @@ -0,0 +1,18 @@ +package main + +import ( + "bufio" + "os" +) + +func parseInput() Input { + stdinReader := bufio.NewReader(os.Stdin) + scanner := bufio.NewScanner(stdinReader) + scanner.Split(bufio.ScanWords) + + var input Input + for scanner.Scan() { + input.HelmChartModifiedDirectories = append(input.HelmChartModifiedDirectories, scanner.Text()) + } + return input +} diff --git a/scripts/generate-gha-matrix/process.go b/scripts/generate-gha-matrix/process.go new file mode 100644 index 0000000000..cf60b65a71 --- /dev/null +++ b/scripts/generate-gha-matrix/process.go @@ -0,0 +1,30 @@ +package main + +import ( + "helm.sh/helm/v3/pkg/chart/loader" +) + +func processInputs(input Input) ([]Output, error) { + var output []Output + for _, modifiedDir := range input.HelmChartModifiedDirectories { + chart, err := loader.Load("../../" + modifiedDir) + if err != nil { + return nil, ProcessErrorf("failed to load chart for modified dir %s : %s", modifiedDir, err) + } + + version := getCamundaVersion(chart) + previousVersionMajor, err := getPreviousHelmChartVersion(chart, version) + if err != nil { + return nil, ProcessErrorf("failed to get previous helm chart metadata for version %s : %s", version, err) + } + + matrixRunVector := Output{ + CamundaVersion: version, + HelmChartVersion: chart.Metadata.Version, + PreviousHelmChartVersion: previousVersionMajor, + } + + output = append(output, matrixRunVector) + } + return output, nil +} diff --git a/scripts/generate-gha-matrix/process_test.go b/scripts/generate-gha-matrix/process_test.go new file mode 100644 index 0000000000..8504ffaef9 --- /dev/null +++ b/scripts/generate-gha-matrix/process_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestProcess(t *testing.T) { + inputs := Input{ + HelmChartModifiedDirectories: []string{"charts/camunda-platform-8.8"}, + } + + outputs, err := processInputs(inputs) + require.NoError(t, err) + + require.Len(t, outputs, 1) + require.Contains(t, outputs[0].CamundaVersion, "8.8") + require.Contains(t, outputs[0].HelmChartVersion, "13") + require.Contains(t, outputs[0].PreviousHelmChartVersion, "12") +} + +func TestProcessInvalidChartDir(t *testing.T) { + inputs := Input{ + HelmChartModifiedDirectories: []string{"charts/invalid"}, + } + + outputs, err := processInputs(inputs) + require.NotNil(t, err) + require.Nil(t, outputs) +} diff --git a/scripts/generate-gha-matrix/types.go b/scripts/generate-gha-matrix/types.go new file mode 100644 index 0000000000..58f15a023a --- /dev/null +++ b/scripts/generate-gha-matrix/types.go @@ -0,0 +1,11 @@ +package main + +type Input struct { + HelmChartModifiedDirectories []string +} + +type Output struct { + CamundaVersion string `json:"version"` + HelmChartVersion string `json:"chartVersion"` + PreviousHelmChartVersion string `json:"previousHelmVersion"` +} diff --git a/scripts/generate-gha-matrix/versions.go b/scripts/generate-gha-matrix/versions.go new file mode 100644 index 0000000000..a296b82e05 --- /dev/null +++ b/scripts/generate-gha-matrix/versions.go @@ -0,0 +1,38 @@ +package main + +import ( + "golang.org/x/mod/semver" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "strconv" + "strings" +) + +func getCamundaVersion(chart *chart.Chart) string { + var version string + if chart.Values["zeebe"] == nil { + version = "v" + chart.Values["core"].(map[string]interface{})["image"].(map[string]interface{})["tag"].(string) + } else { + version = "v" + chart.Values["zeebe"].(map[string]interface{})["image"].(map[string]interface{})["tag"].(string) + } + camundaVersionParsed := semver.MajorMinor(version) + camundaVersionParsed = strings.TrimPrefix(camundaVersionParsed, "v") + return camundaVersionParsed +} + +func getPreviousHelmChartVersion(chart *chart.Chart, version string) (string, error) { + camundaVersionFloat, err := strconv.ParseFloat(version, 64) + if err != nil { + return "", VersionParsingErrorf("failed to parse version from input: %s", err) + } + previousVersionFloat := (camundaVersionFloat*10 - 1) / 10 + previousCamundaVersion := strconv.FormatFloat(previousVersionFloat, 'f', -1, 64) + previousChart, err := loader.Load("../../charts/camunda-platform-" + previousCamundaVersion) + if err != nil { + return "", VersionParsingErrorf("failed to load the previous chart: %s", err) + } + previousChartVersionSemver := "v" + previousChart.Metadata.Version + previousVersionMajor := semver.Major(previousChartVersionSemver) + previousVersionMajor = strings.TrimPrefix(previousVersionMajor, "v") + return previousVersionMajor, nil +} diff --git a/scripts/generate-gha-matrix/versions_test.go b/scripts/generate-gha-matrix/versions_test.go new file mode 100644 index 0000000000..2dc4569c45 --- /dev/null +++ b/scripts/generate-gha-matrix/versions_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "github.com/stretchr/testify/require" + "helm.sh/helm/v3/pkg/chart/loader" + "regexp" + "testing" +) + +var validVersionRegex = regexp.MustCompile(`^\d+\.\d+$`) + +func TestGetCamundaVersionPre88(t *testing.T) { + chart, err := loader.Load("../../charts/camunda-platform-8.6") + require.NoError(t, err) + + version := getCamundaVersion(chart) + valid := validVersionRegex.MatchString(version) + + require.True(t, valid) +} + +func TestGetCamundaVersionPost88(t *testing.T) { + chart, err := loader.Load("../../charts/camunda-platform-8.8") + require.NoError(t, err) + + version := getCamundaVersion(chart) + valid := validVersionRegex.MatchString(version) + + require.True(t, valid) +} + +func TestGetPreviousChart(t *testing.T) { + chart, err := loader.Load("../../charts/camunda-platform-8.8") + require.NoError(t, err) + + version := getCamundaVersion(chart) + previousVersion, err := getPreviousHelmChartVersion(chart, version) + require.NoError(t, err) + require.Equal(t, "12", previousVersion) +} + +func TestGetPreviousChartInvalidVersion(t *testing.T) { + chart, err := loader.Load("../../charts/camunda-platform-8.8") + require.NoError(t, err) + + version := "badversion" + _, err = getPreviousHelmChartVersion(chart, version) + require.NotNil(t, err) + require.Contains(t, err.Error(), "invalid syntax") +} + +func TestGetPreviousChartChartTooOld(t *testing.T) { + chart, err := loader.Load("../../charts/camunda-platform-8.2") + require.NoError(t, err) + + version := getCamundaVersion(chart) + _, err = getPreviousHelmChartVersion(chart, version) + require.NotNil(t, err) + require.Contains(t, err.Error(), "failed to load the previous chart") +} From 18b94e451509f8ac7b8afba0f3aeb5a37ea79c04 Mon Sep 17 00:00:00 2001 From: Jesse Simpson Date: Fri, 18 Apr 2025 14:40:30 -0400 Subject: [PATCH 3/7] fix: run --force on upgrades from 8.4 -> 8.5 --- .../workflows/test-chart-version-template.yaml | 1 - .../scenarios/lib/chart-upgrade-taskfile.yaml | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-chart-version-template.yaml b/.github/workflows/test-chart-version-template.yaml index 79d5cd8c9d..485842935d 100644 --- a/.github/workflows/test-chart-version-template.yaml +++ b/.github/workflows/test-chart-version-template.yaml @@ -78,7 +78,6 @@ jobs: flows: "upgrade" camunda-helm-dir: "camunda-platform-${{ inputs.chart-version }}" camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}" - # TODO: find a way to auto bump camunda-helm-upgrade-version: "${{ inputs.previous-helm-version }}" caller-git-ref: "${{ github.event.pull_request.head.sha }}" diff --git a/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml b/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml index a7581c3374..e574eeb32e 100644 --- a/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml +++ b/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml @@ -20,8 +20,23 @@ tasks: exec: cmds: - | + set +e -x + # if the upgrade is from 8.4 -> 8.5, we need to add --force + FORCE="" + helm history -n {{ .TEST_NAMESPACE }} -o json integration | jq '.[].app_version' | grep 8.4 + if [ $? -eq 0 ]; then + helm show chart {{ .TEST_CHART_NAME }} | yq .appVersion | grep 8.5 + if [ $? -eq 0 ]; then + kubectl delete deployment -n {{ .TEST_NAMESPACE }} integration-identity + kubectl wait -n {{ .TEST_NAMESPACE }} --for='delete' deployment/integration-identity + FORCE="--force --install" + fi + fi + set -e +x + helm upgrade integration {{ .TEST_CHART_NAME }} \ --namespace {{ .TEST_NAMESPACE }} \ --values {{ .TEST_VALUES_BASE_DIR }}/common/values-integration-test.yaml \ --timeout 20m0s \ + $FORCE \ --wait {{ .TEST_HELM_EXTRA_ARGS }} From 61414c9461ebb6fd37f95202e4f2d00d30d12b01 Mon Sep 17 00:00:00 2001 From: Jesse Simpson Date: Fri, 18 Apr 2025 15:33:03 -0400 Subject: [PATCH 4/7] fix: at the moment I only want to test 8.5 --- charts/camunda-platform-8.3/Chart.yaml | 1 - charts/camunda-platform-8.4/Chart.yaml | 1 - charts/camunda-platform-8.6/Chart.yaml | 1 - charts/camunda-platform-8.7/Chart.yaml | 1 - 4 files changed, 4 deletions(-) diff --git a/charts/camunda-platform-8.3/Chart.yaml b/charts/camunda-platform-8.3/Chart.yaml index e00bedc321..c84cd53ef9 100644 --- a/charts/camunda-platform-8.3/Chart.yaml +++ b/charts/camunda-platform-8.3/Chart.yaml @@ -70,4 +70,3 @@ annotations: - kind: fixed description: "Changing helper function for identityURL" camunda.io/helmCLIVersion: 3.17.2 - test: test diff --git a/charts/camunda-platform-8.4/Chart.yaml b/charts/camunda-platform-8.4/Chart.yaml index d12372bf06..90a358ad39 100644 --- a/charts/camunda-platform-8.4/Chart.yaml +++ b/charts/camunda-platform-8.4/Chart.yaml @@ -72,4 +72,3 @@ annotations: - kind: fixed description: "Changing helper function for identityURL" camunda.io/helmCLIVersion: 3.17.2 - test: test diff --git a/charts/camunda-platform-8.6/Chart.yaml b/charts/camunda-platform-8.6/Chart.yaml index 89ef4ae9e0..d7ba6d1700 100644 --- a/charts/camunda-platform-8.6/Chart.yaml +++ b/charts/camunda-platform-8.6/Chart.yaml @@ -80,4 +80,3 @@ annotations: description: "Refactored unit tests for identity and operate in 8.6" - kind: fixed description: "Refactor unit tests in 8.6/test/unit/console to new style" - test: test diff --git a/charts/camunda-platform-8.7/Chart.yaml b/charts/camunda-platform-8.7/Chart.yaml index f7803ba8ce..21451a6d6b 100644 --- a/charts/camunda-platform-8.7/Chart.yaml +++ b/charts/camunda-platform-8.7/Chart.yaml @@ -96,4 +96,3 @@ annotations: description: "Refactored zeebe gateway unit tests to new style" - kind: fixed description: "Refactor of connector unit tests to new style in 8.7" - test: test From 133ff88934c867798b5885424019f5ecc32740bf Mon Sep 17 00:00:00 2001 From: Jesse Simpson Date: Mon, 21 Apr 2025 14:12:46 -0400 Subject: [PATCH 5/7] fix: use values.yaml for previous version during minor version upgrade --- .../test-chart-version-template.yaml | 5 +++++ .github/workflows/test-chart-version.yaml | 1 + .../workflows/test-integration-template.yaml | 19 ++++++++++++++++--- scripts/generate-gha-matrix/process.go | 5 +++-- scripts/generate-gha-matrix/types.go | 6 ++++++ scripts/generate-gha-matrix/versions.go | 14 +++++++++----- scripts/generate-gha-matrix/versions_test.go | 3 ++- 7 files changed, 42 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-chart-version-template.yaml b/.github/workflows/test-chart-version-template.yaml index 485842935d..80eccda6c4 100644 --- a/.github/workflows/test-chart-version-template.yaml +++ b/.github/workflows/test-chart-version-template.yaml @@ -16,6 +16,10 @@ on: description: Pull Request number. Required due to a github bug that is not giving the concurrency group the pull_request number from the github event required: true type: string + previous-helm-dir: + description: base directory of the previous helm version to upgrade from for references to test values.yamls + required: false + type: string concurrency: group: ${{ github.workflow }}-${{ inputs.pr_number }}-${{ inputs.chart-version }} @@ -79,5 +83,6 @@ jobs: camunda-helm-dir: "camunda-platform-${{ inputs.chart-version }}" camunda-helm-git-ref: "${{ github.event.pull_request.head.sha }}" camunda-helm-upgrade-version: "${{ inputs.previous-helm-version }}" + camunda-upgrade-helm-dir: "${{ inputs.previous-helm-dir }}" caller-git-ref: "${{ github.event.pull_request.head.sha }}" diff --git a/.github/workflows/test-chart-version.yaml b/.github/workflows/test-chart-version.yaml index b2b50964e2..4ae03a10eb 100644 --- a/.github/workflows/test-chart-version.yaml +++ b/.github/workflows/test-chart-version.yaml @@ -50,3 +50,4 @@ jobs: chart-version: "${{ matrix.version }}" previous-helm-version: "${{ matrix.previousHelmVersion }}" pr_number: "${{ github.event.pull_request.number }}" # Required as there is a github bug that prevent the concurrency group on the sub workflow getting the pull_request number from the github event + previous-helm-dir: "${{ matrix.previousHelmDir }}" diff --git a/.github/workflows/test-integration-template.yaml b/.github/workflows/test-integration-template.yaml index a258cf7e4d..923bc7f10c 100644 --- a/.github/workflows/test-integration-template.yaml +++ b/.github/workflows/test-integration-template.yaml @@ -17,6 +17,12 @@ on: required: false default: camunda-platform-8.8 type: string + # directory to get values.yaml from during minor upgrade flow + camunda-upgrade-helm-dir: + required: false + description: During upgrade, the path towards the chart base dir that contains values.yamls + default: "" + type: string camunda-helm-git-ref: required: false default: main @@ -240,12 +246,14 @@ jobs: id: test-type-vars uses: ./.github/actions/test-type-vars with: - chart-dir: "${{ inputs.camunda-helm-dir }}" + chart-dir: "${{ inputs.camunda-upgrade-helm-dir || inputs.camunda-helm-dir }}" - name: Add Helm repos and dependencies run: | - export chartPath="charts/${{ inputs.camunda-helm-dir }}" + export chartPath="charts/$CHART_PATH" make helm.repos-add make helm.dependency-update + env: + CHART_PATH: ${{ inputs.camunda-upgrade-helm-dir || inputs.camunda-helm-dir }} - name: Create test namespace run: | echo $TEST_NAMESPACE @@ -332,6 +340,11 @@ jobs: if: matrix.scenario.flow == 'upgrade' run: | task -d ${CI_TASKS_BASE_DIR}/chart-full-setup upgrade.pre + - name: Set test type vars + id: test-type-vars-2 + uses: ./.github/actions/test-type-vars + with: + chart-dir: "${{ inputs.camunda-helm-dir }}" - name: 🌟 Upgrade Camunda chart 🌟 if: matrix.scenario.flow == 'upgrade' env: @@ -339,7 +352,7 @@ jobs: TEST_HELM_EXTRA_ARGS: >- ${{ env.TEST_HELM_EXTRA_ARGS_UPGRADE }} --set global.ingress.host=${{ steps.vars.outputs.ingress-host }} - --values ${{ steps.test-type-vars.outputs.valuesBaseDir }}/infra/values-infra-${{ inputs.infra-type }}.yaml + --values ${{ steps.test-type-vars-2.outputs.valuesBaseDir }}/infra/values-infra-${{ inputs.infra-type }}.yaml --values /tmp/extra-values-file.yaml run: | task -d ${CI_TASKS_BASE_DIR}/chart-full-setup upgrade.exec diff --git a/scripts/generate-gha-matrix/process.go b/scripts/generate-gha-matrix/process.go index cf60b65a71..3f6b00517c 100644 --- a/scripts/generate-gha-matrix/process.go +++ b/scripts/generate-gha-matrix/process.go @@ -13,7 +13,7 @@ func processInputs(input Input) ([]Output, error) { } version := getCamundaVersion(chart) - previousVersionMajor, err := getPreviousHelmChartVersion(chart, version) + previousVersion, err := getPreviousHelmChartVersion(chart, version) if err != nil { return nil, ProcessErrorf("failed to get previous helm chart metadata for version %s : %s", version, err) } @@ -21,7 +21,8 @@ func processInputs(input Input) ([]Output, error) { matrixRunVector := Output{ CamundaVersion: version, HelmChartVersion: chart.Metadata.Version, - PreviousHelmChartVersion: previousVersionMajor, + PreviousHelmChartVersion: previousVersion.Version, + PreviousHelmChartDir: previousVersion.Dir, } output = append(output, matrixRunVector) diff --git a/scripts/generate-gha-matrix/types.go b/scripts/generate-gha-matrix/types.go index 58f15a023a..0a1a702e0e 100644 --- a/scripts/generate-gha-matrix/types.go +++ b/scripts/generate-gha-matrix/types.go @@ -8,4 +8,10 @@ type Output struct { CamundaVersion string `json:"version"` HelmChartVersion string `json:"chartVersion"` PreviousHelmChartVersion string `json:"previousHelmVersion"` + PreviousHelmChartDir string `json:"previousHelmDir"` +} + +type ChartVersion struct { + Version string + Dir string } diff --git a/scripts/generate-gha-matrix/versions.go b/scripts/generate-gha-matrix/versions.go index a296b82e05..87fd7930d6 100644 --- a/scripts/generate-gha-matrix/versions.go +++ b/scripts/generate-gha-matrix/versions.go @@ -20,19 +20,23 @@ func getCamundaVersion(chart *chart.Chart) string { return camundaVersionParsed } -func getPreviousHelmChartVersion(chart *chart.Chart, version string) (string, error) { +func getPreviousHelmChartVersion(chart *chart.Chart, version string) (*ChartVersion, error) { camundaVersionFloat, err := strconv.ParseFloat(version, 64) if err != nil { - return "", VersionParsingErrorf("failed to parse version from input: %s", err) + return nil, VersionParsingErrorf("failed to parse version from input: %s", err) } previousVersionFloat := (camundaVersionFloat*10 - 1) / 10 previousCamundaVersion := strconv.FormatFloat(previousVersionFloat, 'f', -1, 64) - previousChart, err := loader.Load("../../charts/camunda-platform-" + previousCamundaVersion) + previousChartDir := "camunda-platform-" + previousCamundaVersion + previousChart, err := loader.Load("../../charts/" + previousChartDir) if err != nil { - return "", VersionParsingErrorf("failed to load the previous chart: %s", err) + return nil, VersionParsingErrorf("failed to load the previous chart: %s", err) } previousChartVersionSemver := "v" + previousChart.Metadata.Version previousVersionMajor := semver.Major(previousChartVersionSemver) previousVersionMajor = strings.TrimPrefix(previousVersionMajor, "v") - return previousVersionMajor, nil + return &ChartVersion{ + Version: previousVersionMajor, + Dir: previousChartDir, + }, nil } diff --git a/scripts/generate-gha-matrix/versions_test.go b/scripts/generate-gha-matrix/versions_test.go index 2dc4569c45..44b7ac104c 100644 --- a/scripts/generate-gha-matrix/versions_test.go +++ b/scripts/generate-gha-matrix/versions_test.go @@ -36,7 +36,8 @@ func TestGetPreviousChart(t *testing.T) { version := getCamundaVersion(chart) previousVersion, err := getPreviousHelmChartVersion(chart, version) require.NoError(t, err) - require.Equal(t, "12", previousVersion) + require.Equal(t, "12", previousVersion.Version) + require.Equal(t, "camunda-platform-8.7", previousVersion.Dir) } func TestGetPreviousChartInvalidVersion(t *testing.T) { From 7d17fd8f10838c22b5d7bfec0fd84a12c5af5061 Mon Sep 17 00:00:00 2001 From: "distro-ci[bot]" <122795778+distro-ci[bot]@users.noreply.github.com> Date: Mon, 21 Apr 2025 18:45:27 +0000 Subject: [PATCH 6/7] chore: chart chores --- charts/camunda-platform-8.5/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/camunda-platform-8.5/README.md b/charts/camunda-platform-8.5/README.md index bda22720fb..1f7f7f2384 100644 --- a/charts/camunda-platform-8.5/README.md +++ b/charts/camunda-platform-8.5/README.md @@ -1658,7 +1658,7 @@ Please see the corresponding [release guide](../../docs/release.md) to find out | `postgresql.enabled` | if true, a PostgreSQL database will be deployed as part of the Helm release by using the dependency chart | `false` | | `postgresql.nameOverride` | defines the name of the Postgres resources (names will be prefixed with the release name), see https://github.com/bitnami/charts/tree/main/bitnami/postgresql#common-parameters | `postgresql-web-modeler` | | `postgresql.image.repository` | PostgreSQL repo | `bitnami/postgresql` | -| `postgresql.image.tag` | PostgreSQL image tag | `14.17.0-debian-12-r10` | +| `postgresql.image.tag` | PostgreSQL image tag | `14.17.0-debian-12-r15` | | `postgresql.auth` | configuration of the database authentication | | | `postgresql.auth.username` | defines the name of the database user to be created for WebModeler | `web-modeler` | | `postgresql.auth.password` | can be used to provide the database user's password; a random password will be generated if left empty / ignored if `postgresql.auth.existingSecret` is set | `""` | From 2b9ae5918cd67ddb808281ba699196282ca868c8 Mon Sep 17 00:00:00 2001 From: Jesse Simpson Date: Mon, 21 Apr 2025 19:08:45 -0400 Subject: [PATCH 7/7] fix: response to global.postgresql.auth.password' must not be empty --- test/integration/scenarios/lib/chart-upgrade-taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml b/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml index e574eeb32e..0aeaafcb27 100644 --- a/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml +++ b/test/integration/scenarios/lib/chart-upgrade-taskfile.yaml @@ -29,7 +29,7 @@ tasks: if [ $? -eq 0 ]; then kubectl delete deployment -n {{ .TEST_NAMESPACE }} integration-identity kubectl wait -n {{ .TEST_NAMESPACE }} --for='delete' deployment/integration-identity - FORCE="--force --install" + FORCE="--force --install --set=global.postgresql.auth.existingSecret=integration-test-credentials" fi fi set -e +x