[fix] Retool internal versions are not semver compliant, so ensure that our helm chart is able to support all version formats (#196)

Author: jjlgao

.github/kubeconform.sh

@@ -21,8 +21,12 @@ for CHART_DIR in ${CHART_DIRS}; do
   echo "Running kubeconform for folder: '$CHART_DIR'"
   helm dep up "${CHART_DIR}"
   for VALUES_FILE in $(find "${CHART_DIR}/ci" -name '*values.yaml'); do
-    echo "== Checking values file: ${VALUES_FILE}"
     helm template --kube-version "${KUBERNETES_VERSION#v}" --values "${VALUES_FILE}" "${CHART_DIR}" \
       | ./kubeconform --strict --summary --kubernetes-version "${KUBERNETES_VERSION#v}"
+    for OPTION_FILE in $(find "${CHART_DIR}/ci" -name '*option.yaml'); do
+      echo "== Checking values file: ${VALUES_FILE} and option file: ${OPTION_FILE}"
+      helm template --kube-version "${KUBERNETES_VERSION#v}" --values "${VALUES_FILE}" --values "${OPTION_FILE}" "${CHART_DIR}" \
+        | ./kubeconform --strict --summary --kubernetes-version "${KUBERNETES_VERSION#v}"
+    done
   done
 done

.github/workflows/ci.yaml

@@ -27,7 +27,7 @@ jobs:
           version: v3.6.3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.12
       - name: Set up chart-testing
         uses: helm/[email protected]
       - name: Run chart-testing (list-changed)
@@ -64,7 +64,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.12
       - name: Set up chart-testing
         uses: helm/[email protected]
       - name: Run chart-testing (lint)

charts/retool/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: retool
 description: A Helm chart for Kubernetes
 type: application
-version: 6.2.14
+version: 6.2.15
 maintainers:
   - name: Retool Engineering
     email: [email protected]

charts/retool/ci/test-edge-release-option.yaml

@@ -0,0 +1,5 @@
+image:
+  tag: "100.100.0-edge"
+dbconnector:
+  java:
+    enabled: true

charts/retool/ci/test-invalid-semver-option.yaml

@@ -0,0 +1,2 @@
+image:
+  tag: "i.am.not-a.real-version"

charts/retool/ci/test-stable-release-option.yaml

@@ -0,0 +1,2 @@
+image:
+  tag: "3.0.0-stable"

charts/retool/templates/_helpers.tpl

@@ -50,7 +50,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{/*
 Selector labels for main backend. Note changes here will require deployment
 recreation and incur downtime. The "app.kubernetes.io/instance" label should
-also be included in all deployments, so telemetry knows how to find logs. 
+also be included in all deployments, so telemetry knows how to find logs.
 */}}
 {{- define "retool.selectorLabels" -}}
 app.kubernetes.io/name: {{ include "retool.name" . }}
@@ -200,6 +200,8 @@ Usage: (include "retool.workflows.enabled" .)
 */}}
 {{- define "retool.workflows.enabled" -}}
 {{- $output := "" -}}
+{{- $valid_retool_version_regexp := "([0-9]+\\.[0-9]+(\\.[0-9]+)?(-[a-zA-Z0-9]+)?)" }}
+{{- $retool_version_with_workflows := ( and ( regexMatch $valid_retool_version_regexp $.Values.image.tag ) ( semverCompare ">= 3.6.11-0" ( regexFind $valid_retool_version_regexp $.Values.image.tag ) ) ) }}
 {{- if or
     (eq (toString .Values.workflows.enabled) "true")
     (eq (toString .Values.workflows.enabled) "false")
@@ -213,7 +215,7 @@ Usage: (include "retool.workflows.enabled" .)
   {{- $output = "" -}}
 {{- else if eq .Values.image.tag "latest" -}}
   {{- $output = "1" -}}
-{{- else if semverCompare ">= 3.6.11-0" .Values.image.tag -}}
+{{- else if $retool_version_with_workflows -}}
   {{- $output = "1" -}}
 {{- else -}}
   {{- $output = "" -}}
@@ -227,6 +229,8 @@ Usage: (include "retool.codeExecutor.enabled" .)
 */}}
 {{- define "retool.codeExecutor.enabled" -}}
 {{- $output := "" -}}
+{{- $valid_retool_version_regexp := "([0-9]+\\.[0-9]+(\\.[0-9]+)?(-[a-zA-Z0-9]+)?)" }}
+{{- $retool_version_with_ce := ( and ( regexMatch $valid_retool_version_regexp (include "retool.codeExecutor.image.tag" .) ) ( semverCompare ">= 3.20.15-0" ( regexFind $valid_retool_version_regexp (include "retool.codeExecutor.image.tag" .) ) ) ) }}
 {{- if or
     (eq (toString .Values.codeExecutor.enabled) "true")
     (eq (toString .Values.codeExecutor.enabled) "false")
@@ -240,7 +244,7 @@ Usage: (include "retool.codeExecutor.enabled" .)
   {{- $output = "" -}}
 {{- else if (or (contains "stable" (include "retool.codeExecutor.image.tag" .)) (contains "edge" (include "retool.codeExecutor.image.tag" .))) -}}
   {{- $output = "1" -}}
-{{- else if semverCompare ">= 3.20.15-0" (include "retool.codeExecutor.image.tag" .) -}}
+{{- else if $retool_version_with_ce -}}
   {{- $output = "1" -}}
 {{- else -}}
   {{- $output = "" -}}
@@ -319,11 +323,13 @@ Usage: (template "retool.codeExecutor.image.tag" .)
 {{- if .Values.codeExecutor.image.tag -}}
   {{- .Values.codeExecutor.image.tag -}}
 {{- else if .Values.image.tag  -}}
+  {{- $valid_retool_version_regexp := "([0-9]+\\.[0-9]+(\\.[0-9]+)?(-[a-zA-Z0-9]+)?)" }}
+  {{- $retool_version_with_ce := ( and ( regexMatch $valid_retool_version_regexp $.Values.image.tag ) ( semverCompare ">= 3.20.15-0" ( regexFind $valid_retool_version_regexp $.Values.image.tag ) ) ) }}
   {{- if and (eq .Values.image.tag "latest") (eq (toString .Values.codeExecutor.enabled) "true") -}}
     {{- fail "If using image.tag=latest (not recommended, select an explicit tag instead) and enabling codeExecutor, explicitly set codeExecutor.image.tag" }}
   {{- else if (eq .Values.image.tag "latest") -}}
     {{- "" -}}
-  {{- else if semverCompare ">= 3.20.15-0" .Values.image.tag -}}
+  {{- else if $retool_version_with_ce -}}
     {{- .Values.image.tag -}}
   {{- else -}}
     {{- "1.1.0" -}}
@@ -332,3 +338,16 @@ Usage: (template "retool.codeExecutor.image.tag" .)
   {{- fail "Please set a value for .Values.image.tag" }}
 {{- end -}}
 {{- end -}}
+
+{{- define "retool_version_with_java_dbconnector_opt_out" -}}
+{{- $output := "" -}}
+{{- $valid_retool_version_regexp := "([0-9]+\\.[0-9]+(\\.[0-9]+)?(-[a-zA-Z0-9]+)?)" }}
+{{- if not ( regexMatch $valid_retool_version_regexp .Values.image.tag ) -}}
+  {{- $output = "1" -}}
+{{- else if semverCompare ">= 3.93.0-0" ( regexFind $valid_retool_version_regexp .Values.image.tag ) -}}
+  {{- $output = "1" -}}
+{{- else -}}
+  {{- $output = "" -}}
+{{- end -}}
+{{- $output -}}
+{{- end -}}

charts/retool/templates/deployment_backend.yaml

@@ -77,21 +77,27 @@ spec:
             value: {{ template "retool.deploymentTemplateVersion" . }}
           - name: NODE_ENV
             value: production
-          {{- if include "retool.jobRunner.enabled" . }}
-          {{ if ( not $.Values.dbconnector.java.enabled ) }}
-          - name: DISABLE_JAVA_DBCONNECTOR
-            value: "true"
-          {{ end }}
+          {{- $serviceType := list "MAIN_BACKEND" "DB_CONNECTOR" "DB_SSH_CONNECTOR" }}
+          {{- /*
+            JAVA_DBCONNECTOR in the service type only applies before the version of Retool that changes it to opt-out (3.93.0-edge),
+            and only if the Java dbconnector is enabled in values.yaml.
+          */}}
+          {{- if and ( not ( include "retool_version_with_java_dbconnector_opt_out" . ) ) ( $.Values.dbconnector.java.enabled ) }}
+            {{- $serviceType = append $serviceType "JAVA_DBCONNECTOR" }}
+          {{- end }}
+          {{- /*
+            It may seem counterintuitive to add the JOBS_RUNNER service type only without a jobs runner.
+            The reason for this is that the backend needs to act as a jobs runner, if the jobs runner is not enabled.
+          */}}
+          {{- if not .Values.jobRunner.enabled }}
+            {{- $serviceType = append $serviceType "JOBS_RUNNER" }}
+          {{- end }}
           - name: SERVICE_TYPE
-            value: MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR
-          {{- else }}
-          {{ if ( not $.Values.dbconnector.java.enabled ) }}
+            value: {{ join "," $serviceType }}
+          {{ if and ( include "retool_version_with_java_dbconnector_opt_out" . ) ( not $.Values.dbconnector.java.enabled ) }}
           - name: DISABLE_JAVA_DBCONNECTOR
             value: "true"
           {{ end }}
-          - name: SERVICE_TYPE
-            value: MAIN_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR,JOBS_RUNNER
-          {{- end }}
           - name: CLIENT_ID
             value: {{ default "" .Values.config.auth.google.clientId }}
           - name: COOKIE_INSECURE

charts/retool/templates/deployment_workflows.yaml

@@ -66,12 +66,20 @@ spec:
             value: {{ template "retool.deploymentTemplateVersion" . }}
           - name: NODE_ENV
             value: production
-          {{ if ( not $.Values.dbconnector.java.enabled ) }}
+          {{- $serviceType := list "WORKFLOW_BACKEND" "DB_CONNECTOR" "DB_SSH_CONNECTOR" }}
+          {{- /*
+            JAVA_DBCONNECTOR in the service type only applies before the version of Retool that changes it to opt-out (3.93.0-edge),
+            and only if the Java dbconnector is enabled in values.yaml.
+          */}}
+          {{- if and ( not ( include "retool_version_with_java_dbconnector_opt_out" . ) ) ( $.Values.dbconnector.java.enabled ) }}
+            {{- $serviceType = append $serviceType "JAVA_DBCONNECTOR" }}
+          {{- end }}
+          - name: SERVICE_TYPE
+            value: {{ join "," $serviceType }}
+          {{ if and ( include "retool_version_with_java_dbconnector_opt_out" . ) ( not $.Values.dbconnector.java.enabled ) }}
           - name: DISABLE_JAVA_DBCONNECTOR
             value: "true"
           {{ end }}
-          - name: SERVICE_TYPE
-            value: WORKFLOW_BACKEND,DB_CONNECTOR,DB_SSH_CONNECTOR
           - name: DBCONNECTOR_POSTGRES_POOL_MAX_SIZE
             value: "100"
           - name: DBCONNECTOR_QUERY_TIMEOUT_MS