[CI] Added testing framework

.github/workflows/ci.yaml

@@ -20,7 +20,7 @@ jobs:
       run: make repo-add update-req
 
     - name: Lint
-      run: make lint
+      run: make lint test-charts
 
     - name: Test stable
       run: make stable

.github/workflows/publish.yml

@@ -16,10 +16,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4  #v4.3.1
-        with:
-          version: "v3.13.3"
 
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
 
       - name: Configure Git Author as Workflow Actor
         run: |

Makefile

@@ -332,6 +332,11 @@ lint:
 	@echo "Linting all charts"
 	@HELM=$(HELM) ./hack/scripts/lint.sh
 
+.PHONY: test-charts
+test-charts: check-helm
+	@echo "Running chart tests"
+	@bash tests/test_runner.sh
+
 .PHONY: repo-add
 repo-add:
 	helm repo add stable https://charts.helm.sh/stable

stable/mlrun/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: mlrun
-version: 0.11.13
+version: 0.11.14
 appVersion: 1.10.0
 description: Machine Learning automation and tracking
 sources:

stable/mlrun/templates/_helpers.tpl

@@ -187,17 +187,32 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 Determine MySQL image tag based on MLRun API image tag.
 - If db.image.tag is explicitly set in values, use that (allows override)
 - Otherwise, use MySQL 8.0 for api.image.tag < 1.11.0, and MySQL 8.4 for api.image.tag >= 1.11.0
+- If api.image.tag is not semver-compatible, fall back to appVersion:
+  - appVersion 1.10.x -> MySQL 8.0
+  - appVersion 1.11.x -> MySQL 8.4
 */}}
 {{- define "mlrun.db.mysqlTag" -}}
 {{- if .Values.db.image.tag -}}
-{{- .Values.db.image.tag -}}
+  {{- .Values.db.image.tag -}}
 {{- else -}}
-{{- if semverCompare "<1.11.0" .Values.api.image.tag -}}
-{{- print "8.0" -}}
-{{- else -}}
-{{- print "8.4" -}}
-{{- end -}}
-{{- end -}}
+  {{- $apiTag := .Values.api.image.tag -}}
+  {{- $isSemver := regexMatch "^v?[0-9]+\\.[0-9]+\\.[0-9]+(-rc[0-9]+\\.*)?" $apiTag -}}
+  {{- if not $isSemver -}}
+  {{- /* Non-semver tag, use appVersion */ -}}
+    {{- if semverCompare "<1.11.0" .Chart.AppVersion -}}
+    {{- print "8.0" -}}
+    {{- else -}}
+    {{- print "8.4" -}}
+    {{- end -}}
+  {{- else -}}
+    {{- /* Valid semver tag, use API tag */ -}}
+      {{- if semverCompare "<1.11.0" $apiTag -}}
+      {{- print "8.0" -}}
+      {{- else -}}
+      {{- print "8.4" -}}
+      {{- end -}}
+    {{- end -}}
+  {{- end -}}
 {{- end -}}
 
 

stable/mlrun/templates/api-chief-ingress.yaml

@@ -1,5 +1,4 @@
 {{- if "mlrun.api.worker.minReplicas" -}}
-{{- if semverCompare ">=1.1.0-X" .Values.api.image.tag -}}
 {{- if .Values.api.chief.ingress.enabled -}}
 {{- $fullName := include "mlrun.api.chief.fullname" . -}}
 {{- $svcPort := .Values.api.chief.service.port -}}
@@ -42,4 +41,3 @@ spec:
   {{- end }}
 {{- end }}
 {{- end }}
-{{- end }}

stable/mlrun/templates/api-chief-service.yaml

@@ -1,5 +1,4 @@
 {{- if "mlrun.api.worker.minReplicas" -}}
-{{- if semverCompare ">=1.1.0-X" .Values.api.image.tag -}}
 apiVersion: v1
 kind: Service
 metadata:
@@ -39,4 +38,3 @@ spec:
   selector:
     {{- include "mlrun.api.chief.selectorLabels" . | nindent 4 }}
 {{- end -}}
-{{- end -}}

stable/mlrun/templates/api-worker-deployment.yaml

@@ -1,5 +1,4 @@
 {{- if "mlrun.api.worker.minReplicas" -}}
-{{- if semverCompare ">=1.1.0-X" .Values.api.image.tag -}}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -283,4 +282,3 @@ spec:
       priorityClassName: {{ .Values.api.priorityClassName | quote }}
     {{- end }}
 {{- end -}}
-{{- end -}}

tests/README.md

@@ -0,0 +1,35 @@
+# Helm Charts Test Framework
+
+Generic test framework for Helm charts in this repository.
+
+## Prerequisites
+
+- Helm 3.x
+- Chart dependencies installed (`helm dependency update` in chart directory)
+
+
+## Structure
+
+```
+tests/
+├── test_runner.sh           # Top-level runner (discovers all chart tests)
+├── lib/
+│   └── common.sh            # Shared test utilities
+└── <chart-name>/            # Chart-specific tests
+    └── test_<name>.sh       # Individual test cases
+```
+
+## Usage
+
+```bash
+# Run all chart tests
+make test-charts
+
+# Run a specific test
+./tests/<chart-name>/test_<name>.sh
+```
+
+## Adding Tests for a New Chart
+
+1. Create chart test directory: `tests/<chart-name>/`
+2. Create test cases: `cp tests/mlrun/test_mysql_tag.sh tests/<chart-name>/test_<name>.sh`

tests/lib/common.sh

@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+# Common functions and utilities for chart tests
+
+# Colors for output (use regular variables to avoid readonly conflicts when sourced multiple times)
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Test framework variables
+TEST_COUNT=0
+PASS_COUNT=0
+FAIL_COUNT=0
+TEST_START_TIME=0
+
+# Get the root directory of the repository
+get_repo_root() {
+    local script_dir
+    script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    # Go up from tests/lib to repo root
+    echo "$(cd "$script_dir/../.." && pwd)"
+}
+
+# Get the chart directory by name
+get_chart_dir() {
+    chart_name="$1"
+    local repo_root
+    repo_root=$(get_repo_root)
+    echo "$repo_root/stable/$chart_name"
+}
+
+# Check if helm is available
+check_helm() {
+    local helm="${HELM:-helm}"
+    if ! command -v "$helm" &> /dev/null; then
+        echo -e "${RED}Error: helm command not found. Please install Helm 3.x${NC}" >&2
+        exit 1
+    fi
+    echo "$helm"
+}
+
+# Render the chart with given values to output directory
+# Returns the path to the output directory
+render_chart() {
+    local helm="$1"
+    local chart_dir="$2"
+    shift 2
+    local set_args=("$@")
+
+    # Create a temporary directory for output
+    local output_dir
+    output_dir=$(mktemp -d -t helm-test-XXXXXX)
+
+    # Render chart to output directory
+    local helm_stderr
+    local helm_exit_code
+
+    if ! helm_stderr=$("$helm" template test-release "$chart_dir" \
+        --output-dir "$output_dir" \
+        "${set_args[@]}" 2>&1); then
+        echo -e "${RED}ERROR: helm template command failed${NC}" >&2
+        echo "$helm_stderr" >&2
+        rm -rf "$output_dir"
+        return 1
+    fi
+
+    # Return the output directory path
+    echo "$output_dir"
+}
+
+# Assert that a value equals expected
+assert_equal() {
+    local actual="$1"
+    local expected="$2"
+    local message="${3:-}"
+
+    if [ "$actual" == "$expected" ]; then
+        return 0
+    else
+        if [ -n "$message" ]; then
+            echo -e "${RED}Assertion failed: $message${NC}" >&2
+        fi
+        echo -e "${RED}  Expected: '$expected'${NC}" >&2
+        echo -e "${RED}  Actual: '$actual'${NC}" >&2
+        return 1
+    fi
+}
+
+# Assert that a value is not empty
+assert_not_empty() {
+    local value="$1"
+    local message="${2:-Value should not be empty}"
+
+    if [ -z "$value" ]; then
+        echo -e "${RED}Assertion failed: $message${NC}" >&2
+        return 1
+    fi
+    return 0
+}
+
+# Assert that a pattern exists in output
+assert_contains() {
+    local output="$1"
+    local pattern="$2"
+    local message="${3:-Pattern not found}"
+
+    if echo "$output" | grep -q "$pattern"; then
+        return 0
+    else
+        echo -e "${RED}Assertion failed: $message${NC}" >&2
+        echo -e "${RED}  Pattern: '$pattern'${NC}" >&2
+        return 1
+    fi
+}
+
+# Start a test
+test_start() {
+    TEST_COUNT=$((TEST_COUNT + 1))
+    TEST_START_TIME=$(date +%s)
+    echo -e "${YELLOW}[TEST $TEST_COUNT]${NC} $1"
+}
+
+# Mark test as passed
+test_pass() {
+    PASS_COUNT=$((PASS_COUNT + 1))
+    local duration=0
+    if [ $TEST_START_TIME -gt 0 ]; then
+        duration=$(($(date +%s) - TEST_START_TIME))
+    fi
+    echo -e "${GREEN}  ✓ PASSED${NC} (${duration}s)"
+    return 0
+}
+
+# Mark test as failed
+test_fail() {
+    FAIL_COUNT=$((FAIL_COUNT + 1))
+    local duration=0
+    if [ $TEST_START_TIME -gt 0 ]; then
+        duration=$(($(date +%s) - TEST_START_TIME))
+    fi
+    echo -e "${RED}  ✗ FAILED${NC} (${duration}s)"
+    return 1
+}
+
+# Print test summary
+print_test_summary() {
+    echo ""
+    echo "=========================================="
+    echo "Test Summary"
+    echo "=========================================="
+    echo "Total tests: $TEST_COUNT"
+    echo -e "${GREEN}Passed: $PASS_COUNT${NC}"
+    if [ $FAIL_COUNT -gt 0 ]; then
+        echo -e "${RED}Failed: $FAIL_COUNT${NC}"
+        return 1
+    else
+        echo -e "${GREEN}Failed: $FAIL_COUNT${NC}"
+        return 0
+    fi
+}