diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5677594de..aaf01d6ac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,7 @@ jobs: run: make repo-add update-req - name: Lint - run: make lint + run: make lint test-charts - name: Test stable run: make stable diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ff5dde795..1f0193b5a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,10 +16,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 #v4.3.1 - with: - version: "v3.13.3" - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Configure Git Author as Workflow Actor run: | diff --git a/Makefile b/Makefile index 4f7a6ee21..d1a6939b0 100644 --- a/Makefile +++ b/Makefile @@ -332,6 +332,11 @@ lint: @echo "Linting all charts" @HELM=$(HELM) ./hack/scripts/lint.sh +.PHONY: test-charts +test-charts: check-helm + @echo "Running chart tests" + @bash tests/test_runner.sh + .PHONY: repo-add repo-add: helm repo add stable https://charts.helm.sh/stable diff --git a/stable/mlrun/Chart.yaml b/stable/mlrun/Chart.yaml index 7af324200..d176a0f1e 100644 --- a/stable/mlrun/Chart.yaml +++ b/stable/mlrun/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v1 name: mlrun -version: 0.11.13 +version: 0.11.14 appVersion: 1.10.0 description: Machine Learning automation and tracking sources: diff --git a/stable/mlrun/templates/_helpers.tpl b/stable/mlrun/templates/_helpers.tpl index 71135fdab..8628eae8a 100644 --- a/stable/mlrun/templates/_helpers.tpl +++ b/stable/mlrun/templates/_helpers.tpl @@ -187,17 +187,32 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this Determine MySQL image tag based on MLRun API image tag. - If db.image.tag is explicitly set in values, use that (allows override) - Otherwise, use MySQL 8.0 for api.image.tag < 1.11.0, and MySQL 8.4 for api.image.tag >= 1.11.0 +- If api.image.tag is not semver-compatible, fall back to appVersion: + - appVersion 1.10.x -> MySQL 8.0 + - appVersion 1.11.x -> MySQL 8.4 */}} {{- define "mlrun.db.mysqlTag" -}} {{- if .Values.db.image.tag -}} -{{- .Values.db.image.tag -}} + {{- .Values.db.image.tag -}} {{- else -}} -{{- if semverCompare "<1.11.0" .Values.api.image.tag -}} -{{- print "8.0" -}} -{{- else -}} -{{- print "8.4" -}} -{{- end -}} -{{- end -}} + {{- $apiTag := .Values.api.image.tag -}} + {{- $isSemver := regexMatch "^v?[0-9]+\\.[0-9]+\\.[0-9]+(-rc[0-9]+\\.*)?" $apiTag -}} + {{- if not $isSemver -}} + {{- /* Non-semver tag, use appVersion */ -}} + {{- if semverCompare "<1.11.0" .Chart.AppVersion -}} + {{- print "8.0" -}} + {{- else -}} + {{- print "8.4" -}} + {{- end -}} + {{- else -}} + {{- /* Valid semver tag, use API tag */ -}} + {{- if semverCompare "<1.11.0" $apiTag -}} + {{- print "8.0" -}} + {{- else -}} + {{- print "8.4" -}} + {{- end -}} + {{- end -}} + {{- end -}} {{- end -}} diff --git a/stable/mlrun/templates/api-chief-ingress.yaml b/stable/mlrun/templates/api-chief-ingress.yaml index 56ed55e2a..2dddd053f 100644 --- a/stable/mlrun/templates/api-chief-ingress.yaml +++ b/stable/mlrun/templates/api-chief-ingress.yaml @@ -1,5 +1,4 @@ {{- if "mlrun.api.worker.minReplicas" -}} -{{- if semverCompare ">=1.1.0-X" .Values.api.image.tag -}} {{- if .Values.api.chief.ingress.enabled -}} {{- $fullName := include "mlrun.api.chief.fullname" . -}} {{- $svcPort := .Values.api.chief.service.port -}} @@ -42,4 +41,3 @@ spec: {{- end }} {{- end }} {{- end }} -{{- end }} diff --git a/stable/mlrun/templates/api-chief-service.yaml b/stable/mlrun/templates/api-chief-service.yaml index 0eb090369..49cf30c38 100644 --- a/stable/mlrun/templates/api-chief-service.yaml +++ b/stable/mlrun/templates/api-chief-service.yaml @@ -1,5 +1,4 @@ {{- if "mlrun.api.worker.minReplicas" -}} -{{- if semverCompare ">=1.1.0-X" .Values.api.image.tag -}} apiVersion: v1 kind: Service metadata: @@ -39,4 +38,3 @@ spec: selector: {{- include "mlrun.api.chief.selectorLabels" . | nindent 4 }} {{- end -}} -{{- end -}} diff --git a/stable/mlrun/templates/api-worker-deployment.yaml b/stable/mlrun/templates/api-worker-deployment.yaml index 993d6ba63..49153f21a 100644 --- a/stable/mlrun/templates/api-worker-deployment.yaml +++ b/stable/mlrun/templates/api-worker-deployment.yaml @@ -1,5 +1,4 @@ {{- if "mlrun.api.worker.minReplicas" -}} -{{- if semverCompare ">=1.1.0-X" .Values.api.image.tag -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -283,4 +282,3 @@ spec: priorityClassName: {{ .Values.api.priorityClassName | quote }} {{- end }} {{- end -}} -{{- end -}} diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 000000000..7e48c6fdf --- /dev/null +++ b/tests/README.md @@ -0,0 +1,35 @@ +# Helm Charts Test Framework + +Generic test framework for Helm charts in this repository. + +## Prerequisites + +- Helm 3.x +- Chart dependencies installed (`helm dependency update` in chart directory) + + +## Structure + +``` +tests/ +├── test_runner.sh # Top-level runner (discovers all chart tests) +├── lib/ +│ └── common.sh # Shared test utilities +└── / # Chart-specific tests + └── test_.sh # Individual test cases +``` + +## Usage + +```bash +# Run all chart tests +make test-charts + +# Run a specific test +./tests//test_.sh +``` + +## Adding Tests for a New Chart + +1. Create chart test directory: `tests//` +2. Create test cases: `cp tests/mlrun/test_mysql_tag.sh tests//test_.sh` diff --git a/tests/lib/common.sh b/tests/lib/common.sh new file mode 100755 index 000000000..ea6e36b06 --- /dev/null +++ b/tests/lib/common.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +# Common functions and utilities for chart tests + +# Colors for output (use regular variables to avoid readonly conflicts when sourced multiple times) +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Test framework variables +TEST_COUNT=0 +PASS_COUNT=0 +FAIL_COUNT=0 +TEST_START_TIME=0 + +# Get the root directory of the repository +get_repo_root() { + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Go up from tests/lib to repo root + echo "$(cd "$script_dir/../.." && pwd)" +} + +# Get the chart directory by name +get_chart_dir() { + chart_name="$1" + local repo_root + repo_root=$(get_repo_root) + echo "$repo_root/stable/$chart_name" +} + +# Check if helm is available +check_helm() { + local helm="${HELM:-helm}" + if ! command -v "$helm" &> /dev/null; then + echo -e "${RED}Error: helm command not found. Please install Helm 3.x${NC}" >&2 + exit 1 + fi + echo "$helm" +} + +# Render the chart with given values to output directory +# Returns the path to the output directory +render_chart() { + local helm="$1" + local chart_dir="$2" + shift 2 + local set_args=("$@") + + # Create a temporary directory for output + local output_dir + output_dir=$(mktemp -d -t helm-test-XXXXXX) + + # Render chart to output directory + local helm_stderr + local helm_exit_code + + if ! helm_stderr=$("$helm" template test-release "$chart_dir" \ + --output-dir "$output_dir" \ + "${set_args[@]}" 2>&1); then + echo -e "${RED}ERROR: helm template command failed${NC}" >&2 + echo "$helm_stderr" >&2 + rm -rf "$output_dir" + return 1 + fi + + # Return the output directory path + echo "$output_dir" +} + +# Assert that a value equals expected +assert_equal() { + local actual="$1" + local expected="$2" + local message="${3:-}" + + if [ "$actual" == "$expected" ]; then + return 0 + else + if [ -n "$message" ]; then + echo -e "${RED}Assertion failed: $message${NC}" >&2 + fi + echo -e "${RED} Expected: '$expected'${NC}" >&2 + echo -e "${RED} Actual: '$actual'${NC}" >&2 + return 1 + fi +} + +# Assert that a value is not empty +assert_not_empty() { + local value="$1" + local message="${2:-Value should not be empty}" + + if [ -z "$value" ]; then + echo -e "${RED}Assertion failed: $message${NC}" >&2 + return 1 + fi + return 0 +} + +# Assert that a pattern exists in output +assert_contains() { + local output="$1" + local pattern="$2" + local message="${3:-Pattern not found}" + + if echo "$output" | grep -q "$pattern"; then + return 0 + else + echo -e "${RED}Assertion failed: $message${NC}" >&2 + echo -e "${RED} Pattern: '$pattern'${NC}" >&2 + return 1 + fi +} + +# Start a test +test_start() { + TEST_COUNT=$((TEST_COUNT + 1)) + TEST_START_TIME=$(date +%s) + echo -e "${YELLOW}[TEST $TEST_COUNT]${NC} $1" +} + +# Mark test as passed +test_pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + local duration=0 + if [ $TEST_START_TIME -gt 0 ]; then + duration=$(($(date +%s) - TEST_START_TIME)) + fi + echo -e "${GREEN} ✓ PASSED${NC} (${duration}s)" + return 0 +} + +# Mark test as failed +test_fail() { + FAIL_COUNT=$((FAIL_COUNT + 1)) + local duration=0 + if [ $TEST_START_TIME -gt 0 ]; then + duration=$(($(date +%s) - TEST_START_TIME)) + fi + echo -e "${RED} ✗ FAILED${NC} (${duration}s)" + return 1 +} + +# Print test summary +print_test_summary() { + echo "" + echo "==========================================" + echo "Test Summary" + echo "==========================================" + echo "Total tests: $TEST_COUNT" + echo -e "${GREEN}Passed: $PASS_COUNT${NC}" + if [ $FAIL_COUNT -gt 0 ]; then + echo -e "${RED}Failed: $FAIL_COUNT${NC}" + return 1 + else + echo -e "${GREEN}Failed: $FAIL_COUNT${NC}" + return 0 + fi +} diff --git a/tests/mlrun/test_mysql_tag.sh b/tests/mlrun/test_mysql_tag.sh new file mode 100755 index 000000000..8f0b77e1f --- /dev/null +++ b/tests/mlrun/test_mysql_tag.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# Test case: MySQL tag selection based on MLRun API image tag +# This test verifies that the MySQL image tag is correctly selected +# based on the MLRun API image tag version. + +set -euo pipefail + +# Source common functions +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../lib/common.sh" + +# Chart name (derived from test directory name) +CHART_NAME="${CHART_NAME:-$(basename "$SCRIPT_DIR")}" + +# Test name +TEST_NAME="MySQL Tag Selection" + +# Function to extract MySQL image tag from output directory +extract_mysql_tag() { + local output_dir="$1" + + # Find the db deployment YAML file + local db_deployment_file + db_deployment_file=$(find "$output_dir" -name "*db-deployment.yaml" -o -name "*db-deployment.yml" | head -1) + + if [ -z "$db_deployment_file" ] || [ ! -f "$db_deployment_file" ]; then + echo -e "${RED}ERROR: Could not find db deployment file in output directory${NC}" >&2 + return 1 + fi + + # Extract MySQL tag from the deployment file + # Look for the init container with name "init-mysql" and extract the image tag + if command -v yq &> /dev/null; then + yq eval '.spec.template.spec.initContainers[] | select(.name == "init-mysql") | .image | split(":")[1]' "$db_deployment_file" 2>/dev/null + elif command -v jq &> /dev/null && command -v yaml2json &> /dev/null; then + yaml2json < "$db_deployment_file" | jq -r '.spec.template.spec.initContainers[]? | select(.name == "init-mysql") | .image | split(":")[1]' 2>/dev/null + else + # Fallback: use grep/sed + grep -A 2 "name: init-mysql" "$db_deployment_file" | grep "image:" | sed -E 's/.*image:[[:space:]]*"mysql:([^"]+)".*/\1/' | head -1 + fi +} + +# Test case function +run_test() { + local helm + helm=$(check_helm) + + local chart_dir + chart_dir=$(get_chart_dir "$CHART_NAME") + + # Only show verbose output if not suppressed + if [ "${SUPPRESS_SUMMARY:-0}" != "1" ]; then + echo "" + echo "==========================================" + echo "$TEST_NAME" + echo "==========================================" + echo "" + fi + + # Test 1: MLRun 1.10.0 should yield MySQL 8.0 + test_start "MLRun 1.10.0 -> MySQL 8.0" + local output_dir + output_dir=$(render_chart "$helm" "$chart_dir" \ + --set httpDB.dbType=mysql \ + --set api.image.tag="1.10.0" \ + --set db.image.tag="") + + if [ $? -ne 0 ] || [ -z "$output_dir" ]; then + echo -e " ${RED}Failed to render chart${NC}" + test_fail + return + fi + + local mysql_tag + mysql_tag=$(extract_mysql_tag "$output_dir") + local extract_exit=$? + + # Clean up output directory + rm -rf "$output_dir" + + if [ $extract_exit -ne 0 ] || [ -z "$mysql_tag" ]; then + echo -e " ${RED}Could not extract MySQL tag${NC}" + test_fail + return + fi + + if assert_equal "$mysql_tag" "8.0" "MySQL tag should be 8.0 for API tag 1.10.0"; then + test_pass + else + test_fail + fi + + # Test 2: MLRun 1.11.0-rc5 should yield MySQL 8.4 + test_start "MLRun 1.11.0-rc5 -> MySQL 8.4" + output_dir=$(render_chart "$helm" "$chart_dir" \ + --set httpDB.dbType=mysql \ + --set api.image.tag="1.11.0-rc5" \ + --set db.image.tag="") + + if [ $? -ne 0 ] || [ -z "$output_dir" ]; then + echo -e " ${RED}Failed to render chart${NC}" + test_fail + return + fi + + mysql_tag=$(extract_mysql_tag "$output_dir") + extract_exit=$? + + # Clean up output directory + rm -rf "$output_dir" + + if [ $extract_exit -ne 0 ] || [ -z "$mysql_tag" ]; then + echo -e " ${RED}Could not extract MySQL tag${NC}" + test_fail + return + fi + + if assert_equal "$mysql_tag" "8.4" "MySQL tag should be 8.4 for API tag 1.11.0-rc5"; then + test_pass + else + test_fail + fi + + # Test 3: Explicit MySQL tag should override conditional logic + test_start "Explicit MySQL tag 9.0 with MLRun 1.10.0 -> MySQL 9.0" + output_dir=$(render_chart "$helm" "$chart_dir" \ + --set httpDB.dbType=mysql \ + --set api.image.tag="1.10.0" \ + --set db.image.tag="9.0") + + if [ $? -ne 0 ] || [ -z "$output_dir" ]; then + echo -e " ${RED}Failed to render chart${NC}" + test_fail + return + fi + + mysql_tag=$(extract_mysql_tag "$output_dir") + extract_exit=$? + + # Clean up output directory + rm -rf "$output_dir" + + if [ $extract_exit -ne 0 ] || [ -z "$mysql_tag" ]; then + echo -e " ${RED}Could not extract MySQL tag${NC}" + test_fail + return + fi + + # Even though MLRun 1.10.0 would normally select MySQL 8.0, + # the explicit tag "9.0" should be used instead + if assert_equal "$mysql_tag" "9.0" "MySQL tag should be 9.0 when explicitly set, overriding conditional logic"; then + test_pass + else + test_fail + fi + + # Test 4: Non-semver API tag with appVersion 1.10.0 should yield MySQL 8.0 + test_start "Non-semver API tag 'latest' with appVersion 1.10.0 -> MySQL 8.0" + output_dir=$(render_chart "$helm" "$chart_dir" \ + --set httpDB.dbType=mysql \ + --set api.image.tag="latest" \ + --set db.image.tag="") + + if [ $? -ne 0 ] || [ -z "$output_dir" ]; then + echo -e " ${RED}Failed to render chart${NC}" + test_fail + return + fi + + mysql_tag=$(extract_mysql_tag "$output_dir") + extract_exit=$? + + # Clean up output directory + rm -rf "$output_dir" + + if [ $extract_exit -ne 0 ] || [ -z "$mysql_tag" ]; then + echo -e " ${RED}Could not extract MySQL tag${NC}" + test_fail + return + fi + + # Non-semver tag should fallback to appVersion (1.10.0) which should yield MySQL 8.0 + if assert_equal "$mysql_tag" "8.0" "MySQL tag should be 8.0 for non-semver API tag when appVersion is 1.10.0"; then + test_pass + else + test_fail + fi + + # Test 5: Non-semver API tag with appVersion 1.11.0 should yield MySQL 8.4 + test_start "Non-semver API tag 'dev' with appVersion 1.11.0 -> MySQL 8.4" + + # Temporarily modify Chart.yaml to set appVersion to 1.11.0 + local chart_yaml="$chart_dir/Chart.yaml" + local chart_yaml_backup + chart_yaml_backup=$(mktemp) + cp "$chart_yaml" "$chart_yaml_backup" + + # Update appVersion to 1.11.0 + if sed -i.bak 's/^appVersion:.*/appVersion: 1.11.0/' "$chart_yaml"; then + output_dir=$(render_chart "$helm" "$chart_dir" \ + --set httpDB.dbType=mysql \ + --set api.image.tag="dev" \ + --set db.image.tag="") + + local render_exit=$? + + # Restore Chart.yaml + mv "$chart_yaml_backup" "$chart_yaml" + rm -f "${chart_yaml}.bak" + + if [ $render_exit -ne 0 ] || [ -z "$output_dir" ]; then + echo -e " ${RED}Failed to render chart${NC}" + test_fail + return + fi + + mysql_tag=$(extract_mysql_tag "$output_dir") + extract_exit=$? + + # Clean up output directory + rm -rf "$output_dir" + + if [ $extract_exit -ne 0 ] || [ -z "$mysql_tag" ]; then + echo -e " ${RED}Could not extract MySQL tag${NC}" + test_fail + return + fi + + # Non-semver tag should fallback to appVersion (1.11.0) which should yield MySQL 8.4 + if assert_equal "$mysql_tag" "8.4" "MySQL tag should be 8.4 for non-semver API tag when appVersion is 1.11.0"; then + test_pass + else + test_fail + fi + else + # Restore Chart.yaml on error + mv "$chart_yaml_backup" "$chart_yaml" + rm -f "${chart_yaml}.bak" + echo -e " ${RED}Failed to modify Chart.yaml${NC}" + test_fail + fi + + # Summary is handled by the test runner, no need to print here +} + +# Run the test +run_test diff --git a/tests/test_runner.sh b/tests/test_runner.sh new file mode 100755 index 000000000..ff435eb5f --- /dev/null +++ b/tests/test_runner.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Generic test runner for Helm chart tests +# Discovers and runs all test_*.sh files in chart directories under tests/ + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TESTS_DIR="$SCRIPT_DIR" + +# Source common functions +source "$TESTS_DIR/lib/common.sh" + +# Find all chart test directories +find_chart_test_dirs() { + find "$TESTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name ".*" ! -name "lib" | sort +} + +# Find all test files in a chart directory +find_chart_tests() { + local chart_dir="$1" + find "$chart_dir" -maxdepth 1 -name "test_*.sh" -type f | sort +} + +# Run a single test file +run_test_file() { + local test_file="$1" + local chart_name="$2" + local test_name + test_name=$(basename "$test_file" .sh) + + # Make sure test file is executable + chmod +x "$test_file" + + # Run the test with CHART_NAME set + local test_output + local test_exit_code + test_output=$(CHART_NAME="$chart_name" bash "$test_file" 2>&1) + test_exit_code=$? + + if [ $test_exit_code -eq 0 ]; then + return 0 + else + echo -e " ${RED}✗ ${test_name} failed${NC}" + # Show error output (filter out successful test messages) + echo "$test_output" | grep -E "(ERROR|FAILED|Assertion failed)" | sed 's/^/ /' || true + return 1 + fi +} + +# Run tests for a specific chart +run_chart_tests() { + local chart_test_dir="$1" + local chart_name + chart_name=$(basename "$chart_test_dir") + + # Find all test files in this chart directory + local test_files + test_files=$(find_chart_tests "$chart_test_dir") + + if [ -z "$test_files" ]; then + return 0 # Skip directories without test files + fi + + echo -e "${BLUE}→ Testing chart: ${chart_name}${NC}" + + local total_tests=0 + local passed_tests=0 + local failed_tests=0 + + # Run each test file + while IFS= read -r test_file; do + total_tests=$((total_tests + 1)) + if run_test_file "$test_file" "$chart_name"; then + passed_tests=$((passed_tests + 1)) + else + failed_tests=$((failed_tests + 1)) + fi + done <<< "$test_files" + + if [ $failed_tests -gt 0 ]; then + echo -e "${RED} ✗ ${chart_name} failed (${failed_tests}/${total_tests} tests)${NC}" + echo "" + return 1 + else + echo -e "${GREEN} ✓ ${chart_name} passed (${total_tests} tests)${NC}" + echo "" + return 0 + fi +} + +# Main execution +main() { + local chart_dirs + chart_dirs=$(find_chart_test_dirs) + + if [ -z "$chart_dirs" ]; then + echo -e "${YELLOW}No chart test directories found in $TESTS_DIR${NC}" + exit 0 + fi + + echo "Running Helm Chart Tests" + echo "" + + local total_charts=0 + local passed_charts=0 + local failed_charts=0 + + # Run tests for each chart + while IFS= read -r chart_dir; do + if run_chart_tests "$chart_dir"; then + passed_charts=$((passed_charts + 1)) + else + failed_charts=$((failed_charts + 1)) + fi + total_charts=$((total_charts + 1)) + done <<< "$chart_dirs" + + # Print final summary + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + if [ $failed_charts -gt 0 ]; then + echo -e "${RED}✗ FAILED${NC} - $failed_charts of $total_charts chart(s) failed" + exit 1 + else + echo -e "${GREEN}✓ PASSED${NC} - All $total_charts chart(s) passed" + exit 0 + fi +} + +# Run main if script is executed directly +if [ "${BASH_SOURCE[0]}" == "${0}" ]; then + main "$@" +fi