diff --git a/.github/.env.base b/.github/.env.base index 2c655ee..1c2b651 100644 --- a/.github/.env.base +++ b/.github/.env.base @@ -193,7 +193,7 @@ GO_COVERAGE_LOG_ENABLED=true # Redis Service Control ENABLE_REDIS_SERVICE=false # Enable Redis service container for tests/benchmarks -REDIS_SERVICE_MODE=auto # Options: auto, always, never (auto = enabled if redis tests detected) +REDIS_SERVICE_MODE=never # Options: auto, always, never (auto = enabled if redis tests detected) # Redis Version Configuration REDIS_VERSION=7-alpine # Redis Docker image version (7-alpine, 6-alpine, latest) diff --git a/.github/actions/cache-redis-image/action.yml b/.github/actions/cache-redis-image/action.yml new file mode 100644 index 0000000..7fac275 --- /dev/null +++ b/.github/actions/cache-redis-image/action.yml @@ -0,0 +1,316 @@ +# ------------------------------------------------------------------------------------ +# Cache Redis Image Composite Action (GoFortress) +# +# Purpose: Cache Redis Docker images to accelerate service container startup. +# Provides intelligent caching similar to Go module and build caching, pinned to +# Redis version for proper cache invalidation. +# +# Features: +# - Docker image caching using actions/cache +# - Version-pinned cache keys tied to REDIS_VERSION +# - Cross-platform image caching support +# - Performance tracking and metrics +# - Automatic fallback to Docker Hub on cache miss +# - Cache compression for storage efficiency +# +# Usage: +# - uses: ./.github/actions/cache-redis-image +# with: +# redis-version: ${{ env.REDIS_VERSION }} +# runner-os: ${{ runner.os }} +# cache-mode: "restore-save" # restore, save, restore-save +# +# Maintainer: @mrz1836 +# +# ------------------------------------------------------------------------------------ + +name: "Cache Redis Image" +description: "Cache Redis Docker images for faster service container startup with version-pinned keys" + +inputs: + redis-version: + description: "Redis Docker image version (e.g., 7-alpine, 6-alpine)" + required: true + runner-os: + description: "Operating system for cache keys (e.g., Linux, macOS)" + required: true + cache-mode: + description: "Cache operation mode: restore, save, restore-save" + required: false + default: "restore-save" + force-pull: + description: "Force pull image even if cache exists (for cache warming)" + required: false + default: "false" + +outputs: + cache-hit: + description: "Whether Redis image was restored from cache (true/false)" + value: ${{ steps.restore-redis-image.outputs.cache-hit }} + image-size: + description: "Size of Redis image in MB" + value: ${{ steps.image-info.outputs.image-size }} + operation-time: + description: "Total operation time in seconds" + value: ${{ steps.operation-summary.outputs.operation-time }} + cache-key: + description: "Cache key used for Redis image" + value: ${{ steps.cache-config.outputs.cache-key }} + image-available: + description: "Whether Redis image is available locally (true/false)" + value: ${{ steps.image-verification.outputs.image-available }} + +runs: + using: "composite" + steps: + # ———————————————————————————————————————————————————————————————— + # Initialize operation tracking + # ———————————————————————————————————————————————————————————————— + - name: ⏱️ Initialize operation tracking + id: operation-start + shell: bash + run: | + echo "🗄️ Starting Redis image caching operation..." + OPERATION_START=$(date +%s) + echo "operation-start=$OPERATION_START" >> $GITHUB_OUTPUT + echo "📋 Configuration:" + echo " • Redis Version: ${{ inputs.redis-version }}" + echo " • Runner OS: ${{ inputs.runner-os }}" + echo " • Cache Mode: ${{ inputs.cache-mode }}" + echo " • Force Pull: ${{ inputs.force-pull }}" + echo "" + + # ———————————————————————————————————————————————————————————————— + # Configure cache settings and keys + # ———————————————————————————————————————————————————————————————— + - name: 🔧 Configure cache settings + id: cache-config + shell: bash + run: | + echo "🔧 Configuring Redis image cache settings..." + + # Generate cache key pinned to Redis version and OS + REDIS_VERSION="${{ inputs.redis-version }}" + RUNNER_OS="${{ inputs.runner-os }}" + + # Normalize Redis version for cache key (remove special characters) + NORMALIZED_VERSION=$(echo "$REDIS_VERSION" | sed 's/[^a-zA-Z0-9.-]/_/g') + + # Create cache key + CACHE_KEY="redis-image-${RUNNER_OS}-${NORMALIZED_VERSION}" + + # Define cache paths + CACHE_DIR="$HOME/.cache/redis-images" + IMAGE_TAR="redis-${NORMALIZED_VERSION}.tar" + CACHE_PATH="${CACHE_DIR}/${IMAGE_TAR}" + + echo "📋 Cache Configuration:" + echo " • Cache Key: $CACHE_KEY" + echo " • Cache Directory: $CACHE_DIR" + echo " • Image Tar: $IMAGE_TAR" + echo " • Cache Path: $CACHE_PATH" + + # Create cache directory + mkdir -p "$CACHE_DIR" + + # Set outputs + echo "cache-key=$CACHE_KEY" >> $GITHUB_OUTPUT + echo "cache-dir=$CACHE_DIR" >> $GITHUB_OUTPUT + echo "image-tar=$IMAGE_TAR" >> $GITHUB_OUTPUT + echo "cache-path=$CACHE_PATH" >> $GITHUB_OUTPUT + echo "normalized-version=$NORMALIZED_VERSION" >> $GITHUB_OUTPUT + + # ———————————————————————————————————————————————————————————————— + # Restore Redis image from cache + # ———————————————————————————————————————————————————————————————— + - name: 💾 Restore Redis image from cache + if: contains(inputs.cache-mode, 'restore') + id: restore-redis-image + uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: ${{ steps.cache-config.outputs.cache-path }} + key: ${{ steps.cache-config.outputs.cache-key }} + + # ———————————————————————————————————————————————————————————————— + # Load cached Redis image into Docker + # ———————————————————————————————————————————————————————————————— + - name: 📦 Load cached Redis image + if: contains(inputs.cache-mode, 'restore') && steps.restore-redis-image.outputs.cache-hit == 'true' && inputs.force-pull != 'true' + id: load-cached-image + shell: bash + run: | + echo "📦 Loading Redis image from cache..." + CACHE_PATH="${{ steps.cache-config.outputs.cache-path }}" + + if [ -f "$CACHE_PATH" ]; then + echo "✅ Cache file found: $CACHE_PATH" + + # Load image from tar file + if docker load < "$CACHE_PATH"; then + echo "✅ Redis image loaded successfully from cache" + echo "image-loaded=true" >> $GITHUB_OUTPUT + else + echo "❌ Failed to load Redis image from cache" + echo "image-loaded=false" >> $GITHUB_OUTPUT + exit 1 + fi + else + echo "❌ Cache file not found: $CACHE_PATH" + echo "image-loaded=false" >> $GITHUB_OUTPUT + exit 1 + fi + + # ———————————————————————————————————————————————————————————————— + # Pull Redis image if not cached or force-pull enabled + # ———————————————————————————————————————————————————————————————— + - name: 📥 Pull Redis image from Docker Hub + if: (contains(inputs.cache-mode, 'restore') && steps.restore-redis-image.outputs.cache-hit != 'true') || inputs.force-pull == 'true' + id: pull-redis-image + shell: bash + run: | + echo "📥 Pulling Redis image from Docker Hub..." + REDIS_IMAGE="redis:${{ inputs.redis-version }}" + + echo "🔍 Pulling image: $REDIS_IMAGE" + + # Pull the image with progress + if docker pull "$REDIS_IMAGE"; then + echo "✅ Redis image pulled successfully: $REDIS_IMAGE" + echo "image-pulled=true" >> $GITHUB_OUTPUT + else + echo "❌ Failed to pull Redis image: $REDIS_IMAGE" + echo "image-pulled=false" >> $GITHUB_OUTPUT + exit 1 + fi + + # ———————————————————————————————————————————————————————————————— + # Save Redis image to cache + # ———————————————————————————————————————————————————————————————— + - name: 💾 Save Redis image to cache + if: contains(inputs.cache-mode, 'save') && (steps.pull-redis-image.outputs.image-pulled == 'true' || inputs.force-pull == 'true') + id: save-redis-image + shell: bash + run: | + echo "💾 Saving Redis image to cache..." + REDIS_IMAGE="redis:${{ inputs.redis-version }}" + CACHE_PATH="${{ steps.cache-config.outputs.cache-path }}" + + echo "🔍 Saving image: $REDIS_IMAGE" + echo "📁 Cache path: $CACHE_PATH" + + # Save image as tar file + if docker save "$REDIS_IMAGE" | gzip > "$CACHE_PATH"; then + echo "✅ Redis image saved to cache successfully" + + # Get file size + if [ -f "$CACHE_PATH" ]; then + FILE_SIZE_BYTES=$(stat -c%s "$CACHE_PATH" 2>/dev/null || stat -f%z "$CACHE_PATH" 2>/dev/null || echo "0") + FILE_SIZE_MB=$((FILE_SIZE_BYTES / 1024 / 1024)) + echo "📊 Cache file size: ${FILE_SIZE_MB}MB" + echo "cache-size-mb=$FILE_SIZE_MB" >> $GITHUB_OUTPUT + fi + + echo "image-saved=true" >> $GITHUB_OUTPUT + else + echo "❌ Failed to save Redis image to cache" + echo "image-saved=false" >> $GITHUB_OUTPUT + exit 1 + fi + + # ———————————————————————————————————————————————————————————————— + # Save cache using actions/cache + # ———————————————————————————————————————————————————————————————— + - name: 🗄️ Save Redis image cache + if: contains(inputs.cache-mode, 'save') && steps.save-redis-image.outputs.image-saved == 'true' + uses: actions/cache/save@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: ${{ steps.cache-config.outputs.cache-path }} + key: ${{ steps.cache-config.outputs.cache-key }} + + # ———————————————————————————————————————————————————————————————— + # Verify Redis image availability + # ———————————————————————————————————————————————————————————————— + - name: 🔍 Verify Redis image availability + id: image-verification + shell: bash + run: | + echo "🔍 Verifying Redis image availability..." + REDIS_IMAGE="redis:${{ inputs.redis-version }}" + + # Check if image exists locally + if docker image inspect "$REDIS_IMAGE" >/dev/null 2>&1; then + echo "✅ Redis image available locally: $REDIS_IMAGE" + echo "image-available=true" >> $GITHUB_OUTPUT + else + echo "❌ Redis image not available locally: $REDIS_IMAGE" + echo "image-available=false" >> $GITHUB_OUTPUT + exit 1 + fi + + # ———————————————————————————————————————————————————————————————— + # Gather image information and metrics + # ———————————————————————————————————————————————————————————————— + - name: 📊 Gather image information + id: image-info + shell: bash + run: | + echo "📊 Gathering Redis image information..." + REDIS_IMAGE="redis:${{ inputs.redis-version }}" + + # Get image size + IMAGE_SIZE_BYTES=$(docker image inspect "$REDIS_IMAGE" --format='{{.Size}}' 2>/dev/null || echo "0") + IMAGE_SIZE_MB=$((IMAGE_SIZE_BYTES / 1024 / 1024)) + + # Get image ID and created date + IMAGE_ID=$(docker image inspect "$REDIS_IMAGE" --format='{{.Id}}' 2>/dev/null | cut -d: -f2 | head -c 12) + IMAGE_CREATED=$(docker image inspect "$REDIS_IMAGE" --format='{{.Created}}' 2>/dev/null || echo "unknown") + + echo "📋 Image Information:" + echo " • Image: $REDIS_IMAGE" + echo " • Size: ${IMAGE_SIZE_MB}MB" + echo " • ID: $IMAGE_ID" + echo " • Created: $IMAGE_CREATED" + + # Set outputs + echo "image-size=$IMAGE_SIZE_MB" >> $GITHUB_OUTPUT + echo "image-id=$IMAGE_ID" >> $GITHUB_OUTPUT + echo "image-created=$IMAGE_CREATED" >> $GITHUB_OUTPUT + + # ———————————————————————————————————————————————————————————————— + # Operation summary and timing + # ———————————————————————————————————————————————————————————————— + - name: ✅ Operation summary + id: operation-summary + shell: bash + run: | + echo "✅ Redis image caching operation completed" + + # Calculate operation time + OPERATION_START="${{ steps.operation-start.outputs.operation-start }}" + OPERATION_END=$(date +%s) + OPERATION_TIME=$((OPERATION_END - OPERATION_START)) + + # Determine cache status + CACHE_HIT="${{ steps.restore-redis-image.outputs.cache-hit || 'false' }}" + IMAGE_AVAILABLE="${{ steps.image-verification.outputs.image-available }}" + + echo "📊 Operation Summary:" + echo " • Cache Hit: $CACHE_HIT" + echo " • Image Available: $IMAGE_AVAILABLE" + echo " • Operation Time: ${OPERATION_TIME}s" + echo " • Cache Key: ${{ steps.cache-config.outputs.cache-key }}" + + # Set outputs + echo "operation-time=$OPERATION_TIME" >> $GITHUB_OUTPUT + echo "cache-hit-final=$CACHE_HIT" >> $GITHUB_OUTPUT + + if [[ "$IMAGE_AVAILABLE" == "true" ]]; then + if [[ "$CACHE_HIT" == "true" ]]; then + echo "🚀 Redis image restored from cache in ${OPERATION_TIME}s" + else + echo "📥 Redis image pulled and cached in ${OPERATION_TIME}s" + fi + else + echo "❌ Redis image operation failed after ${OPERATION_TIME}s" + exit 1 + fi diff --git a/.github/actions/collect-cache-stats/action.yml b/.github/actions/collect-cache-stats/action.yml index 2e4c8f9..15a221f 100644 --- a/.github/actions/collect-cache-stats/action.yml +++ b/.github/actions/collect-cache-stats/action.yml @@ -62,6 +62,22 @@ inputs: description: "Include golangci-lint cache statistics in output" required: false default: "false" + redis-enabled: + description: "Whether Redis cache statistics should be included" + required: false + default: "false" + redis-cache-hit: + description: "Whether Redis image was restored from cache" + required: false + default: "false" + redis-image-size: + description: "Size of Redis image in MB" + required: false + default: "0" + redis-operation-time: + description: "Time taken for Redis cache operations in seconds" + required: false + default: "0" outputs: stats-file: @@ -120,6 +136,14 @@ runs: echo " \"cache_size_golangci_lint\": \"$GOLANGCI_SIZE\"," >> "$OUTPUT_FILE" fi + # Include Redis cache statistics if requested + if [[ "${{ inputs.redis-enabled }}" == "true" ]]; then + echo " \"redis_enabled\": \"${{ inputs.redis-enabled }}\"," >> "$OUTPUT_FILE" + echo " \"redis_cache_hit\": \"${{ inputs.redis-cache-hit }}\"," >> "$OUTPUT_FILE" + echo " \"redis_image_size_mb\": \"${{ inputs.redis-image-size }}\"," >> "$OUTPUT_FILE" + echo " \"redis_operation_time_s\": \"${{ inputs.redis-operation-time }}\"," >> "$OUTPUT_FILE" + fi + echo ' "workflow": "${{ inputs.workflow-name }}",' >> "$OUTPUT_FILE" echo ' "job_name": "${{ inputs.job-name }}",' >> "$OUTPUT_FILE" echo " \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"" >> "$OUTPUT_FILE" @@ -140,3 +164,6 @@ runs: if [[ "${{ inputs.include-golangci }}" == "true" ]]; then echo " - golangci-lint: $GOLANGCI_SIZE (hit: ${{ inputs.golangci-cache-hit }})" fi + if [[ "${{ inputs.redis-enabled }}" == "true" ]]; then + echo " - Redis: ${{ inputs.redis-image-size }}MB (hit: ${{ inputs.redis-cache-hit }}, ${{ inputs.redis-operation-time }}s)" + fi diff --git a/.github/actions/setup-redis-service/action.yml b/.github/actions/setup-redis-service/action.yml index 220610e..90a2d41 100644 --- a/.github/actions/setup-redis-service/action.yml +++ b/.github/actions/setup-redis-service/action.yml @@ -30,6 +30,10 @@ inputs: redis-enabled: description: "Whether Redis service is enabled (true/false)" required: true + redis-version: + description: "Redis Docker image version for caching" + required: false + default: "7-alpine" redis-host: description: "Redis host address" required: false @@ -46,6 +50,10 @@ inputs: description: "Seconds between retry attempts" required: false default: "2" + use-cache: + description: "Whether to use cached Redis image (true/false)" + required: false + default: "true" outputs: redis-available: @@ -60,6 +68,15 @@ outputs: installation-method: description: "How redis-cli was obtained: pre-existing, installed, or unavailable" value: ${{ steps.redis-verification.outputs.installation-method }} + cache-hit: + description: "Whether Redis image was restored from cache (true/false)" + value: ${{ steps.cache-redis.outputs.cache-hit }} + image-size: + description: "Size of Redis image in MB" + value: ${{ steps.cache-redis.outputs.image-size }} + cache-operation-time: + description: "Time taken for cache operations in seconds" + value: ${{ steps.cache-redis.outputs.operation-time }} runs: using: "composite" @@ -77,6 +94,18 @@ runs: echo "connection-time=0" >> $GITHUB_OUTPUT echo "installation-method=disabled" >> $GITHUB_OUTPUT + # ———————————————————————————————————————————————————————————————— + # Cache Redis Docker image for faster startup + # ———————————————————————————————————————————————————————————————— + - name: 🗄️ Cache Redis Docker Image + if: inputs.redis-enabled == 'true' && inputs.use-cache == 'true' + id: cache-redis + uses: ./.github/actions/cache-redis-image + with: + redis-version: ${{ inputs.redis-version }} + runner-os: ${{ runner.os }} + cache-mode: "restore" + # ———————————————————————————————————————————————————————————————— # Verify Redis service and configure environment # ———————————————————————————————————————————————————————————————— @@ -195,10 +224,27 @@ runs: echo "connection-time=$CONNECTION_TIME" >> $GITHUB_OUTPUT echo "installation-method=$INSTALLATION_METHOD" >> $GITHUB_OUTPUT - # Final status + # Final status with cache metrics + CACHE_HIT="${{ steps.cache-redis.outputs.cache-hit || 'false' }}" + CACHE_TIME="${{ steps.cache-redis.outputs.operation-time || '0' }}" + IMAGE_SIZE="${{ steps.cache-redis.outputs.image-size || '0' }}" + if [ "$REDIS_AVAILABLE" = "true" ]; then - echo "✅ Redis service verification completed successfully (${CONNECTION_TIME}s)" + echo "✅ Redis service verification completed successfully" + echo "📊 Performance Summary:" + echo " • Connection Time: ${CONNECTION_TIME}s" + echo " • Cache Hit: $CACHE_HIT" + echo " • Cache Operation Time: ${CACHE_TIME}s" + echo " • Image Size: ${IMAGE_SIZE}MB" + if [[ "$CACHE_HIT" == "true" ]]; then + echo "🚀 Redis image restored from cache - faster startup achieved!" + else + echo "📥 Redis image pulled from Docker Hub - consider cache warming" + fi else echo "❌ Redis service verification failed after ${CONNECTION_TIME}s" + if [[ "$CACHE_HIT" == "true" ]]; then + echo "ℹ️ Cache was available but Redis service connection failed" + fi exit 1 fi diff --git a/.github/actions/warm-cache/action.yml b/.github/actions/warm-cache/action.yml index e09a7b0..8f26963 100644 --- a/.github/actions/warm-cache/action.yml +++ b/.github/actions/warm-cache/action.yml @@ -34,6 +34,14 @@ inputs: env-json: description: "JSON string of environment variables" required: true + redis-enabled: + description: "Whether Redis caching is enabled" + required: false + default: "false" + redis-versions: + description: "Comma-separated list of Redis versions to warm" + required: false + default: "7-alpine" runs: using: "composite" @@ -91,6 +99,18 @@ runs: magex-version: ${{ env.MAGE_X_VERSION }} runner-os: ${{ inputs.matrix-os }} + # ———————————————————————————————————————————————————————————————— + # Warm Redis Docker image cache + # ———————————————————————————————————————————————————————————————— + - name: 🗄️ Warm Redis Cache + if: inputs.redis-enabled == 'true' + id: warm-redis + uses: ./.github/actions/warm-redis-cache + with: + redis-versions: ${{ inputs.redis-versions }} + runner-os: ${{ inputs.matrix-os }} + force-pull: "true" + # ———————————————————————————————————————————————————————————————— # Ensure go.sum exists and download modules # ———————————————————————————————————————————————————————————————— @@ -178,6 +198,14 @@ runs: run: | STATS_FILE="cache-stats-${{ inputs.matrix-os }}-${{ inputs.go-version }}.json" + # Redis cache statistics + REDIS_ENABLED="${{ inputs.redis-enabled }}" + REDIS_VERSIONS_WARMED="${{ steps.warm-redis.outputs.versions-warmed || '0' }}" + REDIS_TOTAL_CACHE_SIZE="${{ steps.warm-redis.outputs.total-cache-size || '0' }}" + REDIS_OPERATION_TIME="${{ steps.warm-redis.outputs.operation-time || '0' }}" + REDIS_WARMING_STATUS="${{ steps.warm-redis.outputs.warming-status || 'disabled' }}" + REDIS_CACHE_KEYS="${{ steps.warm-redis.outputs.cache-keys || '' }}" + cat > "$STATS_FILE" << EOF { "os": "${{ inputs.matrix-os }}", @@ -186,6 +214,12 @@ runs: "gobuild_cache_hit": ${{ steps.setup-go.outputs.build-cache-hit == 'true' && 'true' || 'false' }}, "cache_size_gomod": "$(du -sh $GOMODCACHE 2>/dev/null | cut -f1 || echo '0')", "cache_size_gobuild": "$(du -sh $GOCACHE 2>/dev/null | cut -f1 || echo '0')", + "redis_enabled": "$REDIS_ENABLED", + "redis_versions_warmed": "$REDIS_VERSIONS_WARMED", + "redis_cache_size_mb": "$REDIS_TOTAL_CACHE_SIZE", + "redis_operation_time_s": "$REDIS_OPERATION_TIME", + "redis_warming_status": "$REDIS_WARMING_STATUS", + "redis_cache_keys": "$REDIS_CACHE_KEYS", "workflow": "cache-warm", "job_name": "warm-cache", "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)" @@ -195,6 +229,20 @@ runs: echo "📊 Cache statistics:" jq . "$STATS_FILE" + # Additional Redis warming report + if [[ "$REDIS_ENABLED" == "true" ]]; then + echo "" + echo "🗄️ Redis Cache Warming Summary:" + echo " • Versions Warmed: $REDIS_VERSIONS_WARMED" + echo " • Total Cache Size: ${REDIS_TOTAL_CACHE_SIZE}MB" + echo " • Operation Time: ${REDIS_OPERATION_TIME}s" + echo " • Status: $REDIS_WARMING_STATUS" + if [[ -n "$REDIS_CACHE_KEYS" ]]; then + echo " • Cache Keys:" + echo "$REDIS_CACHE_KEYS" | tr ',' '\n' | sed 's/^/ - /' + fi + fi + # ———————————————————————————————————————————————————————————————— # Upload cache statistics # ———————————————————————————————————————————————————————————————— diff --git a/.github/actions/warm-redis-cache/action.yml b/.github/actions/warm-redis-cache/action.yml new file mode 100644 index 0000000..8524c66 --- /dev/null +++ b/.github/actions/warm-redis-cache/action.yml @@ -0,0 +1,339 @@ +# ------------------------------------------------------------------------------------ +# Warm Redis Cache Composite Action (GoFortress) +# +# Purpose: Proactively warm Redis Docker image cache to ensure fast startup times +# for test workflows. This action pulls and caches Redis images on a schedule +# to maintain optimal cache hit rates. +# +# Features: +# - Proactive Redis image cache warming +# - Support for multiple Redis versions +# - Force pull to ensure latest images +# - Cache performance metrics and reporting +# - Integration with scheduled warm cache workflows +# - Configurable Redis version lists +# +# Usage: +# - uses: ./.github/actions/warm-redis-cache +# with: +# redis-versions: "7-alpine,6-alpine,latest" +# runner-os: ${{ runner.os }} +# force-pull: "true" +# +# Maintainer: @mrz1836 +# +# ------------------------------------------------------------------------------------ + +name: "Warm Redis Cache" +description: "Proactively warm Redis Docker image cache for faster test workflows" + +inputs: + redis-versions: + description: "Comma-separated list of Redis versions to cache (e.g., '7-alpine,6-alpine')" + required: false + default: "7-alpine" + runner-os: + description: "Operating system for cache keys (e.g., Linux, macOS)" + required: true + force-pull: + description: "Force pull images even if cache exists" + required: false + default: "true" + primary-version: + description: "Primary Redis version (will be warmed first)" + required: false + default: "7-alpine" + +outputs: + versions-warmed: + description: "Number of Redis versions successfully warmed" + value: ${{ steps.warm-summary.outputs.versions-warmed }} + total-cache-size: + description: "Total cache size across all versions in MB" + value: ${{ steps.warm-summary.outputs.total-cache-size }} + operation-time: + description: "Total warming operation time in seconds" + value: ${{ steps.warm-summary.outputs.operation-time }} + cache-keys: + description: "Comma-separated list of cache keys created" + value: ${{ steps.warm-summary.outputs.cache-keys }} + warming-status: + description: "Overall warming status: success, partial, failed" + value: ${{ steps.warm-summary.outputs.warming-status }} + +runs: + using: "composite" + steps: + # ———————————————————————————————————————————————————————————————— + # Initialize warming operation + # ———————————————————————————————————————————————————————————————— + - name: ⏱️ Initialize Redis cache warming + id: warming-start + shell: bash + run: | + echo "🗄️ Starting Redis cache warming operation..." + WARMING_START=$(date +%s) + echo "warming-start=$WARMING_START" >> $GITHUB_OUTPUT + + # Parse Redis versions + REDIS_VERSIONS="${{ inputs.redis-versions }}" + PRIMARY_VERSION="${{ inputs.primary-version }}" + + echo "📋 Warming Configuration:" + echo " • Redis Versions: $REDIS_VERSIONS" + echo " • Primary Version: $PRIMARY_VERSION" + echo " • Runner OS: ${{ inputs.runner-os }}" + echo " • Force Pull: ${{ inputs.force-pull }}" + echo "" + + # Convert comma-separated versions to array + IFS=',' read -ra VERSION_ARRAY <<< "$REDIS_VERSIONS" + + # Create unique list and prioritize primary version + UNIQUE_VERSIONS=() + if [[ -n "$PRIMARY_VERSION" && "$PRIMARY_VERSION" != "null" ]]; then + UNIQUE_VERSIONS+=("$PRIMARY_VERSION") + fi + + for version in "${VERSION_ARRAY[@]}"; do + version=$(echo "$version" | xargs) # trim whitespace + if [[ -n "$version" && "$version" != "$PRIMARY_VERSION" ]]; then + UNIQUE_VERSIONS+=("$version") + fi + done + + # Convert back to comma-separated for processing + ORDERED_VERSIONS=$(IFS=','; echo "${UNIQUE_VERSIONS[*]}") + echo "📋 Ordered versions (primary first): $ORDERED_VERSIONS" + + echo "ordered-versions=$ORDERED_VERSIONS" >> $GITHUB_OUTPUT + echo "version-count=${#UNIQUE_VERSIONS[@]}" >> $GITHUB_OUTPUT + + # ———————————————————————————————————————————————————————————————— + # Warm primary Redis version first (critical path optimization) + # ———————————————————————————————————————————————————————————————— + - name: 🚀 Warm primary Redis version + id: warm-primary + if: inputs.primary-version != '' && inputs.primary-version != 'null' + uses: ./.github/actions/cache-redis-image + with: + redis-version: ${{ inputs.primary-version }} + runner-os: ${{ inputs.runner-os }} + cache-mode: "restore-save" + force-pull: ${{ inputs.force-pull }} + + # ———————————————————————————————————————————————————————————————— + # Process all Redis versions for warming + # ———————————————————————————————————————————————————————————————— + - name: 🔥 Warm Redis versions + id: warm-versions + shell: bash + run: | + echo "🔥 Warming Redis versions..." + + ORDERED_VERSIONS="${{ steps.warming-start.outputs.ordered-versions }}" + RUNNER_OS="${{ inputs.runner-os }}" + FORCE_PULL="${{ inputs.force-pull }}" + + # Initialize tracking variables + VERSIONS_WARMED=0 + TOTAL_CACHE_SIZE=0 + CACHE_KEYS="" + WARMING_ERRORS=0 + WARMING_SUCCESS=0 + + # Convert to array for processing + IFS=',' read -ra VERSION_ARRAY <<< "$ORDERED_VERSIONS" + + echo "🗂️ Processing ${#VERSION_ARRAY[@]} Redis versions..." + + for i in "${!VERSION_ARRAY[@]}"; do + VERSION=$(echo "${VERSION_ARRAY[$i]}" | xargs) # trim whitespace + + if [[ -z "$VERSION" ]]; then + continue + fi + + echo "" + echo "📦 [$((i+1))/${#VERSION_ARRAY[@]}] Processing Redis version: $VERSION" + + # Skip primary version if already warmed + if [[ "$VERSION" == "${{ inputs.primary-version }}" && "${{ steps.warm-primary.outputs.cache-hit }}" == "true" ]]; then + echo "✅ Primary version already warmed, using existing cache" + VERSIONS_WARMED=$((VERSIONS_WARMED + 1)) + WARMING_SUCCESS=$((WARMING_SUCCESS + 1)) + + # Add cache information from primary warming + PRIMARY_SIZE="${{ steps.warm-primary.outputs.image-size || '0' }}" + PRIMARY_KEY="${{ steps.warm-primary.outputs.cache-key }}" + + TOTAL_CACHE_SIZE=$((TOTAL_CACHE_SIZE + PRIMARY_SIZE)) + if [[ -n "$CACHE_KEYS" ]]; then + CACHE_KEYS="${CACHE_KEYS},${PRIMARY_KEY}" + else + CACHE_KEYS="$PRIMARY_KEY" + fi + continue + fi + + # Set up temporary outputs for this version + VERSION_SUCCESS="false" + VERSION_SIZE=0 + VERSION_KEY="" + + # Use a subshell to isolate the warming operation + ( + echo "🔧 Setting up cache for Redis $VERSION..." + + # Create normalized version for cache key + NORMALIZED_VERSION=$(echo "$VERSION" | sed 's/[^a-zA-Z0-9.-]/_/g') + CACHE_KEY="redis-image-${RUNNER_OS}-${NORMALIZED_VERSION}" + + # Check if image needs to be pulled + REDIS_IMAGE="redis:$VERSION" + NEEDS_PULL="true" + + if [[ "$FORCE_PULL" != "true" ]]; then + # Check if image exists locally + if docker image inspect "$REDIS_IMAGE" >/dev/null 2>&1; then + echo "ℹ️ Image already exists locally: $REDIS_IMAGE" + NEEDS_PULL="false" + fi + fi + + # Pull image if needed + if [[ "$NEEDS_PULL" == "true" ]]; then + echo "📥 Pulling Redis image: $REDIS_IMAGE" + if docker pull "$REDIS_IMAGE"; then + echo "✅ Successfully pulled: $REDIS_IMAGE" + else + echo "❌ Failed to pull: $REDIS_IMAGE" + exit 1 + fi + fi + + # Get image size + IMAGE_SIZE_BYTES=$(docker image inspect "$REDIS_IMAGE" --format='{{.Size}}' 2>/dev/null || echo "0") + IMAGE_SIZE_MB=$((IMAGE_SIZE_BYTES / 1024 / 1024)) + + echo "📊 Image size: ${IMAGE_SIZE_MB}MB" + + # Save results to temporary files for parent shell + echo "true" > "/tmp/version_success_${i}" + echo "$IMAGE_SIZE_MB" > "/tmp/version_size_${i}" + echo "$CACHE_KEY" > "/tmp/version_key_${i}" + + ) && VERSION_SUCCESS="true" || VERSION_SUCCESS="false" + + # Read results from temporary files + if [[ -f "/tmp/version_success_${i}" ]]; then + VERSION_SUCCESS=$(cat "/tmp/version_success_${i}") + fi + if [[ -f "/tmp/version_size_${i}" ]]; then + VERSION_SIZE=$(cat "/tmp/version_size_${i}") + fi + if [[ -f "/tmp/version_key_${i}" ]]; then + VERSION_KEY=$(cat "/tmp/version_key_${i}") + fi + + # Update tracking based on results + if [[ "$VERSION_SUCCESS" == "true" ]]; then + echo "✅ Successfully warmed Redis $VERSION" + VERSIONS_WARMED=$((VERSIONS_WARMED + 1)) + WARMING_SUCCESS=$((WARMING_SUCCESS + 1)) + TOTAL_CACHE_SIZE=$((TOTAL_CACHE_SIZE + VERSION_SIZE)) + + if [[ -n "$CACHE_KEYS" ]]; then + CACHE_KEYS="${CACHE_KEYS},${VERSION_KEY}" + else + CACHE_KEYS="$VERSION_KEY" + fi + else + echo "❌ Failed to warm Redis $VERSION" + WARMING_ERRORS=$((WARMING_ERRORS + 1)) + fi + + # Clean up temporary files + rm -f "/tmp/version_success_${i}" "/tmp/version_size_${i}" "/tmp/version_key_${i}" + done + + # Determine overall warming status + if [[ $WARMING_ERRORS -eq 0 ]]; then + WARMING_STATUS="success" + elif [[ $WARMING_SUCCESS -gt 0 ]]; then + WARMING_STATUS="partial" + else + WARMING_STATUS="failed" + fi + + echo "" + echo "📊 Warming Results:" + echo " • Versions Warmed: $VERSIONS_WARMED" + echo " • Total Cache Size: ${TOTAL_CACHE_SIZE}MB" + echo " • Successful: $WARMING_SUCCESS" + echo " • Errors: $WARMING_ERRORS" + echo " • Status: $WARMING_STATUS" + + # Set outputs + echo "versions-warmed=$VERSIONS_WARMED" >> $GITHUB_OUTPUT + echo "total-cache-size=$TOTAL_CACHE_SIZE" >> $GITHUB_OUTPUT + echo "cache-keys=$CACHE_KEYS" >> $GITHUB_OUTPUT + echo "warming-status=$WARMING_STATUS" >> $GITHUB_OUTPUT + echo "warming-success=$WARMING_SUCCESS" >> $GITHUB_OUTPUT + echo "warming-errors=$WARMING_ERRORS" >> $GITHUB_OUTPUT + + # ———————————————————————————————————————————————————————————————— + # Cache warming summary and performance metrics + # ———————————————————————————————————————————————————————————————— + - name: ✅ Warming operation summary + id: warm-summary + shell: bash + run: | + echo "✅ Redis cache warming operation completed" + + # Calculate operation time + WARMING_START="${{ steps.warming-start.outputs.warming-start }}" + WARMING_END=$(date +%s) + OPERATION_TIME=$((WARMING_END - WARMING_START)) + + # Get results from warming step + VERSIONS_WARMED="${{ steps.warm-versions.outputs.versions-warmed }}" + TOTAL_CACHE_SIZE="${{ steps.warm-versions.outputs.total-cache-size }}" + CACHE_KEYS="${{ steps.warm-versions.outputs.cache-keys }}" + WARMING_STATUS="${{ steps.warm-versions.outputs.warming-status }}" + WARMING_SUCCESS="${{ steps.warm-versions.outputs.warming-success }}" + WARMING_ERRORS="${{ steps.warm-versions.outputs.warming-errors }}" + + echo "🎯 Final Warming Summary:" + echo " • Versions Warmed: $VERSIONS_WARMED" + echo " • Total Cache Size: ${TOTAL_CACHE_SIZE}MB" + echo " • Operation Time: ${OPERATION_TIME}s" + echo " • Status: $WARMING_STATUS" + echo " • Success Rate: $WARMING_SUCCESS/$((WARMING_SUCCESS + WARMING_ERRORS))" + + if [[ -n "$CACHE_KEYS" ]]; then + echo " • Cache Keys Created:" + IFS=',' read -ra KEY_ARRAY <<< "$CACHE_KEYS" + for key in "${KEY_ARRAY[@]}"; do + echo " - $key" + done + fi + + # Set final outputs + echo "versions-warmed=$VERSIONS_WARMED" >> $GITHUB_OUTPUT + echo "total-cache-size=$TOTAL_CACHE_SIZE" >> $GITHUB_OUTPUT + echo "operation-time=$OPERATION_TIME" >> $GITHUB_OUTPUT + echo "cache-keys=$CACHE_KEYS" >> $GITHUB_OUTPUT + echo "warming-status=$WARMING_STATUS" >> $GITHUB_OUTPUT + + # Set appropriate exit code based on warming status + if [[ "$WARMING_STATUS" == "failed" ]]; then + echo "❌ Redis cache warming failed completely" + exit 1 + elif [[ "$WARMING_STATUS" == "partial" ]]; then + echo "⚠️ Redis cache warming partially successful" + echo "Some versions may experience slower startup times" + else + echo "🚀 Redis cache warming completed successfully" + echo "All Redis versions cached and ready for fast startup" + fi diff --git a/.github/workflows/fortress-benchmarks.yml b/.github/workflows/fortress-benchmarks.yml index bfec2af..a5fc4aa 100644 --- a/.github/workflows/fortress-benchmarks.yml +++ b/.github/workflows/fortress-benchmarks.yml @@ -143,14 +143,17 @@ jobs: runner-os: ${{ matrix.os }} # ———————————————————————————————————————————————————————————————— - # Setup Redis service using composite action + # Setup Redis service using composite action with caching # ———————————————————————————————————————————————————————————————— - name: 🗄️ Setup Redis Service + id: setup-redis uses: ./.github/actions/setup-redis-service with: redis-enabled: ${{ inputs.redis-enabled }} + redis-version: ${{ inputs.redis-version }} redis-host: ${{ inputs.redis-host }} redis-port: ${{ inputs.redis-port }} + use-cache: "true" # ———————————————————————————————————————————————————————————————— # Start benchmark timer @@ -320,6 +323,10 @@ jobs: cache-prefix: cache-stats gomod-cache-hit: ${{ steps.setup-go-bench.outputs.module-cache-hit }} gobuild-cache-hit: ${{ steps.setup-go-bench.outputs.build-cache-hit }} + redis-enabled: ${{ inputs.redis-enabled }} + redis-cache-hit: ${{ steps.setup-redis.outputs.cache-hit }} + redis-image-size: ${{ steps.setup-redis.outputs.image-size }} + redis-operation-time: ${{ steps.setup-redis.outputs.cache-operation-time }} # ———————————————————————————————————————————————————————————————— # Upload performance cache statistics diff --git a/.github/workflows/fortress-completion-statistics.yml b/.github/workflows/fortress-completion-statistics.yml index 137b96b..f410bf9 100644 --- a/.github/workflows/fortress-completion-statistics.yml +++ b/.github/workflows/fortress-completion-statistics.yml @@ -192,8 +192,8 @@ jobs: { echo "" echo "### 💾 Cache Statistics" - echo "| Workflow/Job | OS | Go Version | Module Cache | Build Cache | Module Size | Build Size |" - echo "|--------------|----|-----------|--------------|-----------|-----------|------------|" + echo "| Workflow/Job | OS | Go Version | Module Cache | Build Cache | Module Size | Build Size | Redis Cache | Redis Size |" + echo "|--------------|----|-----------|--------------|-----------|-----------|------------|-------------|------------|" } >> statistics-section.md TOTAL_CACHE_HITS=0 @@ -211,9 +211,23 @@ jobs: GOMOD_SIZE=$(jq -r '.cache_size_gomod' "$stats_file") GOBUILD_SIZE=$(jq -r '.cache_size_gobuild' "$stats_file") + # Redis cache statistics + REDIS_ENABLED=$(jq -r '.redis_enabled // "false"' "$stats_file") + REDIS_HIT=$(jq -r '.redis_cache_hit // "false"' "$stats_file") + REDIS_SIZE=$(jq -r '.redis_image_size_mb // "0"' "$stats_file") + GOMOD_ICON=$([[ "$GOMOD_HIT" == "true" ]] && echo "✅ Hit" || echo "❌ Miss") GOBUILD_ICON=$([[ "$GOBUILD_HIT" == "true" ]] && echo "✅ Hit" || echo "❌ Miss") + # Redis cache display + if [[ "$REDIS_ENABLED" == "true" ]]; then + REDIS_ICON=$([[ "$REDIS_HIT" == "true" ]] && echo "✅ Hit" || echo "❌ Miss") + REDIS_SIZE_DISPLAY="${REDIS_SIZE}MB" + else + REDIS_ICON="➖ N/A" + REDIS_SIZE_DISPLAY="➖" + fi + # Create workflow/job identifier if [[ -n "$JOB_NAME" && "$JOB_NAME" != "null" ]]; then WORKFLOW_JOB="${WORKFLOW}/${JOB_NAME}" @@ -221,11 +235,14 @@ jobs: WORKFLOW_JOB="${WORKFLOW}" fi - echo "| $WORKFLOW_JOB | $OS | $GO_VER | $GOMOD_ICON | $GOBUILD_ICON | $GOMOD_SIZE | $GOBUILD_SIZE |" >> statistics-section.md + echo "| $WORKFLOW_JOB | $OS | $GO_VER | $GOMOD_ICON | $GOBUILD_ICON | $GOMOD_SIZE | $GOBUILD_SIZE | $REDIS_ICON | $REDIS_SIZE_DISPLAY |" >> statistics-section.md [[ "$GOMOD_HIT" == "true" ]] && TOTAL_CACHE_HITS=$((TOTAL_CACHE_HITS + 1)) [[ "$GOBUILD_HIT" == "true" ]] && TOTAL_CACHE_HITS=$((TOTAL_CACHE_HITS + 1)) + [[ "$REDIS_ENABLED" == "true" && "$REDIS_HIT" == "true" ]] && TOTAL_CACHE_HITS=$((TOTAL_CACHE_HITS + 1)) + TOTAL_CACHE_ATTEMPTS=$((TOTAL_CACHE_ATTEMPTS + 2)) + [[ "$REDIS_ENABLED" == "true" ]] && TOTAL_CACHE_ATTEMPTS=$((TOTAL_CACHE_ATTEMPTS + 1)) # Track workflows that used cache if [[ "$WORKFLOWS_WITH_CACHE" != *"$WORKFLOW"* ]]; then diff --git a/.github/workflows/fortress-test-matrix.yml b/.github/workflows/fortress-test-matrix.yml index 966ce55..24062bc 100644 --- a/.github/workflows/fortress-test-matrix.yml +++ b/.github/workflows/fortress-test-matrix.yml @@ -193,14 +193,17 @@ jobs: echo "test-count=$TEST_COUNT" >> $GITHUB_OUTPUT # ———————————————————————————————————————————————————————————————— - # Setup Redis service using composite action + # Setup Redis service using composite action with caching # ———————————————————————————————————————————————————————————————— - name: 🗄️ Setup Redis Service + id: setup-redis uses: ./.github/actions/setup-redis-service with: redis-enabled: ${{ inputs.redis-enabled }} + redis-version: ${{ inputs.redis-version }} redis-host: ${{ inputs.redis-host }} redis-port: ${{ inputs.redis-port }} + use-cache: "true" # ———————————————————————————————————————————————————————————————— # Setup test failure detection functions @@ -598,6 +601,40 @@ jobs: cache-prefix: cache-stats gomod-cache-hit: ${{ steps.setup-go-test.outputs.module-cache-hit }} gobuild-cache-hit: ${{ steps.setup-go-test.outputs.build-cache-hit }} + redis-enabled: ${{ inputs.redis-enabled }} + redis-cache-hit: ${{ steps.setup-redis.outputs.cache-hit }} + redis-image-size: ${{ steps.setup-redis.outputs.image-size }} + redis-operation-time: ${{ steps.setup-redis.outputs.cache-operation-time }} + + # ———————————————————————————————————————————————————————————————— + # Report Redis cache performance metrics + # ———————————————————————————————————————————————————————————————— + - name: 📊 Report Redis Cache Performance + if: inputs.redis-enabled == 'true' + run: | + echo "📊 Redis Cache Performance Report" + echo "=================================" + echo "• Redis Enabled: ${{ inputs.redis-enabled }}" + echo "• Redis Version: ${{ inputs.redis-version }}" + echo "• Cache Hit: ${{ steps.setup-redis.outputs.cache-hit }}" + echo "• Image Size: ${{ steps.setup-redis.outputs.image-size }}MB" + echo "• Cache Operation Time: ${{ steps.setup-redis.outputs.cache-operation-time }}s" + echo "• Connection Time: ${{ steps.setup-redis.outputs.connection-time }}s" + echo "• Installation Method: ${{ steps.setup-redis.outputs.installation-method }}" + + # Calculate total Redis setup time + CACHE_TIME="${{ steps.setup-redis.outputs.cache-operation-time || '0' }}" + CONNECTION_TIME="${{ steps.setup-redis.outputs.connection-time || '0' }}" + TOTAL_TIME=$((CACHE_TIME + CONNECTION_TIME)) + + echo "• Total Setup Time: ${TOTAL_TIME}s" + + # Performance assessment + if [[ "${{ steps.setup-redis.outputs.cache-hit }}" == "true" ]]; then + echo "🚀 Performance: Redis cache hit - faster startup achieved!" + else + echo "📥 Performance: Redis pulled from Docker Hub - consider cache warming" + fi # ———————————————————————————————————————————————————————————————— # Upload performance cache statistics for completion report diff --git a/.github/workflows/fortress.yml b/.github/workflows/fortress.yml index 4f2bab5..edea6b9 100644 --- a/.github/workflows/fortress.yml +++ b/.github/workflows/fortress.yml @@ -142,6 +142,8 @@ jobs: with: sparse-checkout: | .github/actions/warm-cache + .github/actions/warm-redis-cache + .github/actions/cache-redis-image .github/.env.base go.mod go.sum @@ -156,9 +158,9 @@ jobs: echo "enable_verbose=$(echo '${{ needs.load-env.outputs.env-json }}' | jq -r '.ENABLE_VERBOSE_TEST_OUTPUT')" >> "$GITHUB_OUTPUT" # ———————————————————————————————————————————————————————————————— - # Warm the Go caches using local action + # Warm the Go and Redis caches using local action # ———————————————————————————————————————————————————————————————— - - name: 🔥 Warm Go Caches + - name: 🔥 Warm Go and Redis Caches uses: ./.github/actions/warm-cache # Might not resolve as it's a composite action with: go-version: ${{ matrix.go-version }} @@ -168,6 +170,8 @@ jobs: go-primary-version: ${{ needs.setup.outputs.go-primary-version }} go-secondary-version: ${{ needs.setup.outputs.go-secondary-version }} env-json: ${{ needs.load-env.outputs.env-json }} + redis-enabled: ${{ needs.setup.outputs.redis-enabled }} + redis-versions: ${{ needs.setup.outputs.redis-version }} # ---------------------------------------------------------------------------------- # Security Scans # ----------------------------------------------------------------------------------