Feature/encryption custom compatibility testing #26
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Encryption.Custom Compatibility Tests | |
| # Manual runs: Go to Actions > Encryption.Custom Compatibility Tests > Run workflow | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - main | |
| paths: | |
| - 'Microsoft.Azure.Cosmos.Encryption.Custom/**' | |
| - '.github/workflows/encryption-custom-compatibility.yml' | |
| push: | |
| branches: | |
| - master | |
| - main | |
| paths: | |
| - 'Microsoft.Azure.Cosmos.Encryption.Custom/**' | |
| schedule: | |
| # Run daily at 2:00 AM UTC for comprehensive version matrix | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| test_versions: | |
| description: 'Comma-separated list of versions to test (e.g., "1.0.0-preview07"). Leave empty to test against all build sources.' | |
| required: false | |
| default: '1.0.0-preview07' | |
| type: string | |
| build_sources: | |
| description: 'JSON array of build sources: [{"repo":"owner/repo","ref":"branch","name":"label"}]. Leave empty for current repo/branch.' | |
| required: false | |
| default: '' | |
| type: string | |
| env: | |
| DOTNET_VERSION: '8.0.x' | |
| jobs: | |
| # Prepare build matrix | |
| prepare-matrix: | |
| name: Prepare Build Matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| build_sources: ${{ steps.setup-matrix.outputs.build_sources }} | |
| steps: | |
| - name: Setup build matrix | |
| id: setup-matrix | |
| run: | | |
| # Debug: Show trigger information | |
| echo "Event name: ${{ github.event_name }}" | |
| echo "Build sources input: '${{ github.event.inputs.build_sources }}'" | |
| # Use custom build sources if provided via workflow_dispatch input | |
| BUILD_SOURCES_INPUT="${{ github.event.inputs.build_sources }}" | |
| if [ -n "$BUILD_SOURCES_INPUT" ] && [ "$BUILD_SOURCES_INPUT" != "" ]; then | |
| BUILD_SOURCES="$BUILD_SOURCES_INPUT" | |
| echo "Using custom build sources from workflow input" | |
| else | |
| # Always test against all sources (current + 5 PRs) for comprehensive validation | |
| # Single-line JSON to avoid formatting issues | |
| BUILD_SOURCES='[{"repository":"${{ github.repository }}","ref":"${{ github.ref }}","source_name":"current"},{"repository":"Azure/azure-cosmos-dotnet-v3","ref":"refs/pull/5385/head","source_name":"pr-5385"},{"repository":"Azure/azure-cosmos-dotnet-v3","ref":"refs/pull/5399/head","source_name":"pr-5399"},{"repository":"Azure/azure-cosmos-dotnet-v3","ref":"refs/pull/5403/head","source_name":"pr-5403"},{"repository":"Azure/azure-cosmos-dotnet-v3","ref":"refs/pull/5417/head","source_name":"pr-5417"},{"repository":"Azure/azure-cosmos-dotnet-v3","ref":"refs/pull/5428/head","source_name":"pr-5428"}]' | |
| echo "Using full build matrix with all PRs" | |
| fi | |
| echo "build_sources=$BUILD_SOURCES" >> $GITHUB_OUTPUT | |
| echo "Build sources matrix:" | |
| echo "$BUILD_SOURCES" | jq '.' | |
| # Build packages from different sources (matrix) | |
| build-packages: | |
| name: Build ${{ matrix.build_source.source_name }} | |
| needs: prepare-matrix | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| build_source: ${{ fromJson(needs.prepare-matrix.outputs.build_sources) }} | |
| outputs: | |
| # Output pattern: package_version_<source_name> | |
| package_version: ${{ steps.build-pkg.outputs.version }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ matrix.build_source.repository }} | |
| ref: ${{ matrix.build_source.ref }} | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: ${{ env.DOTNET_VERSION }} | |
| - name: Build and pack | |
| id: build-pkg | |
| run: | | |
| # Create artifacts/local-packages folder (required by Directory.Build.props RestoreSources) | |
| mkdir -p artifacts/local-packages | |
| # Generate unique version with source identifier | |
| SOURCE_SLUG=$(echo "${{ matrix.build_source.source_name }}" | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]-') | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| VERSION_SUFFIX="pr${{ github.event.pull_request.number }}-${SOURCE_SLUG}-${{ github.run_number }}" | |
| else | |
| VERSION_SUFFIX="${SOURCE_SLUG}-${{ github.run_number }}" | |
| fi | |
| # Get base version from Directory.Build.props | |
| BASE_VERSION=$(grep -oPm1 "(?<=<CustomEncryptionVersion>)[^<]+" Directory.Build.props) | |
| PACKAGE_VERSION="${BASE_VERSION}-${VERSION_SUFFIX}" | |
| echo "=========================================" | |
| echo "Building from: ${{ matrix.build_source.repository }}@${{ matrix.build_source.ref }}" | |
| echo "Source name: ${{ matrix.build_source.source_name }}" | |
| echo "Package version: $PACKAGE_VERSION" | |
| echo "=========================================" | |
| echo "version=$PACKAGE_VERSION" >> $GITHUB_OUTPUT | |
| # Build and pack | |
| dotnet pack Microsoft.Azure.Cosmos.Encryption.Custom/src/Microsoft.Azure.Cosmos.Encryption.Custom.csproj \ | |
| --configuration Release \ | |
| -p:Version=$PACKAGE_VERSION \ | |
| -p:PackageVersion=$PACKAGE_VERSION \ | |
| --output ./packages | |
| BUILD_EXIT_CODE=$? | |
| if [ $BUILD_EXIT_CODE -ne 0 ]; then | |
| echo "=========================================" | |
| echo "⚠️ Build failed for ${{ matrix.build_source.source_name }}" | |
| echo "This source will be skipped in compatibility testing" | |
| echo "=========================================" | |
| exit 1 | |
| fi | |
| PACKAGE_FILE="./packages/Microsoft.Azure.Cosmos.Encryption.Custom.${PACKAGE_VERSION}.nupkg" | |
| echo "path=$PACKAGE_FILE" >> $GITHUB_OUTPUT | |
| echo "=========================================" | |
| echo "✅ Built Package: $PACKAGE_FILE" | |
| ls -lh ./packages/ | |
| echo "=========================================" | |
| - name: Upload package artifact | |
| if: steps.build-pkg.outcome == 'success' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: package-${{ matrix.build_source.source_name }} | |
| path: ./packages/*.nupkg | |
| retention-days: 7 | |
| # Run compatibility tests with version matrix | |
| # Tests built packages against specified published versions | |
| compatibility-tests: | |
| name: Test ${{ matrix.build_source.source_name }} ⟷ ${{ matrix.test_version }} | |
| needs: [prepare-matrix, build-packages] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Each build source will be tested against each test version | |
| build_source: ${{ fromJson(needs.prepare-matrix.outputs.build_sources) }} | |
| test_version: | |
| - '1.0.0-preview07' | |
| # Add more versions here as needed | |
| # - '1.0.0-preview06' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup .NET | |
| uses: actions/setup-dotnet@v4 | |
| with: | |
| dotnet-version: ${{ env.DOTNET_VERSION }} | |
| - name: Cache NuGet packages | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.nuget/packages | |
| key: ${{ runner.os }}-nuget-compat-${{ hashFiles('**/Directory.Packages.props', '**/packages.lock.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-nuget-compat- | |
| ${{ runner.os }}-nuget- | |
| - name: Download built package | |
| id: download-pkg | |
| continue-on-error: true | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: package-${{ matrix.build_source.source_name }} | |
| path: ./local-packages | |
| - name: Check if package was built | |
| id: check-pkg | |
| run: | | |
| if [ "${{ steps.download-pkg.outcome }}" != "success" ]; then | |
| echo "=========================================" | |
| echo "⚠️ Skipping tests for ${{ matrix.build_source.source_name }}" | |
| echo "Package was not built successfully" | |
| echo "=========================================" | |
| echo "skip_tests=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "skip_tests=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Setup test environment | |
| if: steps.check-pkg.outputs.skip_tests == 'false' | |
| run: | | |
| # Create artifacts/local-packages folder (required by Directory.Build.props RestoreSources) | |
| mkdir -p artifacts/local-packages | |
| # Copy built package to artifacts/local-packages | |
| cp ./local-packages/*.nupkg artifacts/local-packages/ | |
| echo "Copied built package to artifacts/local-packages" | |
| ls -lh artifacts/local-packages/ | |
| # Get the actual package version from the filename | |
| BUILT_PACKAGE=$(ls ./local-packages/Microsoft.Azure.Cosmos.Encryption.Custom.*.nupkg | head -1) | |
| BUILT_VERSION=$(basename "$BUILT_PACKAGE" | sed 's/Microsoft.Azure.Cosmos.Encryption.Custom.\(.*\).nupkg/\1/') | |
| echo "BUILT_VERSION=$BUILT_VERSION" >> $GITHUB_ENV | |
| echo "Built package version: $BUILT_VERSION" | |
| - name: Update test configuration | |
| if: steps.check-pkg.outputs.skip_tests == 'false' | |
| run: | | |
| BUILT_VERSION="${{ env.BUILT_VERSION }}" | |
| TEST_VERSION="${{ matrix.test_version }}" | |
| # Create version array for this specific test pair | |
| VERSIONS_JSON="[\"$BUILT_VERSION\",\"$TEST_VERSION\"]" | |
| echo "=========================================" | |
| echo "Test Pair Configuration" | |
| echo "=========================================" | |
| echo "Built Source: ${{ matrix.build_source.source_name }}" | |
| echo "Built Version: $BUILT_VERSION" | |
| echo "Test Version: $TEST_VERSION" | |
| echo "Test versions: $VERSIONS_JSON" | |
| echo "=========================================" | |
| # Update testconfig.json with this specific version pair | |
| cd Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.CompatibilityTests | |
| # Update the versions array in testconfig.json | |
| jq ".versionMatrix.versions = $VERSIONS_JSON" testconfig.json > testconfig.json.tmp | |
| mv testconfig.json.tmp testconfig.json | |
| echo "Updated testconfig.json:" | |
| cat testconfig.json | |
| - name: Restore dependencies | |
| if: steps.check-pkg.outputs.skip_tests == 'false' | |
| run: | | |
| dotnet restore Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.CompatibilityTests/Microsoft.Azure.Cosmos.Encryption.Custom.CompatibilityTests.csproj | |
| - name: Build compatibility tests | |
| if: steps.check-pkg.outputs.skip_tests == 'false' | |
| run: | | |
| dotnet build Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.CompatibilityTests/Microsoft.Azure.Cosmos.Encryption.Custom.CompatibilityTests.csproj \ | |
| --configuration Release \ | |
| --no-restore | |
| - name: Run compatibility tests | |
| if: steps.check-pkg.outputs.skip_tests == 'false' | |
| id: run-tests | |
| continue-on-error: true | |
| run: | | |
| dotnet test Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.CompatibilityTests/Microsoft.Azure.Cosmos.Encryption.Custom.CompatibilityTests.csproj \ | |
| --configuration Release \ | |
| --no-build \ | |
| --no-restore \ | |
| --logger "trx;LogFileName=compatibility-${{ matrix.build_source.source_name }}-vs-${{ matrix.test_version }}.trx" \ | |
| --logger "console;verbosity=normal" \ | |
| --results-directory ./TestResults | |
| - name: Upload test results (TRX) | |
| if: always() && steps.check-pkg.outputs.skip_tests == 'false' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results-${{ matrix.build_source.source_name }}-vs-${{ matrix.test_version }} | |
| path: ./TestResults/*.trx | |
| retention-days: 30 | |
| - name: Publish test results | |
| if: always() && steps.check-pkg.outputs.skip_tests == 'false' | |
| uses: dorny/test-reporter@v1 | |
| with: | |
| name: Compatibility Test Results | |
| path: './TestResults/*.trx' | |
| reporter: dotnet-trx | |
| fail-on-error: true | |
| fail-on-empty: true | |
| - name: Generate test summary | |
| if: always() | |
| run: | | |
| # Generate source link | |
| REPO="${{ matrix.build_source.repository }}" | |
| REF="${{ matrix.build_source.ref }}" | |
| SOURCE_NAME="${{ matrix.build_source.source_name }}" | |
| # Check if it's a PR reference (refs/pull/NUMBER/head) | |
| if [[ "$REF" =~ ^refs/pull/([0-9]+)/head$ ]]; then | |
| PR_NUMBER="${BASH_REMATCH[1]}" | |
| SOURCE_LINK="[${SOURCE_NAME}](https://github.com/${REPO}/pull/${PR_NUMBER})" | |
| else | |
| # It's a branch | |
| SOURCE_LINK="[${SOURCE_NAME}](https://github.com/${REPO}/tree/${REF})" | |
| fi | |
| echo "## 🧪 Compatibility Test Results: ${SOURCE_LINK} ⟷ \`${{ matrix.test_version }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Built Source:** ${SOURCE_LINK}" >> $GITHUB_STEP_SUMMARY | |
| echo "**Repository:** \`${{ matrix.build_source.repository }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "**Ref:** \`${{ matrix.build_source.ref }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "**Test Version:** \`${{ matrix.test_version }}\`" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ steps.check-pkg.outputs.skip_tests }}" == "true" ]; then | |
| echo "⚠️ **Status:** Tests skipped - package build failed" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "This source could not be built successfully, likely due to API incompatibilities." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "👉 Check the build logs for details: ${SOURCE_LINK}" >> $GITHUB_STEP_SUMMARY | |
| exit 0 | |
| fi | |
| TRX_FILE="./TestResults/compatibility-${{ matrix.build_source.source_name }}-vs-${{ matrix.test_version }}.trx" | |
| if [ -f "$TRX_FILE" ]; then | |
| # Parse TRX file for summary (basic parsing) | |
| TOTAL=$(grep -oP '(?<=total=")[^"]+' "$TRX_FILE" | head -1) | |
| PASSED=$(grep -oP '(?<=passed=")[^"]+' "$TRX_FILE" | head -1) | |
| FAILED=$(grep -oP '(?<=failed=")[^"]+' "$TRX_FILE" | head -1) | |
| echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Total Tests | $TOTAL |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ✅ Passed | $PASSED |" >> $GITHUB_STEP_SUMMARY | |
| echo "| ❌ Failed | $FAILED |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ "$FAILED" == "0" ]; then | |
| echo "### ✅ All compatibility tests passed!" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "### ❌ Some compatibility tests failed" >> $GITHUB_STEP_SUMMARY | |
| echo "Check the detailed test report above for failure details." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| else | |
| echo "⚠️ Test results file not found: $TRX_FILE" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "---" >> $GITHUB_STEP_SUMMARY | |
| echo "📦 **Test Pair:** \`${{ matrix.build_source.source_name }}\` ⟷ \`${{ matrix.test_version }}\`" >> $GITHUB_STEP_SUMMARY | |
| - name: Fail workflow if tests failed | |
| if: steps.run-tests.outcome == 'failure' | |
| run: | | |
| echo "❌ Compatibility tests failed" | |
| exit 1 | |
| - name: Display test summary | |
| if: always() | |
| run: | | |
| echo "=========================================" | |
| echo "Compatibility Test Summary" | |
| echo "=========================================" | |
| echo "Built Source: ${{ matrix.build_source.source_name }}" | |
| echo "Test Version: ${{ matrix.test_version }}" | |
| echo "=========================================" | |
| if [ "${{ steps.run-tests.outcome }}" == "success" ]; then | |
| echo "✅ Compatibility test passed for this version pair" | |
| else | |
| echo "❌ Compatibility test failures detected for this version pair" | |
| fi | |
| echo "=========================================" |