Add WCRP compliance check pipeline using cc-plugin-wcrp

.github/scripts/gather_test_outputs.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+# Script to gather all NetCDF test outputs into a centralized location
+# This script is designed to be reusable by other QA pipelines (compliance-checker, etc.)
+#
+# Usage: gather_test_outputs.sh <output_directory>
+#
+# The script:
+# 1. Searches for all NetCDF files generated by tests
+# 2. Copies them to a centralized directory with preserved structure
+# 3. Creates a manifest file listing all gathered files
+# 4. Categorizes files by CMIP version (CMIP6 vs CMIP7)
+
+set -e
+
+# Default output directory if not specified
+OUTPUT_DIR="${1:-tmp/qa_test_outputs}"
+
+echo "=========================================="
+echo "Gathering Test Outputs for QA Validation"
+echo "=========================================="
+echo "Output directory: ${OUTPUT_DIR}"
+echo ""
+
+# Create the centralized output directory
+mkdir -p "${OUTPUT_DIR}"
+mkdir -p "${OUTPUT_DIR}/cmip6"
+mkdir -p "${OUTPUT_DIR}/cmip7"
+mkdir -p "${OUTPUT_DIR}/other"
+
+# Initialize counters
+total_files=0
+cmip6_files=0
+cmip7_files=0
+other_files=0
+
+# Create manifest file
+MANIFEST="${OUTPUT_DIR}/manifest.txt"
+echo "# Test Output Manifest" > "${MANIFEST}"
+echo "# Generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "${MANIFEST}"
+echo "# " >> "${MANIFEST}"
+
+echo "Searching for NetCDF files in test directories..."
+echo ""
+
+# Function to categorize and copy files
+gather_files() {
+    local search_dir=$1
+    local label=$2
+
+    if [ ! -d "${search_dir}" ]; then
+        echo "Warning: Directory ${search_dir} not found, skipping"
+        return
+    fi
+
+    echo "Gathering from: ${search_dir} (${label})"
+
+    # Find all .nc files
+    while IFS= read -r -d '' ncfile; do
+        if [ -f "${ncfile}" ]; then
+            # Determine CMIP version from file path or content
+            cmip_version="other"
+            if [[ "${ncfile}" == *"CMIP6"* ]]; then
+                cmip_version="cmip6"
+                cmip6_files=$((cmip6_files + 1))
+            elif [[ "${ncfile}" == *"CMIP7"* ]]; then
+                cmip_version="cmip7"
+                cmip7_files=$((cmip7_files + 1))
+            else
+                # Try to detect from file attributes using ncdump
+                if command -v ncdump &> /dev/null; then
+                    if ncdump -h "${ncfile}" 2>/dev/null | grep -q 'mip_era = "CMIP7"'; then
+                        cmip_version="cmip7"
+                        cmip7_files=$((cmip7_files + 1))
+                    elif ncdump -h "${ncfile}" 2>/dev/null | grep -q 'mip_era = "CMIP6"'; then
+                        cmip_version="cmip6"
+                        cmip6_files=$((cmip6_files + 1))
+                    else
+                        other_files=$((other_files + 1))
+                    fi
+                else
+                    other_files=$((other_files + 1))
+                fi
+            fi
+
+            # Create unique filename to avoid collisions
+            basename=$(basename "${ncfile}")
+            # Add a hash of the full path to ensure uniqueness
+            path_hash=$(echo "${ncfile}" | md5sum | cut -c1-8)
+            unique_name="${path_hash}_${basename}"
+
+            # Copy to categorized directory
+            dest_file="${OUTPUT_DIR}/${cmip_version}/${unique_name}"
+            cp "${ncfile}" "${dest_file}"
+
+            # Add to manifest
+            echo "${ncfile} -> ${cmip_version}/${unique_name}" >> "${MANIFEST}"
+
+            total_files=$((total_files + 1))
+
+            echo "  found: ${ncfile}"
+            echo "    -> ${cmip_version}/${unique_name}"
+        fi
+    done < <(find "${search_dir}" -name "*.nc" -type f -print0 2>/dev/null)
+}
+
+# Gather files from different test output directories
+gather_files "fremorizer/tests/test_files/outdir" "CMIP6 basic tests"
+gather_files "fremorizer/tests/test_files/outdir_ppan_only" "Extended test examples"
+
+# Add summary to manifest
+echo "" >> "${MANIFEST}"
+echo "# Summary" >> "${MANIFEST}"
+echo "# Total files: ${total_files}" >> "${MANIFEST}"
+echo "# CMIP6 files: ${cmip6_files}" >> "${MANIFEST}"
+echo "# CMIP7 files: ${cmip7_files}" >> "${MANIFEST}"
+echo "# Other files: ${other_files}" >> "${MANIFEST}"
+
+echo ""
+echo "=========================================="
+echo "Gathering Complete"
+echo "=========================================="
+echo "Total files gathered: ${total_files}"
+echo "  - CMIP6: ${cmip6_files}"
+echo "  - CMIP7: ${cmip7_files}"
+echo "  - Other: ${other_files}"
+echo ""
+echo "Output directory: ${OUTPUT_DIR}"
+echo "Manifest file: ${MANIFEST}"
+
+# Exit successfully if we found any files
+if [ ${total_files} -gt 0 ]; then
+    exit 0
+else
+    echo "::warning::No NetCDF files were found in test output directories"
+    exit 1
+fi

.github/workflows/wcrp_compliance_check.yml

@@ -0,0 +1,257 @@
+name: wcrp_compliance_check
+on:
+  pull_request:
+    branches:
+  workflow_dispatch:
+
+# cancel running jobs if theres a newer push
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  wcrp-compliance-check:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - name: Checkout Files
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup Conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: fremorizer-wcrp
+          python-version: "3.12"
+          auto-activate-base: false
+          miniforge-version: latest
+          channels: conda-forge,noaa-gfdl
+
+      - name: Configure Conda
+        run: |
+          echo "removing main and r channels from defaults"
+          conda config --remove channels defaults || true
+          conda config --remove channels main || true
+          conda config --remove channels r || true
+
+          echo "setting strict channel priority"
+          conda config --set channel_priority strict
+
+          echo "printing conda config just in case"
+          conda config --show
+
+      - name: Install dependencies for fremorizer
+        run: |
+          conda install -y \
+            conda-forge::cftime \
+            "conda-forge::click>=8.2" \
+            "conda-forge::cmor>=3.14" \
+            "conda-forge::netcdf4>=1.7" \
+            "conda-forge::numpy>=2" \
+            conda-forge::pyyaml \
+            conda-forge::pytest
+
+      - name: Install fremorizer
+        run: |
+          pip install .
+
+      # ── generate test outputs (moved earlier, most fragile step) ──
+      - name: Run tests to generate output files
+        id: run_tests
+        run: |
+          echo "Running test suite to generate NetCDF outputs..."
+          echo "Using --basetemp=tmp/pytest_basetemp to preserve pytest temp artifacts"
+          echo ""
+
+          echo "=========================================="
+          echo "Running CMIP6 basic tests..."
+          echo "=========================================="
+          pytest fremorizer/tests/test_cmor_run_subtool.py -v \
+            --basetemp=tmp/pytest_basetemp || true
+
+          echo ""
+          echo "=========================================="
+          echo "Running extended test examples..."
+          echo "=========================================="
+          pytest fremorizer/tests/test_cmor_run_subtool_further_examples.py -v \
+            --basetemp=tmp/pytest_basetemp || true
+
+          echo ""
+          echo "Test execution complete."
+
+      # ── gather and categorize outputs ──
+      - name: Gather test outputs into centralized directory
+        id: gather_outputs
+        run: |
+          bash .github/scripts/gather_test_outputs.sh tmp/qa_test_outputs
+
+          echo "output_dir=tmp/qa_test_outputs" >> $GITHUB_OUTPUT
+
+          total_files=$(find tmp/qa_test_outputs -name "*.nc" -type f | wc -l)
+          echo "file_count=${total_files}" >> $GITHUB_OUTPUT
+
+          cmip6_files=$(find tmp/qa_test_outputs/cmip6 -name "*.nc" -type f 2>/dev/null | wc -l || echo "0")
+          cmip7_files=$(find tmp/qa_test_outputs/cmip7 -name "*.nc" -type f 2>/dev/null | wc -l || echo "0")
+          other_files=$(find tmp/qa_test_outputs/other -name "*.nc" -type f 2>/dev/null | wc -l || echo "0")
+
+          echo "cmip6_count=${cmip6_files}" >> $GITHUB_OUTPUT
+          echo "cmip7_count=${cmip7_files}" >> $GITHUB_OUTPUT
+          echo "other_count=${other_files}" >> $GITHUB_OUTPUT
+
+          echo ""
+          echo "Summary: total=${total_files} cmip6=${cmip6_files} cmip7=${cmip7_files} other=${other_files}"
+
+      - name: Verify outputs were gathered
+        if: steps.gather_outputs.outputs.file_count == '0'
+        run: |
+          echo "::warning::No NetCDF files were found after running tests"
+          echo "This may indicate that tests did not run successfully or outputs were not generated"
+          echo ""
+          echo "Checking known test output directories for debugging..."
+          echo "--- fremorizer/tests/test_files/outdir ---"
+          find fremorizer/tests/test_files/outdir -name "*.nc" -type f 2>/dev/null || echo "(empty or missing)"
+          echo "--- fremorizer/tests/test_files/outdir_ppan_only ---"
+          find fremorizer/tests/test_files/outdir_ppan_only -name "*.nc" -type f 2>/dev/null || echo "(empty or missing)"
+          echo "--- tmp/pytest_basetemp ---"
+          find tmp/pytest_basetemp -name "*.nc" -type f 2>/dev/null || echo "(empty or missing)"
+
+      # ── install cc-plugin-wcrp (after tests, since it's not needed until now) ──
+      - name: Install cc-plugin-wcrp
+        if: steps.gather_outputs.outputs.file_count != '0'
+        run: |
+          pip install cc-plugin-wcrp
+
+      - name: Configure esgvoc controlled vocabularies
+        if: steps.gather_outputs.outputs.file_count != '0'
+        run: |
+          esgvoc config add cordex-cmip6
+          esgvoc config add cmip7
+          esgvoc config set project:branch=esgvoc
+          esgvoc install
+
+      - name: Verify cc-plugin-wcrp installation
+        if: steps.gather_outputs.outputs.file_count != '0'
+        run: |
+          echo "Listing available compliance-checker plugins..."
+          compliance-checker -l
+
+      # ── run WCRP compliance checks ──
+      - name: Run WCRP compliance checks on CMIP6 outputs
+        if: steps.gather_outputs.outputs.cmip6_count != '0'
+        run: |
+          echo "Running WCRP CMIP6 compliance checks..."
+          echo "CMIP6 files to check: ${{ steps.gather_outputs.outputs.cmip6_count }}"
+          echo ""
+
+          mkdir -p tmp/wcrp_compliance_reports
+
+          for ncfile in tmp/qa_test_outputs/cmip6/*.nc; do
+            if [ -f "$ncfile" ]; then
+              basename=$(basename "$ncfile" .nc)
+              echo ""
+              echo "=========================================="
+              echo "Checking: ${basename}"
+              echo "=========================================="
+
+              # Run wcrp_cmip6 checker - text report for logs
+              compliance-checker \
+                --test=wcrp_cmip6 \
+                --format=text \
+                --output=tmp/wcrp_compliance_reports/cmip6_${basename}_report.txt \
+                "$ncfile" || true
+
+              # Run wcrp_cmip6 checker - json report for artifact
+              compliance-checker \
+                --test=wcrp_cmip6 \
+                --format=json \
+                --output=tmp/wcrp_compliance_reports/cmip6_${basename}_report.json \
+                "$ncfile" 2>&1 || true
+
+              # Display text report summary in logs
+              echo "Report summary:"
+              head -80 tmp/wcrp_compliance_reports/cmip6_${basename}_report.txt || true
+              echo ""
+            fi
+          done
+
+      - name: Run WCRP compliance checks on CMIP7 outputs
+        if: steps.gather_outputs.outputs.cmip7_count != '0'
+        run: |
+          echo "Running WCRP compliance checks on CMIP7 files..."
+          echo "CMIP7 files to check: ${{ steps.gather_outputs.outputs.cmip7_count }}"
+          echo ""
+
+          mkdir -p tmp/wcrp_compliance_reports
+
+          for ncfile in tmp/qa_test_outputs/cmip7/*.nc; do
+            if [ -f "$ncfile" ]; then
+              basename=$(basename "$ncfile" .nc)
+              echo ""
+              echo "=========================================="
+              echo "Checking: ${basename}"
+              echo "=========================================="
+
+              # Run wcrp_cmip6 checker on CMIP7 files too
+              # (the plugin validates WCRP conventions applicable across versions)
+              compliance-checker \
+                --test=wcrp_cmip6 \
+                --format=text \
+                --output=tmp/wcrp_compliance_reports/cmip7_${basename}_report.txt \
+                "$ncfile" || true
+
+              compliance-checker \
+                --test=wcrp_cmip6 \
+                --format=json \
+                --output=tmp/wcrp_compliance_reports/cmip7_${basename}_report.json \
+                "$ncfile" 2>&1 || true
+
+              echo "Report summary:"
+              head -80 tmp/wcrp_compliance_reports/cmip7_${basename}_report.txt || true
+              echo ""
+            fi
+          done
+
+      # ── reports and artifacts ──
+      - name: Generate summary report
+        if: steps.gather_outputs.outputs.file_count != '0'
+        run: |
+          mkdir -p tmp/wcrp_compliance_reports
+
+          echo "# WCRP Compliance Check Results" > tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "**Generated:** $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "**Plugin:** [cc-plugin-wcrp](https://github.com/ESGF/cc-plugin-wcrp)" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "## Files Checked" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "- **Total:** ${{ steps.gather_outputs.outputs.file_count }} files" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "- **CMIP6:** ${{ steps.gather_outputs.outputs.cmip6_count }} files" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "- **CMIP7:** ${{ steps.gather_outputs.outputs.cmip7_count }} files" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "- **Other:** ${{ steps.gather_outputs.outputs.other_count }} files" >> tmp/wcrp_compliance_reports/SUMMARY.md
+          echo "" >> tmp/wcrp_compliance_reports/SUMMARY.md
+
+          cp tmp/qa_test_outputs/manifest.txt tmp/wcrp_compliance_reports/manifest.txt || true
+
+          cat tmp/wcrp_compliance_reports/SUMMARY.md
+
+      - name: Upload test outputs
+        if: always() && steps.gather_outputs.outputs.file_count != '0'
+        uses: actions/upload-artifact@v4
+        with:
+          name: wcrp-test-outputs
+          path: tmp/qa_test_outputs/
+          retention-days: 30
+
+      - name: Upload WCRP compliance reports
+        if: always() && steps.gather_outputs.outputs.file_count != '0'
+        uses: actions/upload-artifact@v4
+        with:
+          name: wcrp-compliance-reports
+          path: tmp/wcrp_compliance_reports/
+          retention-days: 30