.gitlab-ci.yml

# ==========================================================================
# Main GitLab CI Configuration for the global-workflow Project
# ==========================================================================
#
# This is the primary CI/CD configuration file that orchestrates the GitLab CI
# testing framework for the global-workflow project. It defines pipeline stages
# and includes specialized configuration files for different testing aspects.
#
# The pipeline architecture supports:
#   1. Multiple host environments (hera, gaeac6, etc.)
#   2. Two primary testing modes:
#      - PR validation via standard test cases
#      - CTest-based tests triggered via GitHub API
#
# Included files:
#   - .gitlab-ci-ctests.yml: Handles CMake/CTest-based testing framework
#   - .gitlab-ci-cases.yml: Defines templates for standard experiment cases
#   - .gitlab-ci-hosts.yml: Contains host-specific configurations and test matrices
#                          that can be extended to support additional hosts
#
# The host configuration in .gitlab-ci-hosts.yml is designed to be easily 
# extended with new computing platforms and allows per-host specification 
# of which test cases to run.

stages:
  - build
  - setup_tests
  - run_tests
  - finalize

variables:
  CI_DEBUG_TRACE: "false"    # Enable debug tracing for CI jobs
  CI_DEBUG_SERVICES: "false" # Enable debug services for CI jobs
  PIPELINE_TYPE: pr_cases    # Default to PR Cases if not overwritten by GitLab Scheduled Pipelines
  RUN_ON_MACHINES: all       # Default to all machines (overwritten by triggers)
  # Set GFS_CI_RUN_TYPE=nightly in the GitLab Scheuled Pipelines
  # to run the nightly against the develop branch
  GFS_CI_RUN_TYPE: pr_cases  # default value (overwritten by GitLab Scheuled Pipelines)
  GW_REPO_URL: https://github.com/NOAA-EMC/global-workflow.git
  # Using CI_PIPELINE_ID guarantees uniqueness per pipeline (also used for pslot tags)
  WORKSPACE_ID: ${GFS_CI_RUN_TYPE}_${PR_NUMBER}_${CI_COMMIT_SHORT_SHA}_${CI_PIPELINE_ID}
  # Define a base path that will be used for initial cloning
  GIT_CLONE_PATH: ${CI_BUILDS_DIR}/${WORKSPACE_ID}/global-workflow
  GIT_DEPTH: 10
  RUNNER_SCRIPT_TIMEOUT: 8h
  RUNNER_AFTER_SCRIPT_TIMEOUT: 8h
  PR_NUMBER: 0 # PR number is set by GitHub Actions
               # when the pipeline is triggered from a PR
  GITHUB_COMMIT_SHA: "" # GitHub commit SHA for native GitLab integration

workflow:
  name: "PR ${PR_NUMBER} - ${RUN_ON_MACHINES} (${GFS_CI_RUN_TYPE})"

# Include specialized pipeline configuration files
include:
  - local: 'dev/ci/gitlab-ci-ctests.yml'  # CTest framework configuration
  - local: 'dev/ci/gitlab-ci-cases.yml'   # Standard test pr case templates
  - local: 'dev/ci/gitlab-ci-hosts.yml'   # Host configurations, modality logic, and test lists

.base_config:
  variables:
    badge_GIST_ID: "66059582886cb5c2485ff64bf24e7f93"
    GIT_STRATEGY: none
  before_script:
    - |
      echo "Starting GitLab CI pipeline for global-workflow project"
      PR_NUMBER=${PR_NUMBER:-0}
      if [[ "${PIPELINE_TYPE}" == "ctests" ]]; then
        GFS_CI_RUN_TYPE="ctests"
        WORKSPACE_ID="${GFS_CI_RUN_TYPE}_${PR_NUMBER}_${CI_COMMIT_SHORT_SHA}_${CI_PIPELINE_ID}"
      fi

      export GW_RUN_PATH="${CI_BUILDS_DIR}/${WORKSPACE_ID}"
      export GW_HOMEgfs="${GIT_CLONE_PATH}"  # This is set by GitLab for initial clone
      export RUNTESTS_DIR="${GW_RUN_PATH}/RUNTESTS"

      # Create a date-based directory name for successful nightly runs
      # on success of nightly pipelines will have the final directory name
      # (GFS_CI_RUN_TYPE)_(SHORT_SHA)_(MMDDYY) such as nightly_affeae_061025
      # all other pipelines will have the CI_PIPELINE_ID appended instead
      # (GFS_CI_RUN_TYPE)_(SHORT_SHA)_(CI_PIPELINE_ID) such as pr_cases_affeae_2345

      export DATE="$(date +'%m%d%y')"
      export WORKSPACE_DATE="${GFS_CI_RUN_TYPE}_${CI_COMMIT_SHORT_SHA}_${DATE}"

      echo "WORKSPACE_ID: ${WORKSPACE_ID}"
      echo "WORKSPACE_DATE: ${WORKSPACE_DATE}"
      echo "GW_RUN_PATH: ${GW_RUN_PATH}"
      echo "GIT_CLONE_PATH: ${GIT_CLONE_PATH}"
      echo "GW_HOMEgfs: ${GW_HOMEgfs}"
      echo "PIPELINE_TYPE: ${PIPELINE_TYPE}"
      echo "RUN_ON_MACHINES: ${RUN_ON_MACHINES}"

      # TODO We need to add local python env to support PyGitHub not being in Spack-Stack 1.9
      # this should be part of the spack environment setup in Spack-Stack 2.x
      PYTHONPATH=${PYTHONPATH}:$HOME/.local/lib/python3.11/site-packages
      echo "Updated PYTHONPATH: $PYTHONPATH"

      # Make sure GW_HOMEgfs is properly set and the directory exists
      if [ ! -d "${GW_HOMEgfs}" ]; then
        echo "ERROR: GW_HOMEgfs directory does not exist: ${GW_HOMEgfs}"
        exit 1
      fi

      # Setup GitHub CLI for use throughout the pipeline
      export GH=$(which gh 2>/dev/null || echo "${HOME}/bin/gh")
      if [[ ! -x "${GH}" ]]; then
        echo "ERROR: GitHub CLI (gh) not found or not executable. Please install it."
        exit 1
      fi

      # Configure GitLab native GitHub integration
      if [[ -n "${GITHUB_COMMIT_SHA:-}" && ${PR_NUMBER} -ne 0 ]]; then
        echo "Configuring GitLab native GitHub integration for commit: ${GITHUB_COMMIT_SHA}"
        export CI_COMMIT_SHA="${GITHUB_COMMIT_SHA}"
        echo "GitLab will report status to GitHub for this commit"
      else
        echo "Using GitLab default commit SHA for status reporting"
      fi

# Common build template for all modalities
.build_template:
  extends:
    - .base_config
    - .failure_cleanup_template
  variables:
    GIT_STRATEGY: clone
    GIT_SUBMODULE_STRATEGY: recursive
    GIT_SSL_NO_VERIFY: "true"  # Address potential certificate verification issues
  stage: build
  script:
    - |
      # ====================================================================
      # MAIN BUILD TEMPLATE SECTION
      # ====================================================================
      # This is the core build template that serves as the foundation for
      # all Global Workflow CI/CD build processes across different machines
      # and modalities (GFS, GEFS, SFS, GCAFS). This template provides:
      #   - Environment setup and validation
      #   - Git submodule management and PR checkout logic
      #   - Build execution via ci_utils.sh
      #   - Error handling and status reporting
      #   - Artifact archival and cleanup procedures
      # All machine-specific build jobs inherit from this template and
      # customize behavior through variables and job-specific overrides.
      # ====================================================================

      echo "Setting up build environment for ${machine^}..."
      echo "Using build directory ${GW_HOMEgfs}"

      # Load custom Rocoto path if specified for user control of Rocoto version
      # This allows testing with different Rocoto builds without changing system Rocoto
      source "${GW_HOMEgfs}/dev/ci/platforms/config.${MACHINE_ID}"
      if [[ -n "${GFS_CI_ROCOTO_PATH:-}" && -d "${GFS_CI_ROCOTO_PATH}" ]]; then
        export PATH="${GFS_CI_ROCOTO_PATH}:${PATH}"
        echo "Using custom Rocoto build at: $(which rocotorun 2>/dev/null || echo 'NOT FOUND')"
      else
        echo "GFS_CI_ROCOTO_PATH not set or directory not found, using system Rocoto"
      fi

      git submodule status
      # When GFS_CI_RUN_TYPE is set to "nightly", we use the develop branch
      # It should be set when configuring the scheculed nightly pipeline
      if [[ ${GFS_CI_RUN_TYPE} == "pr_cases" && ${PR_NUMBER} != 0 ]]; then
        echo "pipeline is set to checkout PR_NUMBER=${PR_NUMBER} from ${GW_REPO_URL} on machine ${machine^}"
        ${GH} pr edit ${PR_NUMBER} --repo ${GW_REPO_URL} --add-label "CI-${machine^}-Building" --remove-label "CI-${machine^}-Ready"
        git remote add github "${GW_REPO_URL}"
        git fetch github
        ${GH} pr checkout "${PR_NUMBER}"
        git submodule update --init --recursive -j 8
      fi

      dev/ci/scripts/utils/ci_utils.sh build
      build_status=$?

      if [[ ${build_status} -ne 0 || -f logs/error.logs ]]; then
        echo "Build failed with exit code ${build_status}"
        # Mark the build as failed, but don't update labels here - let after_script handle it
        exit 1
      elif [[ ${GFS_CI_RUN_TYPE} == "pr_cases" && ${PR_NUMBER} != 0 ]]; then
        ${GH} pr edit ${PR_NUMBER} --repo ${GW_REPO_URL} --add-label "CI-${machine^}-Running" --remove-label "CI-${machine^}-Building"
      fi

      sorc/link_workflow.sh
      mkdir -p ${RUNTESTS_DIR}

# Reusable template for failure cleanup in after_script
.failure_cleanup_template:
  after_script:
    - |
      echo "After script is running for job ${CI_JOB_NAME} on ${machine} with status ${CI_JOB_STATUS}"
      export GW_HOMEgfs="${GIT_CLONE_PATH}"
      if [[ ${CI_JOB_STATUS} == "failed" ]]; then
        Machine="${machine^}"
        echo "Job failed, performing cleanup actions for ${Machine}"

        # TODO We need to add local python env to support PyGitHub not being in Spack-Stack 1.9
        # this should be part of the spack environment setup in Spack-Stack 2.x
        PYTHONPATH=${PYTHONPATH}:$HOME/.local/lib/python3.11/site-packages
        echo "Updated PYTHONPATH: $PYTHONPATH"       

        # Re-setup GH if needed
        export GH=$(which gh 2>/dev/null || echo "${HOME}/bin/gh")
        if [ ! -x "${GH}" ]; then
          echo "WARNING: GitHub CLI (gh) not found in after_script, trying to continue anyway"
        fi

        error_log_file="sorc/logs/error.logs"
        if [[ "${GFS_CI_RUN_TYPE}" == "pr_cases" && "${PR_NUMBER}" != 0 ]]; then
          if [[ -f "${error_log_file}" ]]; then
            pr_error_log_links=$(dev/ci/scripts/utils/ci_utils.sh publish_logs_from_file "PR_${PR_NUMBER}" "${error_log_file}")
          else
            pr_error_log_links="(no error.logs file found)"
          fi

          if [[ -s "${error_log_file}" ]]; then
            error_logs_for_gist=""
            error_logs_markdown=""

            while IFS= read -r full_log_path; do
              [[ -n "${full_log_path}" ]] || continue

              if [[ -f "${full_log_path}" && -s "${full_log_path}" ]]; then
                error_logs_for_gist="${error_logs_for_gist} ${full_log_path}"
                error_logs_markdown=$(echo -e "${error_logs_markdown}\n${full_log_path}")
              fi
            done < "${error_log_file}"

            comment_body=$(printf "**BUILD FAILED** on %s (pipeline ID: %s)\n\nIn directory: \`%s/sorc\`\n\n**Error log files:**\n\`\`\`\n%s\n\`\`\`\n\n [View Error Logs](%s)\n\n_This failure was detected automatically by global-workflow's CI/CD Pipeline_" "${Machine}" "${CI_PIPELINE_ID}" "${GW_HOMEgfs}" "${error_logs_markdown}" "${pr_error_log_links}")
        
            "${GH}" pr comment "${PR_NUMBER}" --repo "${GW_REPO_URL}" --body "${comment_body}" || true
          fi

          echo "Updating GitHub labels for PR ${PR_NUMBER} failure on ${Machine}"
          ${GH} pr edit ${PR_NUMBER} --repo ${GW_REPO_URL} \
            --add-label "CI-${Machine}-Failed" \
            --remove-label "CI-${Machine}-Building" \
            --remove-label "CI-${Machine}-Running" || true

        elif [[ "${GFS_CI_RUN_TYPE}" == "nightly" && "${CI_PIPELINE_SOURCE}" == "schedule" ]]; then
          echo "Updating nightly badge for ${Machine} failure with GIST ID ${badge_GIST_ID}"
          badge_img_file="${TMPDIR:-/tmp}/$(uuidgen)/${machine}_pipeline_badge.svg"
          mkdir -p "$(dirname "${badge_img_file}")"
          curl -sSL "https://img.shields.io/badge/${machine}_nightly-failed-red?style=flat-round&logo=gitlab" -o "${badge_img_file}"
          ${GH} gist edit "${badge_GIST_ID}" --add "${badge_img_file}" || true

        else
          echo "No cleanup actions needed for GFS_CI_RUN_TYPE=${GFS_CI_RUN_TYPE}, CI_PIPELINE_SOURCE=${CI_PIPELINE_SOURCE}"
        fi
      else
        echo "Job status: ${CI_JOB_STATUS}, no failure cleanup needed"
      fi