.gitlab-ci.yml

include:
  - local: .gitlab/sanitizers-ci.yml
  - local: .gitlab/benchmarks-ci.yml
  - local: .gitlab/build-ont_core.yml
  - local: .gitlab/mlhub.yml

workflow:
  rules:
    # If a new commit is pushed to the branch but an MR is already opened, don't run a "push" pipeline because
    # the merge request train setting will trigger a "merge_request_event" pipeline anyway.
    # In all other cases, run the pipeline.
    - if: $CI_PIPELINE_SOURCE == 'push' && $CI_OPEN_MERGE_REQUESTS
      when: never
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
    - if: $CI_PIPELINE_SOURCE == 'schedule'
    - if: $CI_PIPELINE_SOURCE == 'push'
    - if: $CI_PIPELINE_SOURCE == 'web'

default:
  interruptible: true

stages:
  - pre-flight
  - build
  - test
  - deploy
  - regression_test

variables:
  BUILD_DIR: build
  BUILD_TARGET: install
  BUILD_TYPE: Release
  GIT_SUBMODULE_STRATEGY: recursive
  CUDA: "12.8"
  WIN_CUDA: "12.8"
  WIN_CUDA_TOOLKIT: "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v${WIN_CUDA}"
  MTL_DEBUG_LAYER: 1
  FF_TIMESTAMPS: true

# Build stage components

.macos_arm_runner:
  tags:
    - osx_arm64
    - xcode-15.3
    - on-prem
  variables:
    MACOSX_DEPLOYMENT_TARGET: "13.0"

.linux_runner:
  tags:
    - ubuntu-20.04
    - nvidia-docker
    - on-prem

.linux_arm64_runner:
  tags:
    - linux_aarch64
    - docker-arm
    - aws

.linux_orin_cuda_12_runner:
  tags:
    - linux-arm64-gpu-12.6
    - nvidia-docker
    - on-prem

.windows_runner:
  tags:
    - windows-10
    - VS2019
    - cuda-${WIN_CUDA}
    - on-prem

.linux_build_dependencies:
  before_script:
    - echo Dorado dependency stage
    - apt-get update
    - apt-get install -y --no-install-recommends samtools curl libssl-dev libsz2 ccache automake

.linux_test_dependencies:
  before_script:
    - echo Dorado dependency stage
    - apt-get update
    - apt-get install -y --no-install-recommends samtools curl tabix

.set_parallel_jobs:
  - |
    if [[ $OSTYPE == 'darwin'* ]]; then
      PARALLEL_JOBS=$(( $(sysctl -n hw.physicalcpu) / 2 ))
    elif [[ $(arch) != 'aarch64' ]]; then
      PARALLEL_JOBS=$(( $(nproc) / 4 ))
    else
      PARALLEL_JOBS=8
    fi
    PARALLEL_JOBS=$(( 8 > PARALLEL_JOBS ? 8 : PARALLEL_JOBS ))
  - echo "PARALLEL_JOBS '${PARALLEL_JOBS}'"

.setup_ccache:
  - if [[ -d '/data/gitlab-ci-cache' ]] && [[ $OSTYPE != 'darwin'* ]] && [[ $OSTYPE != 'msys' ]]; then
  - export CCACHE_DIR=/data/gitlab-ci-cache
  - fi
  - ccache -s || true

.run_dorado_cmake_build:
  - echo Dorado build stage
  - !reference [.set_parallel_jobs]
  - !reference [.setup_ccache]
  - echo "BUILD_OPTIONS '${BUILD_OPTIONS}'"
  - eval cmake ${BUILD_OPTIONS} -S . -B ${BUILD_DIR} -D CMAKE_BUILD_TYPE=${BUILD_TYPE} -D DORADO_ENABLE_BENCHMARK_TESTS=ON
  - cmake --build ${BUILD_DIR} --config ${BUILD_TYPE} --target ${BUILD_TARGET} -j ${PARALLEL_JOBS}

.run_dorado_cmake_ctest:
  - echo Dorado test stage
  - ctest -C ${BUILD_TYPE} --test-dir ${BUILD_DIR} --output-on-failure --verbose

.run_dorado_cmake_cpack:
  - echo Dorado packaging stage
  - cpack --config ${BUILD_DIR}/CPackConfig.cmake

.zip_test_files:
  # Uploading these allows us to avoid checking out the repo in tests.
  - echo Zipping test files
  - |
    zip -r  tests.archive tests/*.sh tests/*.py tests/data tests/cram ||
    tar -cf tests.archive tests/*.sh tests/*.py tests/data tests/cram

.unzip_test_files:
  - echo Unzipping test files
  - unzip tests.archive || tar -xf tests.archive

.build_and_test_dorado:
  stage: build
  script:
    - !reference [.run_dorado_cmake_build]
    - !reference [.run_dorado_cmake_ctest]
    - !reference [.zip_test_files]
  artifacts:
    paths:
      - dist
      - tests.archive
    expire_in: 1 day

.build_dorado_archive:
  stage: build
  script:
    - !reference [.run_dorado_cmake_build]
    - !reference [.run_dorado_cmake_cpack]
  artifacts:
    paths:
      - archive
    expire_in: 1 day

.sign_dorado:macos:
  script:
    # Older versions of macOS (10.15) can't run notarytool through xcrun, but can make
    # use of the standalone notarytool executable if it's copied from a newer system.
    # See https://developer.apple.com/forums/thread/712665.
    - |
      if command -v notarytool > /dev/null; then
        NOTARYTOOL=(notarytool)
      else
        NOTARYTOOL=(xcrun notarytool)
      fi
    # We sign the executable as part of CPack's packaging steps, so the package that it
    # produces should be ready to be notarized. That can be done asynchronously, but
    # doing everything in one pass (--wait flag) saves complicating the jobs.
    - ${NOTARYTOOL[@]} submit archive/dorado-*.zip
      --apple-id ${APPLE_NOTARIZATION_ID}
      --password ${APPLE_NOTARIZATION_PASSWORD}
      --team-id ${APPLE_NOTARIZATION_TEAM}
      --progress --verbose --wait
    # If notarization fails you need to run `xcrun notarytool log <request id>` to get
    # information about what failed.
    #
    # Extract the binary and check that it's notarized, since notarytool doesn't appear
    # to return an error on failure.
    - rm -rf notarization_check
    - unzip archive/dorado-*.zip -d notarization_check
    - spctl -a -t install -vvvv notarization_check/dorado-*/bin/dorado
    #
    # No stapling for now since you can only staple packages and not executables, hence
    # we'd need to switch to a dmg instead and do something like:
    #- xcrun stapler staple dorado-*.dmg

# Run this job manually, but always run on nightly.
.on_nightly_or_manual:
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_PIPELINE_SOURCE != "schedule"
      when: manual
      # See https://forum.gitlab.com/t/specifying-a-pipeline-step-as-manual-causes-its-status-to-always-be-blocked/58333/3
      allow_failure: true

# Pre-flight stages

pre-commit:
  image: ${DORADO_PRECOMMIT}
  variables:
    GIT_SUBMODULE_STRATEGY: none
  stage: pre-flight
  script:
    - echo "Pipeline source is ${CI_PIPELINE_SOURCE}"
    - pip install pre-commit
    - pre-commit run --all-files
  tags:
    - linux

# Build stages

build:linux:x86:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-20.04-cuda-${CUDA}.0:1.1
  variables:
    BUILD_OPTIONS: "-DONT_MM2_EXE=ON"
  extends:
    - .linux_build_dependencies
    - .linux_runner
    - .build_and_test_dorado

build:linux:x86:focal_koi_from_source:
  image: ${DORADO_DOCKER_ROOT}/dorado-20.04-cuda-${CUDA}.0:1.1
  variables:
    BUILD_OPTIONS: "-DBUILD_KOI_FROM_SOURCE=ON -DGITLAB_CI_TOKEN=${CI_JOB_TOKEN} -DONT_MM2_EXE=ON"
  extends:
    - .linux_build_dependencies
    - .linux_runner
    - .build_and_test_dorado
    - .on_nightly_or_manual
  artifacts: null

build:linux:x86:noble:
  image: ${DORADO_DOCKER_ROOT}/dorado-24.04-cuda-${CUDA}.0:1.0
  variables:
    BUILD_OPTIONS: "-DONT_MM2_EXE=ON"
  extends:
    - .linux_build_dependencies
    - .linux_runner
    - .build_and_test_dorado
  before_script:
    - !reference [.linux_build_dependencies, before_script]
    - apt-get install -y build-essential
  artifacts: null

build:linux:x86:focal:debug:
  variables:
    BUILD_TYPE: "Debug"
  extends:
    - build:linux:x86:focal
  rules:
    - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result"
  artifacts: null

build:linux:x86:coverage:
  stage: build
  image: ${DORADO_DOCKER_ROOT}/dorado-20.04-cuda-${CUDA}.0:1.1
  variables:
    BUILD_OPTIONS: "-DGENERATE_TEST_COVERAGE=1"
    BUILD_TARGET: "dorado_test_coverage"
  extends:
    - .linux_build_dependencies
    - .linux_runner
    - .on_nightly_or_manual
  script:
    - pip install gcovr
    - !reference [.run_dorado_cmake_build]
  artifacts:
    paths:
      - build/dorado_test_coverage/*

build:linux:x86:clang_tidy:
  stage: build
  image: ${DORADO_DOCKER_ROOT}/dorado-24.04-cuda-${CUDA}.0:1.0
  extends:
    - .linux_build_dependencies
    - .linux_runner
  script:
    - apt update
    - apt install -y automake clang clang-tidy
    - !reference [.set_parallel_jobs]
    - ./.gitlab/run_clang_tidy.sh -j ${PARALLEL_JOBS}
  artifacts: null

.build:linux:x86:valgrind:
  stage: build
  image: ${DORADO_DOCKER_ROOT}/dorado-20.04-cuda-${CUDA}.0:1.1
  variables:
    # valgrind can't cope with huge executables, so we have to go dynamic.
    BUILD_OPTIONS: "-D TRY_USING_STATIC_TORCH_LIB=NO"
  extends:
    - .linux_runner
  script:
    - !reference [.set_parallel_jobs]
    - apt update
    - apt install -y autoconf autotools-dev automake wget libc6-dbg
    # We need a recent valgrind that includes https://bugs.kde.org/show_bug.cgi?id=402833.
    - VALGRIND_VERSION="3.22.0"
    - wget https://sourceware.org/pub/valgrind/valgrind-${VALGRIND_VERSION}.tar.bz2
    - tar -xvf valgrind-${VALGRIND_VERSION}.tar.bz2
    - pushd valgrind-${VALGRIND_VERSION}
    - ./autogen.sh
    - ./configure
    - make -j ${PARALLEL_JOBS}
    - valgrind=$(pwd)/vg-in-place
    - popd
    # We need symbols otherwise the generated reports are useless.
    - export CFLAGS="${CFLAGS} -g"
    - export CXXFLAGS="${CXXFLAGS} -g"
    # Build dorado.
    - !reference [.run_dorado_cmake_build]
    # Run the ordinary tests directly under valgrind.
    # We don't use `ctest -VV -E smoke -T memcheck` since that requires additional setup.
    - ${valgrind} --error-exitcode=1 --track-origins=yes -- ./${BUILD_DIR}/tests/dorado_tests

build:linux:arm64:jammy:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-r36.4.0:1.0
  variables:
    BUILD_OPTIONS: "-DONT_MM2_EXE=ON"
  extends:
    - .linux_build_dependencies
    - .linux_arm64_runner
    - .build_and_test_dorado

build:linux:x86:focal:no_ccache:
  variables:
    BUILD_OPTIONS: "-D DORADO_DISABLE_CCACHE=ON"
  extends:
    - build:linux:x86:focal
    - .on_nightly_or_manual
  artifacts: null

build:windows:
  extends:
    - .windows_runner
    - .build_and_test_dorado
  before_script:
    - export CUDA_TOOLKIT=$(cygpath -u $(cygpath -d "${WIN_CUDA_TOOLKIT}"))
    - export BUILD_OPTIONS="-A x64 -T cuda=${WIN_CUDA} -DCUDAToolkit_ROOT=${CUDA_TOOLKIT} -D WIN_THREADS=4 -G \"Visual Studio 16 2019\""

build:windows:debug:
  variables:
    BUILD_TYPE: "Debug"
  extends:
    - build:windows
  rules:
    - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result"
  artifacts: null

build:windows_koi_from_source:
  extends:
    - build:windows
    - .on_nightly_or_manual
  before_script:
    - !reference ["build:windows", before_script]
    - export BUILD_OPTIONS="${BUILD_OPTIONS} -DBUILD_KOI_FROM_SOURCE=ON -DGITLAB_CI_TOKEN=${CI_JOB_TOKEN} -G \"Visual Studio 16 2019\""
  artifacts: null

build:windows:no_ccache:
  extends:
    - build:windows
    - .on_nightly_or_manual
  before_script:
    - !reference ["build:windows", before_script]
    - export BUILD_OPTIONS="${BUILD_OPTIONS} -D DORADO_DISABLE_CCACHE=ON -G \"Visual Studio 16 2019\""
  artifacts: null

build:macos:arm:
  variables:
    BUILD_OPTIONS: "-DONT_MM2_EXE=ON"
  extends:
    - .macos_arm_runner
    - .build_and_test_dorado

build:macos:arm:no_ccache:
  variables:
    BUILD_OPTIONS: "-D DORADO_DISABLE_CCACHE=ON"
  extends:
    - build:macos:arm
    - .on_nightly_or_manual
  artifacts: null

build_archive:linux:arm64:jammy:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-r36.4.0:1.0
  extends:
    - .linux_build_dependencies
    - .linux_arm64_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:linux:x86:cuda11:
  image: ${DORADO_DOCKER_ROOT}/dorado-rocky8-cuda-${CUDA}.0:1.0
  variables:
    CUDA: "11.8"
  script:
    - source /opt/rh/gcc-toolset-11/enable
    - !reference [.build_dorado_archive, script]
  extends:
    - .linux_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:linux:x86:cuda12:
  image: ${DORADO_DOCKER_ROOT}/dorado-rocky8-cuda-${CUDA}.0:1.0
  variables:
    BUILD_OPTIONS: ""
  script:
    - source /opt/rh/gcc-toolset-11/enable
    - !reference [.build_dorado_archive, script]
  extends:
    - .linux_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:windows:
  before_script:
    - export CUDA_TOOLKIT=$(cygpath -u $(cygpath -d "${WIN_CUDA_TOOLKIT}"))
    - export BUILD_OPTIONS="-A x64 -T cuda=${WIN_CUDA} -DCUDAToolkit_ROOT=${CUDA_TOOLKIT} -D WIN_THREADS=4 -G \"Visual Studio 16 2019\""
  extends:
    - .windows_runner
    - .build_dorado_archive
    - .on_nightly_or_manual

build_archive:macos:arm:
  extends:
    - .macos_arm_runner
    - .build_dorado_archive
    - .on_nightly_or_manual
  script:
    - !reference [.build_dorado_archive, script]
    - !reference [.sign_dorado:macos, script]

# Test stage components

.test_dorado:
  stage: test
  variables:
    GIT_STRATEGY: none
    MODEL: dna_r10.4.1_e8.2_400bps_hac@v5.0.0
    BATCH: 384
    NO_TEST_DORADO_CORRECT: 0
    NO_TEST_DORADO_POLISH: 0
    NO_TEST_DUPLEX: 0
    VALIDATE_FASTQ: 1
    PYTHON: python3
  script:
    # Do everything inside a newly created folder, since some runners only run
    # tests and hence never perform a git clean.
    - rm -rf test_archive
    - mkdir test_archive
    - cd test_archive
    - mv ../dist ../tests.archive .
    - !reference [.unzip_test_files]
    - bash ./tests/test_simple_basecaller_execution.sh ./dist/bin/dorado ${MODEL} ${BATCH}
    - bash ./tests/test_expected_logging.sh ./dist/bin/dorado ${MODEL} ${BATCH}
    - bash ./tests/test_dorado_correct.sh ./dist/bin/dorado
    - bash ./tests/test_dorado_correct_cram.sh ./dist/bin/dorado
    - bash ./tests/test_dorado_polish.sh ./dist/bin/dorado
    - bash ./tests/test_dorado_variant.sh ./dist/bin/dorado
  artifacts:
    when: on_failure
    paths:
      - test_archive/tests
    expire_in: 1 day

.test_dorado_windows:
  stage: test
  variables:
    GIT_STRATEGY: none
    SAMTOOLS_UNAVAILABLE: sadly_yes
    VALIDATE_FASTQ: 1
    PYTHON: python
  script:
    # Do everything inside a newly created folder, since some runners only run
    # tests and hence never perform a git clean.
    - rm -rf test_archive
    - mkdir test_archive
    - cd test_archive
    - mv ../dist ../tests.archive .
    - !reference [.unzip_test_files]
    - bash ./tests/test_simple_basecaller_execution.sh ./dist/bin/dorado ${MODEL} ${BATCH}
    - bash ./tests/test_expected_logging.sh ./dist/bin/dorado ${MODEL} ${BATCH}
    - bash ./tests/test_dorado_correct.sh ./dist/bin/dorado

.test_archive:
  stage: test
  variables:
    GIT_STRATEGY: none
  script:
    # Do everything inside a newly created folder, since some runners only run
    # tests and hence never perform a git clean.
    - rm -rf test_archive
    - mkdir test_archive
    - cd test_archive
    - mv ../archive .
    - tar -xzf archive/dorado-*.gz || unzip archive/dorado-*.zip
    - ldd ./dorado-*/bin/dorado
    - if [ -e ./dorado-*/lib/libdorado_torch_lib.so ]; then ldd ./dorado-*/lib/libdorado_torch_lib.so; fi
    # We just want to check that dorado will run here, so simply display the verbose version info
    - ./dorado-*/bin/dorado -vv

.enable_ont_python:
    - curl -LfsS ${PYTHON_URL} > ont_python.tar.gz
    - tar -xf ont_python.tar.gz
    - export PATH=${PWD}/ont-python/bin:${PWD}/ont-python:${PATH}

# Test stages

test:windows:
  extends:
    - .windows_runner
    - .test_dorado_windows
  needs:
    - build:windows

test:linux:x86:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-20.04-cuda-${CUDA}.0:1.1
  variables:
    PYTHON_URL: ${ONT_PYTHON_URL_PREFIX}/ont-python-Linux-x86_64-3.12.1-1.tar.gz
  script:
    - !reference [.enable_ont_python]
    - !reference [.test_dorado, script]
  extends:
    - .linux_test_dependencies
    - .linux_runner
    - .test_dorado
  needs:
    - build:linux:x86:focal

test:linux:arm64:jammy:
  image: ${DORADO_DOCKER_ROOT}/dorado-l4t-r36.4.0:1.0
  variables:
    PYTHON_URL: ${ONT_PYTHON_URL_PREFIX}/ont-python-Linux-aarch64-3.12.1-1.tar.gz
  script:
    - !reference [.enable_ont_python]
    - !reference [.test_dorado, script]
  extends:
    - .linux_test_dependencies
    - .linux_orin_cuda_12_runner
    - .test_dorado
  needs:
    - build:linux:arm64:jammy

test:macos:arm:
  variables:
    PYTHON: python3.10
  extends:
    - .macos_arm_runner
    - .test_dorado
  needs:
    - build:macos:arm

# Test that you can run dorado in clean environments
.test_archive:linux:x86:
  extends:
    - .test_archive
  tags:
    - ubuntu-20.04
    - ${RUNNER_TAG}
  image: ${IMAGE}
  parallel:
    matrix:
      # CUDA environment
      - RUNNER_TAG: "nvidia-docker"
        IMAGE:
          - "nvidia/cuda:${CUDA}.0-devel-ubuntu20.04"
          - "nvidia/cuda:${CUDA}.0-devel-ubuntu22.04"
          - "nvidia/cuda:${CUDA}.0-devel-ubuntu24.04"
          - "nvidia/cuda:${CUDA}.0-devel-rockylinux8"
      # CPU environment
      - RUNNER_TAG: "docker-as"
        IMAGE:
          - "${DORADO_DOCKER_EXT}/ubuntu:20.04"
          - "${DORADO_DOCKER_EXT}/ubuntu:22.04"
          - "${DORADO_DOCKER_EXT}/ubuntu:24.04"
          - "${DORADO_DOCKER_EXT}/rockylinux:8"

test_archive:linux:x86:cuda11:
  extends:
    - .test_archive:linux:x86
  needs:
    - build_archive:linux:x86:cuda11

test_archive:linux:x86:cuda12:
  extends:
    - .test_archive:linux:x86
  needs:
    - build_archive:linux:x86:cuda12

# Test that you can run dorado in a clean l4t environment
test_archive:linux:arm64:22.04:
  image: nvcr.io/nvidia/l4t-cuda:12.6.11-runtime
  extends:
    - .linux_orin_cuda_12_runner
    - .test_archive
  needs:
    - build_archive:linux:arm64:jammy

# Test that you can run dorado on a CUDA-less windows machine
test_archive:windows:cpu:
  extends:
    - .test_archive
  tags:
    - windows-11
    - no-gpu
  needs:
    - build_archive:windows

# Test that models can be downloaded and run.
test:model_download:linux:x86:focal:
  image: ${DORADO_DOCKER_ROOT}/dorado-20.04-cuda-${CUDA}.0:1.1
  extends:
    - .linux_test_dependencies
    - .linux_runner
    - .on_nightly_or_manual
  stage: test
  needs:
    - build:linux:x86:focal
  script:
    - ./tests/test_model_download.py --exe ./dist/bin/dorado --data ./tests/data --batchsize 64

test:model_download:macos:arm:
  extends:
    - .macos_arm_runner
    - .on_nightly_or_manual
  stage: test
  needs:
    - build:macos:arm
  script:
    - ./tests/test_model_download.py --exe ./dist/bin/dorado --data ./tests/data
  timeout: 3h