From 159607a03d869dbfaa8d6e3a49e6d89b0fa0a940 Mon Sep 17 00:00:00 2001 From: Soren Rasmussen Date: Tue, 25 Mar 2025 12:49:26 -0600 Subject: [PATCH 1/3] - Unifying apt-get install into one command - using slim debian docker image - git clones of depth=1 so less data is downloaded - only download test datasets for PR --- docker/Dockerfile | 83 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 58 insertions(+), 25 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 979781134..62e7f67b2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,30 +1,59 @@ -FROM debian:12 +FROM debian:12-slim LABEL maintainer="Michael Kavulich " # arguments that can be passed in ARG PR_NUMBER +# build variables +ARG GNU_VERSION=12 +ARG VERBOSE=0 + # Set up base OS environment RUN apt-get -y update # Get "essential" tools and libraries -RUN apt-get -y install build-essential \ - && apt-get -y install cmake cmake-curses-gui curl git file gfortran-12 ksh m4 python3 tcsh time wget vim emacs-nox \ - && apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 \ - && apt-get -y install python3-pip python3.11-venv python3-netcdf4 \ - && apt-get -y install openmpi-bin libopenmpi-dev +RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \ + build-essential \ + cmake \ + cmake-curses-gui \ + curl \ + git \ + file \ + gfortran-$GNU_VERSION \ + ksh \ + m4 \ + tcsh \ + time \ + wget \ + vim \ + emacs-nox \ + python3 \ + python3-pip \ + python3.11-venv \ + python3-netcdf4 \ + libnetcdf-pnetcdf-19 \ + libnetcdff7 \ + libnetcdf-dev \ + libnetcdff-dev \ + libxml2 \ + openmpi-bin \ + libopenmpi-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + RUN ln -s /usr/bin/python3 /usr/bin/python +RUN ln -s /usr/bin/gfortran-$GNU_VERSION /usr/bin/gfortran # Set up python needed packages, preferred Docker method is apt-get but # f90nml can't be installed for debian that way RUN pip install f90nml --break-system-packages #Compiler environment variables -ENV CC=/usr/bin/gcc -ENV FC=/usr/bin/gfortran -ENV CXX=/usr/bin/g++ -ENV F77=/usr/bin/gfortran -ENV F90=/usr/bin/gfortran +ENV CC=gcc +ENV FC=gfortran +ENV CXX=g++ +ENV F77=gfortran +ENV F90=gfortran # Other necessary environment variables ENV LD_LIBRARY_PATH=/usr/lib/ @@ -47,24 +76,24 @@ WORKDIR /home ENV NCEPLIBS_DIR=/comsoftware/nceplibs RUN mkdir -p $NCEPLIBS_DIR/src && cd $NCEPLIBS_DIR/src \ - && git clone -b v2.4.1 --recursive https://github.com/NOAA-EMC/NCEPLIBS-bacio \ - && mkdir NCEPLIBS-bacio/build && cd NCEPLIBS-bacio/build \ - && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ - && make VERBOSE=1 -j \ - && make install + && git clone -b v2.4.1 --depth 1 --recursive https://github.com/NOAA-EMC/NCEPLIBS-bacio \ + && mkdir NCEPLIBS-bacio/build && cd NCEPLIBS-bacio/build \ + && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ + && make VERBOSE=$VERBOSE -j \ + && make install RUN cd $NCEPLIBS_DIR/src \ - && git clone -b v2.3.3 --recursive https://github.com/NOAA-EMC/NCEPLIBS-sp \ + && git clone -b v2.3.3 --depth 1 --recursive https://github.com/NOAA-EMC/NCEPLIBS-sp \ && mkdir NCEPLIBS-sp/build && cd NCEPLIBS-sp/build \ && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ - && make VERBOSE=1 -j \ + && make VERBOSE=$VERBOSE -j \ && make install RUN cd $NCEPLIBS_DIR/src \ - && git clone -b v2.11.0 --recursive https://github.com/NOAA-EMC/NCEPLIBS-w3emc \ + && git clone -b v2.11.0 --depth 1 --recursive https://github.com/NOAA-EMC/NCEPLIBS-w3emc \ && mkdir NCEPLIBS-w3emc/build && cd NCEPLIBS-w3emc/build \ && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ - && make VERBOSE=1 -j \ + && make VERBOSE=$VERBOSE -j \ && make install ENV bacio_ROOT=/comsoftware/nceplibs @@ -81,7 +110,7 @@ RUN if [ -z "$PR_NUMBER" ]; then \ && cd ccpp-scm \ && git fetch origin pull/${PR_NUMBER}/head:test_pr \ && git checkout test_pr \ - && git submodule update --init --recursive; \ + && git submodule update --init --recursive --depth 1; \ fi RUN mkdir /comsoftware/ccpp-scm/scm/bin \ @@ -89,9 +118,6 @@ RUN mkdir /comsoftware/ccpp-scm/scm/bin \ && cmake ../src \ && make -j -RUN cd /comsoftware/ccpp-scm/ \ - && ./contrib/get_all_static_data.sh \ - && ./contrib/get_thompson_tables.sh # The analysis scripts have options for using LaTeX when making figure labels. # If you would like to install LaTeK, uncomment the section below. @@ -109,4 +135,11 @@ ENV SCM_ROOT=/comsoftware/ccpp-scm/ # For interactive use, vim mouse settings are infuriating RUN echo "set mouse=" > ~/.vimrc -ENTRYPOINT ["sh", "-c", "./contrib/get_aerosol_climo.sh && cd /comsoftware/ccpp-scm/scm/bin"] +# only download test data when PR is being tested +ENTRYPOINT ["sh", "-c", "\ +if [ -n \"$PR_NUMBER\" ]; then \ + ./contrib/get_aerosol_climo.sh && \ + ./contrib/get_all_static_data.sh && \ + ./contrib/get_thompson_tables.sh; \ +fi && \ +cd /comsoftware/ccpp-scm/scm/bin"] From 628897b0e64fd2f16d963c7772f8d45f28d3e66c Mon Sep 17 00:00:00 2001 From: Soren Rasmussen Date: Tue, 25 Mar 2025 13:28:47 -0600 Subject: [PATCH 2/3] Don't run when merging PR into main --- .github/workflows/ci_test_docker.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_test_docker.yml b/.github/workflows/ci_test_docker.yml index 05abd8501..7365c9d0d 100644 --- a/.github/workflows/ci_test_docker.yml +++ b/.github/workflows/ci_test_docker.yml @@ -1,6 +1,10 @@ name: build test and push docker -on: [pull_request,workflow_dispatch] +on: + # not listing pull_request closed, since it would run when merged + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: env: TEST_TAG: dtcenter/ccpp-scm:test From 432bfa9f2e8c5181b176d9aba478c9e634b60329 Mon Sep 17 00:00:00 2001 From: Soren Rasmussen Date: Thu, 3 Apr 2025 12:07:02 -0600 Subject: [PATCH 3/3] Run only after RT actions are complete --- .github/workflows/ci_build_scm_ubuntu_nvidia.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_nvidia.yml index 9c3ec2590..36305a219 100644 --- a/.github/workflows/ci_build_scm_ubuntu_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_nvidia.yml @@ -1,11 +1,18 @@ name: build the CCPP-SCM with Nvidia -on: [pull_request,workflow_dispatch] +on: + workflow_run: + workflows: ["build and run SCM regression tests"] + types: + - completed jobs: build_scm: - # The type of runner that the job will run on + # trigger only after the SCM regression tests have completed from a PR or workflow event + if: | + github.event.workflow_run.event == 'pull_request' || + github.event.workflow_run.event == 'workflow_dispatch' runs-on: ubuntu-latest strategy: fail-fast: false