ctsm5.3.049: Preferentially use Podman for ctsm-docs container

.github/workflows/docker-image-common.yml

@@ -87,7 +87,7 @@ jobs:
       - name: Build docs using container
         id: build-docs
         run: |
-          cd doc && ./build_docs -b ${PWD}/_build -c -d -i $IMAGE_TAG
+          cd doc && ./build_docs -b ${PWD}/_build -c -d --container-cli-tool docker -i $IMAGE_TAG
 
 
   check-version:

.github/workflows/docs-build-and-deploy.yml

@@ -51,7 +51,7 @@ jobs:
         run: |
           bin/git-fleximod update -o
           cd doc
-          ./build_docs_to_publish -d --site-root https://escomp.github.io/ctsm
+          ./build_docs_to_publish -d --container-cli-tool docker --site-root https://escomp.github.io/ctsm
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.github/workflows/docs.yml

@@ -56,4 +56,4 @@ jobs:
       - name: Build docs using container
         id: build-docs
         run: |
-          cd doc && ./build_docs -b ${PWD}/_build -c -d
+          cd doc && ./build_docs -b ${PWD}/_build -c -d --container-cli-tool docker

.gitignore

@@ -112,6 +112,7 @@ core.*
 *.pyc
 Depends
 
-# Docs build output
+# Docs build and testing output
 _build*/
 _publish*/
+doc/.coverage

.gitmodules

@@ -124,7 +124,7 @@ fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial
 [submodule "doc-builder"]
 path = doc/doc-builder
 url = https://github.com/ESMCI/doc-builder
-fxtag = v2.1.2
+fxtag = v2.2.3
 fxrequired = ToplevelOptional
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/doc-builder

doc/ctsm-docs_container/Dockerfile

@@ -29,4 +29,4 @@ CMD ["/bin/bash", "-l"]
 
 LABEL org.opencontainers.image.title="Container for building CTSM documentation"
 LABEL org.opencontainers.image.source=https://github.com/ESCOMP/CTSM
-LABEL org.opencontainers.image.version="v1.0.2"
+LABEL org.opencontainers.image.version="v1.0.2b"

doc/ctsm-docs_container/README.md

@@ -1,31 +1,33 @@
 # The ctsm-docs container
-This directory and its Dockerfile are used to build a Docker container for building the CTSM documentation. Unless you're a developer working on the container, you probably don't need to care about anything in here.
+This directory and its Dockerfile are used to build a container for building the CTSM documentation. Unless you're a developer working on the container, you probably don't need to care about anything in here.
 
 ## Introduction
 
-This Readme tells you how to update the ctsm-docs Docker container if a need to do so arises—for example, adding a Python module that brings new functionality in the build. After you've followed all these instructions, you will probably want to push an update to [doc-builder](https://github.com/ESMCI/doc-builder) that updates `DEFAULT_DOCKER_IMAGE` in [build_commands.py](https://github.com/ESMCI/doc-builder/blob/master/doc_builder/build_commands.py) to point to the new tag.
+This Readme tells you how to update the ctsm-docs container if a need to do so arises—for example, adding a Python module that brings new functionality in the build. After you've followed all these instructions, you will probably want to push an update to [doc-builder](https://github.com/ESMCI/doc-builder) that updates `DEFAULT_IMAGE` in [build_commands.py](https://github.com/ESMCI/doc-builder/blob/master/doc_builder/build_commands.py) to point to the new tag.
 
 ## Building
 
-If you actually want to build the container, make sure Docker is running. In the Docker Desktop settings, make sure you've enabled the [`containerd` image store](https://docs.docker.com/desktop/features/containerd/), which allows multi-platform builds. Then do:
+If you actually want to build the container, you will need to have Podman installed. We previously used Docker for this, but had to move away from it due to licensing issues. Note that these issues have to do specifically with Docker Desktop, which is required to get the Docker Engine on some platforms. The Docker Engine itself is open-source, so our ctsm-docs container testing and publishing workflows are fine to continue using it. We are also fine to use the Docker Engine via Podman for local builds, which is what's described here.
+
+Once you have Podman installed, you can do:
 ```shell
-docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/escomp/ctsm/ctsm-docs .
+podman build --no-cache -t ctsm-docs .
 ```
 
-To use your new version for local testing, you'll need to tell doc-builder to use that image. Call `docker images`, which should return something like this:
+To use your new version for local testing, you'll need to tell doc-builder to use that image. Call `podman images`, which should return something like this:
 ```shell
-REPOSITORY                      TAG          IMAGE ID       CREATED         SIZE
-ghcr.io/escomp/ctsm/ctsm-docs   latest       ab51446519a4   3 seconds ago   233MB
+REPOSITORY                TAG            IMAGE ID      CREATED             SIZE
+localhost/ctsm-docs       latest         6464f26339bc  22 seconds ago      241 MB
 ...
 ```
 
-To test, you can tell `build_docs` to use your new version by adding `--docker-image IMAGE_ID` to your call, where in the example above `IMAGE_ID` is `ab51446519a4`.
+To test, you can tell `build_docs` to use your new version by adding `--container-image IMAGE_ID` to your call, where in the example above `IMAGE_ID` is `6464f26339bc`.
 
 ## Publishing automatically
 
 The `docker-image-build-publish.yml` workflow makes it so that new versions of the workflow will be published to the GitHub Container Registry whenever changes to the container setup are merged to CTSM's `master` branch. This will fail (as will a similar, no-publish workflow that happens on PRs) unless you specify exactly one new version number in the Dockerfile. This version number will be used as a tag that can be referenced by, e.g., doc-builder.
 
-Lots of Docker instructions tell you to use the `latest` tag, and indeed the workflow will add that tag automatically. However, actually _using_ `latest` can lead to support headaches as users think they have the right version but actually don't. Instead, you'll make a new version number incremented from the [previous one](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs/versions), in the `vX.Y.Z` format.
+Lots of container instructions tell you to use the `latest` tag, and indeed the workflow will add that tag automatically. However, actually _using_ `latest` can lead to support headaches as users think they have the right version but actually don't. Instead, you'll make a new version number incremented from the [previous one](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs/versions), in the `vX.Y.Z` format.
 
 Here's where you need to specify the version number in the Dockerfile:
 ```docker
@@ -36,33 +38,44 @@ The string there can technically be anything as long as (a) it starts with a low
 You can check the results of the automatic publication on the [container's GitHub page](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs).
 
 ### Updating doc-builder
-After the new version of the container is published, you will probably want to tell [doc-builder](https://github.com/ESMCI/doc-builder) to use the new one. Open a PR where you change the tag (the part after the colon) in the definition of `DEFAULT_DOCKER_IMAGE` in `doc_builder/build_commands.py`. Remember, **use the version number**, not "latest".
+After the new version of the container is published, you will probably want to tell [doc-builder](https://github.com/ESMCI/doc-builder) to use the new one. Open a PR where you change the tag (the part after the colon) in the definition of `DEFAULT_IMAGE` in `doc_builder/build_commands.py`. Remember, **use the version number**, not "latest".
 
 ## Publishing manually (NOT recommended)
 
 It's vastly preferable to let GitHub build and publish the new repo using the `docker-image-build-publish.yml` workflow as described above. However, if you need to publish manually for some reason, here's how.
 
+### Building the multi-architecture version
+
+When publishing our container, we need to make sure it can run on either arm64 or amd64 processor architecture. This requires a special build process:
+```shell
+podman manifest create ctsm-docs-manifest
+podman build --platform linux/amd64,linux/arm64  --manifest ctsm-docs-manifest .
+```
+
 ### Pushing to GitHub Container Registry
 If you want to publish the container, you first need a [GitHub Personal Access Token (Classic)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic) with the `write:packages` permissions. You can see your existing PAT(C)s [here](https://github.com/settings/tokens). If you don't have one with the right permissions, [this link](https://github.com/settings/tokens/new?scopes=write:packages) should start the setup process for you.
 
 Once you have a PAT(C), you can authenticate in your shell session like so:
 
-```shell
-   echo YOUR_PERSONAL_ACCESS_TOKEN_CLASSIC | docker login ghcr.io -u YOUR_USERNAME --password-stdin
+```bash
+# bash: Make it so that, in this session, commands with a leading space are not saved to terminal history
+export HISTCONTROL=ignoreboth
+
+# Include leading spaces so that your secret PAT(C) isn't included in terminal history
+   echo YOUR_PERSONAL_ACCESS_TOKEN_CLASSIC | podman login ghcr.io -u YOUR_USERNAME --password-stdin
 ```
-The leading spaces are intended to prevent this command, which contains your secret PAT(C), from being written to your shell's history file. That at least works in bash... sometimes. To be extra safe, in bash you can do `history -c` and it will clear your entire bash history. That can be pretty disruptive, but fortunately you should only need to authenticate once.
 
 ### Tagging
-You'll next need to tag the image. Lots of Docker instructions tell you to use the `latest` tag, and Docker may actually do that for you. However, `latest` can lead to support headaches as users think they have the right version but actually don't. Instead, you'll make a new version number incremented from the [previous one](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs/versions), in the `vX.Y.Z` format.
+You'll next need to tag the image. Lots of container instructions tell you to use the `latest` tag, and Podman may actually add that for you. However, `latest` can lead to support headaches as users think they have the right version but actually don't. Instead, you'll make a new version number incremented from the [previous one](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs/versions), in the `vX.Y.Z` format.
 
-Copy the relevant image ID (see `docker images` instructions above) and tag it with your version number like so:
+Copy the relevant image ID (see `podman images` instructions above) and tag it with your version number like so:
 ```shell
-docker tag ab51446519a4 ghcr.io/escomp/ctsm/ctsm-docs:vX.Y.Z
+podman tag 6464f26339bc ghcr.io/escomp/ctsm/ctsm-docs:vX.Y.Z
 ```
 
 Push to the repo:
 ```shell
-docker push ghcr.io/escomp/ctsm/ctsm-docs:vX.Y.Z
+podman manifest push --all ctsm-docs-manifest ghcr.io/escomp/ctsm/ctsm-docs:vX.Y.Z
 ```
 
 Then browse to the [container's GitHub page](https://github.com/ESCOMP/CTSM/pkgs/container/ctsm%2Fctsm-docs) to make sure this all worked and the image is public.

doc/source/users_guide/working-with-documentation/building-docs-prereqs-docker-mac.md → doc/source/users_guide/working-with-documentation/building-docs-prereqs-mac.md

@@ -1,4 +1,4 @@
-.. _building-docs-prereqs-docker-mac:
+.. _building-docs-prereqs-mac:
 
 # Initial setup: Mac
 
@@ -63,18 +63,22 @@ If you don't have Conda yet, go on to the next section. Otherwise, continue to :
 ### Installing Conda
 We suggest installing Conda, if needed, via Miniforge:
 
-1. [Download Miniforge](https://conda-forge.org/download/) and install it. (:ref:`what-kind-of-mac-chip`) You can also [install Miniforge via Homebrew](https://formulae.brew.sh/cask/miniforge#default), if you already have that installed.
+1. [Download Miniforge](https://conda-forge.org/download/) and install it. (:ref:`what-kind-of-mac-chip`) You can also [install Miniforge via Homebrew](https://formulae.brew.sh/cask/miniforge#default), if you already have that installed. (:ref:`install-homebrew-mac`)
 2. Activate Conda permanently in your shell by opening a new Terminal window and doing `conda init "$(basename $SHELL)"`.
 
 You should now have `conda` and an up-to-date version of `python3` available, although will need to open another new Terminal window for it to work.
 
 ## Additional requirements
 The remaining software you need to install depends on which documentation-building method(s) you plan to use.
 
-### Docker (recommended)
-Install Docker Desktop through the [download page on Docker's website](https://docs.docker.com/desktop/setup/install/mac-install/). (:ref:`what-kind-of-mac-chip`)
+### Container (recommended)
+We use Podman to build and run our containers. The recommended way to install Podman is with Homebrew. (:ref:`install-homebrew-mac`)
 
-### Non-Docker method
+1. Install Podman with `brew install podman`.
+1. Set up and start a Podman "virtual machine" with `podman machine init --now`.
+1. Test your installation by doing `podman run --rm hello-world`. If it worked, you should see ASCII art of the Podman logo.
+
+### Non-container method
 
 Install Conda, if needed (see :ref:`installing-conda-for-docs`). Then follow the instructions for setting up the `ctsm_pylib` Conda environment in Sect. :numref:`using-ctsm-pylib`.
 
@@ -89,9 +93,8 @@ git --version
 git-lfs --version
 ```
 
-If either of those fail with "command not found," you'll need to install them. The recommended way is with [Homebrew](https://brew.sh/).
+If either of those fail with "command not found," you'll need to install them. The recommended way is with Homebrew. (:ref:`install-homebrew-mac`)
 
-1. [Install Homebrew](https://brew.sh/), if needed. (Check using `brew --version`.) Make sure to follow the instructions for adding Homebrew to your path. (:ref:`what-kind-of-mac-chip`)
 2. Use Homebrew to [install Git](https://formulae.brew.sh/formula/git#default), if needed.
 3. Use Homebrew to [install Git LFS](https://formulae.brew.sh/formula/git-lfs#default), if needed.
 
@@ -100,4 +103,10 @@ If either of those fail with "command not found," you'll need to install them. T
 .. _what-kind-of-mac-chip:
 
 ### What kind of chip does my Mac have?
-For certain steps in this installation process, you may need to know whether your Mac has an Intel (`x86_64`) or an Apple Silicon (`arm64`) chip. If you don't know, visit Apple's [Mac computers with Apple silicon](https://support.apple.com/en-us/116943) page for instructions.
+For certain steps in this installation process, you may need to know whether your Mac has an Intel (`x86_64`) or an Apple Silicon (`arm64`) chip. If you don't know, visit Apple's [Mac computers with Apple silicon](https://support.apple.com/en-us/116943) page for instructions.
+
+.. _install-homebrew-mac:
+
+### How do I install Homebrew?
+1. Install Homebrew using the instructions at https://brew.sh/. Make sure to follow the instructions during this process for adding Homebrew to your path.
+1. Check your installation by making sure that `brew --version` doesn't error.

doc/source/users_guide/working-with-documentation/building-docs-prereqs-docker-windows.md → doc/source/users_guide/working-with-documentation/building-docs-prereqs-windows.md

@@ -1,4 +1,4 @@
-.. _building-docs-prereqs-docker-windows:
+.. _building-docs-prereqs-windows:
 
 # Initial setup: Windows
 
@@ -17,6 +17,5 @@ Next, download and install Ubuntu (a version of Linux) via the Microsoft Store.
 
 Finally, try opening Ubuntu. If you run into an error, you may need to manually enable WSL. To do so: Open Control Panel, go to "Programs and Features," and then "Turn Windows features on or off." Check the box next to "Windows Subsystem for Linux" and click OK.
 
-## Docker Desktop
-1. Install Docker Desktop through the Microsoft Store or the [download page on Docker's website](https://docs.docker.com/desktop/setup/install/windows-install/).
-1. Open Docker Desktop. In Settings > Resources > WSL, Enable Docker to work under Ubuntu.
+## Podman
+Follow the [installation instructions on Podman's website](https://podman.io/docs/installation#ubuntu), entering the commands into your Ubuntu terminal.

doc/source/users_guide/working-with-documentation/docs-intro-and-recommended.md

@@ -5,8 +5,8 @@
 
 ## One-time setup
 In addition to a CTSM checkout in which to work, you will need to have some software installed in order to build the documentation:
-- :ref:`building-docs-prereqs-docker-mac`
-- :ref:`building-docs-prereqs-docker-windows`
+- :ref:`building-docs-prereqs-mac`
+- :ref:`building-docs-prereqs-windows`
 
 ## Editing the documentation
 The CTSM documentation is built from files in the `doc/source/tech_note/` and `doc/source/users_guide/` directories. These files are written in a mixture of what are called "markup languages." You may already be familiar—for better or for worse—with the LaTeX markup language. Fortunately, our documentation is simpler than that. It was originally written entirely in [reStructuredText](http://www.sphinx-doc.org/en/stable/rest.html), and it still mostly is, as you can tell by the predominance of .rst files. However, it's also possible to write Markdown documents (.md), which is nice because it's a much simpler and more widespread format (although see :ref:`tips-for-working-with-markdown`). If you've formatted text on GitHub, for instance, you've used Markdown.
@@ -26,7 +26,7 @@ You will need a place to build the documentation. It's fine if that doesn't exis
 Your CTSM clone, from which you're building the documentation, also needs to be somewhere in your user home directory.
 
 ### Building a preview
-Ensure that Docker Desktop is running. (You do not actually need to do anything in Docker Desktop; it just needs to be running. It's fine for it to be minimized or hidden.) Then all you need to do is
+All you need to do is
 ```shell
 cd doc
 ./build_docs -b $HOME/path/to/build-dir -c -d

doc/source/users_guide/working-with-documentation/index.rst

@@ -10,8 +10,8 @@ Working with CTSM Documentation
    :maxdepth: 1
 
    docs-intro-and-recommended.md
-   building-docs-prereqs-docker-mac.md
-   building-docs-prereqs-docker-windows.md
+   building-docs-prereqs-mac.md
+   building-docs-prereqs-windows.md
    building-docs-multiple-versions.rst
    tips-for-working-with-markdown.md
    tips-for-working-with-rst.md

doc/testing.sh

@@ -4,12 +4,17 @@ set -x
 
 rm -rf _publish*
 
+# For some reason, Podman has trouble on GH Actions runners, so force use of Docker
+if [[ "${GITHUB_ACTIONS}" != "" ]]; then
+    containercli="--container-cli-tool docker"
+fi
+
 # Build all docs using container
 echo "~~~~~ Build all docs using container"
 # Also do a custom --conf-py-path
 rm -rf _build _publish
 d1="$PWD/_publish_container"
-./build_docs_to_publish -r _build -d --site-root "$PWD/_publish"
+./build_docs_to_publish -r _build -d --site-root "$PWD/_publish" ${containercli}
 # VERSION LINKS WILL NOT RESOLVE IN _publish_container
 cp -a _publish "${d1}"
 
@@ -28,8 +33,8 @@ diff -qr "${d1}" "${d2}"
 # Check that -r -v works
 echo "~~~~~ Check that -r -v works"
 # Also do a custom --conf-py-path
-rm -rf _build_docker
-./build_docs -r _build_docker -v latest -d -c --conf-py-path doc-builder/test/conf.py --static-path ../_static --templates-path ../_templates
+rm -rf _build_container
+./build_docs -r _build_container -v latest -d ${containercli} -c --conf-py-path doc-builder/test/conf.py --static-path ../_static --templates-path ../_templates
 
 # Check that Makefile method works
 echo "~~~~~ Check that Makefile method works"
@@ -38,12 +43,12 @@ make SPHINXOPTS="-W --keep-going" BUILDDIR=${PWD}/_build html
 
 # Check that -b works
 echo "~~~~~ Check that -b works"
-rm -rf _build_docker
-./build_docs -b _build_docker -d -c
+rm -rf _build_container
+./build_docs -b _build_container -d ${containercli} -c
 
 # Check that doc-builder tests pass
 # Don't run if on a GitHub runner; failing 🤷. Trust that doc-builder does this test.
-if [[ "${GITHUB_ACTION}" == "" ]]; then
+if [[ "${GITHUB_ACTIONS}" == "" ]]; then
     echo "~~~~~ Check that doc-builder tests pass"
     cd doc-builder/test
     conda run -n ctsm_pylib make test