[CIVIS-9806] ENH update civis-python to v2.4.0; add uv (#99)

* ENH update base image to python 3.12.7

* ENH update core dependency versions

* FIX include awscli v2 in requirements files

* TST expected shell commands should be available

* ENH add uv to image

* MAINT update changelog

* FIX version env vars

* MAINT update readme

* MAINT always build images for dev branches

* TST add an apt-get test

* FIX set env var for uv to use system python

* STY typos

* TST uv

Author: jacksonlee-civis

.circleci/test_image.py

@@ -2,9 +2,12 @@
 
 import os
 import re
+import shutil
+import subprocess
 import unittest
 
 
+# Just use the stdlib's `unittest` rather than needing to install `pytest`.
 class TestImage(unittest.TestCase):
 
     def test_version(self):
@@ -42,6 +45,32 @@ def test_civis_can_import(self):
         import civis.ml  # noqa: F401
         import civis.utils  # noqa: F401
 
+    def test_shell_commands_available(self):
+        """Ensure the main shell commands are available."""
+        # A non-exhaustive list of commands -- we just test those we'd likely use.
+        expected_cmds = "aws civis curl git pip python unzip uv wget".split()
+        for cmd in expected_cmds:
+            self.assertIsNotNone(shutil.which(cmd), f"{cmd} not found in PATH")
+
+    def _test_shell_command(self, cmd: str):
+        """Check if the shell command runs successfully in the image."""
+        try:
+            subprocess.check_call(cmd, shell=True)
+        except subprocess.CalledProcessError as e:
+            self.fail(
+                f"apt-get test failed with return code {e.returncode}\n"
+                f"stdout: {e.stdout}\n"
+                f"stderr: {e.stderr}"
+            )
+
+    def test_apt_get(self):
+        """Ensure that apt-get works in the image."""
+        self._test_shell_command("apt-get update -y && apt-get install -y htop")
+
+    def test_uv(self):
+        """Ensure that uv works in the image."""
+        self._test_shell_command("uv pip install python-iso639")
+
 
 if __name__ == "__main__":
     unittest.main()

CHANGELOG.md

@@ -9,6 +9,19 @@ Version number changes (major.minor.micro) in this package denote the following:
 
 ## Unreleased
 
+## [8.1.0]
+
+- Python version updated to v3.12.7
+- Core dependencies updated to latest versions:
+  * awscli 2.17.37 -> 2.19.5
+  * boto3 1.34.127 -> 1.35.58
+  * civis 2.3.0 -> 2.4.0
+  * numpy 2.0.0 -> 2.1.3
+  * pandas 2.2.2 -> 2.2.3
+  * scikit-learn 1.5.0 -> 1.5.2
+  * scipy 1.13.1 -> 1.14.1
+- uv added to the image
+
 ## [8.0.1]
 
 - Python version updated to v3.12.6

Dockerfile

@@ -1,5 +1,5 @@
 ARG PLATFORM=linux/x86_64
-ARG BASE_IMAGE=python:3.12.6-slim
+ARG BASE_IMAGE=python:3.12.7-slim
 
 # This is the primary build target used for the production image
 FROM --platform=$PLATFORM $BASE_IMAGE AS production
@@ -37,25 +37,24 @@ COPY requirements-full.txt .
 RUN pip install --progress-bar off --no-cache-dir -r requirements-full.txt && \
   rm requirements-full.txt
 
+# Install uv.
+ADD https://astral.sh/uv/0.5.1/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.local/bin/:$PATH" \
+  UV_SYSTEM_PYTHON=1
+
 # Instruct joblib to use disk for temporary files. Joblib defaults to
 # /shm when that directory is present. In the Docker container, /shm is
 # present but defaults to 64 MB.
 # https://github.com/joblib/joblib/blob/0.11/joblib/parallel.py#L328L342
 ENV JOBLIB_TEMP_FOLDER=/tmp
 
-ENV VERSION=8.0.1 \
+ENV VERSION=8.1.0 \
   VERSION_MAJOR=8 \
-  VERSION_MINOR=0 \
-  VERSION_MICRO=1
-
-# Install the AWSCLI for moving match targets in the QC workflow.
-# See https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#cliv2-linux-install
-RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
-    unzip awscliv2.zip && \
-    ./aws/install && \
-    rm -rf aws awscliv2.zip
+  VERSION_MINOR=1 \
+  VERSION_MICRO=0
 
-# This build target is for testing in Circle CI.
+# This build target is for testing in CircleCI.
 FROM --platform=$PLATFORM production AS test
 COPY .circleci/test_image.py .
 COPY CHANGELOG.md .

README.md

@@ -66,8 +66,8 @@ default location for staging temporary files to the /tmp directory.
 The normal default is /shm. /shm is a RAM disk which defaults to a 64 MB size
 in Docker containers, too small for typical scientific computing.
 
-# Updating existing PyPi Package Version
-1. Update version of existing package in `requirements-core.txt`
+# Updating Existing Package Versions
+1. Update versions of existing packages in `requirements-core.txt`
 2. Run script `generate-requirements-full.sh`
 
 # Creating Equivalent Local Environments
@@ -88,7 +88,7 @@ and describe any changes in the [change log](CHANGELOG.md).
 ## For Maintainers
 
 This repo has autobuild enabled. Any PR that is merged to master will
-be built as the `latest` tag on Dockerhub.
+be built as the `latest` tag on DockerHub.
 Once you are ready to create a new version, go to the "releases" tab of the repository and click
 "Draft a new release". GitHub will prompt you to create a new tag, release title, and release
 description. The tag should use semantic versioning in the form "vX.X.X"; "major.minor.micro".

buildspec/push.yaml

@@ -10,9 +10,9 @@ phases:
       - echo $COMMIT_HASH_SHORT
       - echo $BRANCH_NAME
       - docker build --tag ${FIPS_REPOSITORY_URI}:${COMMIT_HASH_SHORT} --tag ${FIPS_REPOSITORY_URI}:${BRANCH_NAME} .
-      # This config tests the codebuild login and the build but does not push dev images.
-      # The following lines can be temporarily uncommented to test a dev image.
-      # - docker image push --all-tags ${FIPS_REPOSITORY_URI}
+      # We have a life cycle policy in place to expire and delete images from dev branches,
+      # so there are no issues with pushing as many of these images as there may be.
+      - docker image push --all-tags ${FIPS_REPOSITORY_URI}
   post_build:
     commands:
       - echo Build completed!

docker-compose.yml

@@ -8,4 +8,4 @@ services:
       - .:/app
     stdin_open: true
     tty: true
-    working_dir: /app
+    working_dir: /app

generate-requirements-full.sh

@@ -2,4 +2,4 @@
 # Run this script to update requirements-core.txt.
 # It uses Docker to ensure that the environment matches what will be used in the production image.
 set -e
-docker compose run --rm pip-tools /bin/sh -c "pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt"
+docker compose run --rm pip-tools /bin/sh -c "apt-get update -y && apt-get install -y git && pip install --upgrade pip-tools && pip-compile --output-file=requirements-full.txt --pip-args='--prefer-binary' --strip-extras --upgrade requirements-core.txt"

requirements-core.txt

@@ -1,7 +1,11 @@
-boto3==1.34.127
-civis==2.3.0
-numpy==2.0.0
-pandas==2.2.2
+# awscli v2 is not officially available on PyPI (https://github.com/aws/aws-cli/issues/4947).
+# Specifying awscli in requirements-core.txt here ensures that it (and its transitive dependencies)
+# are taken into account when generating the final requirements-full.txt file.
+awscli @ git+https://github.com/aws/[email protected]
+boto3==1.35.58
+civis==2.4.0
+numpy==2.1.3
+pandas==2.2.3
 requests==2.32.3
-scikit-learn==1.5.0
-scipy==1.13.1
+scikit-learn==1.5.2
+scipy==1.14.1

requirements-full.txt

@@ -8,26 +8,41 @@ attrs==24.2.0
     # via
     #   jsonschema
     #   referencing
-boto3==1.34.127
+awscli @ git+https://github.com/aws/[email protected]
     # via -r requirements-core.txt
-botocore==1.34.162
+awscrt==0.22.0
+    # via awscli
+boto3==1.35.58
+    # via -r requirements-core.txt
+botocore==1.35.58
     # via
     #   boto3
     #   s3transfer
-certifi==2024.7.4
+certifi==2024.8.30
     # via requests
-charset-normalizer==3.3.2
+cffi==1.17.1
+    # via cryptography
+charset-normalizer==3.4.0
     # via requests
-civis==2.3.0
+civis==2.4.0
     # via -r requirements-core.txt
 click==8.1.7
     # via civis
-cloudpickle==3.0.0
+cloudpickle==3.1.0
     # via civis
-idna==3.8
+colorama==0.4.6
+    # via awscli
+cryptography==43.0.1
+    # via awscli
+distro==1.8.0
+    # via awscli
+docutils==0.19
+    # via awscli
+idna==3.10
     # via requests
 jmespath==1.0.1
     # via
+    #   awscli
     #   boto3
     #   botocore
 joblib==1.4.2
@@ -38,21 +53,26 @@ jsonref==1.1.0
     # via civis
 jsonschema==4.23.0
     # via civis
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2024.10.1
     # via jsonschema
-numpy==2.0.0
+numpy==2.1.3
     # via
     #   -r requirements-core.txt
     #   pandas
     #   scikit-learn
     #   scipy
-pandas==2.2.2
+pandas==2.2.3
     # via -r requirements-core.txt
-python-dateutil==2.9.0.post0
+prompt-toolkit==3.0.38
+    # via awscli
+pycparser==2.22
+    # via cffi
+python-dateutil==2.9.0
     # via
+    #   awscli
     #   botocore
     #   pandas
-pytz==2024.1
+pytz==2024.2
     # via pandas
 pyyaml==6.0.2
     # via civis
@@ -64,15 +84,19 @@ requests==2.32.3
     # via
     #   -r requirements-core.txt
     #   civis
-rpds-py==0.20.0
+rpds-py==0.21.0
     # via
     #   jsonschema
     #   referencing
-s3transfer==0.10.2
+ruamel-yaml==0.17.21
+    # via awscli
+ruamel-yaml-clib==0.2.8
+    # via awscli
+s3transfer==0.10.3
     # via boto3
-scikit-learn==1.5.0
+scikit-learn==1.5.2
     # via -r requirements-core.txt
-scipy==1.13.1
+scipy==1.14.1
     # via
     #   -r requirements-core.txt
     #   scikit-learn
@@ -82,9 +106,12 @@ tenacity==9.0.0
     # via civis
 threadpoolctl==3.5.0
     # via scikit-learn
-tzdata==2024.1
+tzdata==2024.2
     # via pandas
-urllib3==2.2.2
+urllib3==1.26.20
     # via
+    #   awscli
     #   botocore
     #   requests
+wcwidth==0.2.13
+    # via prompt-toolkit