Optimize docker build caching and fix coverage.

Author: MarcoSteinacher

.github/workflows/coverage.yml

@@ -2,7 +2,7 @@ name: Post coverage comment
 
 on:
   workflow_run:
-    workflows: ["Django Tests on PR"]
+    workflows: ["Django Tests"]
     types:
       - completed
 

.github/workflows/test-django.yml

@@ -1,13 +1,11 @@
-name: Django Tests on PR
+name: Django Tests
 
-# trigger: every pull-request
 on:
   push:
   pull_request:
     branches:
       - main
       - develop
-      - fix/ci_test
   workflow_dispatch: # allow manual trigger
 
 permissions:
@@ -61,20 +59,24 @@ jobs:
       - name: Initialize submodules
         run: git submodule update --init --recursive
 
-      # 1. Build only the stage that contains code + deps
       - name: Build test image
         run: |
           cd django && \
           docker build . \
             --target cohiva-platform-django \
             -t cohiva:test
 
-      # 2. Run the test script inside that image
-      - name: Run Django tests
+      - name: Prepare for coverage report
         run: |
           set -e
           cd django
           touch .coverage
+          ln -sf website_example website
+
+      - name: Run Django tests
+        run: |
+          set -e
+          cd django
           docker run --rm \
             --network host \
             -e GITHUB_ACTIONS=true \
@@ -101,29 +103,3 @@ jobs:
           # If you use a different name, update COMMENT_FILENAME accordingly
           path: python-coverage-comment-action.txt
 
-
-#      - name: Upload coverage to Codecov
-#        uses: codecov/codecov-action@v3.1.0
-#        with:
-#          files: ./django/coverage.xml
-#          fail_ci_if_error: true
-#          verbose: true
-#          flags: unittests
-#          name: codecov-umbrella
-#          token: ${{ secrets.CODECOV_TOKEN }}
-#          directory: ./django/
-#          workdir: ./django/
-#
-#      - name: Upload coverage html report
-#        #uses: actions/upload-artifact@v3
-#        uses: actions/upload-pages-artifact@v3
-#        with:
-#          #name: coverage-html-report
-#          path: ./django/htmlcov/
-#
-#
-#      - name: Publish to GitHub pages
-#        #uses: peaceiris/actions-gh-pages@v3
-#        uses: actions/deploy-pages@v3
-#        #with:
-#        #  artifact_name: coverage-html-report

django/.coveragerc

@@ -0,0 +1,12 @@
+[run]
+relative_files = True
+
+[report]
+omit =
+    cohiva/base_config.py
+    cohiva/settings.py
+    cohiva/settings_production.py
+    cohiva/*example*.py
+    scripts/*
+    website_example/*
+    manage.py

django/Dockerfile

@@ -10,15 +10,19 @@ RUN apt-get update && \
 
 # Install python dependencies in /usr/local/
 WORKDIR /cohiva
-COPY . .
-RUN cp /cohiva/requirements-docker.txt /cohiva/requirements.txt   # Use requirements without dev packages
+# Copy only the necessary files to improve build caching
+COPY requirements-docker.txt requirements.txt
+COPY install_dependencies.sh install.py ./
+COPY geno/python-sepa geno/python-sepa
+COPY cohiva/saml2/*.patch cohiva/saml2/
 RUN chmod +x ./install_dependencies.sh
 RUN ./install_dependencies.sh -e docker
 
 # Remove unnecessary stuff from /usr/local/
 RUN find /usr/local/lib -type d -name '__pycache__' -exec rm -rf {} + && \
     find /usr/local/lib -type f -name '*.dist-info' -exec rm -rf {} +
 
+# Runtime stage
 FROM python:3.13-slim-bookworm AS cohiva-platform-os
 
 # Set environment variables to ensure LibreOffice runs in headless mode

django/README.md

@@ -84,13 +84,12 @@ Example for Debian 11 (should work on most Debian/Ubuntu based systems):
     sudo apt install clang
     sudo apt install python3-dev
     sudo apt install python3-venv
-    sudo apt install libmariadb-dev   ## or default-libmysqlclient-dev
+    sudo apt install libmariadb-dev       ## or default-libmysqlclient-dev
     sudo apt install libfreetype-dev
     sudo apt install libjpeg-dev
     sudo apt install libffi-dev
-    sudo apt install xmlsec1          ## for SAML 2.0 IDP
-    sudo apt install poppler-utils    ## (optional, for tests)
-    sudo apt install libreoffice-writer   ## (optional, for PDF generation)
+    sudo apt install xmlsec1              ## for SAML 2.0 IDP
+    sudo apt install libreoffice-writer   ## (or libreoffice-writer-nogui, required for PDF generation)
 
 **Note:** If using Docker for development (recommended), you don't need `redis-server` or `mariadb-server` system packages. The bootstrap script automatically detects and offers to install required packages on macOS.
 

django/docker-compose.dev.yml

@@ -7,12 +7,12 @@ services:
       MYSQL_ROOT_PASSWORD: root
       # Database is created in init script with correct collation
       MYSQL_USER: cohiva
-      MYSQL_PASSWORD: cohiva_password
+      MYSQL_PASSWORD: c0H1v4
     ports:
       - "3306:3306"
     volumes:
       - mariadb_data:/var/lib/mysql
-      #- ./docker/mariadb-init:/docker-entrypoint-initdb.d
+      - ./docker/mariadb-init:/docker-entrypoint-initdb.d
     command: >
       --character-set-server=utf8mb4
       --collation-server=utf8mb4_unicode_ci

django/run-tests.sh

@@ -3,34 +3,38 @@
 # exit on error
 set -e
 
-# if called from CI (GitHub Actions), copy the base_config_for_tests.py to base_config.py
-#if [ -n "$GITHUB_ACTIONS" ] ; then
-# This is done with setup.py in the workflow already...
-#    touch cohiva/settings.py
-#    cp cohiva/base_config_for_tests.py cohiva/base_config.py
-#fi
+## Uncomment to enable coverage
+#COVERAGE="true"
 
 ## Show warnings?
 export PYTHONWARNINGS=always
 
 INSTALL_DIR=$(grep "^INSTALL_DIR = " cohiva/base_config.py | cut -d \" -f 2)
 
+## Normalize flags
+case "${COVERAGE,,}" in
+  true|1|yes|on)
+    COVERAGE="true"
+    ;;
+  *)
+    COVERAGE="false"
+    ;;
+esac
+case "${GITHUB_ACTIONS,,}" in
+  true|1|yes|on)
+    GITHUB_ACTIONS="true"
+    ;;
+  *)
+    GITHUB_ACTIONS="false"
+    ;;
+esac
+
 ## Coverage options
-#COVERAGE="true"  # Uncomment to enable coverage
 COVERAGE_OPTS=()
 #COVERAGE_OPTS=(--source "$INSTALL_DIR/django")
 #COVERAGE_OPTS=(--append)
-if [ -n "$GITHUB_ACTIONS" ] ; then
-    #COVERAGE_OPTS=(--source "/cohiva/")
-    cat <<EOT >.coveragerc
-[run]
-source = /cohiva/
-relative_files = True
-EOT
-
-fi
 
-if [ -n "$COVERAGE" ] ; then
+if [ "$COVERAGE" = "true" ] ; then
     COVERAGE_CMD="coverage run ${COVERAGE_OPTS[*]}"
 else
     COVERAGE_CMD=""
@@ -46,7 +50,7 @@ TEST_OPTS=""
 ## Select test to run (leave emtpy to run all tests)
 SELECTED_TESTS=""
 # Examples:
-SELECTED_TESTS="finance.tests geno.tests.test_documents.DocumentSendTest.test_send_member_bill"
+#SELECTED_TESTS="finance.tests geno.tests.test_documents.DocumentSendTest.test_send_member_bill"
 
 ## Full test suite incl. migrations
 # Run tests and capture the exit code of the test runner even though output is piped to tee.
@@ -57,13 +61,14 @@ TEST_EXIT_CODE=${PIPESTATUS[0]}
 set -e
 
 if [ -n "$COVERAGE_CMD" ] ; then
-  if [ -n "$GITHUB_ACTIONS" ] ; then
-    #coverage html 2>&1 | tee coverage.log
-    # Copy the output to the mounted docker host file
+  if [ "$GITHUB_ACTIONS" = "true" ] ; then
+    # Copy the output to the mounted docker host file because
+    # coverage can't use the mounted file directly in the container.
     cp .coverage .coverage_output_for_github_action
+    #coverage html 2>&1 | tee coverage.log
   else
-    coverage html 2>&1 | tee coverage.log
     #coverage report
+    coverage html 2>&1 | tee coverage.log
   fi
 fi
 

django/scripts/mariadb/init_test.sql

@@ -13,6 +13,3 @@ FLUSH PRIVILEGES;
 CREATE DATABASE IF NOT EXISTS `cohiva_gnucash_test`
     CHARACTER SET utf8mb4
     COLLATE utf8mb4_unicode_ci;
-
--- USE `cohiva_gnucash_test`;
--- SOURCE demo-data/demo-data-gnucash.sql;