Merge pull request #1161 from Aiven-Open/jjaakola-aiven-collect-ci-logs

chore: collect test logs in CI

Author: muralibasani

.github/workflows/tests.yml

@@ -21,11 +21,6 @@ jobs:
     strategy:
       matrix:
         python-version: [ '3.10', '3.11', '3.12' ]
-    env:
-      PYTEST_ADDOPTS: >-
-        --log-dir=/tmp/ci-logs
-        --log-file=/tmp/ci-logs/pytest.log
-        --showlocals
     steps:
     - uses: actions/checkout@v4
       # Need fetch-depth 0 to fetch tags, see https://github.com/actions/checkout/issues/701
@@ -76,7 +71,7 @@ jobs:
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}
         COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4"
+        PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4 --log-file=/opt/test-tmp/test-logs/unit-tests-pytest.${{ matrix.python-version }}.log --showlocals"
 
     - run: make e2e-tests-in-docker
       env:
@@ -85,7 +80,7 @@ jobs:
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}
         COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4"
+        PYTEST_ARGS: "--cov=karapace --cov-append --numprocesses 4 --basetemp /opt/test-tmp/e2e-test-logs --log-file=/opt/test-tmp/test-logs/e2e-tests-pytest.${{ matrix.python-version }}.log --showlocals"
 
     - run: make integration-tests-in-docker
       env:
@@ -94,15 +89,19 @@ jobs:
         RUNNER_UID: ${{ env.RUNNER_UID }}
         RUNNER_GID: ${{ env.RUNNER_GID }}
         COVERAGE_FILE: "/opt/karapace/coverage/.coverage.${{ matrix.python-version }}"
-        PYTEST_ARGS: "--cov=karapace --cov-append --random-order --numprocesses 4"
+        PYTEST_ARGS: "--cov=karapace --cov-append --random-order --numprocesses 4 --basetemp /opt/test-tmp/integration-test-logs --log-file=/opt/test-tmp/test-logs/integration-tests-pytest.${{ matrix.python-version }}.log --showlocals"
+
+    - name: Allow reading of test temporary directory
+      run: sudo chmod -R a+rX test-tmp.${{ matrix.python-version }}
 
     - name: Archive logs
       uses: actions/upload-artifact@v4
       if: ${{ always() }}
       with:
-        name: karapace-integration-test-logs-${{ matrix.python-version }}
-        path: /tmp/ci-logs
-
+        name: karapace-test-logs-${{ matrix.python-version }}
+        path: |
+          test-tmp.${{ matrix.python-version }}
+          !**/popen-*/**
     - name: Archive coverage file
       uses: actions/upload-artifact@v4
       with:

GNUmakefile

@@ -113,8 +113,10 @@ stop-karapace-docker-resources:
 .PHONY: start-karapace-docker-resources
 start-karapace-docker-resources: export KARAPACE_VERSION ?= 4.1.1.dev44+gac20eeed.d20241205
 start-karapace-docker-resources:
-	sudo touch .coverage.3.10 .coverage.3.11 .coverage.3.12
-	sudo chown ${RUNNER_UID}:${RUNNER_GID} .coverage.3.10 .coverage.3.11 .coverage.3.12
+	sudo touch .coverage.${PYTHON_VERSION}
+	sudo chown ${RUNNER_UID}:${RUNNER_GID} .coverage.${PYTHON_VERSION}
+	sudo mkdir -p test-tmp.${PYTHON_VERSION}
+	sudo chown -R ${RUNNER_UID}:${RUNNER_GID} test-tmp.${PYTHON_VERSION}
 	$(DOCKER_COMPOSE) -f container/compose.yml up -d --build --wait --detach
 
 .PHONY: smoke-test-schema-registry

container/compose.yml

@@ -231,9 +231,8 @@ services:
       - ../mypy.ini:/opt/karapace/mypy.ini
       - ../.pre-commit-config.yaml:/opt/karapace/.pre-commit-config.yaml
       - ../.coveragerc:/opt/karapace/.coveragerc
-      - ../.coverage.3.10:/opt/karapace/coverage/.coverage.3.10
-      - ../.coverage.3.11:/opt/karapace/coverage/.coverage.3.11
-      - ../.coverage.3.12:/opt/karapace/coverage/.coverage.3.12
+      - "../.coverage.$PYTHON_VERSION:/opt/karapace/coverage/.coverage.$PYTHON_VERSION"
+      - "../test-tmp.$PYTHON_VERSION:/opt/test-tmp"
     environment:
       - COVERAGE_FILE
       - COVERAGE_RCFILE=/opt/karapace/.coveragerc