Merge pull request #264 from SciCatProject/setup-for-integration-testing

Setup for integration testing

Author: jl-wynen

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   formatting:
     name: Formatting and static analysis
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -24,7 +24,7 @@ jobs:
   type-checking:
     name: Type checking
     needs: formatting
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -35,28 +35,23 @@ jobs:
       - run: tox -e mypy
 
   tests:
-    name: Tests ${{ matrix.os }} ${{ matrix.tox }}
+    name: Tests
     needs: formatting
-    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         include:
-          - {python: '3.13', os: ubuntu-22.04, tox: py313-full}
-          - {python: '3.12', os: ubuntu-22.04, tox: py312-full}
-          - {python: '3.11', os: ubuntu-22.04, tox: py311-full}
-          - {python: '3.10', os: ubuntu-22.04, tox: py310-full}
+          - {python: '3.13', os: ubuntu-24.04, tox: py313-full}
+          - {python: '3.12', os: ubuntu-24.04, tox: py312-full}
+          - {python: '3.11', os: ubuntu-24.04, tox: py311-full}
+          - {python: '3.10', os: ubuntu-24.04, tox: py310-full}
           - {python: '3.10', os: macos-14, tox: py310}
           - {python: '3.10', os: windows-2022, tox: py310}
-    steps:
-      - run: sudo apt install --yes docker-compose
-        if: ${{ contains(matrix.os, 'ubuntu') }}
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python }}
-      - run: python -m pip install --upgrade pip
-      - run: python -m pip install -r requirements/ci.txt
-      - run: tox -e ${{ matrix.tox }}
+    uses: ./.github/workflows/test.yml
+    with:
+      os-variant: ${{ matrix.os }}
+      python-version: ${{ matrix.python }}
+      tox-env: ${{ matrix.tox }}
+    secrets: inherit
 
   docs:
     needs: tests

.github/workflows/docs.yml

@@ -18,7 +18,7 @@ on:
 jobs:
   docs:
     name: Build documentation
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - run: sudo apt install --yes pandoc
       - uses: actions/checkout@v4

.github/workflows/release.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   build_wheels:
     name: Wheels
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - uses: actions/checkout@v4
@@ -31,7 +31,7 @@ jobs:
   upload_packages:
     name: Deploy packages
     needs: [build_wheels]
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     environment:
       name: release
       url: https://pypi.org/p/scitacean/

.github/workflows/test.yml

@@ -0,0 +1,60 @@
+name: Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      os-variant:
+        default: 'ubuntu-24.04'
+        type: string
+        description: 'Operating system'
+      python-version:
+        type: string
+        description: 'Python interpreter version'
+      tox-env:
+        default: 'test'
+        type: string
+        description: 'Tox environment to run'
+      checkout-ref:
+        default: ''
+        type: string
+        description: 'Git ref to checkout'
+      backend-version:
+        default: ''
+        type: string
+        description: 'SciCat backend version, e.g., "v4.8.0"'
+  workflow_call:
+    inputs:
+      os-variant:
+        default: 'ubuntu-24.04'
+        type: string
+      python-version:
+        type: string
+      tox-env:
+        default: 'test'
+        type: string
+      checkout-ref:
+        default: ''
+        type: string
+      backend-version:
+        default: ''
+        type: string
+
+jobs:
+  tests:
+    name: Tests ${{ inputs.os-variant }} ${{ inputs.tox-env }}
+    runs-on: ${{ inputs.os-variant }}
+    steps:
+      - run: sudo apt install --yes docker-compose
+        if: ${{ contains(inputs.os-variant, 'ubuntu') }}
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.checkout-ref }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ inputs.python-version }}
+      - run: python -m pip install --upgrade pip
+      - run: python -m pip install -r requirements/ci.txt
+      - run: tox -e ${{ inputs.tox-env }}
+        if: ${{ !inputs.backend-version }}
+      - run: tox -e ${{ inputs.tox-env }} -- ${{ inputs.backend-version }}
+        if: ${{ inputs.backend-version }}

docs/user-guide/testing.ipynb

@@ -390,21 +390,19 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "29",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "import pytest\n",
-    "from scitacean.testing.backend import add_pytest_option as add_backend_option\n",
+    "from scitacean.testing.backend import add_pytest_options as add_backend_options\n",
     "\n",
     "\n",
     "pytest_plugins = (\n",
     "    \"scitacean.testing.backend.fixtures\",\n",
     ")\n",
     "\n",
     "def pytest_addoption(parser: pytest.Parser) -> None:\n",
-    "    add_backend_option(parser)"
+    "    add_backend_options(parser)"
    ]
   },
   {
@@ -413,7 +411,7 @@
    "metadata": {},
    "source": [
     "The backend will only be launched when the corresponding command line option is given.\n",
-    "By default, this is `--backend-tests` but it can be changed via the `option` argument of `add_pytest_option`.\n",
+    "By default, this is `--backend-tests` but it can be changed via the `option` argument of `add_pytest_options`.\n",
     "\n",
     "### Use SciCat in tests\n",
     "\n",
@@ -424,9 +422,7 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "31",
-   "metadata": {
-    "tags": []
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "def test_something_with_scicat(require_scicat_backend):\n",

src/scitacean/testing/backend/__init__.py

@@ -61,7 +61,7 @@ def test_manual_client(require_scicat_backend, scicat_access):
 .. autosummary::
    :toctree: ../functions
 
-   add_pytest_option
+   add_pytest_options
    backend_enabled
    configure
    skip_if_not_backend
@@ -77,10 +77,10 @@ def test_manual_client(require_scicat_backend, scicat_access):
     stop_backend,
     wait_until_backend_is_live,
 )
-from ._pytest_helpers import add_pytest_option, backend_enabled, skip_if_not_backend
+from ._pytest_helpers import add_pytest_options, backend_enabled, skip_if_not_backend
 
 __all__ = [
-    "add_pytest_option",
+    "add_pytest_options",
     "backend_enabled",
     "config",
     "configure",

src/scitacean/testing/backend/_backend.py

@@ -31,7 +31,9 @@ def _docker_compose_template() -> dict[str, Any]:
 
 
 def _apply_config(
-    template: dict[str, Any], account_config_path: Path
+    template: dict[str, Any],
+    account_config_path: Path,
+    version: str | None,
 ) -> dict[str, Any]:
     res = deepcopy(template)
     scicat = res["services"]["scicat"]
@@ -47,20 +49,29 @@ def _apply_config(
         f"{account_config_path}:/home/node/app/functionalAccounts.json",
     ]
 
+    if version is not None:
+        url = scicat["image"].split(":")[0]
+        scicat["image"] = f"{url}:{version}"
+
     return res
 
 
-def configure(target_path: _PathLike) -> None:
+def configure(target_path: _PathLike, *, version: str | None = None) -> None:
     """Build a docker-compose file for the testing backend.
 
     Parameters
     ----------
     target_path:
         Generate a docker-compose file at this path.
+    version:
+        The backend version to use, e.g., ``"v4.8.0"``.
+        If not provided, the latest version will be used that is known to be compatible.
     """
     account_config_path = Path(target_path).parent / "functionalAccounts.json"
     config.dump_account_config(account_config_path)
-    c = yaml.dump(_apply_config(_docker_compose_template(), account_config_path))
+    c = yaml.dump(
+        _apply_config(_docker_compose_template(), account_config_path, version)
+    )
     if "PLACEHOLDER" in c:
         raise RuntimeError("Incorrect config")
 

src/scitacean/testing/backend/_pytest_helpers.py

@@ -7,15 +7,15 @@
 _COMMAND_LINE_OPTION: str | None = None
 
 
-def add_pytest_option(parser: pytest.Parser, option: str = "--backend-tests") -> None:
-    """Add a command-line option to pytest to toggle backend tests.
+def add_pytest_options(parser: pytest.Parser, option: str = "--backend-tests") -> None:
+    """Add command-line options to pytest to control backend tests.
 
     Parameters
     ----------
     parser:
         Pytest's command-line argument parser.
     option:
-        Name of the command-line option.
+        Name of the command-line option to toggle backend tests.
     """
     parser.addoption(
         option,
@@ -26,6 +26,12 @@ def add_pytest_option(parser: pytest.Parser, option: str = "--backend-tests") ->
     global _COMMAND_LINE_OPTION
     _COMMAND_LINE_OPTION = option
 
+    parser.addoption(
+        "--scitacean-backend-version",
+        default=None,
+        help="Specify a version for the SciCat backend",
+    )
+
 
 def skip_if_not_backend(request: pytest.FixtureRequest) -> None:
     """Mark the current test to be skipped if backend tests are disabled."""

src/scitacean/testing/backend/docker-compose-backend-template.yaml

@@ -3,8 +3,8 @@
 # before it can be used by docker compose.
 #
 # The images are based on the CI setup of the SciCat backend
-# and the latest stable package:
-# https://github.com/SciCatProject/scicat-backend-next/pkgs/container/backend-next/97880429?tag=stable
+# and a released package of the version configured in pyproject.toml:
+# https://github.com/SciCatProject/scicat-backend-next/pkgs/container/backend-next/versions
 #
 # See https://scicatproject.github.io/documentation/Development/v4.x/backend/configuration.html
 # for a list of all env variables.

src/scitacean/testing/backend/fixtures.py

@@ -147,8 +147,12 @@ def scicat_backend(request, tmp_path_factory, scicat_access):
         ):
             yield False
     else:
+        version = request.config.getoption("--scitacean-backend-version")
         with _prepare_with_backend(
-            scicat_access=scicat_access, target_dir=target_dir, counter=counter
+            scicat_access=scicat_access,
+            target_dir=target_dir,
+            counter=counter,
+            version=version,
         ):
             yield True
 
@@ -174,11 +178,12 @@ def _prepare_with_backend(
     scicat_access: SciCatAccess,
     counter: FileCounter | NullCounter,
     target_dir: Path,
+    version: str | None,
 ) -> Generator[None, None, None]:
     try:
         with counter.increment() as count:
             if count == 1:
-                _backend_docker_up(target_dir)
+                _backend_docker_up(target_dir, version)
                 _seed_database(
                     Client, scicat_access=scicat_access, target_dir=target_dir
                 )
@@ -206,15 +211,15 @@ def _seed_database(
     seed.save_seed(target_dir)
 
 
-def _backend_docker_up(target_dir: Path) -> None:
+def _backend_docker_up(target_dir: Path, version: str | None) -> None:
     if _backend_is_running():
         raise RuntimeError("SciCat docker container is already running")
     docker_compose_file = target_dir / "docker-compose.yaml"
     log = logging.getLogger("scitacean.testing")
     log.info(
         "Starting docker container with SciCat backend from %s", docker_compose_file
     )
-    configure(docker_compose_file)
+    configure(docker_compose_file, version=version)
     docker.docker_compose_up(docker_compose_file)
     log.info("Waiting for SciCat docker to become accessible")
     wait_until_backend_is_live(max_time=60, n_tries=40)

tests/conftest.py

@@ -4,7 +4,7 @@
 import hypothesis
 import pytest
 
-from scitacean.testing.backend import add_pytest_option as add_backend_option
+from scitacean.testing.backend import add_pytest_options as add_backend_options
 from scitacean.testing.sftp import add_pytest_option as add_sftp_option
 
 pytest_plugins = (
@@ -26,5 +26,5 @@
 
 
 def pytest_addoption(parser: pytest.Parser) -> None:
-    add_backend_option(parser)
+    add_backend_options(parser)
     add_sftp_option(parser)

tox.ini

@@ -5,8 +5,8 @@ isolated_build = true
 [testenv]
 deps = -r requirements/test.txt
 commands =
-    full: python -m pytest --backend-tests --sftp-tests
-    !full: python -m pytest
+    full: python -m pytest --backend-tests --sftp-tests {posargs}
+    !full: python -m pytest {posargs}
 
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs