Merge pull request #176 from datajoint/fix/plat-341-filepath-records

PLAT-341: debug _DJConnector._fetch_records for filepath attributes

Author: drewyangdev

.github/workflows/development.yaml

@@ -49,22 +49,22 @@ jobs:
         run: |
           export PHARUS_VERSION=$(cat pharus/version.py | grep -oP '\d+\.\d+\.\d+')
           export HOST_UID=$(id -u)
-          docker-compose -f docker-compose-build.yaml up --exit-code-from pharus --build
+          docker compose -f docker-compose-build.yaml up --exit-code-from pharus --build
           IMAGE=$(docker images --filter "reference=datajoint/pharus*" \
             --format "{{.Repository}}")
           TAG=$(docker images --filter "reference=datajoint/pharus*" --format "{{.Tag}}")
           docker save "${IMAGE}:${TAG}" | \
             gzip > "image-pharus-${PHARUS_VERSION}-py${PY_VER}-${DISTRO}.tar.gz"
           echo "PHARUS_VERSION=${PHARUS_VERSION}" >> $GITHUB_ENV
       - name: Add image artifact
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: image-pharus-${{env.PHARUS_VERSION}}-py${{matrix.py_ver}}-${{matrix.distro}}
           path: "image-pharus-${{env.PHARUS_VERSION}}-py${{matrix.py_ver}}-\
             ${{matrix.distro}}.tar.gz"
           retention-days: 1
       - name: Add pip artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: pip-pharus-${{env.PHARUS_VERSION}}-py${{matrix.py_ver}}
           path: dist
@@ -95,14 +95,14 @@ jobs:
           PHARUS_VERSION=$(cat pharus/version.py | grep -oP '\d+\.\d+\.\d+')
           echo "PHARUS_VERSION=${PHARUS_VERSION}" >> $GITHUB_ENV
       - name: Fetch image artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: image-pharus-${{env.PHARUS_VERSION}}-py${{matrix.py_ver}}-${{matrix.distro}}
       - name: Run primary tests
         run: |
           export HOST_UID=$(id -u)
           docker load < "image-pharus-${PHARUS_VERSION}-py${PY_VER}-${DISTRO}.tar.gz"
-          docker-compose -f docker-compose-test.yaml up --exit-code-from pharus
+          docker compose -f docker-compose-test.yaml up --exit-code-from pharus
   publish-release:
     if: |
       github.event_name == 'push' &&
@@ -143,24 +143,24 @@ jobs:
           prerelease: ${{steps.changelog_reader.outputs.status == 'prereleased'}}
           draft: ${{steps.changelog_reader.outputs.status == 'unreleased'}}
       - name: Fetch image artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: image-pharus-${{env.PHARUS_VERSION}}-py3.8-alpine
       - name: Fetch 3.8 pip artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: pip-pharus-${{env.PHARUS_VERSION}}-py3.8
           path: dist
       - name: Fetch 3.9 pip artifacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: pip-pharus-${{env.PHARUS_VERSION}}-py3.8
           path: dist
       - name: Publish pip release
         run: |
           export HOST_UID=$(id -u)
           docker load < "image-pharus-${PHARUS_VERSION}-py3.8-alpine.tar.gz"
-          docker-compose -f docker-compose-build.yaml run \
+          docker compose -f docker-compose-build.yaml run \
             -e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} pharus \
             sh -lc "pip install twine && python -m twine upload dist/*"
       - name: Determine pip artifact paths
@@ -243,7 +243,7 @@ jobs:
           PHARUS_VERSION=$(cat pharus/version.py | grep -oP '\d+\.\d+\.\d+')
           echo "PHARUS_VERSION=${PHARUS_VERSION}" >> $GITHUB_ENV
       - name: Fetch image artifact
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v4
         with:
           name: image-pharus-${{env.PHARUS_VERSION}}-py${{matrix.py_ver}}-${{matrix.distro}}
       - name: Login to DockerHub
@@ -287,5 +287,5 @@ jobs:
       )
     needs: publish-release
     uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main
-    permissions: 
+    permissions:
       contents: write

docker-compose-test.yaml

@@ -15,14 +15,16 @@ services:
       - MYSQL_ROOT_PASSWORD=simple
   pharus:
     <<: *net
+    stdin_open: true # docker run -i
+    tty: true        # docker run -t
     extends:
       file: ./docker-compose-build.yaml
       service: pharus
     environment:
       - FLASK_ENV=development # enables logging to console from Flask
-      - TEST_DB_SERVER=db
-      - TEST_DB_USER=root
-      - TEST_DB_PASS=simple
+      - TEST_DB_SERVER=${TEST_DB_SERVER:-db}
+      - TEST_DB_USER=${TEST_DB_USER:-root}
+      - TEST_DB_PASS=${TEST_DB_PASS:-simple}
       - AS_SCRIPT
       - PHARUS_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml
     volumes:
@@ -39,12 +41,12 @@ services:
           flake8 $${PKG_DIR} --count --select=E9,F63,F7,F82 --show-source --statistics
           flake8 /main/tests --count --select=E9,F63,F7,F82 --show-source --statistics
           echo "------ UNIT TESTS ------"
-          pytest -sv --cov-report term-missing --cov=pharus /main/tests
+          pytest -sv --cov-report term-missing --cov=pharus /main/tests/
           echo "------ STYLE TESTS ------"
-          black $${PKG_DIR} --check -v --extend-exclude "^.*dynamic_api.py$$"
+          black $${PKG_DIR} --required-version '24.8.0' --check -v --extend-exclude "^.*dynamic_api.py$$"
           flake8 $${PKG_DIR} --count --max-complexity=20 --max-line-length=94 --statistics --exclude=*dynamic_api.py --ignore=W503,W605
-          black /main/tests --check -v
-          flake8 /main/tests --count --max-complexity=20 --max-line-length=94 --statistics --ignore=F401,F811,W503
+          black /main/tests --required-version '24.8.0' --check -v
+          flake8 /main/tests --count --max-complexity=20 --max-line-length=94 --statistics --ignore=F401,F811,W503,F403
         else
           echo "=== Running ==="
           echo "Please see 'docker-compose-test.yaml' for detail on running tests."

pharus/interface.py

@@ -176,7 +176,15 @@ def _fetch_records(
             # Loop through each attributes, append to the tuple_to_return with specific
             #   modification based on data type
             for attribute_name, attribute_info in attributes.items():
-                if not attribute_info.is_blob:
+                if attribute_info.is_external:
+                    # Attribute is external type (filepath or attach),
+                    # thus fill it in string instead
+                    (
+                        row.append(non_blobs_row[attribute_name])
+                        if fetch_blobs
+                        else row.append("=FILE=")
+                    )
+                elif not attribute_info.is_blob:
                     if non_blobs_row[attribute_name] is None:
                         # If it is none then just append None
                         row.append(None)

tests/__init__.py

@@ -1,4 +1,5 @@
 import pytest
+from typing import Optional
 from pharus.dynamic_api import app
 from uuid import UUID
 from os import getenv
@@ -13,6 +14,25 @@
 SCHEMA_PREFIX = "test_"
 
 
+def get_db_creds():
+    return dict(
+        host=getenv("TEST_DB_SERVER"),
+        user=getenv("TEST_DB_USER"),
+        password=getenv("TEST_DB_PASS"),
+    )
+
+
+def get_schema_as_vm(
+    schema: str, connection: dj.Connection
+) -> Optional[dj.VirtualModule]:
+    try:
+        return dj.VirtualModule(schema, schema, connection=connection)
+    except dj.DataJointError as e:
+        if "has not yet been declared" in str(e):
+            return None
+        raise
+
+
 @pytest.fixture
 def client():
     """REST client interface."""
@@ -59,11 +79,7 @@ def group1_token(client, connection):
 def connection():
     """Root database connection."""
     dj.config["safemode"] = False
-    connection = dj.Connection(
-        host=getenv("TEST_DB_SERVER"),
-        user=getenv("TEST_DB_USER"),
-        password=getenv("TEST_DB_PASS"),
-    )
+    connection = dj.Connection(**get_db_creds())
     yield connection
     dj.config["safemode"] = True
     connection.close()

tests/conftest.py

@@ -0,0 +1 @@
+from . import *

tests/test_interface.py

@@ -0,0 +1,40 @@
+import os
+from typing import Optional, Generator, Any
+import pytest
+import datajoint as dj
+from pharus.interface import _DJConnector as DJC
+from . import get_schema_as_vm, get_db_creds
+
+
+@pytest.fixture
+def nei_nienborg_model_labeledvideo_file(
+    connection: dj.Connection,
+) -> Generator[Optional[dj.Table], Any, Any]:
+    with pytest.MonkeyPatch.context() as mp:
+        mp.setenv("DJ_SUPPORT_FILEPATH_MANAGEMENT", "TRUE")
+        vm = get_schema_as_vm("nei_nienborg_model", connection)
+        yield None if vm is None else vm.LabeledVideo.File
+
+
+class TestDJConnector:
+
+    def test_can_init(self):
+        djc = DJC()
+        assert djc is not None
+
+    @pytest.mark.skipif(
+        (get_schema_as_vm("nei_nienborg_model", dj.conn(**get_db_creds())) is None),
+        reason="Cannot access schema 'nei_nienborg_model' with these credentials",
+    )
+    def test_can_fetch_filepath_attrs(
+        self, nei_nienborg_model_labeledvideo_file: dj.Table, connection
+    ):
+        """
+        Tests _DJConnector._fetch_records for a table with a filepath attribute
+
+        https://datajoint.atlassian.net/browse/PLAT-341
+        """
+        table = nei_nienborg_model_labeledvideo_file
+        assert os.environ.get("DJ_SUPPORT_FILEPATH_MANAGEMENT").upper() == "TRUE"
+        assert table.fetch("KEY", limit=1, download_path=None)
+        assert table.fetch(limit=1)