Merge pull request #448 from DigitalSlideArchive/selection-algorithm

Add an example slide selection algorithm.

Author: manthey

.pre-commit-config.yaml

@@ -43,7 +43,7 @@ repos:
     files: README.rst
     name: rst-linter of README.rst
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.3.0
+  rev: v2.4.1
   hooks:
   - id: codespell
     args:
@@ -63,13 +63,13 @@ repos:
 #         - './histomicsui/web_client/package.json'
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.8.3
+  rev: v0.12.0
   hooks:
     - id: ruff
       args: [--fix, --exit-non-zero-on-fix]
       types_or: [python, pyi, jupyter]
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.19.0
+  rev: v3.20.0
   hooks:
     - id: pyupgrade
       args:
@@ -80,6 +80,6 @@ repos:
   hooks:
     - id: autopep8
 - repo: https://github.com/PyCQA/flake8
-  rev: 7.1.1
+  rev: 7.2.0
   hooks:
     - id: flake8

README.rst

@@ -29,5 +29,6 @@ Navigating the Documentation
 * `docs/GLOSSARY.rst <docs/GLOSSARY.rst>`_ contains a glossary of technical terms.
 * `docs/ERROR-TABLES.rst <docs/ERROR-TABLES.rst>`_ contains tables of explanations for error messages that users may encounter.
 * `docs/CUSTOMIZING.rst <docs/CUSTOMIZING.rst>`_ contains details on customizing the workflow for different use cases.
+* `docs/TASKS.rst <docs/TASKS.rst>`_ contains an alternative deployment to allow running external tasks, such as a slide selection algorithm.
 * `docs/APIUSAGE.rst <docs/APIUSAGE.rst>`_ describes how to make API called for perform common workflow actions.
 * `docs/rationale.md <docs/rationale.md>`_ contains a rationale of the SEER WSI DeID pilot project.

devops/wsi_deid/docker-compose.with-tasks.yml

@@ -0,0 +1,98 @@
+---
+services:
+  girder:
+    image: dsarchive/wsi_deid:latest
+    restart: unless-stopped
+    # To have full capabilities with S3 assetstores, we use a user file system
+    # (fuse).  This requires some privileges.  This is not needed if only
+    # filesystem assetstores are used.  Instead of privileged mode, fuse can
+    # use specific devices, security_opt, and cap_add:
+    # devices:
+    #   - /dev/fuse:/dev/fuse
+    # security_opt:
+    #   - apparmor:unconfined
+    # cap_add:
+    #   - SYS_ADMIN
+    # but these may be somewhat host specific, so we default to privileged.  If
+    # the docker daemon is being run with --no-new-privileges, fuse may not
+    # work.
+    # See also https://github.com/docker/for-linux/issues/321 for possible
+    # methods to avoid both privileged mode and cap_add SYS_ADMIN.
+    privileged: true
+    # Set DSA_PORT to expose the interface on another port (default 8080).
+    ports:
+      - "${DSA_PORT:-8080}:8080"
+    # Set DSA_USER to a user id that is part of the docker group (e.g.,
+    # `DSA_USER=$(id -u):$(id -g)`).  This makes files in assetstores and logs
+    # owned by that user and provides permissions to manage docker
+    environment:
+      DSA_USER: ${DSA_USER:-}
+      GIRDER_CONFIG: /conf/girder.local.conf
+      PROVISION: tasks
+    volumes:
+      # Needed to use slicer_cli_web to run docker containers
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Default assetstore
+      - fsdata:/assetstore
+      - logs:/logs
+      # Change for local files:
+      # - ./assetstore:/assetstore
+      # - ./logs:/logs
+      # Location of girder.local.conf and provision.py; add to use local
+      # versions
+      # - .:/conf
+      # Add for import and export location.  This can also be done with a
+      # docker-compose.local.yml file.  See the example.
+      # - <some local path>:/import
+      # - <some local path>:/export
+      - ./worker.local.cfg:/venv/lib/python3.11/site-packages/girder_worker/worker.local.cfg
+    depends_on:
+      - mongodb
+      - memcached
+      - rabbitmq
+  mongodb:
+    image: "mongo:latest"
+    restart: unless-stopped
+    volumes:
+      # Location to store database files
+      # Change for local files:
+      # - ./db:/data/db
+      # - ./logs:/var/log/mongodb
+      - dbdata:/data/db
+  memcached:
+    image: memcached
+    command: -m 4096
+    restart: unless-stopped
+  # rabbitmq is used to communicate to the worker to run tasks
+  rabbitmq:
+    image: "rabbitmq:latest"
+    restart: unless-stopped
+    environment:
+      RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER:-}
+      RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS:-}
+    volumes:
+      - ./rabbitmq.advanced.config:/etc/rabbitmq/advanced.config:ro
+  worker:
+    image: dsarchive/wsi_deid:latest
+    # Set DSA_USER to a user id that is part of the docker group (e.g.,
+    # `DSA_USER=$(id -u):$(id -g)`).  This provides permissions to manage
+    # docker
+    environment:
+      DSA_USER: ${DSA_USER:-}
+      PROVISION: worker
+      C_FORCE_ROOT: true
+    restart: unless-stopped
+    volumes:
+      # Needed to use slicer_cli_web to run docker containers
+      - /var/run/docker.sock:/var/run/docker.sock
+      # Needed to allow transferring data to slicer_cli_web docker containers
+      - ${TMPDIR:-/tmp}:${TMPDIR:-/tmp}
+      - ./worker.local.cfg:/venv/lib/python3.11/site-packages/girder_worker/worker.local.cfg
+    depends_on:
+      - rabbitmq
+    command: bash -c "C_FORCE_ROOT=true PROVISION=worker python /conf/provision.py && python -m girder_worker --concurrency=1 -Ofair --prefetch-multiplier=1"
+
+volumes:
+  dbdata:
+  fsdata:
+  logs:

devops/wsi_deid/provision.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import girder.utility.config
 from girder.models import getDbConnection
@@ -91,6 +92,22 @@ def provision():  # noqa
             folder['name'] = folderName
             folder['public'] = public
             Folder().save(folder)
+    if os.environ.get('PROVISION') == 'tasks':
+        taskCollName = 'Tasks'
+        if Collection().findOne({'lowerName': taskCollName.lower()}) is None:
+            Collection().createCollection(taskCollName, adminUser)
+        taskCollection = Collection().findOne({'lowerName': taskCollName.lower()})
+        taskFolder = Folder().createFolder(
+            taskCollection, 'Slicer CLI Web Tasks', parentType='collection',
+            public=True, creator=adminUser, reuseExisting=True)
+        Setting().set('slicer_cli_web.task_folder', str(taskFolder['_id']))
+        Setting().set('worker.broker', 'amqp://guest:guest@rabbitmq')
+        Setting().set('worker.backend', 'rpc://guest:guest@rabbitmq')
+        Setting().set('worker.api_url', 'http://girder:8080/api/v1')
+        try:
+            os.chmod('/var/run/docker.sock', 0o777)
+        except Exception:
+            pass
     # Set default import/export paths
     if not Setting().get(PluginSettings.WSI_DEID_IMPORT_PATH):
         Setting().set(PluginSettings.WSI_DEID_IMPORT_PATH, '/import')
@@ -127,6 +144,12 @@ def provision():  # noqa
 
 
 if __name__ == '__main__':
+    if os.environ.get('PROVISION') == 'worker':
+        try:
+            os.chmod('/var/run/docker.sock', 0o777)
+        except Exception:
+            pass
+        sys.exit(0)
     # This loads plugins, allowing setting validation
     configureServer()
     provision()

devops/wsi_deid/rabbitmq.advanced.config

@@ -0,0 +1 @@
+[ {rabbit, [ {consumer_timeout, undefined} ]} ].

devops/wsi_deid/worker.local.cfg

@@ -0,0 +1,10 @@
+[celery]
+app_main=girder_worker
+broker=amqp://guest:guest@rabbitmq/
+backend=rpc://guest:guest@rabbitmq/
+
+[girder_worker]
+
+[logging]
+level = debug
+format = [%%(asctime)s] %%(levelname)s: %%(message)s

docs/TASKS.rst

@@ -0,0 +1,48 @@
+Installation for Running Tasks
+==============================
+
+See `INSTALL.rst <./INSTALL.rst>`_ for basic installation.  This has the same
+prerequisites, but adds running custom tasks, like a third-party slide 
+selection algorithm.
+
+.. contents:: Table of Contents
+    :depth: 1
+    :local:
+    :backlinks: none
+
+Differences in Start
+--------------------
+
+Compared to starting without task support, we need to use a different docker-compose.yml file, and, for non-Windows systems, we need to specify the user that is running the process which grants permission to run other docker containers.
+
+Pulling images, stopping, and updating are the same.
+
+The start command changes to::
+
+    docker-compose -f docker-compose.with-tasks.yml -f docker-compose.local.yml up -d
+
+As before, the system will be available from a web browser on http://localhost:8080.
+
+Adding a Task
+-------------
+
+Tasks can be added by going to ``Collections`` -> ``Tasks`` -> ``Slicer CLI Web Tasks`` and clicking the upload CLI button.
+
+.. image:: screenshots/upload_cli_button.png
+   :alt: The upload CLI button
+
+We need an example task to demonstrate.  This can be built from the command line by going to ``utilities/sample-algorithm`` directory, and running ``docker build --force-rm -t dsarchive/example_ssa .``.  At the end of this process, there will be a docker image named ``dsarchive/example_ssa:latest``.
+
+Type ``dsarchive/example_ssa:latest`` in the Upload CLI Docker Images dialog and click ``Import Image``.  After a short amount of time, the new task will be available. 
+
+Running a Task
+--------------
+
+Navigate to the task, e.g., ``Collections`` -> ``Tasks`` -> ``Slicer CLI Web Tasks`` -> ``dsarchive/example_ssa`` -> ``latest`` -> ``Slide Selection``.  Select the appropriate parameters as you desire and click ``Run Task``.
+
+.. image:: screenshots/sample_task.png
+   :alt: A sample task
+
+The ``Jobs`` link will show the progress of the task.
+
+The sample slide selection algorithm will create a collection called ``Selected Slides`` that defaults to having a folder with the date that the task was run containing the selected slides.

docs/screenshots/sample_task.png

@@ -44,8 +44,9 @@ def prerelease_local_scheme(version):
         'easyocr',
         'girder>=3.2.8',
         'girder-homepage',
+        'girder-worker[girder]',
         'histomicsui',
-        'large-image[tiff,ometiff,openslide,memcached,converter]>=1.32.7',
+        'large-image[tiff,ometiff,openslide,memcached,converter]>=1.32.10',
         'large-image-source-tiff[all]',
         'lxml',
         'openpyxl',