From cfcf82d51df44d5b3f1e753ccc2fec571b59cd9c Mon Sep 17 00:00:00 2001 From: Zach Mullen Date: Fri, 14 Apr 2023 12:52:52 -0400 Subject: [PATCH 1/7] Client script for uploading tasks This changes the ingestion process for tasks. Rather than parsing the CLI specs on the server, the client can use a script to parse them locally and send up the output to the server. --- README.rst | 22 +++++++++-- setup.py | 8 +++- slicer_cli_web/girder_plugin.py | 2 - slicer_cli_web/upload_slicer_cli_task.py | 48 ++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 7 deletions(-) create mode 100755 slicer_cli_web/upload_slicer_cli_task.py diff --git a/README.rst b/README.rst index 7452313..93b8851 100644 --- a/README.rst +++ b/README.rst @@ -33,7 +33,7 @@ To use Girder Worker: .. code-block:: bash - pip install girder-slicer-cli-web[worker] + pip install 'girder-slicer-cli-web[worker]' GW_DIRECT_PATHS=true girder_worker -l info -Ofair --prefetch-multiplier=1 The first time you start Girder, you'll need to configure it with at least one user and one assetstore (see the Girder_ documentation). Additionally, it is recommended that you install some dockerized tasks, such as the HistomicsTK_ algorithms. This can be done going to the Admin Console, Plugins, Slicer CLI Web settings. Set a default task upload folder, then import the `dsarchive/histomicstk:latest` docker image. @@ -44,9 +44,24 @@ Girder Plugin Importing Docker Images ======================= -When installed in Girder, an admin user can go to the Admin Console -> Plugins -> Slicer CLI Web to add Docker images. Select a Docker image and an existing folder and then select Import Image. Slicer CLI Web will pull the Docker image if it is not available on the Girder machine. +Once a docker image has been created and pushed to Docker Hub, you can register the image's CLI as a set of tasks on the server. To do so, +use the client upload script bundled with this tool. To install it, run: + +.. code-block:: bash + + pip install 'girder-slicer-cli-web[client]' + +Create an API key with the "Manage Slicer CLI tasks" scope, and set it in your environment and run a command like this example: + +.. code-block:: bash + + GIRDER_API_KEY=my_key_vale upload-slicer-cli-task https://my-girder-host.com/api/v1 641b8578cdcf8f129805524b my-slicer-cli-image:latest + +The first argument of this command is the API URL of the server, the second is a Girder folder ID where the tasks will live, and the +last argument is the docker image identifier. (If the image does not exist locally it will be pulled.) If you just want to create a +single CLI task rather than all tasks from ``--list_cli``, you can pass ``--cli=CliName``. If you wish to replace the existing tasks +with the latest specifications, also pass the ``--replace`` flag to the command. -For each docker image that is imported, a folder is created with the image tag. Within this folder, a subfolder is created with the image version. The subfolder will have one item per CLI that the Docker image reports. These items can be moved after they have been imported, just like standard Girder items. Running CLIs ============ @@ -195,4 +210,3 @@ If the local (server) environment has any environment variables that begin with .. _Girder: http://girder.readthedocs.io/en/latest/ .. _Girder Worker: https://girder-worker.readthedocs.io/en/latest/ .. _HistomicsTK: https://github.com/DigitalSlideArchive/HistomicsTK - diff --git a/setup.py b/setup.py index 72e8cf0..a7a2600 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,6 @@ def prerelease_local_scheme(version): ], extras_require={ 'girder': [ - 'docker>=2.6.0', 'girder>=3.0.4', 'girder-jobs>=3.0.3', 'girder-worker[girder]>=0.6.0', @@ -74,6 +73,10 @@ def prerelease_local_scheme(version): 'worker': [ 'docker>=2.6.0', 'girder-worker[worker]>=0.6.0', + ], + 'client': [ + 'click', + 'girder-client', ] }, entry_points={ @@ -82,6 +85,9 @@ def prerelease_local_scheme(version): ], 'girder_worker_plugins': [ 'slicer_cli_web = slicer_cli_web.girder_worker_plugin:SlicerCLIWebWorkerPlugin' + ], + 'console_scripts': [ + 'upload-slicer-cli-task = slicer_cli_web.upload_slicer_cli_task:upload_slicer_cli_task' ] }, python_requires='>=3.6', diff --git a/slicer_cli_web/girder_plugin.py b/slicer_cli_web/girder_plugin.py index 1e0458a..924d536 100644 --- a/slicer_cli_web/girder_plugin.py +++ b/slicer_cli_web/girder_plugin.py @@ -23,7 +23,6 @@ from girder_jobs.constants import JobStatus from girder_jobs.models.job import Job -from . import worker_tools from .docker_resource import DockerResource from .models import DockerImageItem @@ -79,4 +78,3 @@ def load(self, info): pass if count: logger.info('Marking %d old job(s) as cancelled' % count) - worker_tools.start() diff --git a/slicer_cli_web/upload_slicer_cli_task.py b/slicer_cli_web/upload_slicer_cli_task.py new file mode 100755 index 0000000..0f8ed3e --- /dev/null +++ b/slicer_cli_web/upload_slicer_cli_task.py @@ -0,0 +1,48 @@ +import base64 +import json +import os +import subprocess +from typing import Optional + +import click +from girder_client import GirderClient + + +def upload_cli(gc: GirderClient, image_name: str, replace: bool, cli_name: str, folder_id: str): + output = subprocess.check_output(['docker', 'run', image_name, cli_name, '--xml']) + gc.post(f'slicer_cli_web/task/{folder_id}', data={ + 'cli': base64.b64encode(output), + 'image': image_name, + 'name': cli_name, + 'replace': str(replace), + }) + + +@click.command() +@click.argument('api_url') +@click.argument('folder_id') +@click.argument('image_name') +@click.option('--cli', description='Push a single CLI with the given name', default=None) +@click.option('--replace', is_flag=True, description='Replace existing item if it exists', default=False) +def upload_slicer_cli_task(api_url: str, folder_id: str, image_name: str, cli: Optional[str], replace: bool): + if 'GIRDER_API_KEY' not in os.environ: + raise Exception('Please set GIRDER_API_KEY in your environment.') + + gc = GirderClient(apiUrl=api_url) + gc.authenticate(apiKey=os.environ['GIRDER_API_KEY']) + + output = subprocess.check_output(['docker', 'run', image_name, '--list_cli']) + cli_list_json: dict = json.loads(output) + + # The keys are the names of each CLI in the image + if cli: # upload one + if cli not in cli_list_json: + raise ValueError('Invalid CLI name, not found in image CLI list.') + upload_cli(gc, image_name, replace, cli, folder_id) + else: # upload all + for cli_name in cli_list_json: + upload_cli(gc, image_name, replace, cli_name, folder_id) + + +if __name__ == '__main__': + upload_slicer_cli_task() From 6c29a69d9cd9acae9ef06a21fb674d650d15f979 Mon Sep 17 00:00:00 2001 From: Zach Mullen Date: Mon, 17 Apr 2023 14:06:11 -0400 Subject: [PATCH 2/7] Server-side handler for Slicer CLI task creation --- slicer_cli_web/__init__.py | 3 +- slicer_cli_web/docker_resource.py | 43 ++++++++++++++++++++++-- slicer_cli_web/girder_plugin.py | 7 +++- slicer_cli_web/upload_slicer_cli_task.py | 5 +-- 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/slicer_cli_web/__init__.py b/slicer_cli_web/__init__.py index b95f648..566e42d 100644 --- a/slicer_cli_web/__init__.py +++ b/slicer_cli_web/__init__.py @@ -37,5 +37,6 @@ # package is not installed pass - __license__ = 'Apache 2.0' + +TOKEN_SCOPE_MANAGE_TASKS = 'slicer_cli_web.manage_tasks' diff --git a/slicer_cli_web/docker_resource.py b/slicer_cli_web/docker_resource.py index 18dd6fc..83e1446 100644 --- a/slicer_cli_web/docker_resource.py +++ b/slicer_cli_web/docker_resource.py @@ -15,16 +15,18 @@ ############################################################################# +from base64 import b64decode import json import os import re from girder.api import access from girder.api.describe import Description, autoDescribeRoute, describeRoute -from girder.api.rest import setRawResponse, setResponseHeader +from girder.api.rest import setRawResponse, setResponseHeader, filtermodel from girder.api.v1.resource import Resource, RestException from girder.constants import AccessType, SortDir from girder.exceptions import AccessException +from girder.models.folder import Folder from girder.models.item import Item from girder.utility import path as path_util from girder.utility.model_importer import ModelImporter @@ -32,8 +34,9 @@ from girder_jobs.constants import JobStatus from girder_jobs.models.job import Job +from . import TOKEN_SCOPE_MANAGE_TASKS from .config import PluginSettings -from .models import CLIItem, DockerImageItem, DockerImageNotFoundError +from .models import CLIItem, DockerImageItem, DockerImageNotFoundError, parser from .rest_slicer_cli import genRESTEndPointsForSlicerCLIsForItem @@ -51,6 +54,7 @@ def __init__(self, name): self.resourceName = name self.jobType = 'slicer_cli_web_job' self.route('PUT', ('docker_image',), self.setImages) + self.route('PUT', ('cli',), self.createOrReplaceCli) self.route('DELETE', ('docker_image',), self.deleteImage) self.route('GET', ('docker_image',), self.getDockerImages) @@ -62,6 +66,7 @@ def __init__(self, name): self.route('GET', ('path_match', ), self.getMatchingResource) + self._generateAllItemEndPoints() @access.public @@ -183,7 +188,7 @@ def parseImageNameList(self, param): raise RestException('Image %s does not have a tag or digest' % img) return nameList - @access.admin + @access.admin(scope=TOKEN_SCOPE_MANAGE_TASKS) @describeRoute( Description('Add one or a list of images') .notes('Must be a system administrator to call this.') @@ -210,6 +215,38 @@ def setImages(self, params): raise RestException('no upload folder given or defined by default') return self._createPutImageJob(nameList, folder, params.get('pull', None)) + @access.admin(scope=TOKEN_SCOPE_MANAGE_TASKS) + @filtermodel(Item) + @autoDescribeRoute( + Description('Add or replace an item task.') + .notes('Must be a system administrator to call this.') + .modelParam('folder', 'The folder ID to upload the task to.', paramType='formData', + level=AccessType.WRITE) + .param('image', 'The docker image identifier.') + .param('name', 'The name of the item to create or replace.') + .param('replace', 'Whether to replace an existing item with this name.', dataType='boolean') + .param('spec', 'Base64-encoded XML spec of the CLI.') + .errorResponse('You are not a system administrator.', 403) + ) + def createOrReplaceCli(self, folder: dict, image: str, name: str, replace: bool, spec: str): + try: + spec = b64decode(spec) + except ValueError: + raise RestException('The CLI spec must be base64-encoded.') + + metadata = dict( + slicerCLIType='task', + type='Unknown', # TODO does "type" matter behaviorally? If so get it from the client + digest=None, # TODO should we support this? + **parser._parse_xml_desc(spec) + ) + + item = Item().createItem( + name, creator=self.getCurrentUser(), folder=folder, + description='Slicer CLI generated CLI command item', reuseExisting=replace + ) + return Item().setMetadata(item, metadata) + def _createPutImageJob(self, nameList, baseFolder, pull=False): job = Job().createLocalJob( module='slicer_cli_web.image_job', diff --git a/slicer_cli_web/girder_plugin.py b/slicer_cli_web/girder_plugin.py index 924d536..3bd4727 100644 --- a/slicer_cli_web/girder_plugin.py +++ b/slicer_cli_web/girder_plugin.py @@ -18,11 +18,12 @@ import json from girder import events, logger -from girder.constants import AccessType +from girder.constants import AccessType, TokenScope from girder.plugin import GirderPlugin, getPlugin from girder_jobs.constants import JobStatus from girder_jobs.models.job import Job +from . import TOKEN_SCOPE_MANAGE_TASKS from .docker_resource import DockerResource from .models import DockerImageItem @@ -54,6 +55,10 @@ def load(self, info): except Exception: logger.info('Girder working is unavailable') + TokenScope.describeScope( + TOKEN_SCOPE_MANAGE_TASKS, name='Manage Slicer CLI tasks', + description='Create / edit Slicer CLI docker tasks', admin=True) + DockerImageItem.prepare() # resource name must match the attribute added to info[apiroot] diff --git a/slicer_cli_web/upload_slicer_cli_task.py b/slicer_cli_web/upload_slicer_cli_task.py index 0f8ed3e..e6d9620 100755 --- a/slicer_cli_web/upload_slicer_cli_task.py +++ b/slicer_cli_web/upload_slicer_cli_task.py @@ -10,11 +10,12 @@ def upload_cli(gc: GirderClient, image_name: str, replace: bool, cli_name: str, folder_id: str): output = subprocess.check_output(['docker', 'run', image_name, cli_name, '--xml']) - gc.post(f'slicer_cli_web/task/{folder_id}', data={ - 'cli': base64.b64encode(output), + gc.post(f'slicer_cli_web/cli', data={ + 'folder': folder_id, 'image': image_name, 'name': cli_name, 'replace': str(replace), + 'spec': base64.b64encode(output), }) From b191428b58513f774d858ab31a574520d59f46f9 Mon Sep 17 00:00:00 2001 From: Zach Mullen Date: Tue, 18 Apr 2023 14:45:21 -0400 Subject: [PATCH 3/7] Fix HTTP method consistency, and use correct kwarg in click --- slicer_cli_web/docker_resource.py | 2 +- slicer_cli_web/upload_slicer_cli_task.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/slicer_cli_web/docker_resource.py b/slicer_cli_web/docker_resource.py index 83e1446..1a70c26 100644 --- a/slicer_cli_web/docker_resource.py +++ b/slicer_cli_web/docker_resource.py @@ -54,7 +54,7 @@ def __init__(self, name): self.resourceName = name self.jobType = 'slicer_cli_web_job' self.route('PUT', ('docker_image',), self.setImages) - self.route('PUT', ('cli',), self.createOrReplaceCli) + self.route('POST', ('cli',), self.createOrReplaceCli) self.route('DELETE', ('docker_image',), self.deleteImage) self.route('GET', ('docker_image',), self.getDockerImages) diff --git a/slicer_cli_web/upload_slicer_cli_task.py b/slicer_cli_web/upload_slicer_cli_task.py index e6d9620..dac3af6 100755 --- a/slicer_cli_web/upload_slicer_cli_task.py +++ b/slicer_cli_web/upload_slicer_cli_task.py @@ -23,8 +23,8 @@ def upload_cli(gc: GirderClient, image_name: str, replace: bool, cli_name: str, @click.argument('api_url') @click.argument('folder_id') @click.argument('image_name') -@click.option('--cli', description='Push a single CLI with the given name', default=None) -@click.option('--replace', is_flag=True, description='Replace existing item if it exists', default=False) +@click.option('--cli', help='Push a single CLI with the given name', default=None) +@click.option('--replace', is_flag=True, help='Replace existing item if it exists', default=False) def upload_slicer_cli_task(api_url: str, folder_id: str, image_name: str, cli: Optional[str], replace: bool): if 'GIRDER_API_KEY' not in os.environ: raise Exception('Please set GIRDER_API_KEY in your environment.') From 568525ce98e3ae491871d7b80d1f163fdf561f69 Mon Sep 17 00:00:00 2001 From: Zach Mullen Date: Tue, 18 Apr 2023 15:40:28 -0400 Subject: [PATCH 4/7] Must specify model type in modelParam --- slicer_cli_web/docker_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slicer_cli_web/docker_resource.py b/slicer_cli_web/docker_resource.py index 1a70c26..6371b92 100644 --- a/slicer_cli_web/docker_resource.py +++ b/slicer_cli_web/docker_resource.py @@ -221,7 +221,7 @@ def setImages(self, params): Description('Add or replace an item task.') .notes('Must be a system administrator to call this.') .modelParam('folder', 'The folder ID to upload the task to.', paramType='formData', - level=AccessType.WRITE) + model=Folder, level=AccessType.WRITE) .param('image', 'The docker image identifier.') .param('name', 'The name of the item to create or replace.') .param('replace', 'Whether to replace an existing item with this name.', dataType='boolean') From 68bd462a220590a2ad6ff4da6d71df0e3e7804d1 Mon Sep 17 00:00:00 2001 From: Zach Mullen Date: Tue, 18 Apr 2023 15:59:09 -0400 Subject: [PATCH 5/7] Call parse XML function correctly --- slicer_cli_web/docker_resource.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/slicer_cli_web/docker_resource.py b/slicer_cli_web/docker_resource.py index 6371b92..06622ee 100644 --- a/slicer_cli_web/docker_resource.py +++ b/slicer_cli_web/docker_resource.py @@ -234,16 +234,14 @@ def createOrReplaceCli(self, folder: dict, image: str, name: str, replace: bool, except ValueError: raise RestException('The CLI spec must be base64-encoded.') + item = Item().createItem( + name, creator=self.getCurrentUser(), folder=folder, reuseExisting=replace + ) metadata = dict( slicerCLIType='task', type='Unknown', # TODO does "type" matter behaviorally? If so get it from the client digest=None, # TODO should we support this? - **parser._parse_xml_desc(spec) - ) - - item = Item().createItem( - name, creator=self.getCurrentUser(), folder=folder, - description='Slicer CLI generated CLI command item', reuseExisting=replace + **parser._parse_xml_desc(item, self.getCurrentUser(), spec) ) return Item().setMetadata(item, metadata) From cf20e1beed3c81a06986c371a65abdfcdd3406b2 Mon Sep 17 00:00:00 2001 From: Zach Mullen Date: Wed, 19 Apr 2023 09:27:38 -0400 Subject: [PATCH 6/7] Decode spec string --- slicer_cli_web/docker_resource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/slicer_cli_web/docker_resource.py b/slicer_cli_web/docker_resource.py index 06622ee..702d161 100644 --- a/slicer_cli_web/docker_resource.py +++ b/slicer_cli_web/docker_resource.py @@ -230,9 +230,9 @@ def setImages(self, params): ) def createOrReplaceCli(self, folder: dict, image: str, name: str, replace: bool, spec: str): try: - spec = b64decode(spec) + spec = b64decode(spec).decode() except ValueError: - raise RestException('The CLI spec must be base64-encoded.') + raise RestException('The CLI spec must be base64-encoded UTF-8.') item = Item().createItem( name, creator=self.getCurrentUser(), folder=folder, reuseExisting=replace From 3eeb4faf2eea19ce54fcee86c27bccef3f68718e Mon Sep 17 00:00:00 2001 From: Zach Mullen Date: Thu, 20 Apr 2023 11:15:46 -0400 Subject: [PATCH 7/7] Add image into task item metadata --- slicer_cli_web/docker_resource.py | 1 + 1 file changed, 1 insertion(+) diff --git a/slicer_cli_web/docker_resource.py b/slicer_cli_web/docker_resource.py index 702d161..f1c46c6 100644 --- a/slicer_cli_web/docker_resource.py +++ b/slicer_cli_web/docker_resource.py @@ -241,6 +241,7 @@ def createOrReplaceCli(self, folder: dict, image: str, name: str, replace: bool, slicerCLIType='task', type='Unknown', # TODO does "type" matter behaviorally? If so get it from the client digest=None, # TODO should we support this? + image=image, **parser._parse_xml_desc(item, self.getCurrentUser(), spec) ) return Item().setMetadata(item, metadata)