Fix bug in console_scripts file

Add validate_model_submission method for validating repository submissions
Add description and contact_email fields to the model schema

Author: langmm

HISTORY.rst

@@ -17,13 +17,14 @@ History
 * Added methods for waiting on a function to return True
 * Refactored multitasking classes to use __slots__ for improved memory performance
 * Added tools for building yggdrasil docker containers and documentation on those tools
-* Added repository_url parameter to model YAML schema that allows for a repository to be specified in the YAML (in addition to via the command line as before)
+* Added repository_url, description, and contact_email parameters to model YAML schema that allows for a repository to be specified in the YAML (in addition to via the command line as before)
 * Added model_only and model_submission options to yggdrasil.yamfile.parse_yaml method and YAML validation CLI
 * Added generated documentation on command line utilities
 * Added a glossary of terms
 * Included downloadable versions of the schemas in the documentation
 * Moved console_scripts list into a text file
 * Note deprecation of the old GUI in the docs for the 2018 & 2019 hackathons
+* Added validate_model_submission method for validating model repository submissions
 
 1.7.0 (2021-08-26) Support for MPI communicators, MPI execution, and pika >= 1.0.0
 ------------------

MANIFEST.in

@@ -4,6 +4,7 @@ include *.svg
 include *.txt
 recursive-include yggdrasil *.cfg
 recursive-include yggdrasil *.yml
+recursive-include yggdrasil *.yaml
 recursive-include yggdrasil *.sh
 recursive-include yggdrasil *.m
 recursive-include yggdrasil *.R

console_scripts.txt

@@ -23,4 +23,4 @@ ygginstall=yggdrasil.command_line:ygginstall
 yggclean=yggdrasil.command_line:yggclean
 yggmodelform=yggdrasil.command_line:yggmodelform
 yggdevup=yggdrasil.command_line:yggdevup
-ygggha=yggdrasil.command_line:generate_gha_workflow'
+ygggha=yggdrasil.command_line:generate_gha_workflow

yggdrasil/.ygg_schema.yml

@@ -181,8 +181,8 @@ definitions:
           type: array
       required:
       - datatype
-      - commtype
       - name
+      - commtype
       title: comm_base
       type: object
     - anyOf:
@@ -370,8 +370,8 @@ definitions:
             - $ref: '#/definitions/transform'
           type: array
       required:
-      - outputs
       - inputs
+      - outputs
       title: connection_base
       type: object
     - anyOf:
@@ -705,8 +705,8 @@ definitions:
           type: string
       required:
       - filetype
-      - working_dir
       - name
+      - working_dir
       title: file_base
       type: object
     - anyOf:
@@ -1319,6 +1319,10 @@ definitions:
           description: Build type/configuration that should be built. Defaults to
             'Release'.
           type: string
+        contact_email:
+          description: Email address that should be used to contact the maintainer
+            of the model. This parameter is only used in the model repository.
+          type: string
         copies:
           default: 1
           description: The number of copies of the model that should be created. Defaults
@@ -1332,6 +1336,10 @@ definitions:
             use any of the files located there since OSR always assumes the included
             file paths are relative. Defaults to False.
           type: boolean
+        description:
+          description: Description of the model. This parameter is only used in the
+            model repository or when providing the model as a service.
+          type: string
         driver:
           description: '[DEPRECATED] Name of driver class that should be used.'
           type: string
@@ -1793,10 +1801,10 @@ definitions:
             is used.
           type: string
       required:
-      - language
       - args
-      - working_dir
       - name
+      - working_dir
+      - language
       title: model_base
       type: object
     - anyOf:

yggdrasil/drivers/ModelDriver.py

@@ -173,6 +173,12 @@ class ModelDriver(Driver):
             the model source code. If provided, relative paths in the model
             YAML definition will be considered relative to the repository root
             directory.
+        description (str, optional): Description of the model. This parameter
+            is only used in the model repository or when providing the model
+            as a service.
+        contact_email (str, optional): Email address that should be used to
+            contact the maintainer of the model. This parameter is only used
+            in the model repository.
         **kwargs: Additional keyword arguments are passed to parent class.
 
     Class Attributes:
@@ -281,6 +287,12 @@ class ModelDriver(Driver):
             source code. If provided, relative paths in the model YAML
             definition will be considered relative to the repository root
             directory.
+        description (str): Description of the model. This parameter is only
+            used in the model repository or when providing the model as a
+            service.
+        contact_email (str): Email address that should be used to contact the
+            maintainer of the model. This parameter is only used in the model
+            repository.
 
     Raises:
         RuntimeError: If both with_strace and with_valgrind are True.
@@ -380,7 +392,9 @@ class ModelDriver(Driver):
         'logging_level': {'type': 'string', 'default': ''},
         'allow_threading': {'type': 'boolean'},
         'copies': {'type': 'integer', 'default': 1, 'minimum': 1},
-        'repository_url': {'type': 'string'}}
+        'repository_url': {'type': 'string'},
+        'description': {'type': 'string'},
+        'contact_email': {'type': 'string'}}
     _schema_excluded_from_class = ['name', 'language', 'args', 'working_dir']
     _schema_excluded_from_class_validation = ['inputs', 'outputs']
 

yggdrasil/schema.py

@@ -886,6 +886,46 @@ def form_schema(self):
         out = convert_extended2base(out)
         return out
 
+    @property
+    def model_form_schema_props(self):
+        r"""dict: Information about how properties should be modified for the
+        model form schema."""
+        prop = {
+            # 'add': {},
+            'replace': {
+                'comm': {
+                    'transform': {
+                        "type": "array",
+                        "items": {"$ref": "#/definitions/transform"}}}},
+            'required': {
+                'model': ['args', 'inputs', 'outputs', 'description',
+                          'repository_url']},
+            'remove': {
+                'comm': ['is_default', 'length_map', 'serializer',
+                         'address', 'dont_copy', 'for_service',
+                         'send_converter', 'recv_converter', 'client_id',
+                         'cookies', 'host', 'params', 'port'],
+                'file': ['is_default', 'length_map',
+                         'wait_for_creation', 'working_dir',
+                         'read_meth', 'in_temp',
+                         'serializer', 'datatype',
+                         'address', 'dont_copy', 'for_service',
+                         'send_converter', 'recv_converter', 'client_id',
+                         'cookies', 'host', 'params', 'port'],
+                'model': ['client_of', 'is_server', 'preserve_cache',
+                          'products', 'source_products', 'working_dir',
+                          'overwrite', 'skip_interpreter', 'copies',
+                          'timesync', 'with_strace', 'with_valgrind',
+                          'valgrind_flags', 'additional_variables',
+                          'aggregation', 'interpolation', 'synonyms',
+                          'driver']},
+            'order': {
+                'model': ['name', 'repository_url', 'contact_email',
+                          'language', 'description', 'args', 'inputs',
+                          'outputs']},
+        }
+        return prop
+        
     @property
     def model_form_schema(self):
         r"""dict: Schema for generating a model YAML form."""
@@ -934,61 +974,30 @@ def model_form_schema(self):
                 out['definitions'][x]['properties'][k].pop('oneOf', None)
                 out['definitions'][x]['properties'][k].update(
                     {"$ref": "#/definitions/serializer"})
-        prop_add = {
-            'model': {'contact_email': {'type': 'string'}}}
-        prop_replace = {
-            'comm': {
-                'transform': {
-                    "type": "array",
-                    "items": {"$ref": "#/definitions/transform"}}}}
-        prop_required = {
-            'model': ['args', 'inputs', 'outputs', 'repository_url']}
-        prop_remove = {
-            'comm': ['is_default', 'length_map', 'serializer',
-                     'address', 'dont_copy', 'for_service',
-                     'send_converter', 'recv_converter', 'client_id',
-                     'cookies', 'host', 'params', 'port'],
-            'file': ['is_default', 'length_map',
-                     'wait_for_creation', 'working_dir',
-                     'read_meth', 'in_temp',
-                     'serializer', 'datatype',
-                     'address', 'dont_copy', 'for_service',
-                     'send_converter', 'recv_converter', 'client_id',
-                     'cookies', 'host', 'params', 'port'],
-            'model': ['client_of', 'is_server', 'preserve_cache',
-                      'products', 'source_products', 'working_dir',
-                      'overwrite', 'skip_interpreter', 'copies',
-                      'timesync', 'with_strace', 'with_valgrind',
-                      'valgrind_flags', 'additional_variables',
-                      'aggregation', 'interpolation', 'synonyms',
-                      'driver']}
-        prop_order = {
-            'model': ['name', 'repository_url', 'contact_email', 'language',
-                      'args', 'inputs', 'outputs']
-        }
+        prop = self.model_form_schema_props
         for k in ['inputs', 'outputs']:
             out['definitions']['model']['properties'][k].pop('default', None)
             desc = out['definitions']['model']['properties'][k]['description'].split(
                 ' A full description')[0]
             out['definitions']['model']['properties'][k]['description'] = desc
         out['definitions']['model']['properties']['args']['minItems'] = 1
-        for k, rlist in prop_remove.items():
+        for k, rlist in prop['remove'].items():
             for p in rlist:
                 out['definitions'][k]['properties'].pop(p, None)
-        for k, rdict in prop_replace.items():
+        for k, rdict in prop['replace'].items():
             for r in rdict.keys():
                 if 'description' in out['definitions'][k]['properties'][r]:
                     rdict[r]['description'] = (
                         out['definitions'][k]['properties'][r]['description'])
             out['definitions'][k]['properties'].update(rdict)
-        for k, rlist in prop_required.items():
+        for k, rlist in prop['required'].items():
             out['definitions'][k].setdefault('required', [])
             for p in rlist:
                 if p not in out['definitions'][k]['required']:
                     out['definitions'][k]['required'].append(p)
-        for k, adict in prop_add.items():
-            out['definitions'][k]['properties'].update(adict)
-        for k, rlist in prop_order.items():
+        # for k, adict in prop['add'].items():
+        #     out['definitions'][k]['properties'].update(adict)
+        for k, rlist in prop['order'].items():
             for i, p in enumerate(rlist):
                 out['definitions'][k]['properties'][p]['propertyOrder'] = i
         out.update(out['definitions'].pop('model'))

yggdrasil/services.py

@@ -1112,3 +1112,28 @@ def add(self, name, yamls=None, **kwargs):
                                  f"    Registry:\n{old}\n    New:\n{new}")
             registry[k] = v
         self.save(registry)
+
+
+def validate_model_submission(fname):
+    r"""Validate a YAML file according to the standards for submission to
+    the yggdrasil model repository.
+
+    Args:
+        fname (str): YAML file to validate.
+
+    """
+    import glob
+    from yggdrasil import yamlfile, runner
+    # 1-2. YAML syntax and schema
+    yml = yamlfile.parse_yaml(fname, model_submission=True)
+    # 3a. LICENSE
+    repo_dir = yml['models'][0]['working_dir']
+    patterns = ['LICENSE', 'LICENSE.*']
+    for x in patterns:
+        if ((glob.glob(os.path.join(repo_dir, x.upper()))
+             or glob.glob(os.path.join(repo_dir, x.lower())))):
+            break
+    else:
+        raise RuntimeError("Model repository does not contain a LICENSE file.")
+    # 4. Run
+    runner.run(fname)

yggdrasil/tests/test_services.py

@@ -8,7 +8,8 @@
 import logging
 from contextlib import contextmanager
 from yggdrasil.services import (
-    IntegrationServiceManager, create_service_manager_class, ServerError)
+    IntegrationServiceManager, create_service_manager_class, ServerError,
+    validate_model_submission)
 from yggdrasil.examples import yamls as ex_yamls
 from yggdrasil.tests import assert_raises, requires_language
 from yggdrasil import runner, import_as_function, platform
@@ -331,3 +332,18 @@ def test_calling_service_as_function(self, running_service):
             fmodel.stop()
         finally:
             cli.registry.remove(name)
+
+
+def test_validate_model_submission():
+    r"""Test validate_model_submission"""
+    import git
+    try:
+        fname = os.path.join(os.path.dirname(__file__), 'yamls',
+                             'FakePlant.yaml')
+        validate_model_submission(fname)
+        os.remove(os.path.join('cropsinsilico', 'example-fakemodel',
+                               'LICENSE'))
+        assert_raises(RuntimeError, validate_model_submission, fname)
+    finally:
+        if os.path.isfile('cropsinsilico/example-fakemodel/fakemodel.yml'):
+            git.rmtree("cropsinsilico")

yggdrasil/tests/yamls/FakePlant.yaml

@@ -0,0 +1,24 @@
+ model:
+  name: FakePlant
+  args:
+  - src/fakemodel.py
+  - --yggdrasil
+  language: python
+  inputs:
+  - name: photosynthesis_rate
+    commtype: default
+    datatype:
+      type: bytes
+    default_file:
+      name: ./Input/input.txt
+      filetype: table
+  outputs:
+  - name: growth_rate
+    commtype: default
+    datatype:
+      type: bytes
+    default_file:
+      name: ./Output/output.txt
+      filetype: table
+  repository_url: https://github.com/cropsinsilico/example-fakemodel
+  description: Example model submission

yggdrasil/yamlfile.py

@@ -295,7 +295,7 @@ def parse_yaml(files, complete_partial=False, partial_commtype=None,
             existing = parse_component(yml, k[:-1], existing=existing)
     # Exit early
     if model_only:
-        return existing
+        return yml_norm
     # Add stand-in model that uses unpaired channels
     if complete_partial:
         existing = complete_partial_integration(