Merge pull request #84 from cropsinsilico/topic/cloud

Topic/cloud

Author: langmm

.github/workflows/test-install.yml

@@ -987,7 +987,7 @@ jobs:
         test-flags1:
         - --long-running --languages python R matlab
         test-flags2:
-        - ''
+        - tests/test_services.py --nocapture
         install-c:
         - true
         install-mpi:
@@ -1644,3 +1644,31 @@ jobs:
         user: __token__
         password: ${{ secrets.pypi }}
         skip_existing: true
+  docker:
+    name: Build and Publish Docker images
+    needs:
+    - test_pip
+    - test_conda
+    - test_rmq_pip
+    - test_rmq_conda
+    - test_stripped
+    - docs
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Login to DockerHub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: Build and push environment docker images for the new release
+      run: python utils/build_docker.py --push
+    - name: Build and push executable docker images for the new release
+      run: python utils/build_docker.py --push executable

.gitignore

@@ -29,7 +29,10 @@ docs/source/modules.rst
 docs/source/examples/*.rst
 docs/source/tables
 docs/source/temp
+docs/source/schema
 docs/source/advanced/YggInterface_R.rst
+docs/source/cli.rst
+docs/source/cli.py
 
 # Dev setup
 requirements_pruned.txt

HISTORY.rst

@@ -2,6 +2,30 @@
 History
 =======
 
+1.8.0 (2021-09-15) Support for REST API based communicators, running integrations as services, and connecting to remote integration services
+------------------
+
+* Added option to return printStatus string for YggClass subclasses via return_str keyword
+* Added classes for managing models as services via Flask or RabbitMQ
+* Transitioned from as_function to complete_partial keyword in parse_yaml that can be used for services as well as functions
+* Added client side instrumentation for connecting to remote model services and tests
+* Added support for registering integrations that can be called locally as services
+* Added 'integration-service-manager' CLI for managing service managers
+* Added configuration options for services
+* Added communicator for use with REST API
+* Added ValueEvent class for returning a value with the event
+* Added methods for waiting on a function to return True
+* Refactored multitasking classes to use __slots__ for improved memory performance
+* Added tools for building yggdrasil docker containers and documentation on those tools
+* Added repository_url, description, and contact_email parameters to model YAML schema that allows for a repository to be specified in the YAML (in addition to via the command line as before)
+* Added model_only and model_submission options to yggdrasil.yamfile.parse_yaml method and YAML validation CLI
+* Added generated documentation on command line utilities
+* Added a glossary of terms
+* Included downloadable versions of the schemas in the documentation
+* Moved console_scripts list into a text file
+* Note deprecation of the old GUI in the docs for the 2018 & 2019 hackathons
+* Added validate_model_submission method for validating model repository submissions
+
 1.7.0 (2021-08-26) Support for MPI communicators, MPI execution, and pika >= 1.0.0
 ------------------
 

MANIFEST.in

@@ -4,6 +4,7 @@ include *.svg
 include *.txt
 recursive-include yggdrasil *.cfg
 recursive-include yggdrasil *.yml
+recursive-include yggdrasil *.yaml
 recursive-include yggdrasil *.sh
 recursive-include yggdrasil *.m
 recursive-include yggdrasil *.R
@@ -23,6 +24,7 @@ recursive-include yggdrasil/demos *.py
 recursive-include yggdrasil/tests/scripts *.py
 recursive-include yggdrasil/examples Makefile_linux
 recursive-include yggdrasil/examples Makefile_windows
+include yggdrasil/templates/*.html
 include yggdrasil/examples/gs_lesson4/src/Makefile_linux
 include yggdrasil/examples/gs_lesson4/src/Makefile_windows
 recursive-include yggdrasil/languages install.py

console_scripts.txt

@@ -0,0 +1,26 @@
+yggdrasil=yggdrasil.command_line:main
+ygginfo=yggdrasil.command_line:ygginfo
+yggrun=yggdrasil.command_line:yggrun
+cisrun=yggdrasil.command_line:yggrun
+yggcompile=yggdrasil.command_line:yggcompile
+yggcc=yggdrasil.command_line:yggcc
+yggcctool=yggdrasil.command_line:cc_toolname
+yggldtool=yggdrasil.command_line:ld_toolname
+yggccflags=yggdrasil.command_line:cc_flags
+yggldflags=yggdrasil.command_line:ld_flags
+yggtest=yggdrasil.command_line:run_tsts
+yggmetaschema=yggdrasil.command_line:regen_metaschema
+yggschema=yggdrasil.command_line:regen_schema
+yggbuildapi_c=yggdrasil.command_line:rebuild_c_api
+yggconfig=yggdrasil.command_line:update_config
+yggtime_comm=yggdrasil.command_line:yggtime_comm
+yggtime_lang=yggdrasil.command_line:yggtime_lang
+yggtime_os=yggdrasil.command_line:yggtime_os
+yggtime_py=yggdrasil.command_line:yggtime_py
+yggtime_paper=yggdrasil.command_line:yggtime_paper
+yggvalidate=yggdrasil.command_line:validate_yaml
+ygginstall=yggdrasil.command_line:ygginstall
+yggclean=yggdrasil.command_line:yggclean
+yggmodelform=yggdrasil.command_line:yggmodelform
+yggdevup=yggdrasil.command_line:yggdevup
+ygggha=yggdrasil.command_line:generate_gha_workflow

create_coveragerc.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import site
 PY_MAJOR_VERSION = sys.version_info[0]
 IS_WINDOWS = (sys.platform in ['win32', 'cygwin'])
 _on_gha = bool(os.environ.get('GITHUB_ACTIONS', False))
@@ -14,6 +15,7 @@
         from configparser import RawConfigParser as HandyConfigParser
     except ImportError:
         HandyConfigParser = None
+_package_dir = os.path.join(site.getsitepackages()[0], 'yggdrasil')
 
 
 def add_excl_rule(excl_list, new_rule):
@@ -142,6 +144,24 @@ def create_coveragerc(installed_languages):
             excl_list = rm_excl_rule(excl_list, 'pragma: no %s' % k)
     # Add new rules
     cp.set('report', 'exclude_lines', '\n' + '\n'.join(excl_list))
+    # Set include path so that filenames in the report are absolute
+    if os.path.isdir(_package_dir):
+        PACKAGE_DIR = _package_dir
+    else:
+        PACKAGE_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                                   'yggdrasil')
+    section = 'source'
+    if section == 'include':
+        section_path = os.path.join(PACKAGE_DIR, '*')
+    else:
+        section_path = PACKAGE_DIR
+    if not cp.has_section('run'):
+        cp.add_section('run')
+    incl_list = []
+    if cp.has_option('run', section):
+        incl_list = cp.get('run', section).strip().split('\n')
+    incl_list.append(section_path)
+    cp.set('run', section, '\n' + '\n'.join(incl_list))
     # Write
     with open(covrc, 'w') as fd:
         cp.write(fd)
@@ -151,8 +171,11 @@ def create_coveragerc(installed_languages):
 def run():
     r"""Run coverage creation function after getting a list of installed
     languages."""
-    LANG_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-                             'yggdrasil', 'languages')
+    if os.path.isdir(_package_dir):
+        LANG_PATH = os.path.join(_package_dir, 'languages')
+    else:
+        LANG_PATH = os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                                 'yggdrasil', 'languages')
     sys.path.insert(0, LANG_PATH)
     try:
         import install_languages

docs/Makefile

@@ -46,6 +46,8 @@ roxy:
 scripts:
 	python $(SOURCEDIR)/examples/make_example_rst.py
 	python $(SOURCEDIR)/make_tables_rst.py
+	python $(SOURCEDIR)/build_schema.py
+	python $(SOURCEDIR)/create_cli_docs.py
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).

docs/source/advanced/metaschema.rst

@@ -4,7 +4,7 @@
 JSON Metaschema
 ###############
 
-|yggdrasil| uses the JSON metaschema below for evaluating schemas including
+|yggdrasil| uses :download:`this JSON metaschema <../schema/metaschema.json>` (shown below) for evaluating schemas including
 type definitions. This metaschema is an expansion of the default metaschema defined by
 `JSON schema <https://json-schema.org/>`_ with the addition of several more
 specific ``simpleTypes`` that allow validation of more complex/specific data types.
@@ -40,6 +40,6 @@ metaschema include:
     Unicode objects that will be encoded as bytes using utf-8 encoding.
 
 
-.. literalinclude:: /../../yggdrasil/.ygg_metaschema.json
+.. literalinclude:: ../schema/metaschema.json
    :language: json
    :linenos:

docs/source/advanced/schema.rst

@@ -1,18 +1,29 @@
 .. _schema_rst:
 
-######################
+Schemas
+#######
+
 YAML Validation Schema
-######################
+======================
 
 
-|yggdrasil| uses the schema below for validating the YAML specification files 
+|yggdrasil| uses :download:`this schema <../schema/integration.json>` (shown below) for validating the YAML specification files 
 used to define integration networks. Users should not need to interact with the
 schema directly. Instead |yggdrasil| provides a command line utility ``yggvalidate``
 for using the schema to validate a provided list of YAML specification files
 defining a run.
 
 
-.. literalinclude:: /../../yggdrasil/.ygg_schema.yml
-   :language: yaml
+.. literalinclude:: ../schema/integration.json
+   :language: json
    :linenos:
 
+
+Additional Schemas
+==================
+
+:download:`model_form.json <../schema/model_form.json>`
+    Pure JSON schema (without |yggdrasil| metaschema extensions) for validating models submitted to the :term:`Yggdrasil Model Repository`.
+
+:download:`integration_strict.json <../schema/integration_strict.json>`
+    Pure JSON schema (without |yggdrasil| metaschema extensions) for validating integration objects.

docs/source/build_schema.py

@@ -0,0 +1,21 @@
+import os
+import shutil
+from yggdrasil import schema, metaschema
+
+
+schema_dir = os.path.join(os.path.dirname(__file__), 'schema')
+if not os.path.isdir(schema_dir):
+    os.mkdir(schema_dir)
+
+
+indent = '    '
+s = schema.get_schema()
+shutil.copy(metaschema._metaschema_fname,
+            os.path.join(schema_dir, 'metaschema.json'))
+with open(os.path.join(schema_dir, 'integration.json'), 'w') as fd:
+    metaschema.encoder.encode_json(s.schema, fd=fd, indent=indent,
+                                   sort_keys=False)
+schema.get_json_schema(os.path.join(schema_dir, 'integration_strict.json'),
+                       indent=indent)
+schema.get_model_form_schema(os.path.join(schema_dir, 'model_form.json'),
+                             indent=indent)