Merge pull request #98 from cropsinsilico/topic/cropps

Topic/cropps

Author: langmm

.github/workflows/test-install.yml

@@ -4,16 +4,20 @@
 env:
   BUILDDOCS: 0
   INSTALLC: 1
+  INSTALLEXCEL: 1
   INSTALLFORTRAN: 1
+  INSTALLIMAGES: 1
   INSTALLJULIA: 1
   INSTALLLPY: 0
   INSTALLMATLAB: 0
   INSTALLMPI: 1
   INSTALLOMP: 1
   INSTALLPYGMENTS: 1
+  INSTALLPYTORCH: 1
   INSTALLR: 1
   INSTALLRMQ: 0
   INSTALLSBML: 0
+  INSTALLSEQ: 1
   INSTALLTRIMESH: 1
   INSTALLZMQ: 1
   NUMPY: numpy
@@ -276,6 +280,8 @@ jobs:
         == 1
       name: Install R
       uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: 4.2.3
     - if: startsWith(runner.os, 'Linux') && matrix.install-method == 'pip'
       name: (LINUX,PIP) Cache pip
       uses: actions/cache@v3
@@ -607,6 +613,8 @@ jobs:
         == 1
       name: Install R
       uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: 4.2.3
     - if: startsWith(runner.os, 'Linux') && matrix.install-method == 'pip'
       name: (LINUX,PIP) Cache pip
       uses: actions/cache@v3
@@ -930,6 +938,8 @@ jobs:
         == 1
       name: Install R
       uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: 4.2.3
     - if: startsWith(runner.os, 'Linux') && matrix.install-method == 'pip'
       name: (LINUX,PIP) Cache pip
       uses: actions/cache@v3
@@ -1236,6 +1246,8 @@ jobs:
         == 1
       name: Install R
       uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: 4.2.3
     - if: startsWith(runner.os, 'Linux') && matrix.install-method == 'pip'
       name: (LINUX,PIP) Cache pip
       uses: actions/cache@v3
@@ -1541,6 +1553,8 @@ jobs:
         == 1
       name: Install R
       uses: r-lib/actions/setup-r@v2
+      with:
+        r-version: 4.2.3
     - if: startsWith(runner.os, 'Linux') && matrix.install-method == 'pip'
       name: (LINUX,PIP) Cache pip
       uses: actions/cache@v3

.gitignore

@@ -65,6 +65,8 @@ tmp/
 *.lib
 *.exe
 
+yggdrasil/examples/pytorch1/Input/data/
+
 # Compiled things - not managed
 *.pyc
 *.o

HISTORY.rst

@@ -2,6 +2,20 @@
 History
 =======
 
+1.10.0 (2023-05-31) Added support for PyTorch models, sequence files, image files, and Excel files
+-------------------
+
+* Added support for running trained PyTorch models as part of integrations
+* Added support for connecting integrations to sequence files in FASTA and FASTQ formats via BioPython
+* Added support for connecting integrations to sequence files in SAM, BAM and CRAM formats via pysam
+* Added support for connecting integrations to sequence files in BCF and VCF formats via pysam
+* Added support for connecting integrations to JPEG, PNG, EPS, BMP, GIF and TIFF formats via Pillow
+* Added support for connecting integrations to Excel files via Pandas
+* Refactor serialization in preparation of moving communication into C++
+* Added CLI for converting files between compatible formats
+* Added support for creating Ply and ObjWavefront objects from explicit meshes
+* Fix bug in R interface
+
 1.9.0 (2023-04-14) Migrate to using extended version of rapidjson/python-rapidjson for serialization and add support for models written in Julia
 ------------------
 

MANIFEST.in

@@ -22,6 +22,8 @@ recursive-include yggdrasil *.xml
 recursive-include yggdrasil *.jl
 recursive-include yggdrasil/examples *.py
 recursive-include yggdrasil/examples *.rst
+recursive-include yggdrasil/examples *.png
+recursive-include yggdrasil/examples *.pth
 recursive-include yggdrasil/demos *.py
 recursive-include yggdrasil/tests/scripts *.py
 recursive-include yggdrasil/examples Makefile_linux
@@ -44,6 +46,7 @@ include yggdrasil/regex/CMakeLists.txt
 include yggdrasil/tests/scripts/CMakeLists.txt
 include yggdrasil/tests/scripts/Makefile_linux
 include yggdrasil/tests/scripts/Makefile_windows
+include yggdrasil/tests/data/example.vcf
 graft docs
 include versioneer.py
 include yggdrasil/_version.py

docs/source/development/todo.rst

@@ -28,6 +28,7 @@ Refactor
 * Split drivers into separate directories for model drivers and connection drivers
 * Change how CLI arguments are added to the arg parser for the language installer to use subparsers
 * Use doxygen linter/parser for parsing functions
+* Add parameter to files/serializers to allow incremental or complete read on file types where that is valid
 
 New feature/example
 -------------------
@@ -41,8 +42,9 @@ New feature/example
 * Assign meanings to error codes and implement across languages (e.g. missing comm class)
 * Add parameters constraining valid values for inputs/outputs (e.g. range) using JSON paramaters
 * Allow users to select from list of multiple possiblities when locating libraries to avoid conflict
-* Write C/C++ as extension to rapidjson and wrap in Python
 * Add 'shell' option for executable models on Windows to allow calling different shell types.
+* Move communication to C++
+* Allow models to be embedded natively
 
 Testing
 -------

docs/source/examples/make_example_rst.py

@@ -20,8 +20,10 @@
               'sbml': 'SBML',
               'osr': 'OpenSimRoot',
               'dummy': 'Dummy',
-              'timesync': 'Timesync'}
-_default_lang = ['python', 'cpp', 'c', 'R', 'fortran', 'matlab', 'sbml']
+              'timesync': 'Timesync',
+              'pytorch': 'PyTorch'}
+_default_lang = ['python', 'cpp', 'c', 'R', 'fortran', 'matlab',
+                 'sbml', 'pytorch']
 
 
 def get_file(fname, local=False):

recipe/meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "yggdrasil" %}
-{% set version = "1.9.1" %}
+{% set version = "1.10.0" %}
 {% set PY_VER_MAJOR = PY_VER.split('.')[0] %}
 {% set PY_VER_MINOR = PY_VER.split('.')[1] %}
 
@@ -187,6 +187,53 @@ outputs:
       run:
         - {{ pin_subpackage('yggdrasil', exact=True) }}
         - pika >=1.0.0
+  - name: yggdrasil.images
+    build:
+      string: py{{ PY_VER_MAJOR }}{{ PY_VER_MINOR }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}
+      run_exports:
+        - {{ pin_subpackage('yggdrasil.images') }}
+    requirements:
+      host:
+        - python
+      run:
+        - {{ pin_subpackage('yggdrasil', exact=True) }}
+        - ghostscript
+        - pillow
+  - name: yggdrasil.seq
+    build:
+      string: py{{ PY_VER_MAJOR }}{{ PY_VER_MINOR }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}
+      run_exports:
+        - {{ pin_subpackage('yggdrasil.seq') }}
+    requirements:
+      host:
+        - python
+      run:
+        - {{ pin_subpackage('yggdrasil', exact=True) }}
+        - biopython
+  - name: yggdrasil.pytorch
+    build:
+      string: py{{ PY_VER_MAJOR }}{{ PY_VER_MINOR }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}
+      run_exports:
+        - {{ pin_subpackage('yggdrasil.pytorch') }}
+    requirements:
+      host:
+        - python
+      run:
+        - {{ pin_subpackage('yggdrasil', exact=True) }}
+        - pytorch  # [not win]
+        - torchvision  # [not win]
+  - name: yggdrasil.excel
+    build:
+      string: py{{ PY_VER_MAJOR }}{{ PY_VER_MINOR }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}
+      run_exports:
+        - {{ pin_subpackage('yggdrasil.excel') }}
+    requirements:
+      host:
+        - python
+      run:
+        - {{ pin_subpackage('yggdrasil', exact=True) }}
+        - openpyxl
+        - pandas
   - name: yggdrasil.testing
     build:
       string: py{{ PY_VER_MAJOR }}{{ PY_VER_MINOR }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}

tests/__init__.py

@@ -280,12 +280,13 @@ def python_class(self, component_type, component_subtype):
     def options(self):
         r"""Arguments that should be provided when getting testing options."""
         return {}
-
+    
     @pytest.fixture(scope="class")
-    def testing_options(self, python_class, options):
+    def testing_options(self, python_class, options, testdir):
         r"""Testing options."""
         if 'explicit_testing_options' in options:
             return copy.deepcopy(options['explicit_testing_options'])
+        options['test_dir'] = testdir
         return python_class.get_testing_options(**options)
 
     @pytest.fixture

tests/communication/test_CommBase.py

@@ -3,7 +3,7 @@
 import copy
 import pytest
 from yggdrasil import platform
-from yggdrasil.communication import new_comm, get_comm
+from yggdrasil.communication import new_comm, get_comm, AddressError
 from yggdrasil.tools import get_supported_comm
 from tests import TestComponentBase
 
@@ -78,6 +78,7 @@ def get_send_comm_kwargs(self, commtype, use_async,
         kws = dict(commtype=commtype, reverse_names=True,
                    direction='send', use_async=use_async)
         kws.update(testing_options['kwargs'])
+        kws.update(testing_options.get('send_kwargs', {}))
         kws.update(kwargs)
         return kws
 
@@ -176,7 +177,14 @@ def wrapped_do_send_recv(send_comm, recv_comm, message=None,
                     recv_comm, recv_params.get('method', 'recv'))(
                         **recv_params.get('kwargs', {'timeout': 0}))
                 assert flag == recv_params.get('flag', True)
-                assert nested_approx(recv_params['message']) == msg
+                try:
+                    assert nested_approx(recv_params['message']) == msg
+                except BaseException:
+                    print("EXPECTED:")
+                    print(recv_params['message'])
+                    print("ACTUAL:")
+                    print(msg)
+                    raise
             if not send_params.get('skip_wait', False):
                 wait_on_function(
                     lambda: send_comm.is_closed or (send_comm.n_msg_send == 0))
@@ -354,7 +362,7 @@ def test_empty_obj_recv(self, run_once, global_comm):
 
     def test_error_name(self, python_class):
         r"""Test error on missing address."""
-        with pytest.raises(RuntimeError):
+        with pytest.raises(AddressError):
             python_class('test%s' % uuid.uuid4())
 
     def test_error_send(self, monkeypatch, send_comm, testing_options,