Merge pull request #106 from jverswijver/dynamic_rest_api

Dynamic rest api

Author: guzman-raphael

.gitignore

@@ -8,3 +8,5 @@ dist
 build
 docs/_build
 *.tar.gz
+pharus/dynamic_api.py
+pharus/dynamic_api_spec.yaml

Dockerfile

@@ -2,9 +2,9 @@ ARG PY_VER
 ARG DISTRO
 ARG IMAGE
 FROM datajoint/${IMAGE}:py${PY_VER}-${DISTRO}
-COPY --chown=dja:anaconda ./README.rst ./requirements.txt ./setup.py \
+COPY --chown=anaconda:anaconda ./README.rst ./requirements.txt ./setup.py \
     /main/
-COPY --chown=dja:anaconda ./pharus/*.py /main/pharus/
+COPY --chown=anaconda:anaconda ./pharus/*.py /main/pharus/
 RUN \
     cd /main && \
     pip install . && \
@@ -23,4 +23,4 @@ WORKDIR /main
 # CMD ["pharus"]
 
 # production service
-CMD ["sh", "-lc", "gunicorn --bind 0.0.0.0:${PHARUS_PORT} pharus.server:app"]
+CMD ["sh", "-c", "gunicorn --bind 0.0.0.0:${PHARUS_PORT} pharus.server:app"]

docker-compose-deploy.yaml

@@ -18,6 +18,7 @@ services:
     # environment: # configurable values with defaults
     #   - PHARUS_PORT=5000
     #   - PHARUS_PREFIX=/
+    #   - API_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml # for dynamic api spec
   fakeservices.datajoint.io:
     <<: *net
     image: datajoint/nginx:v0.0.16

docker-compose-dev.yaml

@@ -25,12 +25,13 @@ services:
       service: pharus
     environment:
       - FLASK_ENV=development # enables logging to console from Flask
+      - API_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml # for dynamic api spec
     volumes:
       - ./pharus:/opt/conda/lib/python3.8/site-packages/pharus
     command: pharus
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.0.16
+    image: datajoint/nginx:v0.0.18
     environment:
       - ADD_pharus_TYPE=REST
       - ADD_pharus_ENDPOINT=pharus:5000

docker-compose-test.yaml

@@ -23,6 +23,7 @@ services:
       - TEST_DB_USER=root
       - TEST_DB_PASS=simple
       - AS_SCRIPT
+      - API_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml
     volumes:
       - ./requirements_test.txt:/tmp/pip_requirements.txt
       - ./pharus:/opt/conda/lib/python3.8/site-packages/pharus

pharus/__init__.py

@@ -1,2 +1,16 @@
+from . import dynamic_api_gen
 from .version import __version__
-__version__
+from os import path, environ
+
+try:
+    if path.exists(environ.get('API_SPEC_PATH')):
+        dynamic_api_gen.populate_api()
+except TypeError:
+    print('No Dynamic API path found')
+
+try:
+    from .dynamic_api import app
+except ImportError:
+    from .server import app
+
+__all__ = ['__version__', 'app']

pharus/dynamic_api_gen.py

@@ -0,0 +1,48 @@
+from textwrap import indent
+from pathlib import Path
+import os
+import yaml
+
+
+def populate_api():
+    header_template = """
+# Auto-generated rest api
+from .server import app, protected_route
+from .interface import _DJConnector, dj
+import json
+import numpy as np
+
+
+class NumpyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return json.JSONEncoder.default(self, obj)
+"""
+    route_template = """
+
+@app.route('{route}', methods=['GET'])
+@protected_route
+def {method_name}(jwt_payload: dict) -> dict:
+
+{query}
+    djconn = _DJConnector._set_datajoint_config(jwt_payload)
+    vm_dict = {{s: dj.VirtualModule(s, s, connection=djconn) for s in dj.list_schemas()}}
+    query, fetch_args = dj_query(vm_dict)
+    return json.dumps(query.fetch(**fetch_args), cls=NumpyEncoder)
+"""
+
+    spec_path = os.environ.get('API_SPEC_PATH')
+    api_path = 'pharus/dynamic_api.py'
+    with open(Path(api_path), 'w') as f, open(Path(spec_path), 'r') as y:
+        f.write(header_template)
+        values_yaml = yaml.load(y, Loader=yaml.FullLoader)
+        pages = values_yaml['SciViz']['pages']
+
+        # Crawl through the yaml file for the routes in the components
+        for page in pages.values():
+            for grid in page['grids'].values():
+                for comp in grid['components'].values():
+                    f.write(route_template.format(route=comp['route'],
+                            method_name=comp['route'].replace('/', ''),
+                            query=indent(comp['dj_query'], '    ')))

requirements.txt

@@ -3,3 +3,4 @@ gunicorn
 pyjwt[crypto]
 datajoint>=0.13.0
 datajoint_connection_hub
+pyyaml

tests/__init__.py

@@ -1,5 +1,5 @@
 import pytest
-from pharus.server import app
+from pharus.dynamic_api import app
 from uuid import UUID
 from os import getenv
 import datajoint as dj

tests/init/test_dynamic_api_spec.yaml

@@ -0,0 +1,83 @@
+version: 'v0.0.0'
+LabBook: null
+Test: "hello world"
+SciViz: # top level tab
+  auth:
+    mode: database # reuse from DJ LabBook
+  pages: # individual pages
+    page2:
+      route: /session1 # dev, be careful of name collisions
+      args:
+        - subject_name
+        - session_number
+      hidden: true
+      grids:
+        grid2:
+          components:
+            component1:
+              route: /query1
+              row_span: 0
+              column_span: 0
+              type: plot:png
+              restriction: >
+                def restriction(**args):
+                  return dict(**args)
+              dj_query: >
+                def dj_query(vms):
+                    TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
+                    return TableA * TableB, dict(order_by='b_number')
+        grid1:
+          components:
+            component1:
+              route: /query2
+              row_span: 0
+              column_span: 0
+              type: plot:png
+              restriction: >
+                def restriction(**args):
+                  return dict(**args)
+              dj_query: >
+                def dj_query(vms):
+                    TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
+                    return TableA * TableB, dict(order_by='b_number')
+          
+    page1:
+      route: /session2 # dev, be careful of name collisions
+      args:
+        - subject_name
+        - session_number
+      hidden: true
+      grids:
+        grid1:
+          type: fixed
+          max_component_per_page: 20
+          rows: 1
+          columns: 3
+          components:
+            component1:
+              route: /query3
+              row_span: 0
+              column_span: 0
+              type: plot:png
+              restriction: >
+                def restriction(**args):
+                  return dict(**args)
+              dj_query: >
+                def dj_query(vms):
+                    TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
+                    return TableA * TableB, dict(order_by='b_number')
+            component2:
+              route: /query4
+              row_span: 0
+              column_span: 1
+              type: plot:plotly1
+              restriction: >
+                def restriction(**args):
+                  return dict(**args)
+              dj_query: >
+                def dj_query(vms):
+                    TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
+                    return TableA * TableB, dict(order_by='b_number')
+              diff_checker: >
+                def diff_checker(**args):
+                  return TrainingStatsPlotly.proj(hash='trial_mean_hash')