Merge pull request #109 from jverswijver/update_dynamic_api

Update dynamic api to support attribute route, and paging sorting filtering ect.

Author: guzman-raphael

.gitignore

@@ -9,4 +9,4 @@ build
 docs/_build
 *.tar.gz
 pharus/dynamic_api.py
-pharus/dynamic_api_spec.yaml
+/specs

CHANGELOG.md

@@ -2,6 +2,13 @@
 
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
 
+## [0.2.0] - 2021-11-02
+### Added
+- Dynamic api generation from spec sheet.(#103, #104, #105, #107, #108, #110) PR #106, #109
+- `dynamic_api_gen.py` Python script that generates `dynamic_api.py`.
+- Add Tests for the new dynamic api.
+- `server.py` now loads the routes generated dynamically from `dynamic_api.py` when it is present.
+
 ## [0.1.0] - 2021-03-31
 ### Added
 - Local database instance pre-populated with sample data for `dev` Docker Compose environment. PR #99
@@ -69,6 +76,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 - Support for DataJoint attribute types: `varchar`, `int`, `float`, `datetime`, `date`, `time`, `decimal`, `uuid`.
 - Check dependency utility to determine child table references.
 
+[0.2.0]: https://github.com/datajoint/pharus/compare/0.1.0...0.2.0
 [0.1.0]: https://github.com/datajoint/pharus/compare/0.1.0b2...0.1.0
 [0.1.0b2]: https://github.com/datajoint/pharus/compare/0.1.0b0...0.1.0b2
 [0.1.0b0]: https://github.com/datajoint/pharus/compare/0.1.0a5...0.1.0b0

pharus/dynamic_api_gen.py

@@ -5,19 +5,12 @@
 
 
 def populate_api():
-    header_template = """
-# Auto-generated rest api
+    header_template = """# Auto-generated rest api
 from .server import app, protected_route
 from .interface import _DJConnector, dj
-import json
-import numpy as np
-
-
-class NumpyEncoder(json.JSONEncoder):
-    def default(self, obj):
-        if isinstance(obj, np.ndarray):
-            return obj.tolist()
-        return json.JSONEncoder.default(self, obj)
+from flask import request
+from json import loads
+from base64 import b64decode
 """
     route_template = """
 
@@ -26,10 +19,44 @@ def default(self, obj):
 def {method_name}(jwt_payload: dict) -> dict:
 
 {query}
-    djconn = _DJConnector._set_datajoint_config(jwt_payload)
-    vm_dict = {{s: dj.VirtualModule(s, s, connection=djconn) for s in dj.list_schemas()}}
-    query, fetch_args = dj_query(vm_dict)
-    return json.dumps(query.fetch(**fetch_args), cls=NumpyEncoder)
+{restriction}
+    if request.method in {{'GET'}}:
+        try:
+            djconn = _DJConnector._set_datajoint_config(jwt_payload)
+            vm_dict = {{s: dj.VirtualModule(s, s, connection=djconn)
+                       for s in dj.list_schemas()}}
+            query, fetch_args = dj_query(vm_dict)
+            query = query & restriction()
+            record_header, table_tuples, total_count = _DJConnector._fetch_records(
+                query=query,
+                **{{k: (int(v) if k in ('limit', 'page')
+                   else (v.split(',') if k == 'order'
+                   else loads(b64decode(v.encode('utf-8')).decode('utf-8'))))
+                   for k, v in request.args.items()}},
+                )
+            return dict(recordHeader=record_header, records=table_tuples,
+                        totalCount=total_count)
+        except Exception as e:
+            return str(e), 500
+
+
+@app.route('{route}/attributes', methods=['GET'])
+@protected_route
+def {method_name}_attributes(jwt_payload: dict) -> dict:
+
+{query}
+    if request.method in {{'GET'}}:
+        try:
+            djconn = _DJConnector._set_datajoint_config(jwt_payload)
+            vm_dict = {{s: dj.VirtualModule(s, s, connection=djconn)
+                       for s in dj.list_schemas()}}
+            query, fetch_args = dj_query(vm_dict)
+            attributes_meta = _DJConnector._get_attributes(query)
+
+            return dict(attributeHeaders=attributes_meta['attribute_headers'],
+                        attributes=attributes_meta['attributes'])
+        except Exception as e:
+            return str(e), 500
 """
 
     spec_path = os.environ.get('API_SPEC_PATH')
@@ -45,4 +72,5 @@ def {method_name}(jwt_payload: dict) -> dict:
                 for comp in grid['components'].values():
                     f.write(route_template.format(route=comp['route'],
                             method_name=comp['route'].replace('/', ''),
-                            query=indent(comp['dj_query'], '    ')))
+                            query=indent(comp['dj_query'], '    '),
+                            restriction=indent(comp['restriction'], '    ')))

pharus/interface.py

@@ -98,19 +98,14 @@ def _list_tables(jwt_payload: dict, schema_name: str) -> dict:
         return tables_dict_list
 
     @staticmethod
-    def _fetch_records(jwt_payload: dict, schema_name: str, table_name: str,
+    def _fetch_records(query,
                        restriction: list = [], limit: int = 1000, page: int = 1,
                        order=['KEY ASC']) -> tuple:
         """
-        Get records from table.
+        Get records from query.
 
-        :param jwt_payload: Dictionary containing databaseAddress, username, and password
-            strings
-        :type jwt_payload: dict
-        :param schema_name: Name of schema
-        :type schema_name: str
-        :param table_name: Table name under the given schema; must be in camel case
-        :type table_name: str
+        :param query: any datajoint object related to QueryExpression
+        :type query: datajoint ``QueryExpression`` or related object
         :param restriction: Sequence of filters as ``dict`` with ``attributeName``,
             ``operation``, ``value`` keys defined, defaults to ``[]``
         :type restriction: list, optional
@@ -125,20 +120,17 @@ def _fetch_records(jwt_payload: dict, schema_name: str, table_name: str,
             can be paged
         :rtype: tuple
         """
-        _DJConnector._set_datajoint_config(jwt_payload)
-
-        schema_virtual_module = dj.create_virtual_module(schema_name, schema_name)
 
         # Get table object from name
-        table = _DJConnector._get_table_object(schema_virtual_module, table_name)
-        attributes = table.heading.attributes
+        attributes = query.heading.attributes
         # Fetch tuples without blobs as dict to be used to create a
         #   list of tuples for returning
-        query = table & dj.AndList([
+        query_restricted = query & dj.AndList([
             _DJConnector._filter_to_restriction(f, attributes[f['attributeName']].type)
             for f in restriction])
-        non_blobs_rows = query.fetch(*table.heading.non_blobs, as_dict=True, limit=limit,
-                                     offset=(page-1)*limit, order_by=order)
+        non_blobs_rows = query_restricted.fetch(*query.heading.non_blobs, as_dict=True,
+                                                limit=limit, offset=(page-1)*limit,
+                                                order_by=order)
 
         # Buffer list to be return
         rows = []
@@ -181,45 +173,34 @@ def _fetch_records(jwt_payload: dict, schema_name: str, table_name: str,
 
             # Add the row list to tuples
             rows.append(row)
-        return list(attributes.keys()), rows, len(query)
+        return list(attributes.keys()), rows, len(query_restricted)
 
     @staticmethod
-    def _get_table_attributes(jwt_payload: dict, schema_name: str, table_name: str) -> dict:
+    def _get_attributes(query) -> dict:
         """
-        Method to get primary and secondary attributes of a table.
+        Method to get primary and secondary attributes of a query.
 
-        :param jwt_payload: Dictionary containing databaseAddress, username, and password
-            strings
-        :type jwt_payload: dict
-        :param schema_name: Name of schema to list all tables from
-        :type schema_name: str
-        :param table_name: Table name under the given schema; must be in camel case
-        :type table_name: str
+        :param query: any datajoint object related to QueryExpression
+        :type query: datajoint ``QueryExpression`` or related object
         :return: Dict with keys ``attribute_headers`` and ``attributes`` containing
             ``primary``, ``secondary`` which each contain a
             ``list`` of ``tuples`` specifying: ``attribute_name``, ``type``, ``nullable``,
             ``default``, ``autoincrement``.
         :rtype: dict
         """
-        _DJConnector._set_datajoint_config(jwt_payload)
-        local_values = locals()
-        local_values[schema_name] = dj.VirtualModule(schema_name, schema_name)
-
-        # Get table object from name
-        table = _DJConnector._get_table_object(local_values[schema_name], table_name)
 
-        table_attributes = dict(primary=[], secondary=[])
-        for attribute_name, attribute_info in table.heading.attributes.items():
+        query_attributes = dict(primary=[], secondary=[])
+        for attribute_name, attribute_info in query.heading.attributes.items():
             if attribute_info.in_key:
-                table_attributes['primary'].append((
+                query_attributes['primary'].append((
                     attribute_name,
                     attribute_info.type,
                     attribute_info.nullable,
                     attribute_info.default,
                     attribute_info.autoincrement
                     ))
             else:
-                table_attributes['secondary'].append((
+                query_attributes['secondary'].append((
                     attribute_name,
                     attribute_info.type,
                     attribute_info.nullable,
@@ -229,7 +210,7 @@ def _get_table_attributes(jwt_payload: dict, schema_name: str, table_name: str)
 
         return dict(attribute_headers=['name', 'type', 'nullable',
                                        'default', 'autoincrement'],
-                    attributes=table_attributes)
+                    attributes=query_attributes)
 
     @staticmethod
     def _get_table_definition(jwt_payload: dict, schema_name: str, table_name: str) -> str:
@@ -270,7 +251,7 @@ def _insert_tuple(jwt_payload: dict, schema_name: str, table_name: str,
         """
         _DJConnector._set_datajoint_config(jwt_payload)
 
-        schema_virtual_module = dj.create_virtual_module(schema_name, schema_name)
+        schema_virtual_module = dj.VirtualModule(schema_name, schema_name)
         getattr(schema_virtual_module, table_name).insert(tuple_to_insert)
 
     @staticmethod
@@ -326,7 +307,7 @@ def _update_tuple(jwt_payload: dict, schema_name: str, table_name: str,
         """
         conn = _DJConnector._set_datajoint_config(jwt_payload)
 
-        schema_virtual_module = dj.create_virtual_module(schema_name, schema_name)
+        schema_virtual_module = dj.VirtualModule(schema_name, schema_name)
         with conn.transaction:
             [getattr(schema_virtual_module, table_name).update1(t) for t in tuple_to_update]
 
@@ -351,7 +332,7 @@ def _delete_records(jwt_payload: dict, schema_name: str, table_name: str,
         """
         _DJConnector._set_datajoint_config(jwt_payload)
 
-        schema_virtual_module = dj.create_virtual_module(schema_name, schema_name)
+        schema_virtual_module = dj.VirtualModule(schema_name, schema_name)
 
         # Get table object from name
         table = _DJConnector._get_table_object(schema_virtual_module, table_name)

pharus/server.py

@@ -1,6 +1,6 @@
 """Exposed REST API."""
 from os import environ
-from .interface import _DJConnector
+from .interface import _DJConnector, dj
 from . import __version__ as version
 from typing import Callable
 from functools import wraps
@@ -605,10 +605,15 @@ def record(jwt_payload: dict, schema_name: str, table_name: str) -> Union[dict,
     """)
     if request.method in {'GET', 'HEAD'}:
         try:
+            _DJConnector._set_datajoint_config(jwt_payload)
+
+            schema_virtual_module = dj.VirtualModule(schema_name, schema_name)
+
+            # Get table object from name
+            dj_table = _DJConnector._get_table_object(schema_virtual_module, table_name)
+
             record_header, table_tuples, total_count = _DJConnector._fetch_records(
-                jwt_payload=jwt_payload,
-                schema_name=schema_name,
-                table_name=table_name,
+                query=dj_table,
                 **{k: (int(v) if k in ('limit', 'page')
                        else (v.split(',') if k == 'order' else loads(
                         b64decode(v.encode('utf-8')).decode('utf-8'))))
@@ -895,8 +900,14 @@ def attribute(jwt_payload: dict, schema_name: str, table_name: str) -> dict:
     """
     if request.method in {'GET', 'HEAD'}:
         try:
-            attributes_meta = _DJConnector._get_table_attributes(jwt_payload, schema_name,
-                                                                 table_name)
+            _DJConnector._set_datajoint_config(jwt_payload)
+            local_values = locals()
+            local_values[schema_name] = dj.VirtualModule(schema_name, schema_name)
+
+            # Get table object from name
+            dj_table = _DJConnector._get_table_object(local_values[schema_name], table_name)
+
+            attributes_meta = _DJConnector._get_attributes(dj_table)
             return dict(attributeHeaders=attributes_meta['attribute_headers'],
                         attributes=attributes_meta['attributes'])
         except Exception as e:

pharus/version.py

@@ -1,2 +1,2 @@
 """Package metadata."""
-__version__ = '0.1.0'
+__version__ = '0.2.0'

tests/init/test_dynamic_api_spec.yaml

@@ -12,30 +12,30 @@ SciViz: # top level tab
         - session_number
       hidden: true
       grids:
-        grid2:
+        grid1:
           components:
             component1:
               route: /query1
               row_span: 0
               column_span: 0
               type: plot:png
               restriction: >
-                def restriction(**args):
-                  return dict(**args)
+                def restriction(**kwargs):
+                    return dict(**kwargs)
               dj_query: >
                 def dj_query(vms):
                     TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
                     return TableA * TableB, dict(order_by='b_number')
-        grid1:
+        grid2:
           components:
             component1:
               route: /query2
               row_span: 0
               column_span: 0
               type: plot:png
               restriction: >
-                def restriction(**args):
-                  return dict(**args)
+                def restriction(**kwargs):
+                    return dict(**kwargs)
               dj_query: >
                 def dj_query(vms):
                     TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
@@ -60,8 +60,8 @@ SciViz: # top level tab
               column_span: 0
               type: plot:png
               restriction: >
-                def restriction(**args):
-                  return dict(**args)
+                def restriction(**kwargs):
+                    return dict(**kwargs)
               dj_query: >
                 def dj_query(vms):
                     TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
@@ -72,12 +72,28 @@ SciViz: # top level tab
               column_span: 1
               type: plot:plotly1
               restriction: >
-                def restriction(**args):
-                  return dict(**args)
+                def restriction(**kwargs):
+                    return dict(**kwargs)
+              dj_query: >
+                def dj_query(vms):
+                    TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
+                    return TableA * TableB, dict(order_by='b_number')
+              diff_checker: >
+                def diff_checker(**args):
+                  return TrainingStatsPlotly.proj(hash='trial_mean_hash')
+            component3:
+              route: /query5
+              row_span: 0
+              column_span: 1
+              type: plot:plotly1
+              restriction: >
+                def restriction(**kwargs):
+                    return dict(a_id=0, **kwargs)
               dj_query: >
                 def dj_query(vms):
                     TableA, TableB = (vms['test_group1_simple'].TableA, vms['test_group1_simple'].TableB)
                     return TableA * TableB, dict(order_by='b_number')
               diff_checker: >
                 def diff_checker(**args):
-                  return TrainingStatsPlotly.proj(hash='trial_mean_hash')
+                  return TrainingStatsPlotly.proj(hash='trial_mean_hash')
+