Merge pull request #115 from jverswijver/update_dynamic_api

Update dynamic api to support plots

Author: guzman-raphael

CHANGELOG.md

@@ -1,14 +1,18 @@
 # Changelog
 
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
+## [0.2.3] - 2021-11-18
+### Added
+- Support for plot component PR #155
+- Fetch argument specification in `dj_query` PR #155
 
 ## [0.2.2] - 2021-11-10
 ### Fixed
-- Optimize dynamic api virtual modules.
+- Optimize dynamic api virtual modules. PR #113
 
 ## [0.2.1] - 2021-11-08
 ### Fixed
-- Error with retrieving the module's installation root path.
+- Error with retrieving the module's installation root path. PR #112
 
 ## [0.2.0] - 2021-11-02
 ### Added
@@ -84,6 +88,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 - Support for DataJoint attribute types: `varchar`, `int`, `float`, `datetime`, `date`, `time`, `decimal`, `uuid`.
 - Check dependency utility to determine child table references.
 
+[0.2.3]: https://github.com/datajoint/pharus/compare/0.2.2...0.2.3
 [0.2.2]: https://github.com/datajoint/pharus/compare/0.2.1...0.2.2
 [0.2.1]: https://github.com/datajoint/pharus/compare/0.2.0...0.2.1
 [0.2.0]: https://github.com/datajoint/pharus/compare/0.1.0...0.2.0

pharus/dynamic_api_gen.py

@@ -27,10 +27,10 @@ def {method_name}(jwt_payload: dict) -> dict:
             djconn = _DJConnector._set_datajoint_config(jwt_payload)
             vm_list = [dj.VirtualModule(s, s, connection=djconn)
                        for s in inspect.getfullargspec(dj_query).args]
-            query, fetch_args = dj_query(*vm_list)
-            query = query & restriction()
+            djdict = dj_query(*vm_list)
+            djdict['query'] = djdict['query'] & restriction()
             record_header, table_tuples, total_count = _DJConnector._fetch_records(
-                query=query,
+                query=djdict['query'], fetch_args=djdict['fetch_args'],
                 **{{k: (int(v) if k in ('limit', 'page')
                    else (v.split(',') if k == 'order'
                    else loads(b64decode(v.encode('utf-8')).decode('utf-8'))))
@@ -52,15 +52,37 @@ def {method_name}_attributes(jwt_payload: dict) -> dict:
             djconn = _DJConnector._set_datajoint_config(jwt_payload)
             vm_list = [dj.VirtualModule(s, s, connection=djconn)
                        for s in inspect.getfullargspec(dj_query).args]
-            query, fetch_args = dj_query(*vm_list)
-            attributes_meta = _DJConnector._get_attributes(query)
+            djdict = dj_query(*vm_list)
+            attributes_meta = _DJConnector._get_attributes(djdict['query'])
 
             return dict(attributeHeaders=attributes_meta['attribute_headers'],
                         attributes=attributes_meta['attributes'])
         except Exception as e:
             return str(e), 500
 """
 
+    plot_route_template = '''
+
+@app.route('{route}', methods=['GET'])
+@protected_route
+def {method_name}(jwt_payload: dict) -> dict:
+
+{query}
+{restriction}
+    if request.method in {{'GET'}}:
+        try:
+            djconn = _DJConnector._set_datajoint_config(jwt_payload)
+            vm_list = [dj.VirtualModule(s, s, connection=djconn)
+                       for s in inspect.getfullargspec(dj_query).args]
+            djdict = dj_query(*vm_list)
+            djdict['query'] = djdict['query'] & restriction()
+            record_header, table_tuples, total_count = _DJConnector._fetch_records(
+                fetch_args=djdict['fetch_args'], query=djdict['query'], fetch_blobs=True)
+            return dict(table_tuples[0][0])
+        except Exception as e:
+            return str(e), 500
+'''
+
     pharus_root = f"{pkg_resources.get_distribution('pharus').module_path}/pharus"
     api_path = f'{pharus_root}/dynamic_api.py'
     spec_path = os.environ.get('API_SPEC_PATH')
@@ -79,3 +101,8 @@ def {method_name}_attributes(jwt_payload: dict) -> dict:
                                 method_name=comp['route'].replace('/', ''),
                                 query=indent(comp['dj_query'], '    '),
                                 restriction=indent(comp['restriction'], '    ')))
+                    if comp['type'] == 'plot:plotly:stored_json':
+                        f.write(plot_route_template.format(route=comp['route'],
+                                method_name=comp['route'].replace('/', ''),
+                                query=indent(comp['dj_query'], '    '),
+                                restriction=indent(comp['restriction'], '    ')))

pharus/interface.py

@@ -100,7 +100,7 @@ def _list_tables(jwt_payload: dict, schema_name: str) -> dict:
     @staticmethod
     def _fetch_records(query,
                        restriction: list = [], limit: int = 1000, page: int = 1,
-                       order=['KEY ASC']) -> tuple:
+                       order=['KEY ASC'], fetch_blobs=False, fetch_args=[]) -> tuple:
         """
         Get records from query.
 
@@ -128,7 +128,15 @@ def _fetch_records(query,
         query_restricted = query & dj.AndList([
             _DJConnector._filter_to_restriction(f, attributes[f['attributeName']].type)
             for f in restriction])
-        non_blobs_rows = query_restricted.fetch(*query.heading.non_blobs, as_dict=True,
+
+        if fetch_blobs and not fetch_args:
+            fetch_args = [*query.heading.attributes]
+        elif not fetch_args:
+            fetch_args = query.heading.non_blobs
+        else:
+            attributes = {k: v for k, v in attributes.items() if k in fetch_args}
+
+        non_blobs_rows = query_restricted.fetch(*fetch_args, as_dict=True,
                                                 limit=limit, offset=(page-1)*limit,
                                                 order_by=order)
 
@@ -169,8 +177,8 @@ def _fetch_records(query,
                             row.append(non_blobs_row[attribute_name])
                 else:
                     # Attribute is blob type thus fill it in string instead
-                    row.append('=BLOB=')
-
+                    (row.append(non_blobs_row[attribute_name])
+                     if fetch_blobs else row.append('=BLOB='))
             # Add the row list to tuples
             rows.append(row)
         return list(attributes.keys()), rows, len(query_restricted)

pharus/version.py

@@ -1,2 +1,2 @@
 """Package metadata."""
-__version__ = '0.2.2'
+__version__ = '0.2.3'

tests/__init__.py

@@ -107,6 +107,23 @@ class TableC(dj.Lookup):
         """
         contents = [(0, 10, 100, -8), (0, 11, 200, -9,), (0, 11, 300, -7,)]
 
+    @group1_simple
+    class PlotlyTable(dj.Lookup):
+        definition = """
+        p_id: int
+        ---
+        plot: longblob
+        """
+        contents = [(2, dict(data=[dict(x=[1, 2, 3],
+                                        y=[2, 6, 3],
+                                        type='scatter',
+                                        mode='lines+markers',
+                                        marker=dict(color='red')),
+                                   dict(type='bar',
+                                        x=[1, 2, 3],
+                                        y=[2, 5, 3])],
+                             layout=dict(title='A Fancy Plot')))]
+
     yield group1_simple, group2_simple
 
     group2_simple.drop()

tests/init/test_dynamic_api_spec.yaml

@@ -25,7 +25,9 @@ SciViz: # top level tab
               dj_query: >
                 def dj_query(test_group1_simple):
                     TableA, TableB = (test_group1_simple.TableA, test_group1_simple.TableB)
-                    return TableA * TableB, dict(order_by='b_number')
+                    q = TableA * TableB
+                    f = []
+                    return dict(query=q, fetch_args=f)
         grid2:
           components:
             component1:
@@ -39,7 +41,9 @@ SciViz: # top level tab
               dj_query: >
                 def dj_query(test_group1_simple):
                     TableA, TableB = (test_group1_simple.TableA, test_group1_simple.TableB)
-                    return TableA * TableB, dict(order_by='b_number')
+                    q = TableA * TableB
+                    f = []
+                    return dict(query=q, fetch_args=f)
           
     page1:
       route: /session2 # dev, be careful of name collisions
@@ -65,7 +69,9 @@ SciViz: # top level tab
               dj_query: >
                 def dj_query(test_group1_simple):
                     TableA, TableB = (test_group1_simple.TableA, test_group1_simple.TableB)
-                    return TableA * TableB, dict(order_by='b_number')
+                    q = TableA * TableB
+                    f = []
+                    return dict(query=q, fetch_args=f)
             component2:
               route: /query4
               row_span: 0
@@ -77,7 +83,9 @@ SciViz: # top level tab
               dj_query: >
                 def dj_query(test_group1_simple):
                     TableA, TableB = (test_group1_simple.TableA, test_group1_simple.TableB)
-                    return TableA * TableB, dict(order_by='b_number')
+                    q = TableA * TableB
+                    f = []
+                    return dict(query=q, fetch_args=f)
               diff_checker: >
                 def diff_checker(**args):
                   return TrainingStatsPlotly.proj(hash='trial_mean_hash')
@@ -92,8 +100,24 @@ SciViz: # top level tab
               dj_query: >
                 def dj_query(test_group1_simple):
                     TableA, TableB = (test_group1_simple.TableA, test_group1_simple.TableB)
-                    return TableA * TableB, dict(order_by='b_number')
+                    q = TableA * TableB
+                    f = []
+                    return dict(query=q, fetch_args=f)
               diff_checker: >
                 def diff_checker(**args):
                   return TrainingStatsPlotly.proj(hash='trial_mean_hash')
-                  
+            plot_test:
+              route: /plot1
+              type: plot:plotly:stored_json
+              x: 0
+              y: 0
+              height: 1
+              width: 1
+              restriction: >
+                def restriction(**kwargs):
+                    return dict(**kwargs, p_id=2)
+              dj_query: >
+                def dj_query(test_group1_simple):
+                    PlotlyTable = test_group1_simple.PlotlyTable
+                    return dict(query=PlotlyTable(), fetch_args=['plot'])
+                

tests/test_api_gen.py

@@ -22,6 +22,20 @@ def test_auto_generated_route(token, client, schemas_simple):
     assert expected_json == json.dumps(REST_response4.get_json(force=True), sort_keys=True)
 
 
+def test_get_full_plot(token, client, schemas_simple):
+    REST_response1 = client.get('/plot1', headers=dict(Authorization=f'Bearer {token}'))
+    expected_json = json.dumps(dict(data=[dict(x=[1, 2, 3],
+                                               y=[2, 6, 3],
+                                               type='scatter',
+                                               mode='lines+markers',
+                                               marker=dict(color='red')),
+                                          dict(type='bar',
+                                               x=[1, 2, 3],
+                                               y=[2, 5, 3])],
+                                    layout=dict(title='A Fancy Plot')), sort_keys=True)
+    assert expected_json == json.dumps(REST_response1.get_json(force=True), sort_keys=True)
+
+
 def test_get_attributes(token, client, schemas_simple):
     REST_response = client.get('/query1/attributes',
                                 headers=dict(Authorization=f'Bearer {token}'))