Merge pull request #79 from zitrosolrac/implementingPythonBackEndFeature

Addressing python backend pagination, node backend pagination, and front end pagination.

Author: guzman-raphael

backend/iblapi.py

@@ -172,10 +172,11 @@ def do_req(subpath):
     # construct kwargs
     kwargs = {'as_dict': True}
     limit = int(request.values['__limit']) if '__limit' in values else None
-    order = request.values['__order'] if '__order' in values else None
+    order = request.values['__order'] if '__order' in values else 'KEY ASC'
+    page = int(request.values['__page']) if '__page' in values else 1
     proj = json.loads(request.values['__proj']) if '__proj' in values else None
-    special_fields = ['__json', '__limit', '__order', '__proj', '__json_kwargs']
-    for a in (v for v in values if v not in special_fields):
+    special_fields = ['__json', '__limit', '__order', '__proj', '__json_kwargs', '__page']
+    for a in (k for k, v in values.items() if k not in special_fields and v):
         # HACK: 'uuid' attrs -> UUID type (see also: datajoint-python #594)
         postargs[a] = UUID(values[a]) if 'uuid' in a else values[a]
     args = [postargs] if len(postargs) else []
@@ -186,9 +187,14 @@ def do_req(subpath):
     if '__json_kwargs' in values:
         json_kwargs = json.loads(request.values['__json_kwargs'])
     args = {} if not args else dj.AndList(args)
-    kwargs = {k: v for k, v in (('as_dict', True,),
-                                   ('limit', limit,),
+    if limit == None:
+        kwargs = {k: v for k, v in (('as_dict', True,),
                                    ('order_by', order,)) if v is not None}
+    else:
+        kwargs = {k: v for k, v in (('as_dict', True,),
+                                    ('limit', limit,),
+                                    ('order_by', order,),
+                                    ('offset', (page-1)*limit)) if v is not None}
     # 2) and dispatch
     app.logger.debug("args: '{}', kwargs: {}".format(args, kwargs))
     if obj not in reqmap:
@@ -216,8 +222,6 @@ def handle_q(subpath, args, proj, fetch_args=None, **kwargs):
         ((session * subject * lab * user) & arg).proj(flist)
     '''
     app.logger.info("handle_q: subpath: '{}', args: {}".format(subpath, args))
-    app.logger.info('key words: {}'.format(kwargs))
-
     fetch_args = {} if fetch_args is None else fetch_args
     ret = []
     post_process = None
@@ -266,10 +270,12 @@ def handle_q(subpath, args, proj, fetch_args=None, **kwargs):
 
         q = ((acquisition.Session() * sess_proj * psych_curve * ephys_data * subject.Subject() *
               subject.SubjectLab() * subject.SubjectUser() * trainingStatus) & args & brain_restriction)
-        
+        q = q.proj(*proj) if proj else q
         dj.conn().query("SET SESSION max_join_size={}".format('18446744073709551615'))
-        q = q.proj(*proj).fetch(**fetch_args) if proj else q.fetch(**fetch_args)
+        ret_count = len(q)
+        ret = q.fetch(**fetch_args) 
         dj.conn().query("SET SESSION max_join_size={}".format(original_max_join_size))
+        return dumps({"records_count": ret_count, "records": ret})
     elif subpath == 'subjpage':
         proj_restr = None
         for e in args:

docker-compose-base.yml

@@ -9,7 +9,7 @@ services:
   iblapi:
     <<: *net
     # image: registry.vathes.com/ibl-navigator/iblapi:v0.5.3 # for internal demo
-    image: registry.vathes.com/ibl-navigator/iblapi:v0.2.1-public # for public demo
+    image: registry.vathes.com/ibl-navigator/iblapi:v0.3.1-public # for public demo
     environment:
     # for internal djcompute
     # - DJ_USER=maho
@@ -33,7 +33,7 @@ services:
   ibl-navigator:
     <<: *net
     # image: registry.vathes.com/ibl-navigator/frontend:v0.4.2 # for internal demo
-    image: registry.vathes.com/ibl-navigator/frontend:v0.2.6-public # for public demo
+    image: registry.vathes.com/ibl-navigator/frontend:v0.3.2-public # for public demo
     healthcheck:
       test: curl --fail http://localhost:9000 || exit 1
       timeout: 3s
@@ -42,7 +42,7 @@ services:
     <<: *net
     platform: linux/amd64
     # image: registry.vathes.com/ibl-navigator/node-server:v0.3.0 # for internal demo
-    image: registry.vathes.com/ibl-navigator/node-server:v0.2.1-public # for public demo
+    image: registry.vathes.com/ibl-navigator/node-server:v0.3.1-public # for public demo
     environment:
     - NODE_ENV=development
     - DEMO_USERNAME=ibluser

ibl-frontend/Dockerfile

@@ -1,6 +1,10 @@
-FROM vathes/angulardev:angcli7.1.4-angbuild0.11.4
+# FROM vathes/angulardev:angcli7.1.4-angbuild0.11.4
 
+FROM node:12-bullseye
 
+RUN \
+    apt update && \
+    apt install nginx -y
 
 WORKDIR /app/dist/pipeline-viewer
 
@@ -12,11 +16,10 @@ COPY ./app.conf /etc/nginx/conf.d/default.conf
 CMD ["nginx", "-g", "daemon off;"]
 
 ADD ./frontend-content/package.json /app/
-ADD ./frontend-content/package-lock.json /app/
+
 RUN \
     cd /app && \
-    npm install --save-dev @angular-devkit/build-angular > /dev/null
-
+    npm install 
 
 ADD ./frontend-content /app
 
@@ -25,7 +28,7 @@ COPY ./frontend-content/src/assets/addons/plotly.js /app/node_modules/plotly.js-
 
 RUN \
     cd /app && \
-    node --max_old_space_size=8192 /usr/local/lib/node_modules/@angular/cli/bin/ng build --prod
+    node --max_old_space_size=5120 /app/node_modules/@angular/cli/bin/ng build --configuration production
 
 
 # CMD ["http-server","-p", "8080" ,"-a","0.0.0.0"]

ibl-frontend/dev.dockerfile

@@ -12,12 +12,10 @@ HEALTHCHECK       \
 COPY ./entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]
-# CMD tail -f /dev/null
 
 WORKDIR /app
 
 ADD ./frontend-content/package.json /app/
-#ADD ./frontend-content/package-lock.json /app/
 RUN \
     npm install && \
     npm install --only=dev
@@ -27,8 +25,8 @@ COPY ./frontend-content/src/assets/addons/indigo-pink-ibl.css /app/node_modules/
 COPY ./frontend-content/src/assets/addons/plotly.js /app/node_modules/plotly.js-dist/
 
 CMD ["node", "--max_old_space_size=5120", "/app/node_modules/@angular/cli/bin/ng", "serve", "--host", "0.0.0.0", "--port", "9000", "--disable-host-check"]
-# /app/node_modules/@angular/cli/bin/ng serve  --host  0.0.0.0  --port  9000  --disable-host-check 
-# /app/node_modules/@angular/cli/bin/ng serve  --host  0.0.0.0  --port  9000  --disable-host-check 1> /app/src/output.log 2> /app/src/error.log
+
+# node --max_old_space_size=5120 /app/node_modules/@angular/cli/bin/ng serve  --host  0.0.0.0  --port  9000  --disable-host-check 1> /app/src/output.log 2> /app/src/error.log
 
 
 

ibl-frontend/frontend-content/angular.json

@@ -31,8 +31,6 @@
                 "input": "src/assets/fonts/css/open-iconic-bootstrap.min.css",
                 "inject": true
               },
-              "node_modules/@angular/material/prebuilt-themes/indigo-pink-ibl.css",
-              "src/assets/fonts/css/open-iconic-bootstrap.min.css",
               "src/styles.css"
             ],
             "scripts": [
@@ -64,7 +62,7 @@
                 {
                   "type": "initial",
                   "maximumWarning": "2mb",
-                  "maximumError": "7mb"
+                  "maximumError": "10mb"
                 },
                 {
                   "type": "anyComponentStyle",

ibl-frontend/frontend-content/src/app/app.module.ts

@@ -64,6 +64,7 @@ import { PsthPlotsComponent } from './cell-list/cell/psth-plots/psth-plots.compo
 import { QualityControlComponent } from './quality-control/quality-control.component';
 import { DriftmapComponent } from './quality-control/driftmap/driftmap.component';
 import { SpinningBrainComponent } from './mouse-list/mouse/spinning-brain/spinning-brain.component';
+import {MatProgressSpinnerModule} from '@angular/material/progress-spinner';
 
 PlotlyModule.plotlyjs = PlotlyJS;
 
@@ -154,7 +155,7 @@ const appRoutes: Routes = [
     FormsModule,
     HttpClientModule,
     RouterModule.forRoot(appRoutes),
-    MatSelectModule, MatAutocompleteModule, MatIconModule, MatInputModule,
+    MatSelectModule, MatAutocompleteModule, MatIconModule, MatInputModule, MatProgressSpinnerModule,
     MatCheckboxModule, MatRadioModule, MatNativeDateModule, MatDatepickerModule, MatMomentDateModule, MatSlideToggleModule,
     MatCardModule, MatButtonModule, MatTableModule, MatPaginatorModule, MatSortModule, MatSliderModule, MatExpansionModule,
     MatDialogModule, ReactiveFormsModule, FlexLayoutModule, MatTreeModule, MatFormFieldModule

ibl-frontend/frontend-content/src/app/session-list/all-sessions.service.ts

@@ -1,12 +1,35 @@
 import { Injectable } from '@angular/core';
-import { Subject } from 'rxjs';
+import { Subject, Observable } from 'rxjs';
 import { HttpClient } from '@angular/common/http';
-
 import { environment } from '../../environments/environment';
 
 
 const BACKEND_API_URL = environment.backend_url;
 
+export interface SessionRecord {
+  mouse_id: number;
+  session_date: string;
+  session_lab: string;
+  subject_nickname: string;
+  subject_birth_date: string;
+  session_start_time: string;
+  task_protocol: string; 
+  subject_line: string; 
+  responsible_user: string;
+  session_uuid: string; 
+  sex: string;
+  subject_uuid: string;
+  nplot: string;
+  nprobe: string;
+  session_project: string;
+  good4bmap: string;
+}
+
+interface SessionApi {
+  records: SessionRecord[];
+  records_count: number;
+}
+
 @Injectable({
   providedIn: 'root'
 })
@@ -36,6 +59,11 @@ export class AllSessionsService {
     return this.http.post(BACKEND_API_URL + '/sessions/', sessionFilters, { responseType: 'json'})
   }
 
+  getSessions(body: Object): Observable<SessionApi> {
+    const requestUrl = BACKEND_API_URL + '/sessions';
+    return this.http.post<SessionApi>(requestUrl, body, { responseType: 'json' });
+  }
+
   getAllSessions() {
     let start = new Date();
     this.http.get(BACKEND_API_URL + `/sessions`)
@@ -88,7 +116,7 @@ export class AllSessionsService {
         (filteredSessionsData) => {
           let end = new Date();
           // console.log(`It took ${Number(end) - Number(start)}ms to retrieve the session list information`)
-          this.retrievedSessions = filteredSessionsData;
+          this.retrievedSessions = filteredSessionsData['records'];
           // console.log('retrievedSessions data are: ');
           // console.log(this.retrievedSessions);
           this.newSessionsLoaded.next(this.retrievedSessions);

ibl-frontend/frontend-content/src/app/session-list/session-list.component.css

@@ -101,6 +101,20 @@ label {
     display: none;
 }
 
+.session-list-loading-shade {
+    position: absolute;
+    /* top: 0; */
+    left: 0;
+    /* bottom: 56px; */
+    right: 0;
+    background: rgba(0, 0, 0, 0.15);
+    z-index: 9;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    height: 100%;
+}
+
 .session-list-loading-message.show {
     display: block;
 }
@@ -122,6 +136,7 @@ label {
 
 .table-container {
     width: 100%;
+    position: relative;
     overflow-x: auto;
 }
 td.mat-cell, 

ibl-frontend/frontend-content/src/app/session-list/session-list.component.html

@@ -31,6 +31,15 @@ <h4>
           </mat-form-field>
 
           <div class="session-date-filter">
+            <mat-form-field class="short" *ngIf="!isSessionDateUsingRange">
+              <label for="">Sessions On</label>
+              <input matInput formControlName="session_start_time" [matDatepicker]="datePicker" [min]="sessionMinDate" [max]="sessionMaxDate" [matDatepickerFilter]="sessionDateFilter" (blur)="updateMenu()" (focus)="stepBackMenu($event)">
+              <mat-datepicker-toggle matSuffix [for]="datePicker"></mat-datepicker-toggle>
+              <mat-datepicker #datePicker></mat-datepicker>
+            </mat-form-field>
+          </div>
+
+          <!-- <div class="session-date-filter">
             <div class="date-range-toggler">
               <label for="isSessionDateUsingRanger">Session Start Date</label>
               <div>
@@ -62,8 +71,7 @@ <h4>
                 <mat-datepicker #datePickerSRE></mat-datepicker>
               </mat-form-field>
             </div>
-          </div>
-          
+          </div> -->
         </div>
         <div class="mouse-field-filters form-section">
           <mat-form-field>
@@ -153,7 +161,7 @@ <h4>
               <label>Brain Regions</label>
               <input matInput (input)="filterChanged($event.target.value.toLowerCase())">
             </mat-form-field>
-            <div>Selected Brain Region:
+           <!-- <div>Selected Brain Region:
               <div class="chip-list">
                 <div class="selected-chip" *ngFor="let item of requested_BR">
                   <div>{{ item }}</div>
@@ -164,16 +172,16 @@ <h4>
 
             <div class="br-selection-area">
               <mat-tree [dataSource]="treeDataSource" [treeControl]="treeControl" class="brain-tree">
-                <!-- // This is the tree node template for leaf nodes -->
+                // This is the tree node template for leaf nodes
                 <mat-nested-tree-node *matTreeNodeDef="let node" matTreeNodeToggle>
                   <li class="mat-tree-node">
-                    <!-- // use a disabled button to provide padding for tree leaf  -->
+                    // use a disabled button to provide padding for tree leaf 
                     <button class="treeChevron" mat-icon-button disabled></button>
                     <mat-checkbox (change)="selectionToggle($event.checked, node)"
                       [checked]="BT_allSelected(node)">{{node.display}}</mat-checkbox>  
                   </li>
                 </mat-nested-tree-node>
-                <!-- // This is the tree node template for expandable nodes -->
+                // This is the tree node template for expandable nodes
                 <mat-nested-tree-node *matTreeNodeDef="let node; when: BT_hasChild">
                   <li>
                     <div class="mat-tree-node">
@@ -193,7 +201,7 @@ <h4>
                   </li>
                 </mat-nested-tree-node>
               </mat-tree>
-            </div>
+            </div> -->
           </div>
 
         </div>
@@ -211,7 +219,7 @@ <h4>
             <button mat-raised-button class="btn btn-refresh" (click)="refreshData()">Refresh Data</button>
           </div>
         </div>
-        <div class="checkbox-filters">
+        <!-- <div class="checkbox-filters">
           <div class="chbox-filter">
             <mat-checkbox class="nplot_status_chbox" [checked]="hideMissingPlots" (change)="toggleNplotStatus()">
               Only show sessions with behavior plots
@@ -227,19 +235,20 @@ <h4>
               Only show sessions with ephys data
             </mat-checkbox>
           </div>
-        </div>
+        </div> -->
       </div>
     </form>
 
     <div [class]="isLoading && initialLoad ? 'loading-icon loading initial': isLoading ? 'loading-icon loading': 'loading-icon'">
       <img src="assets/images/loading_icon.gif">
-      <p [class]="initialLoad ? 'session-list-loading-message show' : 'session-list-loading-message'">
-        Loading for the sessions may take a while depending on your internet connection. We appreciate your patience as we load the full list of sessions.
-      </p>
+      <p [class]="initialLoad ? 'session-list-loading-message show' : 'session-list-loading-message'"></p>
     </div>
     <div class="table-container">
-    <table mat-table class="col-12" [dataSource]="dataSource" matSort (matSortChange)="storeTableInfo($event)">
-
+      <div class="session-list-loading-shade"
+          *ngIf="isLoadingTable">
+        <mat-spinner *ngIf="isLoadingTable"></mat-spinner>
+      </div>
+    <table mat-table class="col-12" [dataSource]="sessionRecords" matSort (matSortChange)="storeTableInfo($event)">
       <ng-container matColumnDef="session_lab">
         <th mat-header-cell *matHeaderCellDef mat-sort-header> Lab </th>
         <td mat-cell *matCellDef="let session"> {{session.session_lab}} </td>
@@ -321,7 +330,8 @@ <h4>
     </table>
     </div>
 
-    <mat-paginator [pageSize]="pageSize"
+    <mat-paginator [length]="sessionRecordLength"
+                [pageSize]="pageSize"
                 [pageSizeOptions]="pageSizeOptions"
                 showFirstLastButtons (page)="storeTableInfo($event)"></mat-paginator>
   </div>