Merge pull request #116 from jverswijver/update_dynamic_api

Update dynamic api to support sci viz templating for real time views

Author: guzman-raphael

.github/workflows/development.yaml

@@ -2,8 +2,6 @@ name: Development
 on:
   pull_request:
   push:
-    tags:
-      - '*.*.*'
 jobs:
   test-changelog:
     runs-on: ubuntu-latest
@@ -40,7 +38,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Compile docs static artifacts
         run: |
-          export PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}')
+          export PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}')
           export HOST_UID=$(id -u)
           docker-compose -f docker-compose-docs.yaml up --exit-code-from pharus-docs --build
           echo "PHARUS_VERSION=${PHARUS_VERSION}" >> $GITHUB_ENV
@@ -69,7 +67,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Compile image
         run: |
-          export PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}')
+          export PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}')
           export HOST_UID=$(id -u)
           docker-compose -f docker-compose-build.yaml up --exit-code-from pharus --build
           IMAGE=$(docker images --filter "reference=datajoint/pharus*" \
@@ -112,7 +110,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Determine package version
         run: |
-          PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}')
+          PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}')
           echo "PHARUS_VERSION=${PHARUS_VERSION}" >> $GITHUB_ENV
       - name: Fetch image artifact
         uses: actions/download-artifact@v2
@@ -124,7 +122,13 @@ jobs:
           docker load < "image-pharus-${PHARUS_VERSION}-py${PY_VER}-${DISTRO}.tar.gz"
           docker-compose -f docker-compose-test.yaml up --exit-code-from pharus
   publish-release:
-    if: github.event_name == 'push'
+    if: |
+      github.event_name == 'push' &&
+      contains(github.ref, 'refs/tags/*.*.*') &&
+      (
+        github.repository_owner == 'datajoint' ||
+        github.repository_owner == 'vathes'
+      )
     needs: test
     runs-on: ubuntu-latest
     env:
@@ -136,7 +140,7 @@ jobs:
       - uses: actions/checkout@v2
       - name: Determine package version
         run: |
-          PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}')
+          PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}')
           echo "PHARUS_VERSION=${PHARUS_VERSION}" >> $GITHUB_ENV
       - name: Get changelog entry
         id: changelog_reader
@@ -203,7 +207,13 @@ jobs:
           asset_name: docker-compose-deploy.yaml
           asset_content_type: application/yaml
   publish-images:
-    if: github.event_name == 'push'
+    if: |
+      github.event_name == 'push' &&
+      contains(github.ref, 'refs/tags/*.*.*') &&
+      (
+        github.repository_owner == 'datajoint' ||
+        github.repository_owner == 'vathes'
+      )
     needs: publish-release
     runs-on: ubuntu-latest
     strategy:
@@ -216,21 +226,23 @@ jobs:
       PY_VER: ${{matrix.py_ver}}
       DISTRO: ${{matrix.distro}}
       IMAGE: ${{matrix.image}}
-      DOCKER_USERNAME: ${{secrets.docker_username}}
-      DOCKER_PASSWORD: ${{secrets.docker_password}}
     steps:
       - uses: actions/checkout@v2
       - name: Determine package version
         run: |
-          PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}')
+          PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}')
           echo "PHARUS_VERSION=${PHARUS_VERSION}" >> $GITHUB_ENV
       - name: Fetch image artifact
         uses: actions/download-artifact@v2
         with:
           name: image-pharus-${{env.PHARUS_VERSION}}-py${{matrix.py_ver}}-${{matrix.distro}}
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{secrets.docker_username}}
+          password: ${{secrets.docker_password}}
       - name: Publish image
         run: |
-          echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
           docker load < "image-pharus-${PHARUS_VERSION}-py${PY_VER}-${DISTRO}.tar.gz"
           IMAGE=$(docker images --filter "reference=datajoint/*" --format "{{.Repository}}")
           TAG=$(docker images --filter "reference=datajoint/*" --format "{{.Tag}}")
@@ -253,14 +265,20 @@ jobs:
             ${{matrix.distro}}.tar.gz"
           asset_content_type: application/gzip
   publish-docs:
-    if: github.event_name == 'push'
+    if: |
+      github.event_name == 'push' &&
+      contains(github.ref, 'refs/tags/*.*.*') &&
+      (
+        github.repository_owner == 'datajoint' ||
+        github.repository_owner == 'vathes'
+      )
     needs: publish-release
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - name: Determine package version
         run: |
-          PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}')
+          PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}')
           echo "PHARUS_VERSION=${PHARUS_VERSION}" >> $GITHUB_ENV
       - name: Fetch docs static artifacts
         uses: actions/download-artifact@v2

CHANGELOG.md

@@ -1,6 +1,20 @@
 # Changelog
 
 Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention.
+
+## [0.3.0] - 2022-01-21
+### Changed
+- Hot-reload mechanism to use `otumat watch` PR #116
+- Renamed environment variable defining spec sheet to `PHARUS_SPEC_PATH` PR #116
+
+### Added
+- Autoformatting strategy using `black` PR #116 
+- Support for sci-viz components `metadata`, `image`, `dynamic grid` PR #116
+- `component interface` for users to be able to load their own custom interface for sci-viz PR #116
+
+### Fixed
+- Various bugs related to datetime PR #116
+
 ## [0.2.3] - 2021-11-18
 ### Added
 - Support for plot component PR #155
@@ -88,6 +102,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and
 - Support for DataJoint attribute types: `varchar`, `int`, `float`, `datetime`, `date`, `time`, `decimal`, `uuid`.
 - Check dependency utility to determine child table references.
 
+[0.3.0]: https://github.com/datajoint/pharus/compare/0.2.3...0.3.0
 [0.2.3]: https://github.com/datajoint/pharus/compare/0.2.2...0.2.3
 [0.2.2]: https://github.com/datajoint/pharus/compare/0.2.1...0.2.2
 [0.2.1]: https://github.com/datajoint/pharus/compare/0.2.0...0.2.1

Dockerfile

@@ -15,13 +15,13 @@ HEALTHCHECK       \
     --retries=5  \
     --interval=15s \
     CMD           \
-        wget --quiet --tries=1 --spider \
-            http://localhost:${PHARUS_PORT}${PHARUS_PREFIX}/version > /dev/null 2>&1 || exit 1
+    wget --quiet --tries=1 --spider \
+    http://localhost:${PHARUS_PORT}${PHARUS_PREFIX}/version > /dev/null 2>&1 || exit 1
 ENV PHARUS_PORT 5000
+# ---TEMP---
+RUN pip install plotly
+# ----------
+COPY --chown=anaconda:anaconda ./reload.sh /tmp/
 WORKDIR /main
 
-# development service
-# CMD ["pharus"]
-
-# production service
-CMD ["sh", "-c", "gunicorn --bind 0.0.0.0:${PHARUS_PORT} pharus.server:app"]
+CMD ["sh", "-c", "otumat watch -f ${PHARUS_SPEC_PATH} -s /tmp/reload.sh -i 5"]

docker-compose-build.yaml

@@ -1,4 +1,4 @@
-# PY_VER=3.8 IMAGE=djbase DISTRO=alpine PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}') HOST_UID=$(id -u) docker-compose -f docker-compose-build.yaml up --exit-code-from pharus --build
+# PY_VER=3.8 IMAGE=djbase DISTRO=alpine PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}') HOST_UID=$(id -u) docker-compose -f docker-compose-build.yaml up --exit-code-from pharus --build
 #
 # Intended for updating dependencies and docker image.
 # Used to build release artifacts.

docker-compose-deploy.yaml

@@ -8,17 +8,18 @@
 # With this config and the configuration below in NGINX, you should be able to verify it is
 # running properly with a `curl https://fakeservices.datajoint.io/api/version`.
 version: "2.4"
-x-net: &net
+x-net:
+  &net
   networks:
-      - main
+    - main
 services:
   pharus:
     <<: *net
     image: datajoint/pharus:${PHARUS_VERSION}
     # environment: # configurable values with defaults
     #   - PHARUS_PORT=5000
     #   - PHARUS_PREFIX=/
-    #   - API_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml # for dynamic api spec
+    #   - PHARUS_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml # for dynamic api spec
   fakeservices.datajoint.io:
     <<: *net
     image: datajoint/nginx:v0.0.16

docker-compose-dev.yaml

@@ -1,4 +1,4 @@
-# PY_VER=3.8 IMAGE=djbase DISTRO=alpine PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}') HOST_UID=$(id -u) docker-compose -f docker-compose-dev.yaml up
+# PY_VER=3.8 IMAGE=djbase DISTRO=alpine PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}') HOST_UID=$(id -u) docker-compose -f docker-compose-dev.yaml up
 #
 # Intended for normal development. Supports hot/live reloading.
 # Note: If requirements or Dockerfile change, will need to add --build flag to docker-compose.
@@ -7,9 +7,10 @@
 # With this config and the configuration below in NGINX, you should be able to verify it is
 # running properly with a `curl https://fakeservices.datajoint.io/api/version`.
 version: "2.4"
-x-net: &net
+x-net:
+  &net
   networks:
-      - main
+    - main
 services:
   local-db:
     <<: *net
@@ -25,7 +26,7 @@ services:
       service: pharus
     environment:
       - FLASK_ENV=development # enables logging to console from Flask
-      - API_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml # for dynamic api spec
+      - PHARUS_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml # for dynamic api spec
     volumes:
       - ./pharus:/opt/conda/lib/python3.8/site-packages/pharus
     command: pharus

docker-compose-test.yaml

@@ -1,11 +1,12 @@
-# PY_VER=3.8 IMAGE=djtest DISTRO=alpine AS_SCRIPT=FALSE PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\' '{print $2}') HOST_UID=$(id -u) docker-compose -f docker-compose-test.yaml up --exit-code-from pharus
+# PY_VER=3.8 IMAGE=djtest DISTRO=alpine AS_SCRIPT=FALSE PHARUS_VERSION=$(cat pharus/version.py | tail -1 | awk -F\" '{print $2}') HOST_UID=$(id -u) docker-compose -f docker-compose-test.yaml up --exit-code-from pharus
 #
 # Intended for running test suite locally.
 # Note: If requirements or Dockerfile change, will need to add --build flag.
 version: "2.4"
-x-net: &net
+x-net:
+  &net
   networks:
-      - main
+    - main
 services:
   db:
     <<: *net
@@ -23,7 +24,7 @@ services:
       - TEST_DB_USER=root
       - TEST_DB_PASS=simple
       - AS_SCRIPT
-      - API_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml
+      - PHARUS_SPEC_PATH=tests/init/test_dynamic_api_spec.yaml
     volumes:
       - ./requirements_test.txt:/tmp/pip_requirements.txt
       - ./pharus:/opt/conda/lib/python3.8/site-packages/pharus
@@ -36,10 +37,14 @@ services:
           echo "------ SYNTAX TESTS ------"
           PKG_DIR=/opt/conda/lib/python3.8/site-packages/pharus
           flake8 $${PKG_DIR} --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 /main/tests --count --select=E9,F63,F7,F82 --show-source --statistics
           echo "------ UNIT TESTS ------"
           pytest -sv --cov-report term-missing --cov=pharus /main/tests
           echo "------ STYLE TESTS ------"
-          flake8 $${PKG_DIR} --count --max-complexity=20 --max-line-length=95 --statistics
+          black $${PKG_DIR} --check -v --extend-exclude "^.*dynamic_api.py$$"
+          flake8 $${PKG_DIR} --count --max-complexity=20 --max-line-length=94 --statistics --exclude=*dynamic_api.py --ignore=W503
+          black /main/tests --check -v
+          flake8 /main/tests --count --max-complexity=20 --max-line-length=94 --statistics --ignore=F401,F811,W503
         else
           echo "=== Running ==="
           echo "Please see 'docker-compose-test.yaml' for detail on running tests."

docs/conf.py

@@ -12,40 +12,41 @@
 #
 import os
 import sys
-sys.path.insert(0, os.path.abspath('..'))
+
+sys.path.insert(0, os.path.abspath(".."))
 
 
 # -- Project information -----------------------------------------------------
 
-project = 'Pharus'
-copyright = '2021, DataJoint Contributors'
-author = 'DataJoint Contributors'
+project = "Pharus"
+copyright = "2021, DataJoint Contributors"
+author = "DataJoint Contributors"
 
 
 # -- General configuration ---------------------------------------------------
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc', 'sphinxcontrib.httpdomain']
+extensions = ["sphinx.ext.autodoc", "sphinxcontrib.httpdomain"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
 
 
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = 'sphinx_rtd_theme'
+html_theme = "sphinx_rtd_theme"
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]

docs/dev_notes.rst

@@ -31,8 +31,8 @@ Run Locally w/ Python
 - For development, use CLI command ``pharus``. This method supports hot-reloading so probably best coupled with ``pip install -e ...``.
 - For production, use ``gunicorn --bind 0.0.0.0:${PHARUS_PORT} pharus.server:app``.
 
-Run Tests for Development w/ Pytest and Flake8
-----------------------------------------------
+Run Tests for Development w/ Pytest, Flake8, Black
+--------------------------------------------------
 
 - Set ``pharus`` testing environment variables:
 
@@ -45,7 +45,12 @@ Run Tests for Development w/ Pytest and Flake8
 
 - For syntax tests, run ``flake8 ${PKG_DIR} --count --select=E9,F63,F7,F82 --show-source --statistics``
 - For pytest integration tests, run ``pytest -sv --cov-report term-missing --cov=${PKG_DIR} /main/tests``
-- For style tests, run ``flake8 ${PKG_DIR} --count --max-complexity=20 --max-line-length=95 --statistics``
+- For style tests, run:
+
+    .. code-block:: bash
+
+        black ${PKG_DIR} --check -v --extend-exclude "^.*dynamic_api.py$"
+        flake8 ${PKG_DIR} --count --max-complexity=20 --max-line-length=94 --statistics --exclude=*dynamic_api.py --ignore=W503
 
 Creating Sphinx Documentation from Scratch
 ------------------------------------------

pharus/__init__.py

@@ -3,14 +3,14 @@
 from os import path, environ
 
 try:
-    if path.exists(environ.get('API_SPEC_PATH')):
+    if path.exists(environ.get("PHARUS_SPEC_PATH")):
         dynamic_api_gen.populate_api()
 except TypeError:
-    print('No Dynamic API path found')
+    print("No Dynamic API path found")
 
 try:
     from .dynamic_api import app
 except ImportError:
     from .server import app
 
-__all__ = ['__version__', 'app']
+__all__ = ["__version__", "app"]