Merge pull request #7 from iamamutt/main

refactor(workflow): 🎨 update workflow to use submodules

Author: Thinh Nguyen

datajoint-workflow/cookiecutter.json

@@ -3,7 +3,6 @@
     "author_email": "{{cookiecutter.author_name|words(1)|lower}}@datajoint.com",
     "organization": "Science Institute",
     "workflow": "Brain Lab",
-    "pkg_type": "SciOps",
     "github_user": [
         "dj-sciops",
         "datajoint",
@@ -19,11 +18,11 @@
         "3.10",
         "3.8"
     ],
-    "_pkg_version": "0.0.1",
+    "_pkg_version": "0.0.0",
     "homepage_url": "https://github.com/{{cookiecutter.github_user}}",
     "docs_url": "https://{{cookiecutter.github_user}}.github.io/{{cookiecutter.github_repo}}",
-    "__project_name": "{{cookiecutter.pkg_type|lower}}-{{cookiecutter.workflow|slugify}}",
-    "__pkg_import_name": "{{cookiecutter.pkg_type|lower}}_{{cookiecutter.workflow|slugify(separator='_')}}",
+    "__project_name": "{{cookiecutter.workflow|slugify}}",
+    "__pkg_import_name": "{{cookiecutter.workflow|slugify(separator='_')}}",
     "_djconfig": "dj_local_conf.json",
     "_copy_without_render": [
         "docs/assets/*.svg",

datajoint-workflow/scripts/retrocookie/retrocookie.sh

@@ -168,7 +168,7 @@ create_commits() {
 		exit 0
 	fi
 	echo -e "\n----------- Adding and committing new files."
-	git add .
+	git add . :^src/*/version.py
 	git commit -m "build(retrocookie): reflect updates to source template from generated project '$(basename "${tmp_proj_dir}")'" --no-verify
 	COMMIT=$(git rev-parse HEAD)
 	rev_num=$((rev_num + 1))

datajoint-workflow/{{cookiecutter.github_repo}}/.github/workflows/main-cicd.yml renamed to datajoint-workflow/{{cookiecutter.github_repo}}/.github/workflows/package-updates.yml

@@ -1,19 +1,23 @@
-# Run workflows that require semantic versioning info
-name: main-cicd
+# Run workflows that require semantic versioning info, updates to python package files, and builds.
+name: package-updates
+
 on:
   workflow_run:
-    workflows: [run-tests]
-    types: [completed]
-    branches: [main]
+    workflows:
+      - run-tests
+    types:
+      - completed
+    branches:
+      - main
 
 jobs:
   call_semantic_versioning:
-    if: ${{"{{"}} github.event.workflow_run.conclusion == 'success' {{"}}"}}
+    if: github.event.workflow_run.conclusion == 'success'
     uses: {{cookiecutter.github_user}}/djsciops-cicd/.github/workflows/semantic-versioning.yml@main
     with:
-      py_ver: "{{cookiecutter.python_version}}"
       create_release: true
-      release_only: "['major','minor','patch']"
+      release_types: major,minor,patch
+      release_branch: main
 
   call_mkdocs_build:
     needs: call_semantic_versioning
@@ -23,6 +27,7 @@ jobs:
       new_tag: ${{"{{"}} needs.call_semantic_versioning.outputs.new_tag {{"}}"}}
       branch_name: gh-pages
       py_ver: "{{cookiecutter.python_version}}"
+      release_branch: main
 
   call_sciops_docker_images:
     needs: call_semantic_versioning
@@ -44,7 +49,7 @@ jobs:
       REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
 
   if_run_tests_failed:
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
+    if: github.event.workflow_run.conclusion == 'failure'
     runs-on: ubuntu-latest
     steps:
       - run: |

datajoint-workflow/{{cookiecutter.github_repo}}/.github/workflows/run-tests.yml

@@ -3,8 +3,10 @@ name: run-tests
 
 on:
   workflow_run:
-    workflows: [pre-commit]
-    types: [completed]
+    workflows:
+      - pre-commit
+    types:
+      - completed
     branches:
       - main
 

datajoint-workflow/{{cookiecutter.github_repo}}/CHANGELOG.md

@@ -1,3 +1 @@
 # Changelog
-
-- [Version {{cookiecutter._pkg_version}}](https://github.com/{{cookiecutter.github_user}}/{{cookiecutter.github_repo}}/releases/tag/v{{cookiecutter._pkg_version}})

datajoint-workflow/{{cookiecutter.github_repo}}/CONTRIBUTING.md

@@ -72,12 +72,11 @@ Format your commit messages so that they help create the appropriate semantic ve
 
 _Conventional Commit Resources_:
 
-- [Conventional commits](https://www.conventionalcommits.org/en/v1.0.0-beta.4/)
-- [More commit examples](https://github.com/conventional-changelog/commitlint/tree/master/%40commitlint/config-conventional)
-- [Angular format](https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#commit-message-format)
+- [`semantic-release`: GitHub Action used with this package](https://github.com/mathieudutour/github-tag-action)
+- Other [conventional](https://www.conventionalcommits.org/en/v1.0.0-beta.4/) commit [examples](https://github.com/conventional-changelog/commitlint/tree/master/%40commitlint/config-conventional) and [resources](https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#commit-message-format)
 
 !!! tip "Commit History"
-    Writing a [descriptive commit message](https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#commit-message-format) is highly recommended. In case of doubt, you can check the commit history to look for recurring communication patterns with:
+    Writing a [descriptive commit message](https://github.com/angular/angular.js/blob/master/DEVELOPERS.md#-git-commit-guidelines) is highly recommended. In case of doubt, you can check the commit history to look for recurring communication patterns with:
 
         git log --graph --decorate --pretty=oneline --abbrev-commit --all
 

datajoint-workflow/{{cookiecutter.github_repo}}/README.md

@@ -4,13 +4,13 @@ _View the latest documentation site here:_ [{{cookiecutter.docs_url}}]({{cookiec
 
 # `{{cookiecutter.__project_name}}`
 
-_A {{cookiecutter.workflow}} {{cookiecutter.pkg_type}} for {{cookiecutter.organization}}_
+_A {{cookiecutter.organization}} Workflow for {{cookiecutter.workflow}}_
 
 ## Description
 
 TODO: finish description.
 
-Welcome to the {{cookiecutter.organization}} {{cookiecutter.pkg_type}} service!
+Welcome to the {{cookiecutter.organization}} {{cookiecutter.workflow}} pipeline!
 
 This service is designed for a user to upload their raw [_modality type_] data acquired with [_description_], which will then be automatically processed with [_..._]. The service also provides online Jupyter notebooks to visualize the results.
 
@@ -21,7 +21,7 @@ This [workflow](https://github.com/{{cookiecutter.github_user}}/{{cookiecutter.g
 
 Please follow the steps listed below to begin working with the platform.
 
-## Using the {{cookiecutter.pkg_type}} Services
+## Using the SciOps Services
 
 ### Account Setup
 
@@ -95,12 +95,12 @@ DataJoint LabBook displays data from your database.
 <!--intro-end-->
 <!--install-start-->
 
-Thank you for using the {{cookiecutter.organization}} {{cookiecutter.pkg_type}} cloud-based platform.
+Thank you for using the {{cookiecutter.organization}}: _{{cookiecutter.workflow}}_ cloud-based platform.
 
 ## Installation
 
 !!! note
-    The following is intended for developers and is not required for users of the {{cookiecutter.pkg_type}} services.
+    The following is intended for developers and is not required for users of the SciOps services.
 
 ### 1. Clone the repository
 
@@ -165,11 +165,19 @@ See the [_Development setup_](./CONTRIBUTING.md#setting-up-a-local-development-e
 ├── figures                 <- Generated plots and figures for sharing, reports or documentation.
 ├── notebooks               <- Jupyter notebooks. Naming convention is a number for
 │                              ordering, the creator's initials, and a description.
-|                              For example, '1.0-fw-initial-data-exploration'.
+│                              For example, '1.0-fw-initial-data-exploration'.
 ├── scripts                 <- Analysis examples or production scripts which rely on
 │                              importing the actual Python package, e.g. running queries.
-├── src
-│   └── {{cookiecutter.__pkg_import_name}}             <- Actual Python package where the main functionality goes.
+├── src/
+│   └── {{cookiecutter.__pkg_import_name}}/          <- Actual Python package where the main functionality goes.
+│       └── pipeline/       <- Main schemas and tables used to run the datajoint pipeline.
+│       └── populate/       <- Code to run `populate` or to ingest data into the database.
+│       └── support/        <- Tables to add functionality to the main pipeline modules.
+│       └── tables/         <- Primary end-user tables defined for easy-of-use.
+│       └── utils/          <- Package utilities.
+│       └── __init__.py     <- Root-level package init file.
+│       └── __main__.py     <- Main package script.
+│       └── version.py      <- Should only contain the current package version number.
 ├── tests                   <- Unit and integration tests which can be run with `pytest` or `nox`.
 ├── .cookiecutter.json      <- Options specified during template generation.
 ├── .gitignore              <- Files and folders to ignore for git.

datajoint-workflow/{{cookiecutter.github_repo}}/configs/axon.yaml

@@ -1,4 +1,4 @@
-
+# template for required information to use Axon.
 version: "0.2.1"
 aws:
   account_id: ""

datajoint-workflow/{{cookiecutter.github_repo}}/docker/README.md

@@ -1,5 +1,10 @@
 # Docker Containers
 
+TODO: docker things
+- test image builds for resuable workflows
+- labhub multi conda envs 
+- actions checkout 
+
 Create a `.env` file and the change any variables as necessary, then source those variables for later use.
 
 ```bash
@@ -56,9 +61,9 @@ cd docker
 docker-compose -f docker-compose-codebook_env.yaml up --detach --force-recreate --remove-orphans --build
 ```
 
-## Standard {{cookiecutter.pkg_type}} Environment
+## Standard Worker Environment
 
-... 
+TODO: add description 
 
 
 ### Docker Compose

datajoint-workflow/{{cookiecutter.github_repo}}/docker/docker-compose-standard_workflow.yaml

@@ -1,4 +1,4 @@
-# docker-compose up --build
+# docker-compose up --build --detach
 version: "2.4"
 services:
   standard_worker: