Merge pull request #97 from AD-SDL/module_about

Add initial support for Module About specification

Author: RyanTheRobothead

.github/workflows/docker-build.yml

@@ -31,6 +31,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
@@ -59,5 +65,6 @@ jobs:
           push: false
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
             #cache-from: type=gha
             #cache-to: type=gha,mode=max

.github/workflows/docker-publish.yml

@@ -31,6 +31,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
@@ -59,5 +65,6 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64,linux/arm64
             #cache-from: type=gha
             #cache-to: type=gha,mode=max

.github/workflows/pypi-publish.yml

@@ -2,9 +2,7 @@ name: PyPi Publish
 
 on:
   push:
-    branches:
-      - main
-    tags: [ '*' ]
+    tags: [ 'v*' ]
 
 jobs:
   publish_pypi:

.github/workflows/tests.yml

@@ -18,8 +18,6 @@ jobs:
           python-version: 3.9
       - name: Build the Project
         run: make init build
-      - name: Start WEI
-        run: make start
       - name: Test with pytest
         run: make test
       - name: Cleanup

Dockerfile

@@ -1,46 +1,46 @@
 FROM python:3.11
 
 LABEL org.opencontainers.image.source=https://github.com/AD-SDL/wei
-LABEL org.opencontainers.image.description="The Workcell Execution Interface (WEI)"
+LABEL org.opencontainers.image.description="The Workflow Execution Interface (WEI)"
 LABEL org.opencontainers.image.licenses=MIT
 
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y gosu; \
+	rm -rf /var/lib/apt/lists/*
+
+# User configuration
 ARG USER_ID=9999
 ARG GROUP_ID=9999
 ARG CONTAINER_USER=app
 
-ADD https://github.com/FooBarWidget/matchhostfsowner/releases/download/v1.0.1/matchhostfsowner-1.0.1-x86_64-linux.gz /sbin/matchhostfsowner.gz
-RUN gunzip /sbin/matchhostfsowner.gz && \
-  chown root: /sbin/matchhostfsowner && \
-  chmod +x /sbin/matchhostfsowner
-
 RUN groupadd -g ${GROUP_ID} ${CONTAINER_USER}
 RUN useradd --create-home -u ${USER_ID} --shell /bin/bash -g ${CONTAINER_USER} ${CONTAINER_USER}
 
-USER ${CONTAINER_USER}
 WORKDIR /home/${CONTAINER_USER}
 
 RUN mkdir -p wei/requirements
-RUN mkdir -p .wei/temp
-RUN mkdir .diaspora
+RUN mkdir -p .wei
+RUN mkdir -p .diaspora
 
 # Install Python Dependencies first, for caching purposes
-COPY --chown=${USER_ID}:${GROUP_ID} requirements/requirements.txt wei/requirements/requirements.txt
-COPY --chown=${USER_ID}:${GROUP_ID} requirements/dev.txt wei/requirements/dev.txt
-RUN --mount=type=cache,target=/home/${CONTAINER_USER}/.cache,uid=${USER_ID},gid=${GROUP_ID} \
+COPY requirements/requirements.txt wei/requirements/requirements.txt
+COPY requirements/dev.txt wei/requirements/dev.txt
+RUN --mount=type=cache,target=/root/.cache \
     pip install -r wei/requirements/requirements.txt
 
 # Copy wei files
-COPY --chown=${USER_ID}:${GROUP_ID} wei wei/wei
-COPY --chown=${USER_ID}:${GROUP_ID} tests wei/tests
-COPY --chown=${USER_ID}:${GROUP_ID} pyproject.toml wei/pyproject.toml
-COPY --chown=${USER_ID}:${GROUP_ID} README.md wei/README.md
-COPY --chown=${USER_ID}:${GROUP_ID} scripts wei/scripts
-COPY --chown=${USER_ID}:${GROUP_ID} workcell_defs wei/workcell_defs
+COPY wei wei/wei
+COPY tests wei/tests
+COPY pyproject.toml wei/pyproject.toml
+COPY README.md wei/README.md
+COPY scripts wei/scripts
+
 
 # Install dependencies and wei
-RUN --mount=type=cache,target=/home/${CONTAINER_USER}/.cache,uid=${USER_ID},gid=${GROUP_ID} \
+RUN --mount=type=cache,target=/root/.cache \
     pip install -e wei
 
-USER root
-
-ENTRYPOINT ["/sbin/matchhostfsowner"]
+COPY wei-entrypoint.sh /wei-entrypoint.sh
+RUN chmod +x /wei-entrypoint.sh
+ENTRYPOINT [ "/wei-entrypoint.sh" ]

Makefile

@@ -29,9 +29,6 @@ $(DOCS): wei/* docs/source/*
 
 publish: build publish-docker publish-python # Publishes the docker image and pypi package for wei
 
-test: # Run Pytests
-	docker compose -f $(COMPOSE_FILE) exec -u app $(APP_NAME) /bin/bash -c "pytest wei"
-
 ################################################################################
 
 # Determine which rules don't correspond to actual files (add rules to NOT_PHONY to exclude)

README.md

@@ -4,7 +4,7 @@
 <!-- [![PyPI version](https://badge.fury.io/py/mdlearn.svg)](https://badge.fury.io/py/mdlearn) -->
 <!-- [![Documentation Status](https://readthedocs.org/projects/mdlearn/badge/?version=latest)](https://mdlearn.readthedocs.io/en/latest/?badge=latest) -->
 
-The Workcell Execution Interface (WEI) for Autonomous Discovery/Self Driving Laboratories (AD/SDLs)
+The Workflow Execution Interface (WEI) for Autonomous Discovery/Self Driving Laboratories (AD/SDLs)
 
 For more details and specific examples of how to use wei, please see our [documentation](https://rpl-wei.readthedocs.io/en/latest/).
 

compose.yaml

@@ -3,9 +3,10 @@ services:
   # Dev App #
   ###########
   test_app:
-    container_name: ${APP_NAME}
+    container_name: test_app
+    image: ${IMAGE_TEST}
     build:
-      context: .
+      context: tests/
       dockerfile: ${DOCKERFILE_TEST}
     volumes:
       - ${WORKCELLS_DIR}:/workcell_defs
@@ -14,16 +15,63 @@ services:
       - ${PROJECT_DIR}:/home/app/wei # for development only
     env_file:
       - .env
+    profiles:
+      - test
+    command: pytest -p no:cacheprovider wei
     depends_on:
+      - synthesis
+      - transfer
+      - measure
       - wei_server
       - wei_engine
       - wei_redis
-    tty: true
+
+  ###########
+  # Modules #
+  ###########
+  synthesis:
+    image: ${IMAGE_TEST}
+    container_name: synthesis
+    ports:
+      - 2000:2000
+    command: 'python3 wei/tests/test_module/test_rest_node.py
+      --port 2000
+      --alias synthesis'
+    env_file:
+      - .env
+  transfer:
+    image: ${IMAGE_TEST}
+    container_name: transfer
+    ports:
+      - 2001:2001
+    command: 'python3 wei/tests/test_module/test_rest_node.py
+      --port 2001
+      --alias transfer'
+    env_file:
+      - .env
+  measure:
+    image: ${IMAGE_TEST}
+    container_name: measure
+    ports:
+      - 2002:2002
+    command: 'python3 wei/tests/test_module/test_rest_node.py
+      --port 2002
+      --alias measure'
+    env_file:
+      - .env
+
   #####################
   # WEI Core Services #
   #####################
   wei_server:
     image: ${IMAGE}
+    build:
+      context: .
+      dockerfile: ${DOCKERFILE}
+      tags:
+        - ${IMAGE}:latest
+        - ${IMAGE}:${PROJECT_VERSION}
+        - ${IMAGE}:dev
     container_name: wei_server
     ports:
       - 8000:8000
@@ -44,6 +92,8 @@ services:
       - ${WORKCELLS_DIR}:/workcell_defs
       - ${WEI_DATA_DIR}:/home/app/.wei
       - ${PROJECT_DIR}:/home/app/wei # for development only
+    environment:
+     - PYTHONUNBUFFERED=1 # Fix weird bug with empty logging
     env_file:
       - .env
     command: python3 -m wei.engine --workcell /workcell_defs/${WORKCELL_FILENAME} --use_diaspora ${USE_DIASPORA}

docs/source/index.rst

@@ -3,7 +3,7 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to the Workcell Execution Interface (WEI) Documentation!
+Welcome to the Workflow Execution Interface (WEI) Documentation!
 ================================================================
 
 .. toctree::

make/config.mk

@@ -7,12 +7,12 @@
 
 # Project Configuration
 PROJECT_DIR := $(abspath $(MAKEFILE_DIR))
-WORKCELLS_DIR := $(PROJECT_DIR)/workcell_defs
+WORKCELLS_DIR := $(PROJECT_DIR)/tests/workcells
 WORKCELL_FILENAME := test_workcell.yaml
 
 # Python Configuration
 PYPROJECT_TOML := $(PROJECT_DIR)/pyproject.toml
-PROJECT_VERSION := $(shell grep -oP '(?<=version = ")[^"]+' $(PYPROJECT_TOML) | head -n 1)
+PROJECT_VERSION := $(shell sed -n 's/version = "\([^"]*\)"/\1/p' $(PYPROJECT_TOML) | head -n 1)
 
 # Docker Configuration
 COMPOSE_FILE := $(PROJECT_DIR)/compose.yaml
@@ -24,11 +24,10 @@ REGISTRY := ghcr.io
 ORGANIZATION := ad-sdl
 IMAGE_NAME := wei
 IMAGE := $(REGISTRY)/$(ORGANIZATION)/$(IMAGE_NAME)
+IMAGE_TEST := $(IMAGE)_test
 
 # This should match the name of your app's service in the compose file
 APP_NAME := test_app
-# This should be the command to run your app in the container
-APP_COMMAND :=
 # This is where the data from the workcell will be stored
 # If these directories don't exist, they will be created
 WEI_DATA_DIR := $(PROJECT_DIR)/.wei
@@ -49,8 +48,6 @@ NOT_PHONY += .env
 	@echo "# THIS FILE IS AUTOGENERATED, CHANGE THE VALUES IN THE MAKEFILE" > $(ENV_FILE)
 	@echo "USER_ID=$(shell id -u)" >> $(ENV_FILE)
 	@echo "GROUP_ID=$(shell id -g)" >> $(ENV_FILE)
-	@echo "MHF_HOST_UID=$(shell id -u)" >> $(ENV_FILE)
-	@echo "MHF_HOST_GID=$(shell id -g)" >> $(ENV_FILE)
 # The following adds every variable in the Makefiles to the .env file,
 # except for everything in ENV_FILTER and ENV_FILTER itself
 	@$(foreach v,\