Enable automated testing for github pull requests

.github/workflows/full_test.yml

@@ -0,0 +1,62 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# This workflow is intended to be used as a validity test for any
+# pull request. That is, this is a minimal functionality that must
+# be successfully executed prior to merging a pull request. Note
+# that this can be overridden by adding '[skip ci]' in the commit
+# name.
+
+name: Run full PDO contract tests
+on: [ workflow_dispatch, pull_request ]
+
+jobs:
+    pdo_ci:
+        if: "!contains(github.event.commits[0].message, '[skip ci]')"
+        name: PDO Contracts Full Test
+        runs-on: ubuntu-22.04
+
+        env:
+          REGISTRY: ghcr.io
+          OWNER: ${{ github.repository_owner }}
+          PDO_USER_UID: 55172
+          PDO_GROUP_UID: 55172
+          PDO_SOURCE_ROOT: private-data-objects
+        steps:
+            - name: Check out repo
+              uses: actions/checkout@v4
+              with:
+                  submodules: recursive
+                  fetch-depth: 0
+                  fetch-tags: true
+
+            - name: Set the version
+              run: |
+                echo "PDO_VERSION=$(${PDO_SOURCE_ROOT}/bin/get_version -f ${PDO_SOURCE_ROOT}/VERSION)" >> $GITHUB_ENV
+                echo "PDO_CONTRACTS_VERSION=$(bin/get_version)" >> $GITHUB_ENV
+
+            - name: Show the version
+              run: |
+                echo "PDO_CONTRACTS_VERSION is $PDO_CONTRACTS_VERSION"
+                echo "PDO_VERSION is $PDO_VERSION"
+
+            - name: Configure PDO and build the base images
+              run: |
+                  if $(docker/require_pdo_images.sh -r ${REGISTRY}/${OWNER} -v ${PDO_VERSION})
+                  then
+                      echo Using pre-built PDO images
+                  else
+                      echo No pre-built PDO images available for version ${PDO_VERSION} from ${REGISTRY}
+                      echo Please populate the registry and retry
+                      exit -1
+                  fi
+
+            - name: Build and run contract tests
+              run: |
+                   git checkout -b ci-test-branch
+                   sudo chown -R $PDO_USER_UID:$PDO_GROUP_UID docker/xfer
+                   sudo chmod -R g+w docker/xfer
+                   make -C docker test \
+                      CONTRACTS_USER_UID=$PDO_USER_UID CONTRACTS_GROUP_UID=$PDO_GROUP_UID \
+                      PDO_VERSION=$PDO_VERSION PDO_REGISTRY=$REGISTRY/$OWNER

docker/Makefile

@@ -26,6 +26,7 @@ all : build_contracts
 #   that the user/group identifiers need to have write access to the xfer directory
 CONTRACTS_USER_UID ?= $(shell id -u)
 CONTRACTS_GROUP_UID ?= $(shell id -g)
+CONTRACTS_USER_NAME ?= $(shell whoami)
 
 DOCKER_COMMAND ?= docker
 
@@ -68,15 +69,19 @@ PDO_VERSION ?= $$(									\
 		echo "latest";								\
 	fi )
 
-PDO_REPSITORY ?= ''
+# This variable ensures that the registry ends with a '/'. The additional
+# variable ensures that the substitution is applied; conditional assignment
+# of the registry appears to prevent early evaluation of the expression.
+PDO_REGISTRY_SAFE := $(if $(PDO_REGISTRY),$(patsubst %/,%,$(PDO_REGISTRY))/)
 
 # DOCKER BUILD
 DOCKER_USERNAME = $(LOGNAME)
+DOCKER_BUILDARGS += --build-arg UNAME=$(CONTRACTS_USER_NAME)
 DOCKER_BUILDARGS += --build-arg UID=$(CONTRACTS_USER_UID)
 DOCKER_BUILDARGS += --build-arg GID=$(CONTRACTS_GROUP_UID)
 DOCKER_BUILDARGS += --build-arg CONTRACTS_VERSION=$(CONTRACTS_VERSION)
 DOCKER_BUILDARGS += --build-arg PDO_VERSION=$(PDO_VERSION)
-DOCKER_BUILDARGS += --build-arg PDO_REPOSITORY=$(PDO_REPOSITORY)
+DOCKER_BUILDARGS += --build-arg PDO_REGISTRY=$(PDO_REGISTRY_SAFE)
 DOCKER_ARGS = $(DOCKER_BUILDARGS)
 
 # The CONTRACT_FAMILIES variable contains a list of contract families
@@ -122,7 +127,6 @@ build_pdo_images : repository
 	PDO_SOURCE_ROOT=$(DOCKER_DIR)/repository/private-data-objects \
 		make -C $(DOCKER_DIR)/repository/private-data-objects/docker clean_repository
 
-
 # -----------------------------------------------------------------
 # -----------------------------------------------------------------
 # For the purposes of testing, we just use the resnet model for
@@ -220,25 +224,25 @@ endif
 TEST_COMPOSE_ARGS += -f test.yaml
 
 test : clean_config clean_repository build_contracts
-	- PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	- PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		$(DOCKER_COMPOSE_COMMAND) $(TEST_COMPOSE_ARGS) $(COMPOSE_PROFILES) up --abort-on-container-exit
-	PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	 PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 	 	$(DOCKER_COMPOSE_COMMAND) $(TEST_COMPOSE_ARGS) down
 
 JUPYTER_COMPOSE_ARGS += -f test_jupyter.yaml
 
 test_jupyter : clean_config clean_repository build_contracts
-	- PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	- PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		$(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) $(COMPOSE_PROFILES) up --abort-on-container-exit
-	PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		$(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) down
 
 JUPYTER_MULTIUSER_COMPOSE_ARGS += -f test_multiuser_jupyter.yaml
 
 test_multiuser_jupyter : clean_config clean_repository build_contracts
-	- PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	- PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		$(DOCKER_COMPOSE_COMMAND) $(JUPYTER_MULTIUSER_COMPOSE_ARGS) $(COMPOSE_PROFILES) up --abort-on-container-exit
-	PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		$(DOCKER_COMPOSE_COMMAND) $(JUPYTER_MULTIUSER_COMPOSE_ARGS) down
 
 # The example target builds and runs only the example contract family. This
@@ -247,18 +251,18 @@ test_multiuser_jupyter : clean_config clean_repository build_contracts
 
 example : CONTRACT_FAMILIES=example-contract
 example : clean_config clean_repository build_contracts
-	- PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	- PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		CONTRACTS_SOURCE_ROOT=$(CONTRACTS_SOURCE_ROOT) \
 		$(DOCKER_COMPOSE_COMMAND) $(TEST_COMPOSE_ARGS) up --abort-on-container-exit
-	PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		$(DOCKER_COMPOSE_COMMAND) $(TEST_COMPOSE_ARGS) down
 
 example_jupyter : CONTRACT_FAMILIES=exchange-contract example-contract
 example_jupyter : clean_config clean_repository build_contracts
-	- PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	- PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		CONTRACTS_SOURCE_ROOT=$(CONTRACTS_SOURCE_ROOT) \
 		$(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) up --abort-on-container-exit
-	PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		$(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) down
 
 # The developer target creates a container with no contracts built or
@@ -270,10 +274,10 @@ example_jupyter : clean_config clean_repository build_contracts
 DEVELOP_COMPOSE_ARGS += -f test_dev.yaml
 developer : CONTRACT_FAMILIES=
 developer : clean_config build_contracts
-	- PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	- PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		CONTRACTS_SOURCE_ROOT=$(CONTRACTS_SOURCE_ROOT) \
 		$(DOCKER_COMPOSE_COMMAND) $(DEVELOP_COMPOSE_ARGS) up --abort-on-container-exit
-	PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
+	PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \
 		$(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) down
 
 # -----------------------------------------------------------------

docker/README.md

@@ -59,7 +59,7 @@ Four configuration variables should be set as necessary:
   user.
 * `PDO_VERSION` -- the version identifier for PDO images; this
   defaults to `latest`.
-* `PDO_REPOSITORY` -- the docker repository from which PDO docker images
+* `PDO_REGISTRY` -- the docker repository from which PDO docker images
   may be retrieved; this defaults to `""` so that local images will be
   used by default.
 * `CONTRACTS_VERSION` -- the version that will be used to tag the PDO

docker/pdo_contracts.dockerfile

@@ -34,27 +34,57 @@
 
 # PURPOSE: develop and test contracts with clean
 
-ARG PDO_REPOSITORY
-ARG PDO_VERSION
-FROM ${PDO_REPOSITORY}pdo_client:${PDO_VERSION}
+ARG PDO_REGISTRY=
+ARG PDO_VERSION=latest
+FROM ${PDO_REGISTRY}pdo_client:${PDO_VERSION}
 
 # -----------------------------------------------------------------
-# Install the necessary dependencies
+# Install the necessary system dependencies
 # -----------------------------------------------------------------
-USER 0:0
-ENV DEBIAN_FRONTEND "noninteractive"
-RUN apt-get update \
+USER root
+ENV DEBIAN_FRONTEND="noninteractive"
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update \
     && apt-get install -y -q \
-    libgl1
+        libgl1
+
+# -----------------------------------------------------------------
+# Create a user account from the specification
+# -----------------------------------------------------------------
+ARG UNAME=pdo_contract
+ENV UNAME=${UNAME}
+
+ARG UID=1000
+ARG GID=${UID}
+
+# Create a group for the UID/GID if it does not already exist, rename
+# it to the correct group if it does exist
+RUN if getent group ${GID} > /dev/null 2>&1 ; then \
+        groupmod -n ${UNAME} $(getent group ${GID} | cut -d: -f1) ; \
+    else \
+        groupadd -f -g ${GID} -o ${UNAME} ; \
+    fi
+
+# Create a user for the UID/GID if it does not already exist, rename
+# it to the correct login if it does exist
+RUN if getent passwd ${UID} > /dev/null 2>&1 ; then \
+        usermod -l ${UNAME} $(getent passwd ${UID} | cut -d: -f1) ; \
+    else \
+        useradd -m -u ${UID} -g ${GID} -d /project/pdo -o -s /bin/bash $UNAME ; \
+    fi
+
+# Reassign ownership of the PDO directory tree to the user:group
+RUN chown --recursive $UNAME:$UNAME /project/pdo
+
+# Use the user account for the rest of the installation process
+USER $UNAME
 
 # Many of the dependencies should be addressed with the installation
 # of the PDO client including the common contracts, the wasi toolkit
 # and all of the WAMR toolchain
 
 # This should set up all we need for the jupyter server
-ARG UID=1000
-ARG GID=${UID}
-USER ${UID}:${GID}
 RUN --mount=type=cache,uid=${UID},gid=${GID},target=/project/pdo/.cache/pip \
     /project/pdo/run/bin/pip install notebook papermill ipywidgets jupytext bash_kernel
 RUN /project/pdo/run/bin/python -m bash_kernel.install
@@ -82,8 +112,6 @@ WORKDIR /project/pdo/tools
 COPY --chown=${UNAME}:${UNAME} tools/*.sh ./
 
 # build it!!!
-ARG UID=1000
-ARG GID=${UID}
 RUN --mount=type=cache,uid=${UID},gid=${GID},target=/project/pdo/.cache/pip \
     /project/pdo/tools/build_contracts.sh
 

docker/require_pdo_images.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# Copyright 2023 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCRIPT_NAME=$(basename ${BASH_SOURCE[-1]} )
+source ${PDO_SOURCE_ROOT}/bin/lib/common.sh
+
+# -----------------------------------------------------------------
+# Process command line arguments
+# -----------------------------------------------------------------
+F_REGISTRY=
+F_VERSION=latest
+
+F_USAGE='-r|--registry [registry] -v|--version [pdo_version]'
+SHORT_OPTS='r:v:'
+LONG_OPTS='registry:,version:'
+
+TEMP=$(getopt -o ${SHORT_OPTS} --long ${LONG_OPTS} -n "${SCRIPT_NAME}" -- "$@")
+if [ $? != 0 ] ; then echo "Usage: ${SCRIPT_NAME} ${F_USAGE}" >&2 ; exit 1 ; fi
+
+eval set -- "$TEMP"
+while true ; do
+    case "$1" in
+        -r|--registry) F_REGISTRY="$2" ; shift 2 ;;
+        -v|--version) F_VERSION="$2" ; shift 2 ;;
+        --help) echo "Usage: ${SCRIPT_NAME} ${F_USAGE}"; exit 0 ;;
+    	--) shift ; break ;;
+    	*) echo "Internal error!" ; exit 1 ;;
+    esac
+done
+
+# if a registry is specified then make sure it has a trailing '/'
+if [ -n "${F_REGISTRY}" ]; then
+   if [[ "${F_REGISTRY}" != */ ]]; then
+       F_REGISTRY=${F_REGISTRY}/
+   fi
+fi
+
+# check for each of the PDO images
+for image in pdo_ccf pdo_services pdo_client ; do
+    # First check to see if there is a local version that matches
+    if docker inspect ${F_REGISTRY}$image:${F_VERSION} > /dev/null 2>&1 ; then
+        continue
+    fi
+
+    # Pull the image if there is a registry and fail if not
+    yell Attempt to pull ${F_REGISTRY}$image:${F_VERSION}
+    docker pull ${F_REGISTRY}$image:${F_VERSION} > /dev/null 2>&1 || \
+        die Unable to find ${F_REGISTRY}$image:${F_VERSION}
+done
+
+exit 0

docker/test.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ------------------------------------------------------------------------------
-version: "3.4"
 
 # Note that we do not need to specify PDO_HOSTNAME or PDO_LEDGER_URL
 # (or the corresponding --inteface or --ledger switches) for the test
@@ -21,15 +20,15 @@ version: "3.4"
 
 services:
   ccf_container:
-    image: ${PDO_REPOSITORY:-}pdo_ccf:${PDO_VERSION:-latest}
+    image: ${PDO_REGISTRY:-}pdo_ccf:${PDO_VERSION:-latest}
     container_name: ccf_container
     network_mode: "host"
     volumes:
       - ./xfer/:/project/pdo/xfer/
     entrypoint: /project/pdo/tools/start_ccf.sh -m build -i localhost --start
 
   services_container:
-    image: ${PDO_REPOSITORY:-}pdo_services:${PDO_VERSION:-latest}
+    image: ${PDO_REGISTRY:-}pdo_services:${PDO_VERSION:-latest}
     container_name: services_container
     depends_on: [ ccf_container ]
     network_mode: "host"

docker/test_dev.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ------------------------------------------------------------------------------
-version: "3.4"
 
 # This compose file provides a means of starting the basic services
 # necessary for developing the contracts image. These can be started
@@ -22,15 +21,15 @@ version: "3.4"
 
 services:
   ccf_container:
-    image: ${PDO_REPOSITORY:-}pdo_ccf:${PDO_VERSION:-latest}
+    image: ${PDO_REGISTRY:-}pdo_ccf:${PDO_VERSION:-latest}
     container_name: ccf_container
     network_mode: "host"
     volumes:
       - ./xfer/:/project/pdo/xfer/
     entrypoint: /project/pdo/tools/start_ccf.sh -m build -i localhost --start
 
   services_container:
-    image: ${PDO_REPOSITORY:-}pdo_services:${PDO_VERSION:-latest}
+    image: ${PDO_REGISTRY:-}pdo_services:${PDO_VERSION:-latest}
     container_name: services_container
     depends_on: [ ccf_container ]
     network_mode: "host"

docker/test_jupyter.yaml

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ------------------------------------------------------------------------------
-version: "3.4"
 
 # This file creates a complete environment for testing the contracts
 # through the jupyter notebook interface.
@@ -24,15 +23,15 @@ version: "3.4"
 
 services:
   ccf_container:
-    image: ${PDO_REPOSITORY:-}pdo_ccf:${PDO_VERSION:-latest}
+    image: ${PDO_REGISTRY:-}pdo_ccf:${PDO_VERSION:-latest}
     container_name: ccf_container
     network_mode: "host"
     volumes:
       - ./xfer/:/project/pdo/xfer/
     entrypoint: /project/pdo/tools/start_ccf.sh -m build -i localhost --start
 
   services_container:
-    image: ${PDO_REPOSITORY:-}pdo_services:${PDO_VERSION:-latest}
+    image: ${PDO_REGISTRY:-}pdo_services:${PDO_VERSION:-latest}
     container_name: services_container
     depends_on: [ ccf_container ]
     network_mode: "host"