diff --git a/.github/workflows/full_test.yml b/.github/workflows/full_test.yml new file mode 100644 index 0000000..575ee51 --- /dev/null +++ b/.github/workflows/full_test.yml @@ -0,0 +1,62 @@ +# +# SPDX-License-Identifier: Apache-2.0 +# + +# This workflow is intended to be used as a validity test for any +# pull request. That is, this is a minimal functionality that must +# be successfully executed prior to merging a pull request. Note +# that this can be overridden by adding '[skip ci]' in the commit +# name. + +name: Run full PDO contract tests +on: [ workflow_dispatch, pull_request ] + +jobs: + pdo_ci: + if: "!contains(github.event.commits[0].message, '[skip ci]')" + name: PDO Contracts Full Test + runs-on: ubuntu-22.04 + + env: + REGISTRY: ghcr.io + OWNER: ${{ github.repository_owner }} + PDO_USER_UID: 55172 + PDO_GROUP_UID: 55172 + PDO_SOURCE_ROOT: private-data-objects + steps: + - name: Check out repo + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + fetch-tags: true + + - name: Set the version + run: | + echo "PDO_VERSION=$(${PDO_SOURCE_ROOT}/bin/get_version -f ${PDO_SOURCE_ROOT}/VERSION)" >> $GITHUB_ENV + echo "PDO_CONTRACTS_VERSION=$(bin/get_version)" >> $GITHUB_ENV + + - name: Show the version + run: | + echo "PDO_CONTRACTS_VERSION is $PDO_CONTRACTS_VERSION" + echo "PDO_VERSION is $PDO_VERSION" + + - name: Configure PDO and build the base images + run: | + if $(docker/require_pdo_images.sh -r ${REGISTRY}/${OWNER} -v ${PDO_VERSION}) + then + echo Using pre-built PDO images + else + echo No pre-built PDO images available for version ${PDO_VERSION} from ${REGISTRY} + echo Please populate the registry and retry + exit -1 + fi + + - name: Build and run contract tests + run: | + git checkout -b ci-test-branch + sudo chown -R $PDO_USER_UID:$PDO_GROUP_UID docker/xfer + sudo chmod -R g+w docker/xfer + make -C docker test \ + CONTRACTS_USER_UID=$PDO_USER_UID CONTRACTS_GROUP_UID=$PDO_GROUP_UID \ + PDO_VERSION=$PDO_VERSION PDO_REGISTRY=$REGISTRY/$OWNER diff --git a/docker/Makefile b/docker/Makefile index 25b3c1d..69ac444 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -26,6 +26,7 @@ all : build_contracts # that the user/group identifiers need to have write access to the xfer directory CONTRACTS_USER_UID ?= $(shell id -u) CONTRACTS_GROUP_UID ?= $(shell id -g) +CONTRACTS_USER_NAME ?= $(shell whoami) DOCKER_COMMAND ?= docker @@ -68,15 +69,19 @@ PDO_VERSION ?= $$( \ echo "latest"; \ fi ) -PDO_REPSITORY ?= '' +# This variable ensures that the registry ends with a '/'. The additional +# variable ensures that the substitution is applied; conditional assignment +# of the registry appears to prevent early evaluation of the expression. +PDO_REGISTRY_SAFE := $(if $(PDO_REGISTRY),$(patsubst %/,%,$(PDO_REGISTRY))/) # DOCKER BUILD DOCKER_USERNAME = $(LOGNAME) +DOCKER_BUILDARGS += --build-arg UNAME=$(CONTRACTS_USER_NAME) DOCKER_BUILDARGS += --build-arg UID=$(CONTRACTS_USER_UID) DOCKER_BUILDARGS += --build-arg GID=$(CONTRACTS_GROUP_UID) DOCKER_BUILDARGS += --build-arg CONTRACTS_VERSION=$(CONTRACTS_VERSION) DOCKER_BUILDARGS += --build-arg PDO_VERSION=$(PDO_VERSION) -DOCKER_BUILDARGS += --build-arg PDO_REPOSITORY=$(PDO_REPOSITORY) +DOCKER_BUILDARGS += --build-arg PDO_REGISTRY=$(PDO_REGISTRY_SAFE) DOCKER_ARGS = $(DOCKER_BUILDARGS) # The CONTRACT_FAMILIES variable contains a list of contract families @@ -122,7 +127,6 @@ build_pdo_images : repository PDO_SOURCE_ROOT=$(DOCKER_DIR)/repository/private-data-objects \ make -C $(DOCKER_DIR)/repository/private-data-objects/docker clean_repository - # ----------------------------------------------------------------- # ----------------------------------------------------------------- # For the purposes of testing, we just use the resnet model for @@ -220,25 +224,25 @@ endif TEST_COMPOSE_ARGS += -f test.yaml test : clean_config clean_repository build_contracts - - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + - PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(TEST_COMPOSE_ARGS) $(COMPOSE_PROFILES) up --abort-on-container-exit - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(TEST_COMPOSE_ARGS) down JUPYTER_COMPOSE_ARGS += -f test_jupyter.yaml test_jupyter : clean_config clean_repository build_contracts - - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + - PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) $(COMPOSE_PROFILES) up --abort-on-container-exit - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) down JUPYTER_MULTIUSER_COMPOSE_ARGS += -f test_multiuser_jupyter.yaml test_multiuser_jupyter : clean_config clean_repository build_contracts - - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + - PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(JUPYTER_MULTIUSER_COMPOSE_ARGS) $(COMPOSE_PROFILES) up --abort-on-container-exit - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(JUPYTER_MULTIUSER_COMPOSE_ARGS) down # The example target builds and runs only the example contract family. This @@ -247,18 +251,18 @@ test_multiuser_jupyter : clean_config clean_repository build_contracts example : CONTRACT_FAMILIES=example-contract example : clean_config clean_repository build_contracts - - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + - PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ CONTRACTS_SOURCE_ROOT=$(CONTRACTS_SOURCE_ROOT) \ $(DOCKER_COMPOSE_COMMAND) $(TEST_COMPOSE_ARGS) up --abort-on-container-exit - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(TEST_COMPOSE_ARGS) down example_jupyter : CONTRACT_FAMILIES=exchange-contract example-contract example_jupyter : clean_config clean_repository build_contracts - - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + - PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ CONTRACTS_SOURCE_ROOT=$(CONTRACTS_SOURCE_ROOT) \ $(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) up --abort-on-container-exit - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) down # The developer target creates a container with no contracts built or @@ -270,10 +274,10 @@ example_jupyter : clean_config clean_repository build_contracts DEVELOP_COMPOSE_ARGS += -f test_dev.yaml developer : CONTRACT_FAMILIES= developer : clean_config build_contracts - - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + - PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ CONTRACTS_SOURCE_ROOT=$(CONTRACTS_SOURCE_ROOT) \ $(DOCKER_COMPOSE_COMMAND) $(DEVELOP_COMPOSE_ARGS) up --abort-on-container-exit - PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ + PDO_REGISTRY=$(PDO_REGISTRY_SAFE) PDO_VERSION=$(PDO_VERSION) CONTRACTS_VERSION=$(CONTRACTS_VERSION) \ $(DOCKER_COMPOSE_COMMAND) $(JUPYTER_COMPOSE_ARGS) down # ----------------------------------------------------------------- diff --git a/docker/README.md b/docker/README.md index c9daaea..07f3bea 100644 --- a/docker/README.md +++ b/docker/README.md @@ -59,7 +59,7 @@ Four configuration variables should be set as necessary: user. * `PDO_VERSION` -- the version identifier for PDO images; this defaults to `latest`. -* `PDO_REPOSITORY` -- the docker repository from which PDO docker images +* `PDO_REGISTRY` -- the docker repository from which PDO docker images may be retrieved; this defaults to `""` so that local images will be used by default. * `CONTRACTS_VERSION` -- the version that will be used to tag the PDO diff --git a/docker/pdo_contracts.dockerfile b/docker/pdo_contracts.dockerfile index 486939e..2cff9af 100644 --- a/docker/pdo_contracts.dockerfile +++ b/docker/pdo_contracts.dockerfile @@ -34,27 +34,57 @@ # PURPOSE: develop and test contracts with clean -ARG PDO_REPOSITORY -ARG PDO_VERSION -FROM ${PDO_REPOSITORY}pdo_client:${PDO_VERSION} +ARG PDO_REGISTRY= +ARG PDO_VERSION=latest +FROM ${PDO_REGISTRY}pdo_client:${PDO_VERSION} # ----------------------------------------------------------------- -# Install the necessary dependencies +# Install the necessary system dependencies # ----------------------------------------------------------------- -USER 0:0 -ENV DEBIAN_FRONTEND "noninteractive" -RUN apt-get update \ +USER root +ENV DEBIAN_FRONTEND="noninteractive" +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update \ && apt-get install -y -q \ - libgl1 + libgl1 + +# ----------------------------------------------------------------- +# Create a user account from the specification +# ----------------------------------------------------------------- +ARG UNAME=pdo_contract +ENV UNAME=${UNAME} + +ARG UID=1000 +ARG GID=${UID} + +# Create a group for the UID/GID if it does not already exist, rename +# it to the correct group if it does exist +RUN if getent group ${GID} > /dev/null 2>&1 ; then \ + groupmod -n ${UNAME} $(getent group ${GID} | cut -d: -f1) ; \ + else \ + groupadd -f -g ${GID} -o ${UNAME} ; \ + fi + +# Create a user for the UID/GID if it does not already exist, rename +# it to the correct login if it does exist +RUN if getent passwd ${UID} > /dev/null 2>&1 ; then \ + usermod -l ${UNAME} $(getent passwd ${UID} | cut -d: -f1) ; \ + else \ + useradd -m -u ${UID} -g ${GID} -d /project/pdo -o -s /bin/bash $UNAME ; \ + fi + +# Reassign ownership of the PDO directory tree to the user:group +RUN chown --recursive $UNAME:$UNAME /project/pdo + +# Use the user account for the rest of the installation process +USER $UNAME # Many of the dependencies should be addressed with the installation # of the PDO client including the common contracts, the wasi toolkit # and all of the WAMR toolchain # This should set up all we need for the jupyter server -ARG UID=1000 -ARG GID=${UID} -USER ${UID}:${GID} RUN --mount=type=cache,uid=${UID},gid=${GID},target=/project/pdo/.cache/pip \ /project/pdo/run/bin/pip install notebook papermill ipywidgets jupytext bash_kernel RUN /project/pdo/run/bin/python -m bash_kernel.install @@ -82,8 +112,6 @@ WORKDIR /project/pdo/tools COPY --chown=${UNAME}:${UNAME} tools/*.sh ./ # build it!!! -ARG UID=1000 -ARG GID=${UID} RUN --mount=type=cache,uid=${UID},gid=${GID},target=/project/pdo/.cache/pip \ /project/pdo/tools/build_contracts.sh diff --git a/docker/require_pdo_images.sh b/docker/require_pdo_images.sh new file mode 100755 index 0000000..cfe60a9 --- /dev/null +++ b/docker/require_pdo_images.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Copyright 2023 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_NAME=$(basename ${BASH_SOURCE[-1]} ) +source ${PDO_SOURCE_ROOT}/bin/lib/common.sh + +# ----------------------------------------------------------------- +# Process command line arguments +# ----------------------------------------------------------------- +F_REGISTRY= +F_VERSION=latest + +F_USAGE='-r|--registry [registry] -v|--version [pdo_version]' +SHORT_OPTS='r:v:' +LONG_OPTS='registry:,version:' + +TEMP=$(getopt -o ${SHORT_OPTS} --long ${LONG_OPTS} -n "${SCRIPT_NAME}" -- "$@") +if [ $? != 0 ] ; then echo "Usage: ${SCRIPT_NAME} ${F_USAGE}" >&2 ; exit 1 ; fi + +eval set -- "$TEMP" +while true ; do + case "$1" in + -r|--registry) F_REGISTRY="$2" ; shift 2 ;; + -v|--version) F_VERSION="$2" ; shift 2 ;; + --help) echo "Usage: ${SCRIPT_NAME} ${F_USAGE}"; exit 0 ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; + esac +done + +# if a registry is specified then make sure it has a trailing '/' +if [ -n "${F_REGISTRY}" ]; then + if [[ "${F_REGISTRY}" != */ ]]; then + F_REGISTRY=${F_REGISTRY}/ + fi +fi + +# check for each of the PDO images +for image in pdo_ccf pdo_services pdo_client ; do + # First check to see if there is a local version that matches + if docker inspect ${F_REGISTRY}$image:${F_VERSION} > /dev/null 2>&1 ; then + continue + fi + + # Pull the image if there is a registry and fail if not + yell Attempt to pull ${F_REGISTRY}$image:${F_VERSION} + docker pull ${F_REGISTRY}$image:${F_VERSION} > /dev/null 2>&1 || \ + die Unable to find ${F_REGISTRY}$image:${F_VERSION} +done + +exit 0 diff --git a/docker/test.yaml b/docker/test.yaml index ae6c25e..c1df690 100644 --- a/docker/test.yaml +++ b/docker/test.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------ -version: "3.4" # Note that we do not need to specify PDO_HOSTNAME or PDO_LEDGER_URL # (or the corresponding --inteface or --ledger switches) for the test @@ -21,7 +20,7 @@ version: "3.4" services: ccf_container: - image: ${PDO_REPOSITORY:-}pdo_ccf:${PDO_VERSION:-latest} + image: ${PDO_REGISTRY:-}pdo_ccf:${PDO_VERSION:-latest} container_name: ccf_container network_mode: "host" volumes: @@ -29,7 +28,7 @@ services: entrypoint: /project/pdo/tools/start_ccf.sh -m build -i localhost --start services_container: - image: ${PDO_REPOSITORY:-}pdo_services:${PDO_VERSION:-latest} + image: ${PDO_REGISTRY:-}pdo_services:${PDO_VERSION:-latest} container_name: services_container depends_on: [ ccf_container ] network_mode: "host" diff --git a/docker/test_dev.yaml b/docker/test_dev.yaml index 4820bef..c1006b8 100644 --- a/docker/test_dev.yaml +++ b/docker/test_dev.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------ -version: "3.4" # This compose file provides a means of starting the basic services # necessary for developing the contracts image. These can be started @@ -22,7 +21,7 @@ version: "3.4" services: ccf_container: - image: ${PDO_REPOSITORY:-}pdo_ccf:${PDO_VERSION:-latest} + image: ${PDO_REGISTRY:-}pdo_ccf:${PDO_VERSION:-latest} container_name: ccf_container network_mode: "host" volumes: @@ -30,7 +29,7 @@ services: entrypoint: /project/pdo/tools/start_ccf.sh -m build -i localhost --start services_container: - image: ${PDO_REPOSITORY:-}pdo_services:${PDO_VERSION:-latest} + image: ${PDO_REGISTRY:-}pdo_services:${PDO_VERSION:-latest} container_name: services_container depends_on: [ ccf_container ] network_mode: "host" diff --git a/docker/test_jupyter.yaml b/docker/test_jupyter.yaml index a29332a..fb4a261 100644 --- a/docker/test_jupyter.yaml +++ b/docker/test_jupyter.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------ -version: "3.4" # This file creates a complete environment for testing the contracts # through the jupyter notebook interface. @@ -24,7 +23,7 @@ version: "3.4" services: ccf_container: - image: ${PDO_REPOSITORY:-}pdo_ccf:${PDO_VERSION:-latest} + image: ${PDO_REGISTRY:-}pdo_ccf:${PDO_VERSION:-latest} container_name: ccf_container network_mode: "host" volumes: @@ -32,7 +31,7 @@ services: entrypoint: /project/pdo/tools/start_ccf.sh -m build -i localhost --start services_container: - image: ${PDO_REPOSITORY:-}pdo_services:${PDO_VERSION:-latest} + image: ${PDO_REGISTRY:-}pdo_services:${PDO_VERSION:-latest} container_name: services_container depends_on: [ ccf_container ] network_mode: "host" diff --git a/docker/test_multiuser_jupyter.yaml b/docker/test_multiuser_jupyter.yaml index 05d45c4..bb789e2 100644 --- a/docker/test_multiuser_jupyter.yaml +++ b/docker/test_multiuser_jupyter.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------ -version: "3.4" # This file creates a complete environment for testing the contracts # through the jupyter notebook interface. @@ -24,7 +23,7 @@ version: "3.4" services: ccf_container: - image: ${PDO_REPOSITORY:-}pdo_ccf:${PDO_VERSION:-latest} + image: ${PDO_REGISTRY:-}pdo_ccf:${PDO_VERSION:-latest} container_name: ccf_container network_mode: "host" volumes: @@ -32,7 +31,7 @@ services: entrypoint: /project/pdo/tools/start_ccf.sh -m build -i localhost --start services_container: - image: ${PDO_REPOSITORY:-}pdo_services:${PDO_VERSION:-latest} + image: ${PDO_REGISTRY:-}pdo_services:${PDO_VERSION:-latest} container_name: services_container depends_on: [ ccf_container ] network_mode: "host"