Add encoding tests for API framework #2104
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run integration and API tests | |
| on: | |
| push: | |
| branches: | |
| - development | |
| - 'rc-[0-9]+.[0-9]+.[0-9]+' | |
| - 'release-[0-9]+.[0-9]+' | |
| - "**-atr" | |
| # We can force an integration/API test run without opening a PR by pushing to a branch name that ends with "-atr" | |
| pull_request: | |
| jobs: | |
| build: | |
| runs-on: ubuntu-24.04 | |
| if: | | |
| ${{ ! contains(github.event.head_commit.message, '#noapitests') && | |
| ( github.event_name == 'push' || | |
| github.event.pull_request.head.repo.full_name == github.event.pull_request.base.repo.full_name | |
| ) | |
| }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Set up Python | |
| uses: actions/setup-python@v7 | |
| with: | |
| python-version: 3.9 | |
| - name: Extract branch name | |
| shell: bash | |
| run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT | |
| id: extract_branch | |
| # For release branches, we use the Docker setup version of that release. | |
| # For rc branches (e.g. rc-2.1.0, rc-2.1.1), we use the Docker setup of the matching | |
| # release branch (e.g. release-2.1), except for ".0" rc branches (e.g. rc-2.1.0), because | |
| # we usually don't have Docker images for a release branch if the first patch release | |
| # is not ready yet. | |
| # For other branches, we use the Docker setup of the development branch. | |
| - name: Determine Yoda repository branch | |
| shell: bash | |
| run: | | |
| branch="${{ steps.extract_branch.outputs.branch }}" | |
| if [[ "$branch" =~ ^rc-([0-9]+\.[0-9]+)\.([0-9]+)$ ]]; then | |
| if [[ "${BASH_REMATCH[2]}" == "0" ]]; then | |
| echo "branch=development" >> $GITHUB_OUTPUT | |
| else | |
| echo "branch=release-${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT | |
| fi | |
| elif [[ "$branch" =~ ^"release-" ]]; then | |
| echo "branch=$branch" >> $GITHUB_OUTPUT | |
| else | |
| echo "branch=development" >> $GITHUB_OUTPUT | |
| fi | |
| id: yoda_repo_branch | |
| - name: Install API test dependencies | |
| run: | | |
| pip3 install --user -r tests/requirements.txt | |
| sudo apt install -y docker-compose | |
| - name: Clone Yoda repo for Docker Setup | |
| run: | | |
| git clone -b "${{ steps.yoda_repo_branch.outputs.branch }}" --single-branch https://github.com/UtrechtUniversity/yoda.git | |
| - name: Prepare hosts file for API tests | |
| run: | | |
| sudo echo "127.0.0.1 portal.yoda eus.yoda data.yoda public.yoda" | sudo tee -a /etc/hosts | |
| - name: Start Dockerized Yoda | |
| run: | | |
| cd yoda/docker/compose | |
| docker-compose pull | |
| ../up.sh -d | |
| - name: Wait until Dockerized setup is ready | |
| shell: bash | |
| run: | | |
| echo "Waiting for portal to be available ..." | |
| until $(curl -k --output /dev/null --silent --head --fail https://portal.yoda:8443 ); do printf '.' ; sleep 1; done | |
| echo "Waiting for EUS to be available ..." | |
| until $(curl -k --output /dev/null --silent --head --fail https://eus.yoda:8444 ); do printf '.' ; sleep 1; done | |
| docker exec provider.yoda sh -c 'while ! pgrep irodsServer > /dev/null ; do echo Waiting for iRODS to start ... ; sleep 1; done' | |
| - name: Pull and install latest version of ruleset | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec provider.yoda sh -c 'set -x ; find /etc/irods/yoda-ruleset -print0 2>/dev/null | sudo xargs -0 -r -I{} bash -c "chown -v irods:irods \"{}\" || true"' | |
| docker exec provider.yoda sh -c 'set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && sudo -u irods git pull && sudo -u irods git status' | |
| docker exec provider.yoda sh -c "set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods git checkout ${{ steps.extract_branch.outputs.branch }} && sudo -u irods python3 -m pip --no-cache-dir install --user --break-system-packages -r /etc/irods/yoda-ruleset/requirements.txt && sudo -u irods make && sudo -u irods make install" | |
| docker exec provider.yoda sh -c 'kill -HUP $(cat /var/run/irods/irods-server.pid)' | |
| - name: Enable code coverage measurements for ruleset | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec provider.yoda sh -c "set -x; sudo perl -pi -e \"s/^(measure_coverage\\s*=\\s*)'false'/\\1'true'/\" /etc/irods/yoda-ruleset/rules_uu.cfg" | |
| # We also run the unit tests in this workflow so that code covered by the unit tests is taken | |
| # into account for the total test coverage. | |
| - name: Run unit tests | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec provider.yoda sudo -u irods bash -c 'set -x; cd /etc/irods/yoda-ruleset/unit-tests; export PYTHONPATH=$(cd ../util ; pwd):$PYTHONPATH; python3 -m coverage run --source=..,../util --omit=test_*.py,unit_tests.py --data-file /tmp/coverage.dat.unittests -m unittest unit_tests' | |
| - name: Pull and install latest version of portal | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec portal.yoda sh -c 'set -x ; cd /var/www/yoda && git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git pull' | |
| docker exec portal.yoda sh -c 'set -x ; cd /var/www/yoda && git checkout ${{ steps.extract_branch.outputs.branch }} || git checkout development' | |
| docker exec portal.yoda sh -c 'set -x ; cd /var/www/yoda && . venv/bin/activate && venv/bin/pip3 install -r requirements.txt' | |
| docker exec portal.yoda sh -c 'set -x ; cd /var/www/yoda && git status' | |
| docker exec portal.yoda sh -c 'set -x ; touch /var/www/yoda/*.wsgi' | |
| - name: Wait until iRODS is ready for integration tests | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| echo "Waiting for iRODS to be available for integration tests ..." | |
| docker exec provider.yoda /bin/bash -c "until nc -vz provider.yoda 1247 >& /dev/null ; do print '.'; sleep 1; done" | |
| echo "iRODS is ready for integration tests." | |
| - name: Run integration tests | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec provider.yoda sh -c 'set -x ; cd /etc/irods/yoda-ruleset && sudo -u irods /usr/bin/irule -r irods_rule_engine_plugin-python-instance -F tools/run-integration-tests.r' > test-results.txt | |
| echo "Integration test results: " | |
| echo | |
| cat test-results.txt | |
| echo | |
| echo "Statistics:" | |
| echo | |
| echo "$(grep -c VERDICT_OK test-results.txt) test(s) OK" | |
| echo "$(grep -c VERDICT_FAILED test-results.txt) test(s) failed with unexpected return value" | |
| echo "$(grep -c VERDICT_EXCEPTION test-results.txt) test(s) failed with exception" | |
| echo "$(grep -c VERDICT_ test-results.txt) test(s) in total" | |
| echo | |
| echo "Overall result: " | |
| if grep -E "VERDICT_EXCEPTION|VERDICT_FAILED" test-results.txt> /dev/null ; then printf "Test failures detected"; exit 1; fi | |
| if grep -E "VERDICT_" test-results.txt > /dev/null ; then printf "Tests succeeded"; else printf "Failure to run tests"; exit 1; fi | |
| - name: Run API tests | |
| shell: bash | |
| run: | | |
| cd tests | |
| nohup bash -c 'while true ; do sleep 5 ; ../yoda/docker/run-cronjob.sh copytovault >> ../copytovault.log 2>&1 ; ../yoda/docker/run-cronjob.sh publication >> ../publication.log 2>&1 ; done' & | |
| test -d mycache || mkdir -p mycache | |
| python3 -m pytest --skip-ui --deposit --datarequest --webdav -o cache_dir=mycache --environment environments/docker.json | |
| cat ../copytovault.log | |
| cat ../publication.log | |
| - name: Print test coverage report | |
| shell: bash | |
| run: | | |
| cd yoda/docker/compose | |
| docker exec provider.yoda sh -c 'cd /tmp; sudo -u irods python3 -m coverage combine -q /tmp/coverage.dat.*' | |
| docker exec provider.yoda sh -c 'cd /tmp; sudo -u irods python3 -m coverage report' | |
| - name: Output rodsLogs | |
| if: failure() | |
| run: | | |
| docker exec provider.yoda sh -c 'cat /var/lib/irods-log/rodsLog*' | |
| - name: Output portal web server logs | |
| if: failure() | |
| run: | | |
| docker exec portal.yoda sh -c 'set -x ; for log in error.log portal_access.log ; do echo "${log}:" ; cat "/var/log/apache2/$log" ; echo; done' | |
| - name: Output EUS server logs | |
| if: failure() | |
| run: | | |
| docker exec eus.yoda sh -c 'set -x ; for log in error.log access.log ; do echo "${log}:" ; cat "/var/log/apache2/$log" ; echo; done' | |
| - name: Output DavRODS server logs | |
| if: failure() | |
| run: | | |
| docker exec data.yoda sh -c 'set -x ; for log in error.log webdav_access.log webdav_anon_access.log; do echo "${log}:" ; cat "/var/log/apache2/$log" ; echo; done' | |
| # Uncomment section below when needed for debugging. | |
| # | |
| # - name: Setup tmate session for debugging | |
| # uses: mxschmitt/action-tmate@v3 | |
| # if: ${{ failure() }} | |
| # with: | |
| # limit-access-to-actor: true |