From 1b4c6f985394471ee36dc43df5d65a0cc9d69704 Mon Sep 17 00:00:00 2001 From: saville Date: Wed, 16 Oct 2024 19:13:42 -0600 Subject: [PATCH 1/2] Remove deploy key usage and instead use temporary SSH key --- .github/workflows/build.yaml | 20 --------------- tests/config-files/dot-buildrunner.yaml | 5 ++-- tests/test-files/test-ssh-buildx.yaml | 20 +++++++++------ tests/test-files/test-ssh.yaml | 22 +++++++++++------ tests/test_buildrunner_files.py | 33 +++++++++++++++++++++++++ 5 files changed, 63 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0684a21f..80db8760 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -34,26 +34,6 @@ jobs: pip install -r test_requirements.txt - name: Pre-commit checks run: pre-commit run --all-files - - name: Write SSH keys - env: - # These were generated with: - # ssh-keygen -t ecdsa -m PEM '' - # and then installed as a read-only deploy key on the buildrunner repository - # - # Since this is a public repository and the key provides the same rights that even anonymous users have, - # this key is rather worthless and can be stored safely in code here. This *could* be used as a secret, - # but since secrets are not available to forks, we cannot test SSH functionality in PRs which defeats - # the purpose somewhat. - # - # Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa - DEPLOY_SSH_KEY: "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW\nQyNTUxOQAAACBasvUoRzAAdHZ5nFWtDR/5DQU+FWtDYNXD0xPGSdjKtwAAAJiLXobki16G\n5AAAAAtzc2gtZWQyNTUxOQAAACBasvUoRzAAdHZ5nFWtDR/5DQU+FWtDYNXD0xPGSdjKtw\nAAAEBcRwB1PEnUHF5aK6q3JYyuOlT+adQ0mcRrIxsmJiiq1Vqy9ShHMAB0dnmcVa0NH/kN\nBT4Va0Ng1cPTE8ZJ2Mq3AAAAEWJ1aWxkcnVubmVyQGFkb2JlAQIDBA==\n-----END OPENSSH PRIVATE KEY-----" - DEPLOY_SSH_KEY_PUB: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFqy9ShHMAB0dnmcVa0NH/kNBT4Va0Ng1cPTE8ZJ2Mq3 buildrunner@adobe" - run: | - mkdir -p ~/.ssh - printf -- "$DEPLOY_SSH_KEY" > ~/.ssh/buildrunner-deploy-id_rsa - printf -- "$DEPLOY_SSH_KEY_PUB" > ~/.ssh/buildrunner-deploy-id_rsa.pub - chmod 700 ~/.ssh - chmod 600 ~/.ssh/buildrunner-deploy-* - name: Test with pytest run: | pytest -v -m "not serial" --numprocesses=auto --junitxml=test-reports/non-serial-test-results.xml diff --git a/tests/config-files/dot-buildrunner.yaml b/tests/config-files/dot-buildrunner.yaml index b16074ec..8c37db49 100644 --- a/tests/config-files/dot-buildrunner.yaml +++ b/tests/config-files/dot-buildrunner.yaml @@ -11,9 +11,10 @@ ssh-keys: aliases: - 'adobe-github' - 'adobe-dma' -- file: ~/.ssh/buildrunner-deploy-id_rsa +# This should match what is in the test_buildrunner_files.py file and is generated before the tests are run +- file: /tmp/buildrunner-test-id_rsa aliases: - - 'buildrunner-deploy' + - 'buildrunner-test-ssh' local-files: tests: ./tests diff --git a/tests/test-files/test-ssh-buildx.yaml b/tests/test-files/test-ssh-buildx.yaml index 15b285df..00db3f59 100644 --- a/tests/test-files/test-ssh-buildx.yaml +++ b/tests/test-files/test-ssh-buildx.yaml @@ -1,5 +1,3 @@ -# Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa -# Look in the .github/workflows/build.yaml file for the SSH private key use-legacy-builder: False steps: clone: @@ -8,12 +6,20 @@ steps: FROM {{ DOCKER_REGISTRY }}/rockylinux:8.5 RUN yum install -y git-core openssh-clients && yum clean all run: - ssh-keys: ['buildrunner-deploy'] + # This key is generated automatically by the test runner + ssh-keys: ['buildrunner-test-ssh'] + services: + ssh-service: + image: {{ DOCKER_REGISTRY }}/linuxserver/openssh-server:latest + wait_for: + - 2222 + env: + # This is set in the test_buildrunner_files.py file + PUBLIC_KEY: "{{ BUILDRUNNER_TEST_SSH_PUB_KEY }}" + LOG_STDOUT: "true" cmds: - mkdir ~/.ssh - - ssh-keyscan github.com > ~/.ssh/known_hosts + - ssh-keyscan -p 2222 ssh-service > ~/.ssh/known_hosts - chmod 700 ~/.ssh - chmod 600 ~/.ssh/known_hosts - # Clone into temp directory since the "buildrunner" directory may already exist - - rm -rf /tmp/test-clone - - git clone git@github.com:adobe/buildrunner.git /tmp/test-clone + - if [ "$(ssh -p 2222 linuxserver.io@ssh-service 'whoami')" != "linuxserver.io" ]; then exit 1; fi diff --git a/tests/test-files/test-ssh.yaml b/tests/test-files/test-ssh.yaml index b9f1e6b0..15730ecd 100644 --- a/tests/test-files/test-ssh.yaml +++ b/tests/test-files/test-ssh.yaml @@ -1,18 +1,24 @@ -# Please note that for the SSH unit tests to pass on a local machine, the private key needs to be placed in ~/.ssh/buildrunner-deploy-id_rsa. -# Look in the .github/workflows/build.yaml file for the SSH private key. steps: - clone: + ssh-test: build: dockerfile: | FROM {{ DOCKER_REGISTRY }}/rockylinux:8.5 RUN yum install -y git-core openssh-clients && yum clean all run: - ssh-keys: ['buildrunner-deploy'] + # This key is generated automatically by the test runner + ssh-keys: ['buildrunner-test-ssh'] + services: + ssh-service: + image: {{ DOCKER_REGISTRY }}/linuxserver/openssh-server:latest + wait_for: + - 2222 + env: + # This is set in the test_buildrunner_files.py file + PUBLIC_KEY: "{{ BUILDRUNNER_TEST_SSH_PUB_KEY }}" + LOG_STDOUT: "true" cmds: - mkdir ~/.ssh - - ssh-keyscan github.com > ~/.ssh/known_hosts + - ssh-keyscan -p 2222 ssh-service > ~/.ssh/known_hosts - chmod 700 ~/.ssh - chmod 600 ~/.ssh/known_hosts - # Clone into temp directory since the "buildrunner" directory may already exist - - rm -rf /tmp/test-clone - - git clone git@github.com:adobe/buildrunner.git /tmp/test-clone + - if [ "$(ssh -p 2222 linuxserver.io@ssh-service 'whoami')" != "linuxserver.io" ]; then exit 1; fi diff --git a/tests/test_buildrunner_files.py b/tests/test_buildrunner_files.py index e29b98bf..818ed736 100644 --- a/tests/test_buildrunner_files.py +++ b/tests/test_buildrunner_files.py @@ -1,11 +1,16 @@ import os import pytest import platform +import subprocess import tempfile +from pathlib import Path from typing import List, Optional, Tuple from tests import test_runner +# This should match what is in the dot-buildrunner.yaml file +TEST_SSH_KEY_FILE = "/tmp/buildrunner-test-id_rsa" + test_dir_path = os.path.realpath(os.path.dirname(__file__)) TEST_DIR = os.path.dirname(__file__) top_dir_path = os.path.realpath(os.path.dirname(test_dir_path)) @@ -18,6 +23,34 @@ ] +@pytest.fixture(autouse=True, scope="session") +def setup_buildrunner_test_ssh_key(): + key_file_path = Path(TEST_SSH_KEY_FILE) + key_file_path.unlink(missing_ok=True) + subprocess.run( + [ + "ssh-keygen", + "-t", + "ecdsa", + "-m", + "PEM", + "-N", + "", + "-f", + TEST_SSH_KEY_FILE, + ], + check=True, + ) + # Set the public key in an environment variable to use in the test buildrunner files + os.environ["BUILDRUNNER_TEST_SSH_PUB_KEY"] = ( + Path(f"{TEST_SSH_KEY_FILE}.pub").read_text().strip() + ) + yield + # Cleanup + del os.environ["BUILDRUNNER_TEST_SSH_PUB_KEY"] + key_file_path.unlink() + + def _get_test_args(file_name: str) -> Optional[List[str]]: if file_name == "test-timeout.yaml": # Set a short timeout here for the timeout test From 146f0c495e17459d05db1e56e25fd1bfa5ac9c2c Mon Sep 17 00:00:00 2001 From: saville Date: Thu, 17 Oct 2024 16:02:19 -0600 Subject: [PATCH 2/2] Remove deploy key usage and instead use temporary SSH key --- .github/workflows/build.yaml | 2 ++ tests/test_buildrunner_files.py | 41 ++++++++++++++++++--------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 80db8760..0018c400 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -35,7 +35,9 @@ jobs: - name: Pre-commit checks run: pre-commit run --all-files - name: Test with pytest + # Create the ssh key file once for all testing run: | + ssh-keygen -t ecdsa -m PEM -N '' -f /tmp/buildrunner-test-id_rsa pytest -v -m "not serial" --numprocesses=auto --junitxml=test-reports/non-serial-test-results.xml pytest -v -m "serial" --junitxml=test-reports/serial-test-results.xml python scripts/combine_xml.py test-reports/serial-test-results.xml test-reports/non-serial-test-results.xml > test-reports/test-result.xml diff --git a/tests/test_buildrunner_files.py b/tests/test_buildrunner_files.py index 818ed736..0957312b 100644 --- a/tests/test_buildrunner_files.py +++ b/tests/test_buildrunner_files.py @@ -26,29 +26,32 @@ @pytest.fixture(autouse=True, scope="session") def setup_buildrunner_test_ssh_key(): key_file_path = Path(TEST_SSH_KEY_FILE) - key_file_path.unlink(missing_ok=True) - subprocess.run( - [ - "ssh-keygen", - "-t", - "ecdsa", - "-m", - "PEM", - "-N", - "", - "-f", - TEST_SSH_KEY_FILE, - ], - check=True, - ) + cleanup_key_file = False + pub_key_file_path = Path(f"{TEST_SSH_KEY_FILE}.pub") + if not key_file_path.exists(): + subprocess.run( + [ + "ssh-keygen", + "-t", + "ecdsa", + "-m", + "PEM", + "-N", + "", + "-f", + TEST_SSH_KEY_FILE, + ], + check=True, + ) + cleanup_key_file = True # Set the public key in an environment variable to use in the test buildrunner files - os.environ["BUILDRUNNER_TEST_SSH_PUB_KEY"] = ( - Path(f"{TEST_SSH_KEY_FILE}.pub").read_text().strip() - ) + os.environ["BUILDRUNNER_TEST_SSH_PUB_KEY"] = pub_key_file_path.read_text().strip() yield # Cleanup del os.environ["BUILDRUNNER_TEST_SSH_PUB_KEY"] - key_file_path.unlink() + if cleanup_key_file: + key_file_path.unlink() + pub_key_file_path.unlink() def _get_test_args(file_name: str) -> Optional[List[str]]: