From 3e5bfbf2b8d8da24b3c789d1ee4946918be6655e Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Fri, 11 Nov 2022 14:45:30 +0000 Subject: [PATCH 01/10] Only leave extended and smoke main pytest targets --- .devcontainer/devcontainer.json | 30 ---------------- .github/workflows/deploy_tre.yml | 4 +-- .github/workflows/pr_comment_bot.yml | 6 +--- Makefile | 4 --- e2e_tests/pytest.ini | 4 +-- ...ioned_health_api.py => test_api_health.py} | 2 +- e2e_tests/test_workspace_services.py | 2 +- e2e_tests/test_workspace_templates.py | 36 ------------------- 8 files changed, 6 insertions(+), 82 deletions(-) rename e2e_tests/{test_provisioned_health_api.py => test_api_health.py} (94%) delete mode 100644 e2e_tests/test_workspace_templates.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 573a093045..3970db6d87 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -89,36 +89,6 @@ "false" ] }, - { - "name": "E2E Extended AAD", - "type": "python", - "request": "launch", - "module": "pytest", - "justMyCode": true, - "cwd": "${workspaceFolder}/e2e_tests/", - "preLaunchTask": "Copy_env_file_for_e2e_debug", - "args": [ - "-m", - "extended_aad", - "--verify", - "false" - ] - }, - { - "name": "E2E Shared Services", - "type": "python", - "request": "launch", - "module": "pytest", - "justMyCode": true, - "cwd": "${workspaceFolder}/e2e_tests/", - "preLaunchTask": "Copy_env_file_for_e2e_debug", - "args": [ - "-m", - "shared_services", - "--verify", - "false" - ] - }, { "name": "E2E Performance", "type": "python", diff --git a/.github/workflows/deploy_tre.yml b/.github/workflows/deploy_tre.yml index fa4b0375b9..6f438f8789 100644 --- a/.github/workflows/deploy_tre.yml +++ b/.github/workflows/deploy_tre.yml @@ -28,9 +28,7 @@ jobs: uses: ./.github/workflows/deploy_tre_reusable.yml with: ciGitRef: ${{ github.ref }} - e2eTestsCustomSelector: >- - ${{ (github.event_name == 'push' && 'extended or extended_aad') - || 'extended or extended_aad' }} + e2eTestsCustomSelector: "extended" environmentName: ${{ github.event.inputs.environment || 'CICD' }} secrets: AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }} diff --git a/.github/workflows/pr_comment_bot.yml b/.github/workflows/pr_comment_bot.yml index 571de510cc..0f856af73b 100644 --- a/.github/workflows/pr_comment_bot.yml +++ b/.github/workflows/pr_comment_bot.yml @@ -135,9 +135,7 @@ jobs: needs: [pr_comment] if: | needs.pr_comment.outputs.command == 'run-tests' || - needs.pr_comment.outputs.command == 'run-tests-extended' || - needs.pr_comment.outputs.command == 'run-tests-extended-aad' || - needs.pr_comment.outputs.command == 'run-tests-shared-services' + needs.pr_comment.outputs.command == 'run-tests-extended' name: Deploy PR uses: ./.github/workflows/deploy_tre_reusable.yml with: @@ -146,8 +144,6 @@ jobs: ciGitRef: ${{ needs.pr_comment.outputs.ciGitRef }} e2eTestsCustomSelector: >- ${{ (needs.pr_comment.outputs.command == 'run-tests-extended' && 'extended') || - (needs.pr_comment.outputs.command == 'run-tests-extended-aad' && 'extended_aad') || - (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_services') || (needs.pr_comment.outputs.command == 'run-tests' && '') }} environmentName: CICD secrets: diff --git a/Makefile b/Makefile index ffe3915575..59a64b8d96 100644 --- a/Makefile +++ b/Makefile @@ -301,10 +301,6 @@ test-e2e-extended: $(call target_title, "Running E2E extended tests") && \ $(MAKE) test-e2e-custom SELECTOR=extended -test-e2e-extended-aad: - $(call target_title, "Running E2E extended AAD tests") && \ - $(MAKE) test-e2e-custom SELECTOR=extended_aad - test-e2e-shared-services: $(call target_title, "Running E2E shared service tests") && \ $(MAKE) test-e2e-custom SELECTOR=shared_services diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index faba35d49f..9614c409ea 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -2,11 +2,11 @@ markers = smoke: marks tests as smoke (run sometimes, relatively fast) extended: marks tests as extended (run less frequently, relatively slow) - extended_aad - shared_services performance: marks tests for performance evaluation timeout: used to set test timeout with pytest-timeout airlock: only airlock related + aad: only related to automatically created AAD configuration + shared_services: only related to shared services asyncio_mode = auto diff --git a/e2e_tests/test_provisioned_health_api.py b/e2e_tests/test_api_health.py similarity index 94% rename from e2e_tests/test_provisioned_health_api.py rename to e2e_tests/test_api_health.py index 14a9806c2d..f2c1e1a52c 100644 --- a/e2e_tests/test_provisioned_health_api.py +++ b/e2e_tests/test_api_health.py @@ -9,7 +9,7 @@ @pytest.mark.smoke -async def test_health() -> None: +async def test_api_health() -> None: async with AsyncClient(verify=False) as client: url = f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_HEALTH}" response = await client.get(url) diff --git a/e2e_tests/test_workspace_services.py b/e2e_tests/test_workspace_services.py index f2cea9bf68..00d445bed9 100644 --- a/e2e_tests/test_workspace_services.py +++ b/e2e_tests/test_workspace_services.py @@ -51,7 +51,7 @@ async def test_create_guacamole_service_into_base_workspace(verify, setup_test_w await disable_and_delete_resource(f'/api{workspace_service_path}', workspace_owner_token, verify) -@pytest.mark.extended_aad +@pytest.mark.aad @pytest.mark.timeout(75 * 60) async def test_create_guacamole_service_into_aad_workspace(verify, setup_test_aad_workspace) -> None: """This test will create a Guacamole service but will create a workspace and automatically register the AAD Application""" diff --git a/e2e_tests/test_workspace_templates.py b/e2e_tests/test_workspace_templates.py deleted file mode 100644 index d2e29fd39a..0000000000 --- a/e2e_tests/test_workspace_templates.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest - -from httpx import AsyncClient -from starlette import status - -import config -from helpers import get_auth_header, get_template -from resources import strings -from helpers import get_admin_token - - -pytestmark = pytest.mark.asyncio - - -workspace_templates = [ - (strings.BASE_WORKSPACE) -] - - -@pytest.mark.smoke -@pytest.mark.parametrize("template_name", workspace_templates) -async def test_get_workspace_templates(template_name, verify) -> None: - async with AsyncClient(verify=verify) as client: - admin_token = await get_admin_token(verify) - response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_TEMPLATES}", headers=get_auth_header(admin_token)) - - template_names = [templates["name"] for templates in response.json()["templates"]] - assert (template_name in template_names), f"No {template_name} template found" - - -@pytest.mark.smoke -@pytest.mark.parametrize("template_name", workspace_templates) -async def test_get_workspace_template(template_name, verify) -> None: - admin_token = await get_admin_token(verify) - async with get_template(template_name, strings.API_WORKSPACE_TEMPLATES, admin_token, verify) as response: - assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} creation failed" From ad08fb3f13f3745b28fa603f8a9679fcd2264d81 Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Fri, 11 Nov 2022 14:46:13 +0000 Subject: [PATCH 02/10] bring back deleted file --- e2e_tests/test_workspace_templates.py | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 e2e_tests/test_workspace_templates.py diff --git a/e2e_tests/test_workspace_templates.py b/e2e_tests/test_workspace_templates.py new file mode 100644 index 0000000000..d2e29fd39a --- /dev/null +++ b/e2e_tests/test_workspace_templates.py @@ -0,0 +1,36 @@ +import pytest + +from httpx import AsyncClient +from starlette import status + +import config +from helpers import get_auth_header, get_template +from resources import strings +from helpers import get_admin_token + + +pytestmark = pytest.mark.asyncio + + +workspace_templates = [ + (strings.BASE_WORKSPACE) +] + + +@pytest.mark.smoke +@pytest.mark.parametrize("template_name", workspace_templates) +async def test_get_workspace_templates(template_name, verify) -> None: + async with AsyncClient(verify=verify) as client: + admin_token = await get_admin_token(verify) + response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_TEMPLATES}", headers=get_auth_header(admin_token)) + + template_names = [templates["name"] for templates in response.json()["templates"]] + assert (template_name in template_names), f"No {template_name} template found" + + +@pytest.mark.smoke +@pytest.mark.parametrize("template_name", workspace_templates) +async def test_get_workspace_template(template_name, verify) -> None: + admin_token = await get_admin_token(verify) + async with get_template(template_name, strings.API_WORKSPACE_TEMPLATES, admin_token, verify) as response: + assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} creation failed" From 52d99cdb80a5deecee5117af5c1a0ce9b7a70f34 Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Fri, 11 Nov 2022 15:00:16 +0000 Subject: [PATCH 03/10] Move test data into data/ folder --- e2e_tests/{ => data}/test_airlock_sample.txt | 0 e2e_tests/test_airlock.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename e2e_tests/{ => data}/test_airlock_sample.txt (100%) diff --git a/e2e_tests/test_airlock_sample.txt b/e2e_tests/data/test_airlock_sample.txt similarity index 100% rename from e2e_tests/test_airlock_sample.txt rename to e2e_tests/data/test_airlock_sample.txt diff --git a/e2e_tests/test_airlock.py b/e2e_tests/test_airlock.py index d2d773e173..fc49ddb41d 100644 --- a/e2e_tests/test_airlock.py +++ b/e2e_tests/test_airlock.py @@ -12,7 +12,7 @@ pytestmark = pytest.mark.asyncio LOGGER = logging.getLogger(__name__) -BLOB_FILE_PATH = "./test_airlock_sample.txt" +BLOB_FILE_PATH = "./data/test_airlock_sample.txt" BLOB_NAME = os.path.basename(BLOB_FILE_PATH) From 8cfc73914ec344e62c9a72a421d31ac0bc7e95a3 Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Fri, 11 Nov 2022 15:01:43 +0000 Subject: [PATCH 04/10] Refactor tests that get templates and move them all into one file --- e2e_tests/test_get_templates.py | 77 +++++++++++++++++++ e2e_tests/test_shared_service_templates.py | 33 -------- e2e_tests/test_workspace_service_templates.py | 61 --------------- e2e_tests/test_workspace_templates.py | 36 --------- 4 files changed, 77 insertions(+), 130 deletions(-) create mode 100644 e2e_tests/test_get_templates.py delete mode 100644 e2e_tests/test_shared_service_templates.py delete mode 100644 e2e_tests/test_workspace_service_templates.py delete mode 100644 e2e_tests/test_workspace_templates.py diff --git a/e2e_tests/test_get_templates.py b/e2e_tests/test_get_templates.py new file mode 100644 index 0000000000..06d9348435 --- /dev/null +++ b/e2e_tests/test_get_templates.py @@ -0,0 +1,77 @@ +import pytest + +from httpx import AsyncClient +from starlette import status + +import config +from helpers import get_auth_header, get_template +from resources import strings +from helpers import get_admin_token + + +pytestmark = pytest.mark.asyncio + + +workspace_templates = [ + (strings.BASE_WORKSPACE) +] + +workspace_service_templates = [ + (strings.AZUREML_SERVICE), + (strings.GUACAMOLE_SERVICE), + (strings.INNEREYE_SERVICE), + (strings.GITEA_SERVICE) +] + +shared_service_templates = [ + (strings.FIREWALL_SHARED_SERVICE), + (strings.GITEA_SHARED_SERVICE), +] + + +@pytest.mark.smoke +@pytest.mark.parametrize("template_name", workspace_templates) +async def test_get_workspace_template(template_name, verify) -> None: + admin_token = await get_admin_token(verify) + # Test that the template is returned in GET request + async with get_template(template_name, strings.API_WORKSPACE_TEMPLATES, admin_token, verify) as response: + assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} creation failed" + + # Test thatt the template is returned in a list GET request + async with AsyncClient(verify=verify) as client: + response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_TEMPLATES}", headers=get_auth_header(admin_token)) + + template_names = [templates["name"] for templates in response.json()["templates"]] + assert (template_name in template_names), f"No {template_name} template found" + + +@pytest.mark.smoke +@pytest.mark.parametrize("template_name", workspace_service_templates) +async def test_get_workspace_service_templates(template_name, verify) -> None: + # Test that the template is returned in GET request + admin_token = await get_admin_token(verify) + async with get_template(template_name, strings.API_WORKSPACE_SERVICE_TEMPLATES, admin_token, verify) as response: + assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} failed" + + # Test that the template is returned in a list GET request + async with AsyncClient(verify=verify) as client: + response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_SERVICE_TEMPLATES}", headers=get_auth_header(admin_token)) + + template_names = [templates["name"] for templates in response.json()["templates"]] + assert (template_name in template_names), f"No {template_name} template found" + + +@pytest.mark.smoke +@pytest.mark.parametrize("template_name", shared_service_templates) +async def test_get_shared_service_templates(template_name, verify) -> None: + # Test that the template is returned in GET request + async with get_template(template_name, strings.API_SHARED_SERVICE_TEMPLATES, admin_token, verify) as response: + assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} failed" + + # Test that the template is returned in a list GET request + async with AsyncClient(verify=verify) as client: + admin_token = await get_admin_token(verify) + response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_SHARED_SERVICE_TEMPLATES}", headers=get_auth_header(admin_token)) + + template_names = [templates["name"] for templates in response.json()["templates"]] + assert (template_name in template_names), f"No {template_name} template found" diff --git a/e2e_tests/test_shared_service_templates.py b/e2e_tests/test_shared_service_templates.py deleted file mode 100644 index 2faafafb6a..0000000000 --- a/e2e_tests/test_shared_service_templates.py +++ /dev/null @@ -1,33 +0,0 @@ -import pytest - -from httpx import AsyncClient -from starlette import status - -import config -from helpers import get_auth_header, get_template -from resources import strings -from helpers import get_admin_token - -shared_service_templates = [ - (strings.FIREWALL_SHARED_SERVICE), - (strings.GITEA_SHARED_SERVICE), -] - - -@pytest.mark.smoke -@pytest.mark.parametrize("template_name", shared_service_templates) -async def test_get_shared_service_templates(template_name, verify) -> None: - async with AsyncClient(verify=verify) as client: - admin_token = await get_admin_token(verify) - response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_SHARED_SERVICE_TEMPLATES}", headers=get_auth_header(admin_token)) - - template_names = [templates["name"] for templates in response.json()["templates"]] - assert (template_name in template_names), f"No {template_name} template found" - - -@pytest.mark.smoke -@pytest.mark.parametrize("template_name", shared_service_templates) -async def test_get_shared_service_template(template_name, verify) -> None: - admin_token = await get_admin_token(verify) - async with get_template(template_name, strings.API_SHARED_SERVICE_TEMPLATES, admin_token, verify) as response: - assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} failed" diff --git a/e2e_tests/test_workspace_service_templates.py b/e2e_tests/test_workspace_service_templates.py deleted file mode 100644 index 1ccb2a8a83..0000000000 --- a/e2e_tests/test_workspace_service_templates.py +++ /dev/null @@ -1,61 +0,0 @@ -import pytest - -from httpx import AsyncClient -from starlette import status - -import config -from helpers import get_auth_header, get_template -from resources import strings -from helpers import get_admin_token - -pytestmark = pytest.mark.asyncio - -workspace_service_templates = [ - (strings.AZUREML_SERVICE), - (strings.GUACAMOLE_SERVICE), - (strings.INNEREYE_SERVICE), - (strings.GITEA_SERVICE) -] - - -@pytest.mark.smoke -@pytest.mark.parametrize("template_name", workspace_service_templates) -async def test_get_workspace_service_templates(template_name, verify) -> None: - async with AsyncClient(verify=verify) as client: - admin_token = await get_admin_token(verify) - response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_SERVICE_TEMPLATES}", headers=get_auth_header(admin_token)) - - template_names = [templates["name"] for templates in response.json()["templates"]] - assert (template_name in template_names), f"No {template_name} template found" - - -@pytest.mark.smoke -@pytest.mark.parametrize("template_name", workspace_service_templates) -async def test_get_workspace_service_template(template_name, verify) -> None: - admin_token = await get_admin_token(verify) - async with get_template(template_name, strings.API_WORKSPACE_SERVICE_TEMPLATES, admin_token, verify) as response: - assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} failed" - - -@pytest.mark.smoke -async def test_create_workspace_service_templates(verify) -> None: - async with AsyncClient(verify=verify) as client: - payload = { - "name": f"{strings.TEST_WORKSPACE_SERVICE_TEMPLATE}", - "version": "0.0.1", - "current": "true", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://github.com/microsoft/AzureTRE/templates/workspaces/myworkspace/workspace_service.json", - "type": "object", - "title": "DONOTUSE", - "description": "DO NOT USE", - "required": [], - "properties": {} - } - } - - admin_token = await get_admin_token(verify) - response = await client.post(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_SERVICE_TEMPLATES}", headers=get_auth_header(admin_token), json=payload) - - assert (response.status_code == status.HTTP_201_CREATED or response.status_code == status.HTTP_409_CONFLICT), "The workspace service template creation service returned unexpected response." diff --git a/e2e_tests/test_workspace_templates.py b/e2e_tests/test_workspace_templates.py deleted file mode 100644 index d2e29fd39a..0000000000 --- a/e2e_tests/test_workspace_templates.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest - -from httpx import AsyncClient -from starlette import status - -import config -from helpers import get_auth_header, get_template -from resources import strings -from helpers import get_admin_token - - -pytestmark = pytest.mark.asyncio - - -workspace_templates = [ - (strings.BASE_WORKSPACE) -] - - -@pytest.mark.smoke -@pytest.mark.parametrize("template_name", workspace_templates) -async def test_get_workspace_templates(template_name, verify) -> None: - async with AsyncClient(verify=verify) as client: - admin_token = await get_admin_token(verify) - response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_TEMPLATES}", headers=get_auth_header(admin_token)) - - template_names = [templates["name"] for templates in response.json()["templates"]] - assert (template_name in template_names), f"No {template_name} template found" - - -@pytest.mark.smoke -@pytest.mark.parametrize("template_name", workspace_templates) -async def test_get_workspace_template(template_name, verify) -> None: - admin_token = await get_admin_token(verify) - async with get_template(template_name, strings.API_WORKSPACE_TEMPLATES, admin_token, verify) as response: - assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} creation failed" From b88d894368e4642eb6bf3fb2a7f8d9453fde7fe4 Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Fri, 11 Nov 2022 15:02:05 +0000 Subject: [PATCH 05/10] Add shared service tests into extended selector --- e2e_tests/test_shared_services.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e_tests/test_shared_services.py b/e2e_tests/test_shared_services.py index 878a789dab..fc575be9ca 100644 --- a/e2e_tests/test_shared_services.py +++ b/e2e_tests/test_shared_services.py @@ -9,6 +9,7 @@ LOGGER = logging.getLogger(__name__) +@pytest.mark.extended @pytest.mark.shared_services async def test_patch_firewall(verify): template_name = strings.FIREWALL_SHARED_SERVICE @@ -91,6 +92,7 @@ async def test_patch_firewall(verify): ] +@pytest.mark.extended @pytest.mark.shared_services @pytest.mark.timeout(65 * 60) @pytest.mark.parametrize("template_name", shared_service_templates_to_create) From 92dd814cce45f3a885504fa96bb117de86c40a8d Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Mon, 14 Nov 2022 13:51:07 +0000 Subject: [PATCH 06/10] wip --- Makefile | 4 -- e2e_tests/config.py | 5 +-- e2e_tests/conftest.py | 41 +++++++++++++++--- e2e_tests/test_performance.py | 64 ++++------------------------ e2e_tests/test_workspace_services.py | 17 ++------ 5 files changed, 49 insertions(+), 82 deletions(-) diff --git a/Makefile b/Makefile index 59a64b8d96..fd9b32c975 100644 --- a/Makefile +++ b/Makefile @@ -301,10 +301,6 @@ test-e2e-extended: $(call target_title, "Running E2E extended tests") && \ $(MAKE) test-e2e-custom SELECTOR=extended -test-e2e-shared-services: - $(call target_title, "Running E2E shared service tests") && \ - $(MAKE) test-e2e-custom SELECTOR=shared_services - test-e2e-custom: $(call target_title, "Running E2E tests with custom selector ${SELECTOR}") \ && . ${MAKEFILE_DIR}/devops/scripts/load_env.sh ${MAKEFILE_DIR}/e2e_tests/.env \ diff --git a/e2e_tests/config.py b/e2e_tests/config.py index f0081d3c63..e41a4316b8 100644 --- a/e2e_tests/config.py +++ b/e2e_tests/config.py @@ -18,11 +18,8 @@ TEST_WORKSPACE_APP_SECRET: str = config("TEST_WORKSPACE_APP_SECRET", default="") TEST_WORKSPACE_APP_PLAN: str = config("WORKSPACE_APP_SERVICE_PLAN_SKU", default="") -# Perf test env vars - set these in private.env if you want to run perf tests and use an existing -# workspace + workspace service for quicker execution. If they're blank the perf test will create + delete them. -PERF_TEST_WORKSPACE_ID: str = config("PERF_TEST_WORKSPACE_ID", default="") -PERF_TEST_WORKSPACE_SERVICE_ID: str = config("PERF_TEST_WORKSPACE_SERVICE_ID", default="") # Set workspace id of an existing workspace to skip creation of a workspace during E2E tests TEST_WORKSPACE_ID: str = config("TEST_WORKSPACE_ID", default="") +TEST_WORKSPACE_SERVICE_ID: str = config("TEST_WORKSPACE_SERVICE_ID", default="") TEST_AAD_WORKSPACE_ID: str = config("TEST_AAD_WORKSPACE_ID", default="") diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 1bcd3c0f2a..9ed6c70ebf 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -6,7 +6,7 @@ from resources.resource import post_resource, disable_and_delete_resource from resources.workspace import get_workspace_auth_details -from resources import strings as resource_strings +from resources import strings from helpers import get_admin_token @@ -37,7 +37,7 @@ async def create_or_get_test_workspace(auth_type: str, verify: bool, pre_created LOGGER.info("Creating workspace") payload = { - "templateName": resource_strings.BASE_WORKSPACE, + "templateName": strings.BASE_WORKSPACE, "properties": { "display_name": "E2E test workspace", "description": "Test workspace for E2E tests", @@ -53,7 +53,7 @@ async def create_or_get_test_workspace(auth_type: str, verify: bool, pre_created payload["properties"]["app_service_plan_sku"] = config.TEST_WORKSPACE_APP_PLAN admin_token = await get_admin_token(verify=verify) - workspace_path, workspace_id = await post_resource(payload, resource_strings.API_WORKSPACES, access_token=admin_token, verify=verify) + workspace_path, workspace_id = await post_resource(payload, strings.API_WORKSPACES, access_token=admin_token, verify=verify) return workspace_path, workspace_id @@ -77,7 +77,7 @@ async def setup_test_workspace(verify) -> Tuple[str, str, str]: yield workspace_path, workspace_id, workspace_owner_token # Tear-down - clean_up_test_workspace(pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify) + await clean_up_test_workspace(pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify) @pytest.fixture(scope="session") @@ -91,4 +91,35 @@ async def setup_test_aad_workspace(verify) -> Tuple[str, str, str]: yield workspace_path, workspace_id, workspace_owner_token # Tear-down - clean_up_test_workspace(pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify) + await clean_up_test_workspace(pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify) + + +@pytest.fixture(scope="session") +async def setup_test_workspace_service(verify, setup_test_workspace) -> Tuple[str, str, str, str, str]: + workspace_path, workspace_id, workspace_owner_token = setup_test_workspace + pre_created_workspace_service_id = config.TEST_WORKSPACE_SERVICE_ID + + if pre_created_workspace_service_id == "": + # create a guac service + service_payload = { + "templateName": strings.GUACAMOLE_SERVICE, + "properties": { + "display_name": "Workspace service test", + "description": "" + } + } + + workspace_service_path, workspace_service_id = await post_resource( + payload=service_payload, + endpoint=f'/api{workspace_path}/{strings.API_WORKSPACE_SERVICES}', + access_token=workspace_owner_token, + verify=verify) + else: + workspace_service_path = f"{workspace_path}/{strings.API_WORKSPACE_SERVICES}/{config.TEST_WORKSPACE_SERVICE_ID}" + workspace_service_id = config.TEST_WORKSPACE_SERVICE_ID + + yield workspace_service_path, workspace_service_id, workspace_path, workspace_id, workspace_owner_token + + if pre_created_workspace_service_id == "": + admin_token = await get_admin_token(verify=verify) + await disable_and_delete_resource(f'/api{workspace_path}', admin_token, verify) diff --git a/e2e_tests/test_performance.py b/e2e_tests/test_performance.py index 4f714cd810..e3401acbba 100644 --- a/e2e_tests/test_performance.py +++ b/e2e_tests/test_performance.py @@ -1,7 +1,6 @@ import asyncio import pytest import config -from resources.workspace import get_workspace_auth_details from resources.resource import disable_and_delete_resource, post_resource from resources import strings @@ -10,13 +9,14 @@ pytestmark = pytest.mark.asyncio +# TODO: tanya +@pytest.mark.skip @pytest.mark.performance @pytest.mark.timeout(3000) async def test_parallel_resource_creations(verify) -> None: """Creates N workspaces in parallel, and creates a workspace service in each, in parallel""" number_workspaces = 2 - tasks = [] for i in range(number_workspaces): @@ -30,6 +30,8 @@ async def test_parallel_resource_creations(verify) -> None: "client_id": f"{config.TEST_WORKSPACE_APP_ID}" } } + if config.TEST_WORKSPACE_APP_PLAN != "": + payload["properties"]["app_service_plan_sku"] = config.TEST_WORKSPACE_APP_PLAN admin_token = await get_admin_token(verify) task = asyncio.create_task(post_resource(payload=payload, endpoint=strings.API_WORKSPACES, access_token=admin_token, verify=verify)) @@ -46,66 +48,25 @@ async def test_parallel_resource_creations(verify) -> None: await asyncio.gather(*tasks) -@pytest.mark.skip +@pytest.mark.extended @pytest.mark.performance @pytest.mark.timeout(3000) -async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify) -> None: +async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify, setup_test_workspace_service) -> None: """Optionally creates a workspace and workspace service, then creates N number of VMs in parallel, patches each, and deletes them""" number_vms = 5 number_updates = 5 - # To avoid creating + deleting a workspace + service in this test, set the vars for existing ones in ./templates/core/.env - # PERF_TEST_WORKSPACE_ID | PERF_TEST_WORKSPACE_SERVICE_ID - workspace_id = config.PERF_TEST_WORKSPACE_ID - - if workspace_id == "": - # create the workspace to use - payload = { - "templateName": strings.BASE_WORKSPACE, - "properties": { - "display_name": "E2E test guacamole service", - "description": "", - "address_space_size": "small", - "auth_type": "Manual", - "client_id": f"{config.TEST_WORKSPACE_APP_ID}", - "client_secret": f"{config.TEST_WORKSPACE_APP_SECRET}" - } - } - - admin_token = await get_admin_token(verify) - workspace_path, workspace_id = await post_resource(payload, strings.API_WORKSPACES, admin_token, verify) - else: - workspace_path = f"/workspaces/{config.PERF_TEST_WORKSPACE_ID}" - - workspace_owner_token, scope_uri = await get_workspace_auth_details(admin_token=admin_token, workspace_id=workspace_id, verify=verify) - - if config.PERF_TEST_WORKSPACE_SERVICE_ID == "": - # create a guac service - service_payload = { - "templateName": strings.GUACAMOLE_SERVICE, - "properties": { - "display_name": "Workspace service test", - "description": "" - } - } - - workspace_service_path, _ = await post_resource( - payload=service_payload, - endpoint=f'/api{workspace_path}/{strings.API_WORKSPACE_SERVICES}', - access_token=workspace_owner_token, - verify=verify) - else: - workspace_service_path = f"{workspace_path}/{strings.API_WORKSPACE_SERVICES}/{config.PERF_TEST_WORKSPACE_SERVICE_ID}" + workspace_service_path, _, _, _, workspace_owner_token = setup_test_workspace_service # Create the VMs in parallel, and wait for them to be created user_resource_payload = { - "templateName": "tre-service-dev-vm", + "templateName": strings.GUACAMOLE_WINDOWS_USER_RESOURCE, "properties": { "display_name": "Perf test VM", "description": "", - "os_image": "Ubuntu 18.04" + "os_image": "Windows 10" } } @@ -145,10 +106,3 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify) -> tasks.append(task) await asyncio.gather(*tasks) - - admin_token = await get_admin_token(verify) - # clear up workspace + service (if we created them) - if config.PERF_TEST_WORKSPACE_SERVICE_ID == "": - await disable_and_delete_resource(f'/api{workspace_service_path}', workspace_owner_token, verify) - if config.PERF_TEST_WORKSPACE_ID == "": - await disable_and_delete_resource(f'/api{workspace_path}', admin_token, verify) diff --git a/e2e_tests/test_workspace_services.py b/e2e_tests/test_workspace_services.py index 00d445bed9..42c661f3d5 100644 --- a/e2e_tests/test_workspace_services.py +++ b/e2e_tests/test_workspace_services.py @@ -10,18 +10,8 @@ @pytest.mark.extended @pytest.mark.timeout(75 * 60) -async def test_create_guacamole_service_into_base_workspace(verify, setup_test_workspace) -> None: - workspace_path, workspace_id, workspace_owner_token = setup_test_workspace - - service_payload = { - "templateName": strings.GUACAMOLE_SERVICE, - "properties": { - "display_name": "Workspace service test", - "description": "Workspace service for E2E test" - } - } - - workspace_service_path, workspace_service_id = await post_resource(service_payload, f'/api{workspace_path}/{strings.API_WORKSPACE_SERVICES}', workspace_owner_token, verify) +async def test_create_guacamole_service_into_base_workspace(verify, setup_test_workspace_service) -> None: + workspace_service_path, workspace_service_id, _, workspace_id, workspace_owner_token = setup_test_workspace_service await ping_guacamole_workspace_service(workspace_id, workspace_service_id, verify) @@ -48,10 +38,9 @@ async def test_create_guacamole_service_into_base_workspace(verify, setup_test_w await disable_and_delete_resource(f'/api{user_resource_path}', workspace_owner_token, verify) - await disable_and_delete_resource(f'/api{workspace_service_path}', workspace_owner_token, verify) - @pytest.mark.aad +@pytest.mark.extended @pytest.mark.timeout(75 * 60) async def test_create_guacamole_service_into_aad_workspace(verify, setup_test_aad_workspace) -> None: """This test will create a Guacamole service but will create a workspace and automatically register the AAD Application""" From 7f2f6b2869016aa3fda70dd19b6fb290de335c11 Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Mon, 14 Nov 2022 14:39:01 +0000 Subject: [PATCH 07/10] Add endopoint logging for debugging --- e2e_tests/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_tests/helpers.py b/e2e_tests/helpers.py index ddb237a68c..99c6dd33e0 100644 --- a/e2e_tests/helpers.py +++ b/e2e_tests/helpers.py @@ -85,7 +85,7 @@ async def check_aad_auth_redirect(endpoint, verify) -> None: while (True): try: response = await client.get(url=endpoint, timeout=TIMEOUT) - LOGGER.info(f"Endpoint Response: {response}") + LOGGER.info(f"Endpoint Response: {endpoint} {response}") if response.status_code in terminal_http_status: break From 61feaea0d2b8f908b4aae88a6c6d54d1e7f91b6f Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Mon, 14 Nov 2022 14:39:41 +0000 Subject: [PATCH 08/10] Fix extended marks --- e2e_tests/test_performance.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/e2e_tests/test_performance.py b/e2e_tests/test_performance.py index e3401acbba..048e72c8c7 100644 --- a/e2e_tests/test_performance.py +++ b/e2e_tests/test_performance.py @@ -9,8 +9,6 @@ pytestmark = pytest.mark.asyncio -# TODO: tanya -@pytest.mark.skip @pytest.mark.performance @pytest.mark.timeout(3000) async def test_parallel_resource_creations(verify) -> None: @@ -48,7 +46,6 @@ async def test_parallel_resource_creations(verify) -> None: await asyncio.gather(*tasks) -@pytest.mark.extended @pytest.mark.performance @pytest.mark.timeout(3000) async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify, setup_test_workspace_service) -> None: From b5a7df47f8ce9285de0ea075ecc659c625488c49 Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Mon, 14 Nov 2022 14:52:30 +0000 Subject: [PATCH 09/10] Run performance tests nightly --- .github/workflows/deploy_tre.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_tre.yml b/.github/workflows/deploy_tre.yml index 6f438f8789..51f2d7f73c 100644 --- a/.github/workflows/deploy_tre.yml +++ b/.github/workflows/deploy_tre.yml @@ -28,7 +28,9 @@ jobs: uses: ./.github/workflows/deploy_tre_reusable.yml with: ciGitRef: ${{ github.ref }} - e2eTestsCustomSelector: "extended" + e2eTestsCustomSelector: >- + ${{ (github.event_name == 'push' && 'extended') + || 'extended or performance' }} environmentName: ${{ github.event.inputs.environment || 'CICD' }} secrets: AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }} From d0ee678b48d4b4d534ec18f1156ce9caa763bbbb Mon Sep 17 00:00:00 2001 From: Tanya Borisova Date: Mon, 14 Nov 2022 14:57:34 +0000 Subject: [PATCH 10/10] fix getting admin token --- e2e_tests/test_get_templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e_tests/test_get_templates.py b/e2e_tests/test_get_templates.py index 06d9348435..ad9ad0e424 100644 --- a/e2e_tests/test_get_templates.py +++ b/e2e_tests/test_get_templates.py @@ -65,6 +65,7 @@ async def test_get_workspace_service_templates(template_name, verify) -> None: @pytest.mark.parametrize("template_name", shared_service_templates) async def test_get_shared_service_templates(template_name, verify) -> None: # Test that the template is returned in GET request + admin_token = await get_admin_token(verify) async with get_template(template_name, strings.API_SHARED_SERVICE_TEMPLATES, admin_token, verify) as response: assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} failed"