E2E: Simplify smoke and extended tests

.devcontainer/devcontainer.json

@@ -89,36 +89,6 @@
             "false"
           ]
         },
-        {
-          "name": "E2E Extended AAD",
-          "type": "python",
-          "request": "launch",
-          "module": "pytest",
-          "justMyCode": true,
-          "cwd": "${workspaceFolder}/e2e_tests/",
-          "preLaunchTask": "Copy_env_file_for_e2e_debug",
-          "args": [
-            "-m",
-            "extended_aad",
-            "--verify",
-            "false"
-          ]
-        },
-        {
-          "name": "E2E Shared Services",
-          "type": "python",
-          "request": "launch",
-          "module": "pytest",
-          "justMyCode": true,
-          "cwd": "${workspaceFolder}/e2e_tests/",
-          "preLaunchTask": "Copy_env_file_for_e2e_debug",
-          "args": [
-            "-m",
-            "shared_services",
-            "--verify",
-            "false"
-          ]
-        },
         {
           "name": "E2E Performance",
           "type": "python",

.github/workflows/deploy_tre.yml

@@ -29,8 +29,8 @@ jobs:
     with:
       ciGitRef: ${{ github.ref }}
       e2eTestsCustomSelector: >-
-        ${{ (github.event_name == 'push' && 'extended or extended_aad')
-        || 'extended or extended_aad' }}
+        ${{ (github.event_name == 'push' && 'extended')
+        || 'extended or performance' }}
       environmentName: ${{ github.event.inputs.environment || 'CICD' }}
     secrets:
       AAD_TENANT_ID: ${{ secrets.AAD_TENANT_ID }}

.github/workflows/pr_comment_bot.yml

@@ -135,9 +135,7 @@ jobs:
     needs: [pr_comment]
     if: |
       needs.pr_comment.outputs.command == 'run-tests' ||
-      needs.pr_comment.outputs.command == 'run-tests-extended' ||
-      needs.pr_comment.outputs.command == 'run-tests-extended-aad' ||
-      needs.pr_comment.outputs.command == 'run-tests-shared-services'
+      needs.pr_comment.outputs.command == 'run-tests-extended'
     name: Deploy PR
     uses: ./.github/workflows/deploy_tre_reusable.yml
     with:
@@ -146,8 +144,6 @@ jobs:
       ciGitRef: ${{ needs.pr_comment.outputs.ciGitRef }}
       e2eTestsCustomSelector: >-
         ${{ (needs.pr_comment.outputs.command == 'run-tests-extended' && 'extended') ||
-        (needs.pr_comment.outputs.command == 'run-tests-extended-aad' && 'extended_aad') ||
-        (needs.pr_comment.outputs.command == 'run-tests-shared-services' && 'shared_services') ||
         (needs.pr_comment.outputs.command == 'run-tests' && '') }}
       environmentName: CICD
     secrets:

Makefile

@@ -301,14 +301,6 @@ test-e2e-extended:
 	$(call target_title, "Running E2E extended tests") && \
 	$(MAKE) test-e2e-custom SELECTOR=extended
 
-test-e2e-extended-aad:
-	$(call target_title, "Running E2E extended AAD tests") && \
-	$(MAKE) test-e2e-custom SELECTOR=extended_aad
-
-test-e2e-shared-services:
-	$(call target_title, "Running E2E shared service tests") && \
-	$(MAKE) test-e2e-custom SELECTOR=shared_services
-
 test-e2e-custom:
 	$(call target_title, "Running E2E tests with custom selector ${SELECTOR}") \
 	&& . ${MAKEFILE_DIR}/devops/scripts/load_env.sh ${MAKEFILE_DIR}/e2e_tests/.env \

e2e_tests/config.py

@@ -18,11 +18,8 @@
 TEST_WORKSPACE_APP_SECRET: str = config("TEST_WORKSPACE_APP_SECRET", default="")
 TEST_WORKSPACE_APP_PLAN: str = config("WORKSPACE_APP_SERVICE_PLAN_SKU", default="")
 
-# Perf test env vars - set these in private.env if you want to run perf tests and use an existing
-# workspace + workspace service for quicker execution. If they're blank the perf test will create + delete them.
-PERF_TEST_WORKSPACE_ID: str = config("PERF_TEST_WORKSPACE_ID", default="")
-PERF_TEST_WORKSPACE_SERVICE_ID: str = config("PERF_TEST_WORKSPACE_SERVICE_ID", default="")
 
 # Set workspace id of an existing workspace to skip creation of a workspace during E2E tests
 TEST_WORKSPACE_ID: str = config("TEST_WORKSPACE_ID", default="")
+TEST_WORKSPACE_SERVICE_ID: str = config("TEST_WORKSPACE_SERVICE_ID", default="")
 TEST_AAD_WORKSPACE_ID: str = config("TEST_AAD_WORKSPACE_ID", default="")

e2e_tests/conftest.py

@@ -6,7 +6,7 @@
 
 from resources.resource import post_resource, disable_and_delete_resource
 from resources.workspace import get_workspace_auth_details
-from resources import strings as resource_strings
+from resources import strings
 from helpers import get_admin_token
 
 
@@ -37,7 +37,7 @@ async def create_or_get_test_workspace(auth_type: str, verify: bool, pre_created
 
     LOGGER.info("Creating workspace")
     payload = {
-        "templateName": resource_strings.BASE_WORKSPACE,
+        "templateName": strings.BASE_WORKSPACE,
         "properties": {
             "display_name": "E2E test workspace",
             "description": "Test workspace for E2E tests",
@@ -53,7 +53,7 @@ async def create_or_get_test_workspace(auth_type: str, verify: bool, pre_created
         payload["properties"]["app_service_plan_sku"] = config.TEST_WORKSPACE_APP_PLAN
 
     admin_token = await get_admin_token(verify=verify)
-    workspace_path, workspace_id = await post_resource(payload, resource_strings.API_WORKSPACES, access_token=admin_token, verify=verify)
+    workspace_path, workspace_id = await post_resource(payload, strings.API_WORKSPACES, access_token=admin_token, verify=verify)
     return workspace_path, workspace_id
 
 
@@ -77,7 +77,7 @@ async def setup_test_workspace(verify) -> Tuple[str, str, str]:
     yield workspace_path, workspace_id, workspace_owner_token
 
     # Tear-down
-    clean_up_test_workspace(pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify)
+    await clean_up_test_workspace(pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify)
 
 
 @pytest.fixture(scope="session")
@@ -91,4 +91,35 @@ async def setup_test_aad_workspace(verify) -> Tuple[str, str, str]:
     yield workspace_path, workspace_id, workspace_owner_token
 
     # Tear-down
-    clean_up_test_workspace(pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify)
+    await clean_up_test_workspace(pre_created_workspace_id=pre_created_workspace_id, workspace_path=workspace_path, verify=verify)
+
+
+@pytest.fixture(scope="session")
+async def setup_test_workspace_service(verify, setup_test_workspace) -> Tuple[str, str, str, str, str]:
+    workspace_path, workspace_id, workspace_owner_token = setup_test_workspace
+    pre_created_workspace_service_id = config.TEST_WORKSPACE_SERVICE_ID
+
+    if pre_created_workspace_service_id == "":
+        # create a guac service
+        service_payload = {
+            "templateName": strings.GUACAMOLE_SERVICE,
+            "properties": {
+                "display_name": "Workspace service test",
+                "description": ""
+            }
+        }
+
+        workspace_service_path, workspace_service_id = await post_resource(
+            payload=service_payload,
+            endpoint=f'/api{workspace_path}/{strings.API_WORKSPACE_SERVICES}',
+            access_token=workspace_owner_token,
+            verify=verify)
+    else:
+        workspace_service_path = f"{workspace_path}/{strings.API_WORKSPACE_SERVICES}/{config.TEST_WORKSPACE_SERVICE_ID}"
+        workspace_service_id = config.TEST_WORKSPACE_SERVICE_ID
+
+    yield workspace_service_path, workspace_service_id, workspace_path, workspace_id, workspace_owner_token
+
+    if pre_created_workspace_service_id == "":
+        admin_token = await get_admin_token(verify=verify)
+        await disable_and_delete_resource(f'/api{workspace_path}', admin_token, verify)

e2e_tests/helpers.py

@@ -85,7 +85,7 @@ async def check_aad_auth_redirect(endpoint, verify) -> None:
         while (True):
             try:
                 response = await client.get(url=endpoint, timeout=TIMEOUT)
-                LOGGER.info(f"Endpoint Response: {response}")
+                LOGGER.info(f"Endpoint Response: {endpoint} {response}")
 
                 if response.status_code in terminal_http_status:
                     break

e2e_tests/pytest.ini

@@ -2,11 +2,11 @@
 markers =
     smoke: marks tests as smoke (run sometimes, relatively fast)
     extended: marks tests as extended (run less frequently, relatively slow)
-    extended_aad
-    shared_services
     performance: marks tests for performance evaluation
     timeout: used to set test timeout with pytest-timeout
     airlock: only airlock related
+    aad: only related to automatically created AAD configuration
+    shared_services: only related to shared services
 
 asyncio_mode = auto
 

e2e_tests/test_airlock.py

@@ -12,7 +12,7 @@
 
 pytestmark = pytest.mark.asyncio
 LOGGER = logging.getLogger(__name__)
-BLOB_FILE_PATH = "./test_airlock_sample.txt"
+BLOB_FILE_PATH = "./data/test_airlock_sample.txt"
 BLOB_NAME = os.path.basename(BLOB_FILE_PATH)
 
 

e2e_tests/test_provisioned_health_api.py → e2e_tests/test_api_health.py

@@ -9,7 +9,7 @@
 
 
 @pytest.mark.smoke
-async def test_health() -> None:
+async def test_api_health() -> None:
     async with AsyncClient(verify=False) as client:
         url = f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_HEALTH}"
         response = await client.get(url)

e2e_tests/test_get_templates.py

@@ -0,0 +1,78 @@
+import pytest
+
+from httpx import AsyncClient
+from starlette import status
+
+import config
+from helpers import get_auth_header, get_template
+from resources import strings
+from helpers import get_admin_token
+
+
+pytestmark = pytest.mark.asyncio
+
+
+workspace_templates = [
+    (strings.BASE_WORKSPACE)
+]
+
+workspace_service_templates = [
+    (strings.AZUREML_SERVICE),
+    (strings.GUACAMOLE_SERVICE),
+    (strings.INNEREYE_SERVICE),
+    (strings.GITEA_SERVICE)
+]
+
+shared_service_templates = [
+    (strings.FIREWALL_SHARED_SERVICE),
+    (strings.GITEA_SHARED_SERVICE),
+]
+
+
+@pytest.mark.smoke
+@pytest.mark.parametrize("template_name", workspace_templates)
+async def test_get_workspace_template(template_name, verify) -> None:
+    admin_token = await get_admin_token(verify)
+    # Test that the template is returned in GET request
+    async with get_template(template_name, strings.API_WORKSPACE_TEMPLATES, admin_token, verify) as response:
+        assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} creation failed"
+
+    # Test thatt the template is returned in a list GET request
+    async with AsyncClient(verify=verify) as client:
+        response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_TEMPLATES}", headers=get_auth_header(admin_token))
+
+        template_names = [templates["name"] for templates in response.json()["templates"]]
+        assert (template_name in template_names), f"No {template_name} template found"
+
+
+@pytest.mark.smoke
+@pytest.mark.parametrize("template_name", workspace_service_templates)
+async def test_get_workspace_service_templates(template_name, verify) -> None:
+    # Test that the template is returned in GET request
+    admin_token = await get_admin_token(verify)
+    async with get_template(template_name, strings.API_WORKSPACE_SERVICE_TEMPLATES, admin_token, verify) as response:
+        assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} failed"
+
+    # Test that the template is returned in a list GET request
+    async with AsyncClient(verify=verify) as client:
+        response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_WORKSPACE_SERVICE_TEMPLATES}", headers=get_auth_header(admin_token))
+
+        template_names = [templates["name"] for templates in response.json()["templates"]]
+        assert (template_name in template_names), f"No {template_name} template found"
+
+
+@pytest.mark.smoke
+@pytest.mark.parametrize("template_name", shared_service_templates)
+async def test_get_shared_service_templates(template_name, verify) -> None:
+    # Test that the template is returned in GET request
+    admin_token = await get_admin_token(verify)
+    async with get_template(template_name, strings.API_SHARED_SERVICE_TEMPLATES, admin_token, verify) as response:
+        assert (response.status_code == status.HTTP_200_OK), f"GET Request for {template_name} failed"
+
+    # Test that the template is returned in a list GET request
+    async with AsyncClient(verify=verify) as client:
+        admin_token = await get_admin_token(verify)
+        response = await client.get(f"https://{config.TRE_ID}.{config.RESOURCE_LOCATION}.cloudapp.azure.com{strings.API_SHARED_SERVICE_TEMPLATES}", headers=get_auth_header(admin_token))
+
+        template_names = [templates["name"] for templates in response.json()["templates"]]
+        assert (template_name in template_names), f"No {template_name} template found"

e2e_tests/test_performance.py

@@ -1,7 +1,6 @@
 import asyncio
 import pytest
 import config
-from resources.workspace import get_workspace_auth_details
 from resources.resource import disable_and_delete_resource, post_resource
 from resources import strings
 
@@ -16,7 +15,6 @@ async def test_parallel_resource_creations(verify) -> None:
     """Creates N workspaces in parallel, and creates a workspace service in each, in parallel"""
 
     number_workspaces = 2
-
     tasks = []
 
     for i in range(number_workspaces):
@@ -30,6 +28,8 @@ async def test_parallel_resource_creations(verify) -> None:
                 "client_id": f"{config.TEST_WORKSPACE_APP_ID}"
             }
         }
+        if config.TEST_WORKSPACE_APP_PLAN != "":
+            payload["properties"]["app_service_plan_sku"] = config.TEST_WORKSPACE_APP_PLAN
 
         admin_token = await get_admin_token(verify)
         task = asyncio.create_task(post_resource(payload=payload, endpoint=strings.API_WORKSPACES, access_token=admin_token, verify=verify))
@@ -46,66 +46,24 @@ async def test_parallel_resource_creations(verify) -> None:
     await asyncio.gather(*tasks)
 
 
-@pytest.mark.skip
 @pytest.mark.performance
 @pytest.mark.timeout(3000)
-async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify) -> None:
+async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify, setup_test_workspace_service) -> None:
     """Optionally creates a workspace and workspace service,
     then creates N number of VMs in parallel, patches each, and deletes them"""
 
     number_vms = 5
     number_updates = 5
 
-    # To avoid creating + deleting a workspace + service in this test, set the vars for existing ones in ./templates/core/.env
-    # PERF_TEST_WORKSPACE_ID | PERF_TEST_WORKSPACE_SERVICE_ID
-    workspace_id = config.PERF_TEST_WORKSPACE_ID
-
-    if workspace_id == "":
-        # create the workspace to use
-        payload = {
-            "templateName": strings.BASE_WORKSPACE,
-            "properties": {
-                "display_name": "E2E test guacamole service",
-                "description": "",
-                "address_space_size": "small",
-                "auth_type": "Manual",
-                "client_id": f"{config.TEST_WORKSPACE_APP_ID}",
-                "client_secret": f"{config.TEST_WORKSPACE_APP_SECRET}"
-            }
-        }
-
-        admin_token = await get_admin_token(verify)
-        workspace_path, workspace_id = await post_resource(payload, strings.API_WORKSPACES, admin_token, verify)
-    else:
-        workspace_path = f"/workspaces/{config.PERF_TEST_WORKSPACE_ID}"
-
-    workspace_owner_token, scope_uri = await get_workspace_auth_details(admin_token=admin_token, workspace_id=workspace_id, verify=verify)
-
-    if config.PERF_TEST_WORKSPACE_SERVICE_ID == "":
-        # create a guac service
-        service_payload = {
-            "templateName": strings.GUACAMOLE_SERVICE,
-            "properties": {
-                "display_name": "Workspace service test",
-                "description": ""
-            }
-        }
-
-        workspace_service_path, _ = await post_resource(
-            payload=service_payload,
-            endpoint=f'/api{workspace_path}/{strings.API_WORKSPACE_SERVICES}',
-            access_token=workspace_owner_token,
-            verify=verify)
-    else:
-        workspace_service_path = f"{workspace_path}/{strings.API_WORKSPACE_SERVICES}/{config.PERF_TEST_WORKSPACE_SERVICE_ID}"
+    workspace_service_path, _, _, _, workspace_owner_token = setup_test_workspace_service
 
     # Create the VMs in parallel, and wait for them to be created
     user_resource_payload = {
-        "templateName": "tre-service-dev-vm",
+        "templateName": strings.GUACAMOLE_WINDOWS_USER_RESOURCE,
         "properties": {
             "display_name": "Perf test VM",
             "description": "",
-            "os_image": "Ubuntu 18.04"
+            "os_image": "Windows 10"
         }
     }
 
@@ -145,10 +103,3 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify) ->
         tasks.append(task)
 
     await asyncio.gather(*tasks)
-
-    admin_token = await get_admin_token(verify)
-    # clear up workspace + service (if we created them)
-    if config.PERF_TEST_WORKSPACE_SERVICE_ID == "":
-        await disable_and_delete_resource(f'/api{workspace_service_path}', workspace_owner_token, verify)
-    if config.PERF_TEST_WORKSPACE_ID == "":
-        await disable_and_delete_resource(f'/api{workspace_path}', admin_token, verify)