From 8c1cea9c7e3e3e71734fab4b2a77e9d8152a6e0b Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Tue, 7 Jan 2025 08:56:40 +0000 Subject: [PATCH 01/38] Consolidate upgrade scripts for Terraform (#4126) --- .../workflows/build_validation_develop.yml | 5 ++ .github/workflows/codeql-analysis.yml | 2 +- CHANGELOG.md | 1 + Makefile | 2 +- core/terraform/upgrade.sh | 13 ---- core/version.txt | 2 +- devops/scripts/upgrade.sh | 61 +++++++++++++++++++ devops/terraform/upgrade.sh | 13 ---- .../admin-vm/terraform/upgrade.sh | 13 ---- .../airlock_notifier/terraform/upgrade.sh | 13 ---- .../certs/terraform/upgrade.sh | 13 ---- .../cyclecloud/terraform/upgrade.sh | 13 ---- .../databricks-auth/terraform/upgrade.sh | 13 ---- .../firewall/terraform/upgrade.sh | 13 ---- .../gitea/terraform/upgrade.sh | 13 ---- .../sonatype-nexus-vm/terraform/upgrade.sh | 13 ---- .../databricks/terraform/upgrade.sh | 13 ---- .../gitea/terraform/upgrade.sh | 13 ---- .../guacamole/terraform/upgrade.sh | 13 ---- .../terraform/upgrade.sh | 13 ---- .../terraform/upgrade.sh | 13 ---- .../terraform/upgrade.sh | 13 ---- .../terraform/upgrade.sh | 13 ---- 23 files changed, 70 insertions(+), 224 deletions(-) delete mode 100644 core/terraform/upgrade.sh create mode 100755 devops/scripts/upgrade.sh delete mode 100644 devops/terraform/upgrade.sh delete mode 100644 templates/shared_services/admin-vm/terraform/upgrade.sh delete mode 100755 templates/shared_services/airlock_notifier/terraform/upgrade.sh delete mode 100644 templates/shared_services/certs/terraform/upgrade.sh delete mode 100644 templates/shared_services/cyclecloud/terraform/upgrade.sh delete mode 100644 templates/shared_services/databricks-auth/terraform/upgrade.sh delete mode 100755 templates/shared_services/firewall/terraform/upgrade.sh delete mode 100644 templates/shared_services/gitea/terraform/upgrade.sh delete mode 100644 templates/shared_services/sonatype-nexus-vm/terraform/upgrade.sh delete mode 100644 templates/workspace_services/databricks/terraform/upgrade.sh delete mode 100644 templates/workspace_services/gitea/terraform/upgrade.sh delete mode 100644 templates/workspace_services/guacamole/terraform/upgrade.sh delete mode 100644 templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/upgrade.sh delete mode 100644 templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/upgrade.sh delete mode 100644 templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/upgrade.sh delete mode 100644 templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/upgrade.sh diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index c44cba4b9d..01be02c3c0 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -52,6 +52,11 @@ jobs: terraform_workspace_services: - templates/workspace_services/**/terraform/**/*.tf + - uses: hashicorp/setup-terraform@v3 + if: ${{ steps.filter.outputs.terraform == 'true' }} + with: + terraform_version: "1.9.8" + - name: Terraform format check if: ${{ steps.filter.outputs.terraform == 'true' }} run: terraform fmt -check -recursive diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 176b3b6a39..2cb40d1fd9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,7 +29,7 @@ concurrency: jobs: analyze: name: Analyze - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: actions: read contents: read diff --git a/CHANGELOG.md b/CHANGELOG.md index df36da2a72..e99edc513f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ENHANCEMENTS: * Split log entries with [Log chunk X of Y] for better readability. ([#3992](https://github.com/microsoft/AzureTRE/issues/3992)) * Expose APP_SERVICE_SKU build variable to allow enablement of App Gateway WAF ([#4111](https://github.com/microsoft/AzureTRE/pull/4111)) * Update Terraform to use Azure AD authentication rather than storage account keys ([#4103](https://github.com/microsoft/AzureTRE/issues/4103)) +* Consolidate Terraform upgrade scripts ([#4099](https://github.com/microsoft/AzureTRE/issues/4099)) * Storage accounts should use infrastructure encryption ([#4001](https://github.com/microsoft/AzureTRE/issues/4001)) * Update obsolete Terraform properties ([#4136](https://github.com/microsoft/AzureTRE/issues/4136)) * Update Guacamole version and dependencies ([#4140](https://github.com/microsoft/AzureTRE/issues/4140)) diff --git a/Makefile b/Makefile index 78e89c76f4..90e7686808 100644 --- a/Makefile +++ b/Makefile @@ -149,7 +149,7 @@ terraform-upgrade: && . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh env \ && . ${MAKEFILE_DIR}/devops/scripts/load_and_validate_env.sh \ && . ${MAKEFILE_DIR}/devops/scripts/load_env.sh ${DIR}/.env \ - && cd ${DIR}/terraform/ && ./upgrade.sh + && ./devops/scripts/upgrade.sh ${DIR} terraform-import: $(call target_title, "Importing ${DIR} with Terraform") \ diff --git a/core/terraform/upgrade.sh b/core/terraform/upgrade.sh deleted file mode 100644 index 861e28d2f1..0000000000 --- a/core/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TF_VAR_tre_resource_id}-core" diff --git a/core/version.txt b/core/version.txt index 87583f1acf..1e6e806534 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.15" +__version__ = "0.11.16" diff --git a/devops/scripts/upgrade.sh b/devops/scripts/upgrade.sh new file mode 100755 index 0000000000..89733492f7 --- /dev/null +++ b/devops/scripts/upgrade.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -e + +# This script is used to upgrade terraform providers in a specified directory + +# Usage: ./upgrade.sh + +DIR=$1 + +# Load environment variables from .env file +if [ -f "$DIR/.env" ]; then + set -a + # shellcheck source=/dev/null + . "$DIR/.env" + set +a +fi + +# Ensure TF_VAR_mgmt_resource_group_name is set +if [ -z "${TF_VAR_mgmt_resource_group_name}" ]; then + echo "Error: TF_VAR_mgmt_resource_group_name is not set." + exit 1 +fi + +# Ensure TF_VAR_mgmt_storage_account_name is set +if [ -z "${TF_VAR_mgmt_storage_account_name}" ]; then + echo "Error: TF_VAR_mgmt_storage_account_name is not set." + exit 1 +fi + +# Ensure TF_VAR_terraform_state_container_name is set +if [ -z "${TF_VAR_terraform_state_container_name}" ]; then + echo "Error: TF_VAR_terraform_state_container_name is not set." + exit 1 +fi + +# Ensure TRE_ID is set +if [ -z "${TRE_ID}" ]; then + echo "Error: TRE_ID is not set." + exit 1 +fi + +# Infer the key from the directory names +PARENT_DIR=$(basename "$(dirname "$DIR")") +GRANDPARENT_DIR=$(basename "$(dirname "$(dirname "$DIR")")") + +if [[ "$GRANDPARENT_DIR" == "workspaces" || "$GRANDPARENT_DIR" == "shared_services" ]]; then + KEY="${TRE_ID?}_${TF_VAR_id?}_${PARENT_DIR}" +elif [[ "$GRANDPARENT_DIR" == "workspace_services" ]]; then + KEY="${TRE_ID?}_${TF_VAR_workspace_id?}_${TF_VAR_id?}_${PARENT_DIR}" +elif [[ "$GRANDPARENT_DIR" == "user_resources" ]]; then + KEY="${TRE_ID?}_${TF_VAR_workspace_id?}_${TF_VAR_workspace_service_id?}_${TF_VAR_id?}_${PARENT_DIR}" +else + KEY="${TRE_ID?}_${PARENT_DIR}" +fi + +# Run terraform init with upgrade and reconfigure options +terraform -chdir="$DIR/terraform" init -upgrade -reconfigure -input=false -backend=true \ + -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ + -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ + -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ + -backend-config="key=${KEY}" diff --git a/devops/terraform/upgrade.sh b/devops/terraform/upgrade.sh deleted file mode 100644 index a05032d544..0000000000 --- a/devops/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TF_VAR_tre_resource_id}-devops" diff --git a/templates/shared_services/admin-vm/terraform/upgrade.sh b/templates/shared_services/admin-vm/terraform/upgrade.sh deleted file mode 100644 index 12597ee325..0000000000 --- a/templates/shared_services/admin-vm/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TF_VAR_tre_resource_id}-shared-adminvm" diff --git a/templates/shared_services/airlock_notifier/terraform/upgrade.sh b/templates/shared_services/airlock_notifier/terraform/upgrade.sh deleted file mode 100755 index 72785516f5..0000000000 --- a/templates/shared_services/airlock_notifier/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TF_VAR_tre_resource_id}-shared-airlock-notifier" diff --git a/templates/shared_services/certs/terraform/upgrade.sh b/templates/shared_services/certs/terraform/upgrade.sh deleted file mode 100644 index aae4c58cef..0000000000 --- a/templates/shared_services/certs/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TRE_ID}-certs" diff --git a/templates/shared_services/cyclecloud/terraform/upgrade.sh b/templates/shared_services/cyclecloud/terraform/upgrade.sh deleted file mode 100644 index 5368cb1a55..0000000000 --- a/templates/shared_services/cyclecloud/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TF_VAR_tre_resource_id}-${ID}" diff --git a/templates/shared_services/databricks-auth/terraform/upgrade.sh b/templates/shared_services/databricks-auth/terraform/upgrade.sh deleted file mode 100644 index 70b3863871..0000000000 --- a/templates/shared_services/databricks-auth/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=tre-workspace-service-gitea-${TF_VAR_id}" diff --git a/templates/shared_services/firewall/terraform/upgrade.sh b/templates/shared_services/firewall/terraform/upgrade.sh deleted file mode 100755 index d690d433a7..0000000000 --- a/templates/shared_services/firewall/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TRE_ID}-shared-service-firewall" diff --git a/templates/shared_services/gitea/terraform/upgrade.sh b/templates/shared_services/gitea/terraform/upgrade.sh deleted file mode 100644 index 6618fcb445..0000000000 --- a/templates/shared_services/gitea/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TRE_ID}-gitea" diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/upgrade.sh b/templates/shared_services/sonatype-nexus-vm/terraform/upgrade.sh deleted file mode 100644 index f15b4a2ee6..0000000000 --- a/templates/shared_services/sonatype-nexus-vm/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TRE_ID:?}-shared-service-sonatype-nexus" diff --git a/templates/workspace_services/databricks/terraform/upgrade.sh b/templates/workspace_services/databricks/terraform/upgrade.sh deleted file mode 100644 index 70b3863871..0000000000 --- a/templates/workspace_services/databricks/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=tre-workspace-service-gitea-${TF_VAR_id}" diff --git a/templates/workspace_services/gitea/terraform/upgrade.sh b/templates/workspace_services/gitea/terraform/upgrade.sh deleted file mode 100644 index 70b3863871..0000000000 --- a/templates/workspace_services/gitea/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=tre-workspace-service-gitea-${TF_VAR_id}" diff --git a/templates/workspace_services/guacamole/terraform/upgrade.sh b/templates/workspace_services/guacamole/terraform/upgrade.sh deleted file mode 100644 index edf24afb36..0000000000 --- a/templates/workspace_services/guacamole/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TRE_ID}${TF_VAR_workspace_id}guacamole" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/upgrade.sh b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/upgrade.sh deleted file mode 100644 index e5bbe41a91..0000000000 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TRE_ID}${TF_VAR_workspace_id}${TF_VAR_parent_service_id}guacamolewindowsvm" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/upgrade.sh b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/upgrade.sh deleted file mode 100644 index e5bbe41a91..0000000000 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TRE_ID}${TF_VAR_workspace_id}${TF_VAR_parent_service_id}guacamolewindowsvm" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/upgrade.sh b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/upgrade.sh deleted file mode 100644 index f6d9bbcefe..0000000000 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TF_VAR_tre_id}${TF_VAR_workspace_id}${TF_VAR_parent_service_id}guacamolelinuxvm" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/upgrade.sh b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/upgrade.sh deleted file mode 100644 index e5bbe41a91..0000000000 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/upgrade.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -e - -# This script is used to install the bundle directly without having to interact with Porter - -# This script assumes you have created an .env from the sample and the variables -# will come from there. -# shellcheck disable=SC2154 -terraform init -upgrade -reconfigure -input=false -backend=true \ - -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ - -backend-config="storage_account_name=${TF_VAR_mgmt_storage_account_name}" \ - -backend-config="container_name=${TF_VAR_terraform_state_container_name}" \ - -backend-config="key=${TRE_ID}${TF_VAR_workspace_id}${TF_VAR_parent_service_id}guacamolewindowsvm" From 97debdc23783e76ce04d6bd230e54713b2fa1f72 Mon Sep 17 00:00:00 2001 From: Yuval Yaron <43217306+yuvalyaron@users.noreply.github.com> Date: Tue, 7 Jan 2025 20:43:09 +0200 Subject: [PATCH 02/38] Add option for forced tunneling through TRE's Firewall (#4238) * Add option for forced tunneling through TRE's Firewall * fix linting issues * refine doc * rename force tunnel route * add variables to schema * fix github actions * add warning to fw force tunnel doc * refine doc * fix linting errors * send firewall props through makefile command * update doc * update config.yaml.sample * remove typo * shorten comment * Update docs/tre-admins/configure-firewall-force-tunneling.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix typo in docs * fix linting issues * fix linting error --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../workflows/build_validation_develop.yml | 21 +- .github/workflows/codeql-analysis.yml | 6 +- CHANGELOG.md | 1 + Makefile | 6 +- config.sample.yaml | 3 +- .../configure-firewall-force-tunneling.md | 21 + mkdocs.yml | 1 + .../shared_services/firewall/parameters.json | 6 + .../shared_services/firewall/porter.yaml | 8 +- .../firewall/template_schema.json | 634 +++++++++--------- .../firewall/terraform/firewall.tf | 4 +- .../firewall/terraform/routetable.tf | 25 + .../firewall/terraform/variables.tf | 5 + 13 files changed, 415 insertions(+), 326 deletions(-) create mode 100644 docs/tre-admins/configure-firewall-force-tunneling.md diff --git a/.github/workflows/build_validation_develop.yml b/.github/workflows/build_validation_develop.yml index 01be02c3c0..a9364293d8 100644 --- a/.github/workflows/build_validation_develop.yml +++ b/.github/workflows/build_validation_develop.yml @@ -1,11 +1,11 @@ --- name: Build Validation -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy pull_request: branches: - main - - 'feature/**' + - "feature/**" # for each ref (branch/pr) run just the most recent, # cancel other pending/running ones @@ -57,6 +57,11 @@ jobs: with: terraform_version: "1.9.8" + - uses: hashicorp/setup-terraform@v3 + if: ${{ steps.filter.outputs.terraform == 'true' }} + with: + terraform_version: "1.9.8" + - name: Terraform format check if: ${{ steps.filter.outputs.terraform == 'true' }} run: terraform fmt -check -recursive @@ -112,7 +117,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VALIDATE_TERRAFORM_TFLINT: true TERRAFORM_TFLINT_CONFIG_FILE: .tflint_core.hcl - FILTER_REGEX_INCLUDE: './core/.*' + FILTER_REGEX_INCLUDE: "./core/.*" - name: Workspace Tags if: ${{ steps.filter.outputs.terraform_workspaces == 'true' }} @@ -123,7 +128,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VALIDATE_TERRAFORM_TFLINT: true TERRAFORM_TFLINT_CONFIG_FILE: .tflint_workspaces.hcl - FILTER_REGEX_INCLUDE: './templates/workspaces/.*' + FILTER_REGEX_INCLUDE: "./templates/workspaces/.*" - name: Workspace Services Tags if: ${{ steps.filter.outputs.terraform_workspace_services == 'true' }} @@ -134,8 +139,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VALIDATE_TERRAFORM_TFLINT: true TERRAFORM_TFLINT_CONFIG_FILE: .tflint_workspace_services.hcl - FILTER_REGEX_INCLUDE: './templates/workspaces/.*' - FILTER_REGEX_EXCLUDE: '.*user_resource.*' + FILTER_REGEX_INCLUDE: "./templates/workspaces/.*" + FILTER_REGEX_EXCLUDE: ".*user_resource.*" - name: User Resources Tags if: ${{ steps.filter.outputs.terraform_workspace_services == 'true' }} @@ -146,7 +151,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VALIDATE_TERRAFORM_TFLINT: true TERRAFORM_TFLINT_CONFIG_FILE: .tflint_user_resources.hcl - FILTER_REGEX_INCLUDE: './templates/workspace_services/.*/user_resources/.*' + FILTER_REGEX_INCLUDE: "./templates/workspace_services/.*/user_resources/.*" - name: Shared Services Tags if: ${{ steps.filter.outputs.terraform_shared_services == 'true' }} @@ -157,4 +162,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} VALIDATE_TERRAFORM_TFLINT: true TERRAFORM_TFLINT_CONFIG_FILE: .tflint_shared_services.hcl - FILTER_REGEX_INCLUDE: './templates/shared_services/.*' + FILTER_REGEX_INCLUDE: "./templates/shared_services/.*" diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2cb40d1fd9..92327b53ff 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -12,13 +12,13 @@ # name: "CodeQL" -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy push: branches: [main] pull_request: branches: [main] schedule: - - cron: '41 3 * * 5' + - cron: "41 3 * * 5" # for each ref (branch/pr) run just the most recent, # cancel other pending/running ones @@ -38,7 +38,7 @@ jobs: strategy: fail-fast: false matrix: - language: ['python', 'java', 'javascript', 'typescript'] + language: ["python", "java", "javascript", "typescript"] steps: - name: Checkout repository diff --git a/CHANGELOG.md b/CHANGELOG.md index e99edc513f..6d19e990a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ BUG FIXES: * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222)) * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)]) * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)]) +* Add option to force tunnel TRE's Firewall ([#4237](https://github.com/microsoft/AzureTRE/issues/4237)) COMPONENTS: diff --git a/Makefile b/Makefile index 90e7686808..58719016b3 100644 --- a/Makefile +++ b/Makefile @@ -309,8 +309,10 @@ deploy-shared-service: && ${MAKEFILE_DIR}/devops/scripts/deploy_shared_service.sh $${PROPS} firewall-install: - $(MAKE) bundle-build bundle-publish bundle-register deploy-shared-service \ - DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service + . ${MAKEFILE_DIR}/devops/scripts/check_dependencies.sh env \ + && $(MAKE) bundle-build bundle-publish bundle-register deploy-shared-service \ + DIR=${MAKEFILE_DIR}/templates/shared_services/firewall/ BUNDLE_TYPE=shared_service \ + PROPS="$${FIREWALL_SKU+--firewall_sku $${FIREWALL_SKU} }$${FIREWALL_FORCE_TUNNEL_IP+--firewall_force_tunnel_ip $${FIREWALL_FORCE_TUNNEL_IP} }" static-web-upload: $(call target_title, "Uploading to static website") \ diff --git a/config.sample.yaml b/config.sample.yaml index 44777e89a3..009f017920 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -10,7 +10,7 @@ management: acr_name: __CHANGE_ME__ # ID of external Key Vault to store CMKs in (only required if enable_cmk_encryption is true) # external_key_store_id: __CHANGE_ME__ - # Name of Key Vault for encryption keys, required only if enable_cmk_encryption is true and not using external_key_store_id + # Name of Key Vault for encryption, required if enable_cmk_encryption is true and external_key_store_id is not set # encryption_kv_name: __CHANGE_ME__ # Azure Resource Manager credentials used for CI/CD pipelines arm_subscription_id: __CHANGE_ME__ @@ -46,6 +46,7 @@ tre: # The TRE Web UI is deployed by default. # Uncomment the following to disable deployment of the Web UI. # deploy_ui: false + # firewall_force_tunnel_ip: __CHANGE_ME__ firewall_sku: Standard app_gateway_sku: Standard_v2 diff --git a/docs/tre-admins/configure-firewall-force-tunneling.md b/docs/tre-admins/configure-firewall-force-tunneling.md new file mode 100644 index 0000000000..68d5886c93 --- /dev/null +++ b/docs/tre-admins/configure-firewall-force-tunneling.md @@ -0,0 +1,21 @@ +# Forced Tunneling to External Firewall in TRE + +Azure TRE deploys and manages an Azure firewall to ensure creation of workspace level rules can be automated when TRE workspaces and other services are created without manual intervention. +It is highly recommended leaving the Azure TRE firewall in place. If there is still the requirement to send all traffic through a centralized enterprise firewall, such as that deployed as part of an Azure landing zone, then forced tunnelling should be used. The centralized firewall will need a superset of rules used by the TRE. + +To setup forced tunneling to an external firewall, follow these steps: + +## 1. Set the firewall_force_tunnel_ip parameter in the config.yaml file +Provide the external firewall's IP address: + +```json +firewall_force_tunnel_ip: 192.168.0.4 +``` +This automatically creates a route table to direct TRE’s traffic to the specified IP. + +## 2. Manually Connect TRE to Your Firewall +Configure connectivity between TRE’s VNet and your external firewall using one of the following methods: + +1. **VNet Peering**: Peer the TRE VNet with your firewall’s VNet. +1. **ExpressRoute**: Use a private connection for firewalls located on-premises. +1. **Site-to-Site VPN**: Establish a VPN connection as an alternative. diff --git a/mkdocs.yml b/mkdocs.yml index d81cd08297..ed2b2c72aa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -140,6 +140,7 @@ nav: - Supported Clouds: tre-admins/supported-clouds.md - Customer Managed Keys: tre-admins/customer-managed-keys.md - Custom Domain Name: tre-admins/custom-domain.md + - Firewall Force Tunneling: tre-admins/configure-firewall-force-tunneling.md - Development: # Docs related to the developing code for the AzureTRE - Local Development: using-tre/local-development/local-development.md diff --git a/templates/shared_services/firewall/parameters.json b/templates/shared_services/firewall/parameters.json index 7883f0aa13..6a1df98da8 100755 --- a/templates/shared_services/firewall/parameters.json +++ b/templates/shared_services/firewall/parameters.json @@ -63,6 +63,12 @@ "source": { "env": "ARM_ENVIRONMENT" } + }, + { + "name": "firewall_force_tunnel_ip", + "source": { + "env": "FIREWALL_FORCE_TUNNEL_IP" + } } ] } diff --git a/templates/shared_services/firewall/porter.yaml b/templates/shared_services/firewall/porter.yaml index d5e7003d14..ffba80504b 100644 --- a/templates/shared_services/firewall/porter.yaml +++ b/templates/shared_services/firewall/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-firewall -version: 1.2.8 +version: 1.3.0 description: "An Azure TRE Firewall shared service" dockerfile: Dockerfile.tmpl registry: azuretre @@ -54,6 +54,9 @@ parameters: default: "graph.microsoft.com" - name: arm_environment type: string + - name: firewall_force_tunnel_ip + type: string + default: "" mixins: - terraform: @@ -69,6 +72,7 @@ install: api_driven_network_rule_collections_b64: ${ bundle.parameters.network_rule_collections } firewall_sku: ${ bundle.parameters.firewall_sku } microsoft_graph_fqdn: ${ bundle.parameters.microsoft_graph_fqdn } + firewall_force_tunnel_ip: ${ bundle.parameters.firewall_force_tunnel_ip } backendConfig: use_azuread_auth: "true" use_oidc: "true" @@ -87,6 +91,7 @@ upgrade: api_driven_network_rule_collections_b64: ${ bundle.parameters.network_rule_collections } firewall_sku: ${ bundle.parameters.firewall_sku } microsoft_graph_fqdn: ${ bundle.parameters.microsoft_graph_fqdn } + firewall_force_tunnel_ip: ${ bundle.parameters.firewall_force_tunnel_ip } backendConfig: use_azuread_auth: "true" use_oidc: "true" @@ -105,6 +110,7 @@ uninstall: api_driven_network_rule_collections_b64: ${ bundle.parameters.network_rule_collections } firewall_sku: ${ bundle.parameters.firewall_sku } microsoft_graph_fqdn: ${ bundle.parameters.microsoft_graph_fqdn } + firewall_force_tunnel_ip: ${ bundle.parameters.firewall_force_tunnel_ip } backendConfig: use_azuread_auth: "true" use_oidc: "true" diff --git a/templates/shared_services/firewall/template_schema.json b/templates/shared_services/firewall/template_schema.json index cd3aab13b3..23b0f8a526 100644 --- a/templates/shared_services/firewall/template_schema.json +++ b/templates/shared_services/firewall/template_schema.json @@ -6,111 +6,127 @@ "description": "Provides Firewall shared service", "required": [], "properties": { + "firewall_sku": { + "type": "string", + "title": "Firewall SKU", + "description": "The SKU that will be used when deploying The Firewall.", + "default": "Standard", + "enum": [ + "Basic", + "Standard", + "Premium" + ] + }, + "firewall_force_tunnel_ip": { + "type": "string", + "title": "Force Tunnel IP", + "description": "Optionally specify an IP address to forward all traffic to" + }, "rule_collections": { - "$id": "#properties/rule_collections", - "title": "application rule collections", - "type": "array", - "default": [], + "$id": "#properties/rule_collections", + "title": "application rule collections", + "type": "array", + "default": [], "updateable": true, - "items":{ - "title": "items", - "type": "object", - "required": [ - "name", - "rules" - ], - "properties": { - "name": { - "title": "name", - "type": "string", - "examples": [ - "my-rule" - ], - "pattern": "^.*$" - }, - "action": { - "title": "action DEPRECATED", - "type": "string", - "examples": [ - "Allow" - ], + "items": { + "title": "items", + "type": "object", + "required": [ + "name", + "rules" + ], + "properties": { + "name": { + "title": "name", + "type": "string", + "examples": [ + "my-rule" + ], + "pattern": "^.*$" + }, + "action": { + "title": "action DEPRECATED", + "type": "string", + "examples": [ + "Allow" + ], "enum": [ "Allow", "Deny" ] }, - "rules": { - "title": "rules", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "title": "name", - "type": "string", - "examples": [ - "rule 1" - ], - "pattern": "^.*$" - }, - "description": { - "title": "description", - "type": "string", - "default": "", - "examples": [ - "My rule description here" - ], - "pattern": "^.*$" - }, - "protocols": { - "title": "protocols", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "object", - "required": [ - "port", - "type" - ], - "properties": { - "port": { - "title": "port", - "type": "string", - "examples": [ - "1234" - ], - "pattern": "^.*$" - }, - "type": { - "title": "type", - "type": "string", + "rules": { + "title": "rules", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "title": "name", + "type": "string", + "examples": [ + "rule 1" + ], + "pattern": "^.*$" + }, + "description": { + "title": "description", + "type": "string", + "default": "", + "examples": [ + "My rule description here" + ], + "pattern": "^.*$" + }, + "protocols": { + "title": "protocols", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "object", + "required": [ + "port", + "type" + ], + "properties": { + "port": { + "title": "port", + "type": "string", + "examples": [ + "1234" + ], + "pattern": "^.*$" + }, + "type": { + "title": "type", + "type": "string", "enum": [ "Http", "Https", "Mssql" ], - "examples": [ - "Http" - ] - } - } - } - }, + "examples": [ + "Http" + ] + } + } + } + }, "fqdn_tags": { - "title": "fqdn tags", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "enum":[ + "title": "fqdn tags", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "enum": [ "AppServiceEnvironment", "AzureBackup", "AzureKubernetesService", @@ -120,246 +136,246 @@ "WindowsUpdate", "WindowsVirtualDesktop" ], - "examples": [ - "AzureKubernetesService" - ] - } - }, - "target_fqdns": { - "title": "destination fqdns", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "one.two.three.com" - ] - } - }, - "source_addresses": { - "title": "source addresses", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "172.196.0.0" - ] - } - }, + "examples": [ + "AzureKubernetesService" + ] + } + }, + "target_fqdns": { + "title": "destination fqdns", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "one.two.three.com" + ] + } + }, + "source_addresses": { + "title": "source addresses", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "172.196.0.0" + ] + } + }, "source_ip_group_ids": { - "title": "source ip group ids", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "some_ip_group_id" - ] - } - }, + "title": "source ip group ids", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "some_ip_group_id" + ] + } + }, "source_ip_groups_in_core": { - "title": "source ip group names in core", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "ip_group_name_in_core_resource_group" - ] - } - } - } - } - } - } - } + "title": "source ip group names in core", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "ip_group_name_in_core_resource_group" + ] + } + } + } + } + } + } + } }, "network_rule_collections": { - "$id": "#properties/network_rule_collections", - "title": "network rule collections", - "type": "array", - "default": [], + "$id": "#properties/network_rule_collections", + "title": "network rule collections", + "type": "array", + "default": [], "updateable": true, - "items":{ - "title": "items", - "type": "object", - "required": [ - "name", - "rules" - ], - "properties": { - "name": { - "title": "name", - "type": "string", - "examples": [ - "my-rule" - ], - "pattern": "^.*$" - }, - "action": { - "title": "action DEPRECATED", - "type": "string", - "examples": [ - "Allow" - ], + "items": { + "title": "items", + "type": "object", + "required": [ + "name", + "rules" + ], + "properties": { + "name": { + "title": "name", + "type": "string", + "examples": [ + "my-rule" + ], + "pattern": "^.*$" + }, + "action": { + "title": "action DEPRECATED", + "type": "string", + "examples": [ + "Allow" + ], "enum": [ "Allow", "Deny" ] }, - "rules": { - "title": "rules", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "title": "name", - "type": "string", - "examples": [ - "rule 1" - ], - "pattern": "^.{5,80}$" - }, - "description": { - "title": "description DEPRECATED", - "type": "string", - "default": "", - "examples": [ - "My rule description here" - ], - "pattern": "^.*$" - }, - "source_addresses": { - "title": "source addresses", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "172.196.0.0" - ] - } - }, + "rules": { + "title": "rules", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "title": "name", + "type": "string", + "examples": [ + "rule 1" + ], + "pattern": "^.{5,80}$" + }, + "description": { + "title": "description DEPRECATED", + "type": "string", + "default": "", + "examples": [ + "My rule description here" + ], + "pattern": "^.*$" + }, + "source_addresses": { + "title": "source addresses", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "172.196.0.0" + ] + } + }, "source_ip_group_ids": { - "title": "source ip group ids", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "some_ip_group_id" - ] - } - }, + "title": "source ip group ids", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "some_ip_group_id" + ] + } + }, "source_ip_groups_in_core": { - "title": "source ip group names in core", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "some_ip_group_name" - ] - } - }, - "destination_addresses": { - "title": "destination addresses", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "172.196.0.0" - ] - } - }, + "title": "source ip group names in core", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "some_ip_group_name" + ] + } + }, + "destination_addresses": { + "title": "destination addresses", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "172.196.0.0" + ] + } + }, "destination_ip_group_ids": { - "title": "destination ip group ids", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "some_ip_group_id" - ] - } - }, + "title": "destination ip group ids", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "some_ip_group_id" + ] + } + }, "destination_fqdns": { - "title": "destination fqdns", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "one.two.three.com" - ] - } - }, + "title": "destination fqdns", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "one.two.three.com" + ] + } + }, "destination_ports": { - "title": "destination ports", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", - "default": "", - "examples": [ - "80", + "title": "destination ports", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", + "default": "", + "examples": [ + "80", "443", "*" - ] - } - }, - "protocols": { - "title": "protocols", - "type": "array", - "default": [], - "items":{ - "title": "items", - "type": "string", + ] + } + }, + "protocols": { + "title": "protocols", + "type": "array", + "default": [], + "items": { + "title": "items", + "type": "string", "enum": [ "Any", "ICMP", "TCP", "UDP" ], - "examples": [ - "TCP" - ] - } - } - } - } - } - } - } + "examples": [ + "TCP" + ] + } + } + } + } + } + } + } } } } diff --git a/templates/shared_services/firewall/terraform/firewall.tf b/templates/shared_services/firewall/terraform/firewall.tf index ae94aecff0..6697a359b6 100644 --- a/templates/shared_services/firewall/terraform/firewall.tf +++ b/templates/shared_services/firewall/terraform/firewall.tf @@ -15,7 +15,7 @@ moved { } resource "azurerm_public_ip" "fwmanagement" { - count = local.effective_firewall_sku == "Basic" ? 1 : 0 + count = (var.firewall_force_tunnel_ip != "" || local.effective_firewall_sku == "Basic") ? 1 : 0 name = "pip-fw-management-${var.tre_id}" resource_group_name = local.core_resource_group_name location = data.azurerm_resource_group.rg.location @@ -42,7 +42,7 @@ resource "azurerm_firewall" "fw" { } dynamic "management_ip_configuration" { - for_each = local.effective_firewall_sku == "Basic" ? [1] : [] + for_each = (var.firewall_force_tunnel_ip != "" || local.effective_firewall_sku == "Basic") ? [1] : [] content { name = "mgmtconfig" subnet_id = data.azurerm_subnet.firewall_management.id diff --git a/templates/shared_services/firewall/terraform/routetable.tf b/templates/shared_services/firewall/terraform/routetable.tf index f0e4388d9b..23202788eb 100644 --- a/templates/shared_services/firewall/terraform/routetable.tf +++ b/templates/shared_services/firewall/terraform/routetable.tf @@ -87,3 +87,28 @@ resource "azurerm_subnet_route_table_association" "rt_airlock_events_subnet_asso azurerm_firewall_policy_rule_collection_group.dynamic_application ] } + +resource "azurerm_route_table" "fw_tunnel_rt" { + count = var.firewall_force_tunnel_ip != "" ? 1 : 0 + name = "rt-fw-tunnel-${var.tre_id}" + resource_group_name = local.core_resource_group_name + location = data.azurerm_resource_group.rg.location + bgp_route_propagation_enabled = true + tags = local.tre_shared_service_tags + + lifecycle { ignore_changes = [tags] } + + route { + name = "ForceTunnelRoute" + address_prefix = "0.0.0.0/0" + next_hop_type = "VirtualAppliance" + next_hop_in_ip_address = var.firewall_force_tunnel_ip + } +} + +resource "azurerm_subnet_route_table_association" "rt_fw_tunnel_subnet_association" { + count = var.firewall_force_tunnel_ip != "" ? 1 : 0 + subnet_id = data.azurerm_subnet.firewall.id + route_table_id = azurerm_route_table.fw_tunnel_rt[0].id +} + diff --git a/templates/shared_services/firewall/terraform/variables.tf b/templates/shared_services/firewall/terraform/variables.tf index a1017e157f..1140c25f40 100644 --- a/templates/shared_services/firewall/terraform/variables.tf +++ b/templates/shared_services/firewall/terraform/variables.tf @@ -27,3 +27,8 @@ variable "firewall_sku" { type = string default = "" } + +variable "firewall_force_tunnel_ip" { + type = string + default = "" +} From 37040028726e999e19a2d80645981dbdfd99aad2 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:19:51 +0200 Subject: [PATCH 03/38] Add EventGrid diagnostic settings (#4258) * add eventgrid diagnostic settings * changelog * core version * add system topics * uncomment --- CHANGELOG.md | 3 +- core/terraform/airlock/airlock_processor.tf | 8 ---- core/terraform/airlock/data.tf | 21 +++++++++ core/terraform/airlock/eventgrid_topics.tf | 52 +++++++++++++++++++-- core/terraform/airlock/identity.tf | 5 -- core/terraform/airlock/locals.tf | 2 + core/terraform/deploy.sh | 2 +- core/version.txt | 2 +- 8 files changed, 74 insertions(+), 21 deletions(-) create mode 100644 core/terraform/airlock/data.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d19e990a0..fcc8d8b3f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,8 @@ ENHANCEMENTS: * Upgrade Python version from 3.8 to 3.12 ([#3949](https://github.com/microsoft/AzureTRE/issues/3949))Upgrade Python version from 3.8 to 3.12 (#3949) * Disable storage account key usage ([[#4227](https://github.com/microsoft/AzureTRE/issues/4227)]) * Update Guacamole dependencies ([[#4232](https://github.com/microsoft/AzureTRE/issues/4232)]) +* Add option to force tunnel TRE's Firewall ([#4237](https://github.com/microsoft/AzureTRE/issues/4237)) +* Add EventGrid diagnostics to identify airlock issues ([#4258](https://github.com/microsoft/AzureTRE/issues/4258)) BUG FIXES: * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) @@ -49,7 +51,6 @@ BUG FIXES: * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222)) * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)]) * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)]) -* Add option to force tunnel TRE's Firewall ([#4237](https://github.com/microsoft/AzureTRE/issues/4237)) COMPONENTS: diff --git a/core/terraform/airlock/airlock_processor.tf b/core/terraform/airlock/airlock_processor.tf index be784443ef..80a6968e97 100644 --- a/core/terraform/airlock/airlock_processor.tf +++ b/core/terraform/airlock/airlock_processor.tf @@ -1,11 +1,3 @@ -data "local_file" "airlock_processor_version" { - filename = "${path.root}/../../airlock_processor/_version.py" -} - -locals { - version = replace(replace(replace(data.local_file.airlock_processor_version.content, "__version__ = \"", ""), "\"", ""), "\n", "") -} - resource "azurerm_service_plan" "airlock_plan" { name = "plan-airlock-${var.tre_id}" resource_group_name = var.resource_group_name diff --git a/core/terraform/airlock/data.tf b/core/terraform/airlock/data.tf new file mode 100644 index 0000000000..73915e6b6a --- /dev/null +++ b/core/terraform/airlock/data.tf @@ -0,0 +1,21 @@ +data "local_file" "airlock_processor_version" { + filename = "${path.root}/../../airlock_processor/_version.py" +} + +data "azurerm_private_dns_zone" "eventgrid" { + name = module.terraform_azurerm_environment_configuration.private_links["privatelink.eventgrid.azure.net"] + resource_group_name = var.resource_group_name +} + +data "azurerm_container_registry" "mgmt_acr" { + name = var.mgmt_acr_name + resource_group_name = var.mgmt_resource_group_name +} + +data "azurerm_monitor_diagnostic_categories" "eventgrid_custom_topics" { + resource_id = azurerm_eventgrid_topic.airlock_notification.id +} + +data "azurerm_monitor_diagnostic_categories" "eventgrid_system_topics" { + resource_id = azurerm_eventgrid_system_topic.export_approved_blob_created.id +} diff --git a/core/terraform/airlock/eventgrid_topics.tf b/core/terraform/airlock/eventgrid_topics.tf index 991325d2e1..2b967a6b79 100644 --- a/core/terraform/airlock/eventgrid_topics.tf +++ b/core/terraform/airlock/eventgrid_topics.tf @@ -1,8 +1,3 @@ -data "azurerm_private_dns_zone" "eventgrid" { - name = module.terraform_azurerm_environment_configuration.private_links["privatelink.eventgrid.azure.net"] - resource_group_name = var.resource_group_name -} - # Below we assign a SYSTEM-assigned identity for the topics. note that a user-assigned identity will not work. # Event grid topics @@ -511,3 +506,50 @@ resource "azurerm_eventgrid_event_subscription" "export_approved_blob_created" { ] } +resource "azurerm_monitor_diagnostic_setting" "eventgrid_custom_topics" { + for_each = merge({ + (azurerm_eventgrid_topic.airlock_notification.name) = azurerm_eventgrid_topic.airlock_notification.id, + (azurerm_eventgrid_topic.step_result.name) = azurerm_eventgrid_topic.step_result.id, + (azurerm_eventgrid_topic.status_changed.name) = azurerm_eventgrid_topic.status_changed.id, + (azurerm_eventgrid_topic.data_deletion.name) = azurerm_eventgrid_topic.data_deletion.id, + }, + var.enable_malware_scanning ? { (azurerm_eventgrid_topic.scan_result[0].name) = azurerm_eventgrid_topic.scan_result[0].id } : null + ) + + name = "${each.key}-diagnostics" + target_resource_id = each.value + log_analytics_workspace_id = var.log_analytics_workspace_id + dynamic "enabled_log" { + for_each = data.azurerm_monitor_diagnostic_categories.eventgrid_custom_topics.log_category_types + content { + category = enabled_log.value + } + } + + metric { + category = "AllMetrics" + } +} + +resource "azurerm_monitor_diagnostic_setting" "eventgrid_system_topics" { + for_each = { + (azurerm_eventgrid_system_topic.import_inprogress_blob_created.name) = azurerm_eventgrid_system_topic.import_inprogress_blob_created.id, + (azurerm_eventgrid_system_topic.import_rejected_blob_created.name) = azurerm_eventgrid_system_topic.import_rejected_blob_created.id, + (azurerm_eventgrid_system_topic.import_blocked_blob_created.name) = azurerm_eventgrid_system_topic.import_blocked_blob_created.id, + (azurerm_eventgrid_system_topic.export_approved_blob_created.name) = azurerm_eventgrid_system_topic.export_approved_blob_created.id, + } + + name = "${each.key}-diagnostics" + target_resource_id = each.value + log_analytics_workspace_id = var.log_analytics_workspace_id + dynamic "enabled_log" { + for_each = data.azurerm_monitor_diagnostic_categories.eventgrid_system_topics.log_category_types + content { + category = enabled_log.value + } + } + + metric { + category = "AllMetrics" + } +} diff --git a/core/terraform/airlock/identity.tf b/core/terraform/airlock/identity.tf index abca21c60d..9711f19ab6 100644 --- a/core/terraform/airlock/identity.tf +++ b/core/terraform/airlock/identity.tf @@ -1,8 +1,3 @@ -data "azurerm_container_registry" "mgmt_acr" { - name = var.mgmt_acr_name - resource_group_name = var.mgmt_resource_group_name -} - resource "azurerm_user_assigned_identity" "airlock_id" { resource_group_name = var.resource_group_name location = var.location diff --git a/core/terraform/airlock/locals.tf b/core/terraform/airlock/locals.tf index 37f7fdead0..3bc09392b6 100644 --- a/core/terraform/airlock/locals.tf +++ b/core/terraform/airlock/locals.tf @@ -1,4 +1,6 @@ locals { + version = replace(replace(replace(data.local_file.airlock_processor_version.content, "__version__ = \"", ""), "\"", ""), "\n", "") + # STorage AirLock EXternal import_external_storage_name = lower(replace("stalimex${var.tre_id}", "-", "")) # STorage AirLock IMport InProgress diff --git a/core/terraform/deploy.sh b/core/terraform/deploy.sh index e71fb14ae1..148cf1aca4 100755 --- a/core/terraform/deploy.sh +++ b/core/terraform/deploy.sh @@ -10,8 +10,8 @@ set -o nounset # shellcheck disable=SC1091 source ./migrate.sh -PLAN_FILE="tfplan$$" TS=$(date +"%s") +PLAN_FILE="${TS}-tre-core.tfplan" LOG_FILE="${TS}-tre-core.log" # This variables are loaded in for us diff --git a/core/version.txt b/core/version.txt index 1e6e806534..d0f18418d1 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.16" +__version__ = "0.11.17" From daef19b114d4d32bba37e48c9b46644defdfe2ed Mon Sep 17 00:00:00 2001 From: Guy Bertental Date: Wed, 15 Jan 2025 07:17:09 +0200 Subject: [PATCH 04/38] Add mermaid diagram for Airlock export request process (#4267) --- docs/azure-tre-overview/airlock.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/azure-tre-overview/airlock.md b/docs/azure-tre-overview/airlock.md index 91b50ee77c..720343a7b9 100644 --- a/docs/azure-tre-overview/airlock.md +++ b/docs/azure-tre-overview/airlock.md @@ -141,6 +141,21 @@ graph LR ``` > Data movement in an Airlock import request +```mermaid +graph LR + subgraph TRE workspace + data(Data to export)-->A + A[(stalexint
export internal)]-->|Request Submitted| B + B[(stalexip
export in-progress)]-->|Security issues found| D[(stalexblocked
export blocked)] + B-->|No security issues found| review{Manual
Approval} + review-->|Rejected| C[(stalexrej
export rejected)] + end + subgraph External + review-->|Approved| E[(stalexapp
export approved)] + end +``` +> Data movement in an Airlock export request + TRE: From 5b99b599cb39d723cf6853a53a86ffa71ec0f249 Mon Sep 17 00:00:00 2001 From: Jonny Rylands Date: Sat, 18 Jan 2025 08:16:11 +0000 Subject: [PATCH 05/38] Surface the Guacamole server-layout parameter (#4266) * Surface the Guacamole server-layout parameter --- CHANGELOG.md | 1 + .../guacamole/guacamole-server/docker/version.txt | 2 +- .../auth/azuretre/connection/ConnectionService.java | 7 +++++++ templates/workspace_services/guacamole/parameters.json | 6 ++++++ templates/workspace_services/guacamole/porter.yaml | 10 +++++++++- .../guacamole/terraform/variables.tf | 4 ++++ .../workspace_services/guacamole/terraform/web_app.tf | 1 + 7 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcc8d8b3f6..72a55a77b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ ENHANCEMENTS: * Update Guacamole dependencies ([[#4232](https://github.com/microsoft/AzureTRE/issues/4232)]) * Add option to force tunnel TRE's Firewall ([#4237](https://github.com/microsoft/AzureTRE/issues/4237)) * Add EventGrid diagnostics to identify airlock issues ([#4258](https://github.com/microsoft/AzureTRE/issues/4258)) +* Surface the server-layout parameter of Guacamole [server-layout](https://guacamole.apache.org/doc/gug/configuring-guacamole.html#session-settings) ([#4234](https://github.com/microsoft/AzureTRE/issues/4234)) BUG FIXES: * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) diff --git a/templates/workspace_services/guacamole/guacamole-server/docker/version.txt b/templates/workspace_services/guacamole/guacamole-server/docker/version.txt index c5981731c5..e94731c0f2 100644 --- a/templates/workspace_services/guacamole/guacamole-server/docker/version.txt +++ b/templates/workspace_services/guacamole/guacamole-server/docker/version.txt @@ -1 +1 @@ -__version__ = "0.9.3" +__version__ = "0.9.4" diff --git a/templates/workspace_services/guacamole/guacamole-server/guacamole-auth-azure/src/main/java/org/apache/guacamole/auth/azuretre/connection/ConnectionService.java b/templates/workspace_services/guacamole/guacamole-server/guacamole-auth-azure/src/main/java/org/apache/guacamole/auth/azuretre/connection/ConnectionService.java index fa1792c82c..26f798ef14 100644 --- a/templates/workspace_services/guacamole/guacamole-server/guacamole-auth-azure/src/main/java/org/apache/guacamole/auth/azuretre/connection/ConnectionService.java +++ b/templates/workspace_services/guacamole/guacamole-server/guacamole-auth-azure/src/main/java/org/apache/guacamole/auth/azuretre/connection/ConnectionService.java @@ -111,6 +111,13 @@ private static void setConfig( config.setParameter("drive-path", System.getenv("GUAC_DRIVE_PATH")); config.setParameter("disable-download", System.getenv("GUAC_DISABLE_DOWNLOAD")); config.setParameter("disable-upload", System.getenv("GUAC_DISABLE_UPLOAD")); + + String serverLayout = System.getenv("GUAC_SERVER_LAYOUT"); + if (serverLayout != null) { + if (!serverLayout.isEmpty()) { + config.setParameter("server-layout", serverLayout); + } + } } private static JSONArray getVMsFromProjectAPI(final AzureTREAuthenticatedUser user) throws GuacamoleException { diff --git a/templates/workspace_services/guacamole/parameters.json b/templates/workspace_services/guacamole/parameters.json index 253f44127d..4e61f8cbb2 100755 --- a/templates/workspace_services/guacamole/parameters.json +++ b/templates/workspace_services/guacamole/parameters.json @@ -94,6 +94,12 @@ "env": "GUAC_ENABLE_DRIVE" } }, + { + "name": "guac_server_layout", + "source": { + "env": "GUAC_SERVER_LAYOUT" + } + }, { "name": "image_tag", "source": { diff --git a/templates/workspace_services/guacamole/porter.yaml b/templates/workspace_services/guacamole/porter.yaml index c7009d1c04..7f6b523b6b 100644 --- a/templates/workspace_services/guacamole/porter.yaml +++ b/templates/workspace_services/guacamole/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole -version: 0.12.6 +version: 0.12.7 description: "An Azure TRE service for Guacamole" dockerfile: Dockerfile.tmpl registry: azuretre @@ -75,6 +75,11 @@ parameters: type: boolean default: true env: GUAC_DISABLE_UPLOAD + - name: guac_server_layout + type: string + default: "" + env: GUAC_SERVER_LAYOUT + description: "Guacamole server keyboard layout (defaults to en-us-qwerty when not set)" - name: is_exposed_externally type: boolean default: true @@ -144,6 +149,7 @@ install: guac_drive_path: ${ bundle.parameters.guac_drive_path } guac_disable_download: ${ bundle.parameters.guac_disable_download } guac_disable_upload: ${ bundle.parameters.guac_disable_upload } + guac_server_layout: ${ bundle.parameters.guac_server_layout } is_exposed_externally: ${ bundle.parameters.is_exposed_externally } tre_resource_id: ${ bundle.parameters.id } aad_authority_url: ${ bundle.parameters.aad_authority_url } @@ -177,6 +183,7 @@ upgrade: guac_drive_path: ${ bundle.parameters.guac_drive_path } guac_disable_download: ${ bundle.parameters.guac_disable_download } guac_disable_upload: ${ bundle.parameters.guac_disable_upload } + guac_server_layout: ${ bundle.parameters.guac_server_layout } is_exposed_externally: ${ bundle.parameters.is_exposed_externally } tre_resource_id: ${ bundle.parameters.id } aad_authority_url: ${ bundle.parameters.aad_authority_url } @@ -210,6 +217,7 @@ uninstall: guac_drive_path: ${ bundle.parameters.guac_drive_path } guac_disable_download: ${ bundle.parameters.guac_disable_download } guac_disable_upload: ${ bundle.parameters.guac_disable_upload } + guac_server_layout: ${ bundle.parameters.guac_server_layout } is_exposed_externally: ${ bundle.parameters.is_exposed_externally } tre_resource_id: ${ bundle.parameters.id } aad_authority_url: ${ bundle.parameters.aad_authority_url } diff --git a/templates/workspace_services/guacamole/terraform/variables.tf b/templates/workspace_services/guacamole/terraform/variables.tf index 26fcbb2f05..b62991b3c5 100644 --- a/templates/workspace_services/guacamole/terraform/variables.tf +++ b/templates/workspace_services/guacamole/terraform/variables.tf @@ -54,6 +54,10 @@ variable "guac_disable_upload" { type = bool description = "Disable upload to the Guacamole workspace" } +variable "guac_server_layout" { + type = string + description = "Server keyboard layout" +} variable "is_exposed_externally" { type = bool description = "Is the Guacamole workspace to be exposed externally?" diff --git a/templates/workspace_services/guacamole/terraform/web_app.tf b/templates/workspace_services/guacamole/terraform/web_app.tf index c92190b9e1..ac6a2ceb05 100644 --- a/templates/workspace_services/guacamole/terraform/web_app.tf +++ b/templates/workspace_services/guacamole/terraform/web_app.tf @@ -55,6 +55,7 @@ resource "azurerm_linux_web_app" "guacamole" { GUAC_DRIVE_PATH = var.guac_drive_path GUAC_DISABLE_DOWNLOAD = var.guac_disable_download GUAC_DISABLE_UPLOAD = var.guac_disable_upload + GUAC_SERVER_LAYOUT = var.guac_server_layout AUDIENCE = "@Microsoft.KeyVault(SecretUri=${data.azurerm_key_vault_secret.workspace_client_id.id})" ISSUER = local.issuer From fd89717d6e674c95cf90b20197b9ad2c478bc155 Mon Sep 17 00:00:00 2001 From: Jonny Rylands Date: Thu, 23 Jan 2025 16:26:06 +0000 Subject: [PATCH 06/38] Allow enablement of Trusted Launch and vTPM for VMs (#4265) --- CHANGELOG.md | 1 + core/terraform/resource_processor/vmss_porter/main.tf | 4 +++- core/version.txt | 2 +- templates/shared_services/admin-vm/porter.yaml | 2 +- .../admin-vm/terraform/admin-jumpbox.tf | 7 ++++++- .../shared_services/sonatype-nexus-vm/porter.yaml | 2 +- .../shared_services/sonatype-nexus-vm/terraform/vm.tf | 7 ++++++- .../guacamole/user_resources/README.md | 6 ++++++ .../guacamole-azure-export-reviewvm/porter.yaml | 4 +++- .../terraform/locals.tf | 2 ++ .../terraform/windowsvm.tf | 7 ++++++- .../guacamole-azure-import-reviewvm/porter.yaml | 6 +++++- .../terraform/locals.tf | 2 ++ .../terraform/windowsvm.tf | 7 ++++++- .../user_resources/guacamole-azure-linuxvm/porter.yaml | 6 +++++- .../guacamole-azure-linuxvm/terraform/linuxvm.tf | 7 ++++++- .../guacamole-azure-linuxvm/terraform/locals.tf | 2 ++ .../guacamole-azure-windowsvm/porter.yaml | 10 +++++++++- .../guacamole-azure-windowsvm/terraform/locals.tf | 2 ++ .../guacamole-azure-windowsvm/terraform/windowsvm.tf | 7 ++++++- 20 files changed, 79 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72a55a77b0..79ed543bd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ ENHANCEMENTS: * Update Guacamole dependencies ([[#4232](https://github.com/microsoft/AzureTRE/issues/4232)]) * Add option to force tunnel TRE's Firewall ([#4237](https://github.com/microsoft/AzureTRE/issues/4237)) * Add EventGrid diagnostics to identify airlock issues ([#4258](https://github.com/microsoft/AzureTRE/issues/4258)) +* Allow enablement of Secure Boot and vTPM for Guacamole VMs ([#4235](https://github.com/microsoft/AzureTRE/issues/4235)) * Surface the server-layout parameter of Guacamole [server-layout](https://guacamole.apache.org/doc/gug/configuring-guacamole.html#session-settings) ([#4234](https://github.com/microsoft/AzureTRE/issues/4234)) BUG FIXES: diff --git a/core/terraform/resource_processor/vmss_porter/main.tf b/core/terraform/resource_processor/vmss_porter/main.tf index 3adaae391b..d1d92c565b 100644 --- a/core/terraform/resource_processor/vmss_porter/main.tf +++ b/core/terraform/resource_processor/vmss_porter/main.tf @@ -82,6 +82,8 @@ resource "azurerm_linux_virtual_machine_scale_set" "vm_linux" { encryption_at_host_enabled = false upgrade_mode = "Automatic" tags = local.tre_core_tags + secure_boot_enabled = true + vtpm_enabled = true extension { auto_upgrade_minor_version = true @@ -127,7 +129,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "vm_linux" { source_image_reference { publisher = "Canonical" offer = "0001-com-ubuntu-server-jammy" - sku = "22_04-lts" + sku = "22_04-lts-gen2" version = "latest" } diff --git a/core/version.txt b/core/version.txt index d0f18418d1..b663def5a3 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.17" +__version__ = "0.11.18" diff --git a/templates/shared_services/admin-vm/porter.yaml b/templates/shared_services/admin-vm/porter.yaml index 4bab3df9cf..8a967f7b5a 100644 --- a/templates/shared_services/admin-vm/porter.yaml +++ b/templates/shared_services/admin-vm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-admin-vm -version: 0.5.1 +version: 0.5.2 description: "An admin vm shared service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/shared_services/admin-vm/terraform/admin-jumpbox.tf b/templates/shared_services/admin-vm/terraform/admin-jumpbox.tf index e89ff05203..2d9a2047b2 100644 --- a/templates/shared_services/admin-vm/terraform/admin-jumpbox.tf +++ b/templates/shared_services/admin-vm/terraform/admin-jumpbox.tf @@ -36,6 +36,8 @@ resource "azurerm_windows_virtual_machine" "jumpbox" { admin_username = "adminuser" admin_password = random_password.password.result tags = local.tre_shared_service_tags + secure_boot_enabled = true + vtpm_enabled = true source_image_reference { publisher = "MicrosoftWindowsDesktop" @@ -51,7 +53,10 @@ resource "azurerm_windows_virtual_machine" "jumpbox" { disk_encryption_set_id = var.enable_cmk_encryption ? azurerm_disk_encryption_set.jumpbox_disk_encryption[0].id : null } - lifecycle { ignore_changes = [tags] } + # ignore changes to secure_boot_enabled and vtpm_enabled as these are destructive + # (may be allowed once https://github.com/hashicorp/terraform-provider-azurerm/issues/25808 is fixed) + # + lifecycle { ignore_changes = [tags, secure_boot_enabled, vtpm_enabled] } } resource "azurerm_disk_encryption_set" "jumpbox_disk_encryption" { diff --git a/templates/shared_services/sonatype-nexus-vm/porter.yaml b/templates/shared_services/sonatype-nexus-vm/porter.yaml index 31ab0a05e4..526b8e5d6e 100644 --- a/templates/shared_services/sonatype-nexus-vm/porter.yaml +++ b/templates/shared_services/sonatype-nexus-vm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-sonatype-nexus -version: 3.3.1 +version: 3.3.2 description: "A Sonatype Nexus shared service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf index 7d3de07039..8bd6d3ff66 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf @@ -103,10 +103,15 @@ resource "azurerm_linux_virtual_machine" "nexus" { admin_username = "adminuser" admin_password = random_password.nexus_vm_password.result tags = local.tre_shared_service_tags + secure_boot_enabled = true + vtpm_enabled = true custom_data = data.template_cloudinit_config.nexus_config.rendered - lifecycle { ignore_changes = [tags] } + # ignore changes to secure_boot_enabled and vtpm_enabled as these are destructive + # (may be allowed once https://github.com/hashicorp/terraform-provider-azurerm/issues/25808 is fixed) + # + lifecycle { ignore_changes = [tags, secure_boot_enabled, vtpm_enabled] } source_image_reference { publisher = "Canonical" diff --git a/templates/workspace_services/guacamole/user_resources/README.md b/templates/workspace_services/guacamole/user_resources/README.md index 623098ad42..608509e00d 100644 --- a/templates/workspace_services/guacamole/user_resources/README.md +++ b/templates/workspace_services/guacamole/user_resources/README.md @@ -37,10 +37,14 @@ custom: apt_sku: 22.04 install_ui: true conda_config: false + secure_boot_enabled: true + vtpm_enabled: true # "Custom Image From Gallery": # source_image_name: your-image # install_ui: true # conda_config: true + # secure_boot_enabled: true + # vtpm_enabled: true ``` The `vm_sizes` section is a map of a custom SKU description to the SKU identifier. @@ -55,6 +59,8 @@ Within the image definition in `image_options` there are a few properties that c | `source_image_reference` | Specify VM image to use by `publisher`, `offer`, `sku` & `version` (e.g. for Azure Marketplace images) | | `install_ui` | (Linux only) Set `true` to install desktop environment | | `conda_config` | Set true to configure conda | +| `secure_boot_enabled` | Set true to enable [Secure Boot](https://learn.microsoft.com/en-us/azure/virtual-machines/trusted-launch#secure-boot). Requires a Requires a [Gen 2](https://learn.microsoft.com/en-us/azure/virtual-machines/generation-2) VM image | +| `vtpm_enabled` | Set true to enable [Secure Boot](https://learn.microsoft.com/en-us/azure/virtual-machines/trusted-launch#vtpm). Requires a [Gen 2](https://learn.microsoft.com/en-us/azure/virtual-machines/generation-2) VM image | When specifying images using `source_image_name`, the image must be stored in an [image gallery](https://learn.microsoft.com/en-us/azure/virtual-machines/azure-compute-gallery). To enable re-using built user resource templates across environments where the image may vary, the image gallery is configured via the `RP_BUNDLE_VALUES` environment variable when deploying the TRE. diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml index 08a2bdf9e0..8a2fa4c0a0 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-export-reviewvm -version: 0.2.1 +version: 0.2.2 description: "An Azure TRE User Resource Template for reviewing Airlock export requests" dockerfile: Dockerfile.tmpl registry: azuretre @@ -18,6 +18,8 @@ custom: sku: winserver-2019 version: latest conda_config: true + secure_boot_enabled: false # dsvm-win-2019 is not a gen2 image + vtpm_enabled: false credentials: - name: azure_tenant_id diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/locals.tf index 1f3ac946bd..33784b6275 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/locals.tf @@ -24,6 +24,8 @@ locals { # selected_image_source_refs is an array to enable easy use of a dynamic block selected_image_source_refs = lookup(local.selected_image, "source_image_reference", null) == null ? [] : [local.selected_image.source_image_reference] selected_image_source_id = lookup(local.selected_image, "source_image_name", null) == null ? null : "${var.image_gallery_id}/images/${local.selected_image.source_image_name}" + secure_boot_enabled = lookup(local.selected_image, "secure_boot_enabled", false) + vtpm_enabled = lookup(local.selected_image, "vtpm_enabled", false) cmk_name = "tre-encryption-${local.workspace_resource_name_suffix}" encryption_identity_name = "id-encryption-${var.tre_id}-${local.short_workspace_id}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/windowsvm.tf index 318ff29761..84724fddbf 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/windowsvm.tf @@ -124,6 +124,8 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { allow_extension_operations = true admin_username = random_string.username.result admin_password = random_password.password.result + secure_boot_enabled = local.secure_boot_enabled + vtpm_enabled = local.vtpm_enabled custom_data = base64encode(data.template_file.download_review_data_script.rendered) @@ -152,7 +154,10 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { tags = local.tre_user_resources_tags - lifecycle { ignore_changes = [tags] } + # ignore changes to secure_boot_enabled and vtpm_enabled as these are destructive + # (may be allowed once https://github.com/hashicorp/terraform-provider-azurerm/issues/25808 is fixed) + # + lifecycle { ignore_changes = [tags, secure_boot_enabled, vtpm_enabled] } } resource "azurerm_disk_encryption_set" "windowsvm_disk_encryption" { diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml index 3e362fc2ef..871eb4d617 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-import-reviewvm -version: 0.3.1 +version: 0.3.2 description: "An Azure TRE User Resource Template for reviewing Airlock import requests" dockerfile: Dockerfile.tmpl registry: azuretre @@ -18,11 +18,15 @@ custom: sku: winserver-2019 version: latest conda_config: true + secure_boot_enabled: false # dsvm-win-2019 is not a gen2 image + vtpm_enabled: false # For information on using custom images, see README.me in the guacamole/user-resources folder # "Custom Image From Gallery": # source_image_name: sltestwin1 # source_image_reference: null, # conda_config: true + # secure_boot_enabled: false + # vtpm_enabled: false credentials: - name: azure_tenant_id diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/locals.tf index 1f3ac946bd..33784b6275 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/locals.tf @@ -24,6 +24,8 @@ locals { # selected_image_source_refs is an array to enable easy use of a dynamic block selected_image_source_refs = lookup(local.selected_image, "source_image_reference", null) == null ? [] : [local.selected_image.source_image_reference] selected_image_source_id = lookup(local.selected_image, "source_image_name", null) == null ? null : "${var.image_gallery_id}/images/${local.selected_image.source_image_name}" + secure_boot_enabled = lookup(local.selected_image, "secure_boot_enabled", false) + vtpm_enabled = lookup(local.selected_image, "vtpm_enabled", false) cmk_name = "tre-encryption-${local.workspace_resource_name_suffix}" encryption_identity_name = "id-encryption-${var.tre_id}-${local.short_workspace_id}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf index a4d250b7f4..e761cca22e 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf @@ -45,6 +45,8 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { allow_extension_operations = true admin_username = random_string.username.result admin_password = random_password.password.result + secure_boot_enabled = local.secure_boot_enabled + vtpm_enabled = local.vtpm_enabled custom_data = base64encode(data.template_file.download_review_data_script.rendered) @@ -73,7 +75,10 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { tags = local.tre_user_resources_tags - lifecycle { ignore_changes = [tags] } + # ignore changes to secure_boot_enabled and vtpm_enabled as these are destructive + # (may be allowed once https://github.com/hashicorp/terraform-provider-azurerm/issues/25808 is fixed) + # + lifecycle { ignore_changes = [tags, secure_boot_enabled, vtpm_enabled] } } resource "azurerm_disk_encryption_set" "windowsvm_disk_encryption" { diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml index 30895442e5..b8ecdc4c79 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-linuxvm -version: 1.2.3 +version: 1.2.4 description: "An Azure TRE User Resource Template for Guacamole (Linux)" dockerfile: Dockerfile.tmpl registry: azuretre @@ -23,11 +23,15 @@ custom: apt_sku: 22.04 install_ui: true conda_config: false + secure_boot_enabled: true + vtpm_enabled: true # For information on using custom images, see README.me in the guacamole/user-resources folder # "Custom Image From Gallery": # source_image_name: your-image # install_ui: true # conda_config: true + # secure_boot_enabled: false + # vtpm_enabled: false credentials: - name: azure_tenant_id diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf index fb2b0b4ce8..ca407e318f 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf @@ -44,6 +44,8 @@ resource "azurerm_linux_virtual_machine" "linuxvm" { disable_password_authentication = false admin_username = random_string.username.result admin_password = random_password.password.result + secure_boot_enabled = local.secure_boot_enabled + vtpm_enabled = local.vtpm_enabled custom_data = data.template_cloudinit_config.config.rendered @@ -72,7 +74,10 @@ resource "azurerm_linux_virtual_machine" "linuxvm" { tags = local.tre_user_resources_tags - lifecycle { ignore_changes = [tags] } + # ignore changes to secure_boot_enabled and vtpm_enabled as these are destructive + # (may be allowed once https://github.com/hashicorp/terraform-provider-azurerm/issues/25808 is fixed) + # + lifecycle { ignore_changes = [tags, secure_boot_enabled, vtpm_enabled] } } resource "azurerm_disk_encryption_set" "linuxvm_disk_encryption" { diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf index a7f326efcf..073adc03ab 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/locals.tf @@ -25,6 +25,8 @@ locals { # selected_image_source_refs is an array to enable easy use of a dynamic block selected_image_source_refs = lookup(local.selected_image, "source_image_reference", null) == null ? [] : [local.selected_image.source_image_reference] selected_image_source_id = lookup(local.selected_image, "source_image_name", null) == null ? null : "${var.image_gallery_id}/images/${local.selected_image.source_image_name}" + secure_boot_enabled = lookup(local.selected_image, "secure_boot_enabled", false) + vtpm_enabled = lookup(local.selected_image, "vtpm_enabled", false) apt_sku = local.selected_image_source_refs[0]["apt_sku"] cmk_name = "tre-encryption-${local.workspace_resource_name_suffix}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index e93de02f38..8587988668 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-windowsvm -version: 1.2.3 +version: 1.2.4 description: "An Azure TRE User Resource Template for Guacamole (Windows 10)" dockerfile: Dockerfile.tmpl registry: azuretre @@ -21,6 +21,8 @@ custom: sku: win10-22h2-pro-g2 version: latest conda_config: false + secure_boot_enabled: true + vtpm_enabled: true "Windows 11": source_image_reference: publisher: microsoftwindowsdesktop @@ -28,6 +30,8 @@ custom: sku: win11-24h2-pro version: latest conda_config: false + secure_boot_enabled: true + vtpm_enabled: true "Server 2019 Data Science VM": source_image_reference: publisher: microsoft-dsvm @@ -35,10 +39,14 @@ custom: sku: winserver-2019 version: latest conda_config: true + secure_boot_enabled: false # dsvm-win-2019 is not a gen2 image + vtpm_enabled: false # For information on using custom images, see README.me in the guacamole/user-resources folder # "Custom Image From Gallery": # source_image_name: your-image # conda_config: true + # secure_boot_enabled: false + # vtpm_enabled: false credentials: - name: azure_tenant_id diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf index 239e304772..d4b784f5d8 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/locals.tf @@ -26,6 +26,8 @@ locals { # selected_image_source_refs is an array to enable easy use of a dynamic block selected_image_source_refs = lookup(local.selected_image, "source_image_reference", null) == null ? [] : [local.selected_image.source_image_reference] selected_image_source_id = lookup(local.selected_image, "source_image_name", null) == null ? null : "${var.image_gallery_id}/images/${local.selected_image.source_image_name}" + secure_boot_enabled = lookup(local.selected_image, "secure_boot_enabled", false) + vtpm_enabled = lookup(local.selected_image, "vtpm_enabled", false) cmk_name = "tre-encryption-${local.workspace_resource_name_suffix}" encryption_identity_name = "id-encryption-${var.tre_id}-${local.short_workspace_id}" diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf index 336293814d..f0de361955 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf @@ -45,6 +45,8 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { allow_extension_operations = true admin_username = random_string.username.result admin_password = random_password.password.result + secure_boot_enabled = local.secure_boot_enabled + vtpm_enabled = local.vtpm_enabled custom_data = base64encode(templatefile( "${path.module}/vm_config.ps1", { @@ -83,7 +85,10 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { tags = local.tre_user_resources_tags - lifecycle { ignore_changes = [tags] } + # ignore changes to secure_boot_enabled and vtpm_enabled as these are destructive + # (may be allowed once https://github.com/hashicorp/terraform-provider-azurerm/issues/25808 is fixed) + # + lifecycle { ignore_changes = [tags, secure_boot_enabled, vtpm_enabled] } } resource "azurerm_disk_encryption_set" "windowsvm_disk_encryption" { From 7c88fe498de3dabc9adb2bfd0122d1f49f246a5b Mon Sep 17 00:00:00 2001 From: Yuval Yaron <43217306+yuvalyaron@users.noreply.github.com> Date: Tue, 28 Jan 2025 18:51:01 +0200 Subject: [PATCH 07/38] Enable encryption at host for vms (#4263) --- CHANGELOG.md | 1 + core/terraform/resource_processor/vmss_porter/main.tf | 2 +- core/terraform/servicebus.tf | 5 +++-- core/version.txt | 2 +- resource_processor/_version.py | 2 +- .../shared_services/admin-vm/terraform/admin-jumpbox.tf | 1 + templates/shared_services/sonatype-nexus-vm/terraform/vm.tf | 1 + .../guacamole-azure-export-reviewvm/terraform/windowsvm.tf | 1 + .../guacamole-azure-import-reviewvm/terraform/windowsvm.tf | 1 + .../guacamole-azure-linuxvm/terraform/linuxvm.tf | 1 + .../guacamole-azure-windowsvm/terraform/windowsvm.tf | 1 + 11 files changed, 13 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ed543bd4..929f97efc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ ENHANCEMENTS: * Add EventGrid diagnostics to identify airlock issues ([#4258](https://github.com/microsoft/AzureTRE/issues/4258)) * Allow enablement of Secure Boot and vTPM for Guacamole VMs ([#4235](https://github.com/microsoft/AzureTRE/issues/4235)) * Surface the server-layout parameter of Guacamole [server-layout](https://guacamole.apache.org/doc/gug/configuring-guacamole.html#session-settings) ([#4234](https://github.com/microsoft/AzureTRE/issues/4234)) +* Add encryption at host for VMs ([#4263](https://github.com/microsoft/AzureTRE/pull/4263)) BUG FIXES: * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) diff --git a/core/terraform/resource_processor/vmss_porter/main.tf b/core/terraform/resource_processor/vmss_porter/main.tf index d1d92c565b..f390be1863 100644 --- a/core/terraform/resource_processor/vmss_porter/main.tf +++ b/core/terraform/resource_processor/vmss_porter/main.tf @@ -79,7 +79,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "vm_linux" { disable_password_authentication = false admin_password = random_password.password.result custom_data = data.template_cloudinit_config.config.rendered - encryption_at_host_enabled = false + encryption_at_host_enabled = true upgrade_mode = "Automatic" tags = local.tre_core_tags secure_boot_enabled = true diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index faef9322d7..f686a8e08e 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -32,8 +32,9 @@ resource "azurerm_servicebus_namespace" "sb" { dynamic "customer_managed_key" { for_each = var.enable_cmk_encryption ? [1] : [] content { - key_vault_key_id = azurerm_key_vault_key.tre_encryption[0].id - identity_id = azurerm_user_assigned_identity.encryption[0].id + key_vault_key_id = azurerm_key_vault_key.tre_encryption[0].id + identity_id = azurerm_user_assigned_identity.encryption[0].id + infrastructure_encryption_enabled = true } } diff --git a/core/version.txt b/core/version.txt index b663def5a3..318bf6c824 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.18" +__version__ = "0.11.19" diff --git a/resource_processor/_version.py b/resource_processor/_version.py index ae6db5f176..fee46bd8ce 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.11.0" +__version__ = "0.11.1" diff --git a/templates/shared_services/admin-vm/terraform/admin-jumpbox.tf b/templates/shared_services/admin-vm/terraform/admin-jumpbox.tf index 2d9a2047b2..97919f81d1 100644 --- a/templates/shared_services/admin-vm/terraform/admin-jumpbox.tf +++ b/templates/shared_services/admin-vm/terraform/admin-jumpbox.tf @@ -36,6 +36,7 @@ resource "azurerm_windows_virtual_machine" "jumpbox" { admin_username = "adminuser" admin_password = random_password.password.result tags = local.tre_shared_service_tags + encryption_at_host_enabled = true secure_boot_enabled = true vtpm_enabled = true diff --git a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf index 8bd6d3ff66..224143937a 100644 --- a/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf +++ b/templates/shared_services/sonatype-nexus-vm/terraform/vm.tf @@ -103,6 +103,7 @@ resource "azurerm_linux_virtual_machine" "nexus" { admin_username = "adminuser" admin_password = random_password.nexus_vm_password.result tags = local.tre_shared_service_tags + encryption_at_host_enabled = true secure_boot_enabled = true vtpm_enabled = true diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/windowsvm.tf index 84724fddbf..aa25d019a9 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-export-reviewvm/terraform/windowsvm.tf @@ -124,6 +124,7 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { allow_extension_operations = true admin_username = random_string.username.result admin_password = random_password.password.result + encryption_at_host_enabled = true secure_boot_enabled = local.secure_boot_enabled vtpm_enabled = local.vtpm_enabled diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf index e761cca22e..792330d7fd 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-import-reviewvm/terraform/windowsvm.tf @@ -45,6 +45,7 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { allow_extension_operations = true admin_username = random_string.username.result admin_password = random_password.password.result + encryption_at_host_enabled = true secure_boot_enabled = local.secure_boot_enabled vtpm_enabled = local.vtpm_enabled diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf index ca407e318f..30b1fdfcb9 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-linuxvm/terraform/linuxvm.tf @@ -44,6 +44,7 @@ resource "azurerm_linux_virtual_machine" "linuxvm" { disable_password_authentication = false admin_username = random_string.username.result admin_password = random_password.password.result + encryption_at_host_enabled = true secure_boot_enabled = local.secure_boot_enabled vtpm_enabled = local.vtpm_enabled diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf index f0de361955..2640c00759 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/windowsvm.tf @@ -45,6 +45,7 @@ resource "azurerm_windows_virtual_machine" "windowsvm" { allow_extension_operations = true admin_username = random_string.username.result admin_password = random_password.password.result + encryption_at_host_enabled = true secure_boot_enabled = local.secure_boot_enabled vtpm_enabled = local.vtpm_enabled From d61965d3ed2955df33637c8f61f7233b0f721673 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Wed, 5 Feb 2025 05:14:01 +0000 Subject: [PATCH 08/38] Fix dev container build failure on mount config and CI fixes (#4290) --- .devcontainer/devcontainer.json | 7 +++++-- CHANGELOG.md | 1 + e2e_tests/config.py | 8 +++----- e2e_tests/conftest.py | 12 +----------- e2e_tests/pytest.ini | 1 + e2e_tests/test_airlock.py | 2 +- e2e_tests/test_performance.py | 2 +- e2e_tests/test_provisioned_health_api.py | 2 +- e2e_tests/test_ui.py | 2 +- e2e_tests/test_workspace_service_templates.py | 2 +- e2e_tests/test_workspace_services.py | 2 +- e2e_tests/test_workspace_templates.py | 2 +- .../workspaces/airlock-import-review/porter.yaml | 2 +- .../airlock-import-review/template_schema.json | 1 + templates/workspaces/base/porter.yaml | 2 +- templates/workspaces/base/template_schema.json | 1 + templates/workspaces/unrestricted/porter.yaml | 2 +- .../workspaces/unrestricted/template_schema.json | 1 + 18 files changed, 24 insertions(+), 28 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9d49130458..8d4eefdebf 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -29,7 +29,7 @@ // Mount docker socket for docker builds "type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock", // Mounts the github cli login details from the host machine to the container (~/.config/gh/hosts.yml) - "type=bind,source=${env:HOME}${env:USERPROFILE}/.config,target=/home/vscode/.config", + "type=bind,source=${env:HOME}${env:USERPROFILE}/.config,target=/home/vscode/.config" ], "remoteUser": "vscode", "containerEnv": { @@ -277,6 +277,8 @@ "ms-python.pylance", "hashicorp.terraform", "github.vscode-pull-request-github", + "gitHub.copilot", + "github.copilot-chat", "getporter.porter-vscode", "davidanson.vscode-markdownlint", "editorconfig.editorconfig", @@ -291,5 +293,6 @@ 8000 ], // Run commands after the container is created. - "postCreateCommand": "./.devcontainer/scripts/post-create.sh" + "postCreateCommand": "./.devcontainer/scripts/post-create.sh", + "initializeCommand": "mkdir -p $HOME/.azure $HOME/.config || true" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 929f97efc4..1f5501078f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ BUG FIXES: * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222)) * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)]) * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)]) +* Fix dev container build failure on missing mount directories, add copilot extensions, and CI fixes ([#4290](https://github.com/microsoft/AzureTRE/pull/4290)) COMPONENTS: diff --git a/e2e_tests/config.py b/e2e_tests/config.py index cd43a78181..31ccb34c37 100644 --- a/e2e_tests/config.py +++ b/e2e_tests/config.py @@ -1,11 +1,9 @@ +import warnings from starlette.config import Config +warnings.filterwarnings("ignore", message="Config file '.env' not found.") -try: - config = Config('.env') -# Workaround needed until FastAPI uses Starlette >= 3.7.1 -except FileNotFoundError: - config = Config() +config = Config('.env') # Resource Info RESOURCE_LOCATION: str = config("RESOURCE_LOCATION", default="") diff --git a/e2e_tests/conftest.py b/e2e_tests/conftest.py index 7195a14588..39589e1696 100644 --- a/e2e_tests/conftest.py +++ b/e2e_tests/conftest.py @@ -12,23 +12,13 @@ LOGGER = logging.getLogger(__name__) -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(loop_scope="session") def pytest_addoption(parser): parser.addoption("--verify", action="store", default="true") -@pytest.fixture(scope="session") -def event_loop(): - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - yield loop - loop.close() - - @pytest.fixture(scope="session") def verify(pytestconfig): if pytestconfig.getoption("verify").lower() == "true": diff --git a/e2e_tests/pytest.ini b/e2e_tests/pytest.ini index 72fe2d3414..3e3cf490e3 100644 --- a/e2e_tests/pytest.ini +++ b/e2e_tests/pytest.ini @@ -10,6 +10,7 @@ markers = workspace_services asyncio_mode = auto +asyncio_default_fixture_loop_scope = session log_cli = 1 log_cli_level = INFO diff --git a/e2e_tests/test_airlock.py b/e2e_tests/test_airlock.py index 43e2df71bb..051a5c9d81 100644 --- a/e2e_tests/test_airlock.py +++ b/e2e_tests/test_airlock.py @@ -14,7 +14,7 @@ from helpers import get_admin_token -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(loop_scope="session") LOGGER = logging.getLogger(__name__) BLOB_FILE_PATH = "./test_airlock_sample.txt" BLOB_NAME = os.path.basename(BLOB_FILE_PATH) diff --git a/e2e_tests/test_performance.py b/e2e_tests/test_performance.py index 6c6d836d9d..f6e7637fe2 100644 --- a/e2e_tests/test_performance.py +++ b/e2e_tests/test_performance.py @@ -8,7 +8,7 @@ from helpers import get_admin_token -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(loop_scope="session") @pytest.mark.performance diff --git a/e2e_tests/test_provisioned_health_api.py b/e2e_tests/test_provisioned_health_api.py index 01c3cbeecc..92636d12dc 100644 --- a/e2e_tests/test_provisioned_health_api.py +++ b/e2e_tests/test_provisioned_health_api.py @@ -5,7 +5,7 @@ from resources import strings -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(loop_scope="session") @pytest.mark.smoke diff --git a/e2e_tests/test_ui.py b/e2e_tests/test_ui.py index 6e34da518b..8d71827fdb 100644 --- a/e2e_tests/test_ui.py +++ b/e2e_tests/test_ui.py @@ -4,7 +4,7 @@ import config -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(loop_scope="session") @pytest.mark.smoke diff --git a/e2e_tests/test_workspace_service_templates.py b/e2e_tests/test_workspace_service_templates.py index 34545d9af1..8ea70e6368 100644 --- a/e2e_tests/test_workspace_service_templates.py +++ b/e2e_tests/test_workspace_service_templates.py @@ -8,7 +8,7 @@ from resources import strings from helpers import get_admin_token -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(loop_scope="session") workspace_service_templates = [ (strings.AZUREML_SERVICE), diff --git a/e2e_tests/test_workspace_services.py b/e2e_tests/test_workspace_services.py index cd48910817..3013fef370 100644 --- a/e2e_tests/test_workspace_services.py +++ b/e2e_tests/test_workspace_services.py @@ -5,7 +5,7 @@ from resources.resource import get_resource, post_resource from resources import strings -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(loop_scope="session") workspace_services = [ strings.AZUREML_SERVICE, diff --git a/e2e_tests/test_workspace_templates.py b/e2e_tests/test_workspace_templates.py index d0ccb1f64e..1fed11daaa 100644 --- a/e2e_tests/test_workspace_templates.py +++ b/e2e_tests/test_workspace_templates.py @@ -11,7 +11,7 @@ from helpers import get_admin_token -pytestmark = pytest.mark.asyncio +pytestmark = pytest.mark.asyncio(loop_scope="session") workspace_templates = [ diff --git a/templates/workspaces/airlock-import-review/porter.yaml b/templates/workspaces/airlock-import-review/porter.yaml index 7e4330cd30..56f90dbc70 100644 --- a/templates/workspaces/airlock-import-review/porter.yaml +++ b/templates/workspaces/airlock-import-review/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-airlock-import-review -version: 0.14.1 +version: 0.14.2 description: "A workspace to do Airlock Data Import Reviews for Azure TRE" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/airlock-import-review/template_schema.json b/templates/workspaces/airlock-import-review/template_schema.json index e05a0d87e7..180e360abc 100644 --- a/templates/workspaces/airlock-import-review/template_schema.json +++ b/templates/workspaces/airlock-import-review/template_schema.json @@ -16,6 +16,7 @@ "description": "The SKU that will be used when deploying an Azure App Service Plan.", "default": "P1v3", "enum": [ + "P0v3", "P1v3", "P1v2", "S1" diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index 89be17e3de..a7e09fa692 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-base -version: 1.9.1 +version: 1.9.2 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/template_schema.json b/templates/workspaces/base/template_schema.json index 77049d7f1c..b456b5f044 100644 --- a/templates/workspaces/base/template_schema.json +++ b/templates/workspaces/base/template_schema.json @@ -28,6 +28,7 @@ "description": "The SKU that will be used when deploying an Azure App Service Plan.", "default": "P1v3", "enum": [ + "P0v3", "P1v3", "P1v2", "S1" diff --git a/templates/workspaces/unrestricted/porter.yaml b/templates/workspaces/unrestricted/porter.yaml index 5d1cc8cfa1..b8bd2becae 100644 --- a/templates/workspaces/unrestricted/porter.yaml +++ b/templates/workspaces/unrestricted/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-unrestricted -version: 0.13.1 +version: 0.13.2 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/unrestricted/template_schema.json b/templates/workspaces/unrestricted/template_schema.json index 3fbaa16a3a..f9c8a807f1 100644 --- a/templates/workspaces/unrestricted/template_schema.json +++ b/templates/workspaces/unrestricted/template_schema.json @@ -28,6 +28,7 @@ "description": "The SKU that will be used when deploying an Azure App Service Plan.", "default": "P1v3", "enum": [ + "P0v3", "P1v3", "P1v2", "S1" From 26621d13296320749e8e3f35e38dd351cd68b3ca Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Wed, 5 Feb 2025 11:44:14 +0200 Subject: [PATCH 09/38] Airlock Azure Function Host Storage Access with Managed Identity (#4276) Airlock function storage to use manage identity --- CHANGELOG.md | 2 +- core/terraform/airlock/airlock_processor.tf | 11 +++++------ core/terraform/airlock/identity.tf | 8 ++++++++ core/version.txt | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f5501078f..11b92d2d74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ ENHANCEMENTS: * Allow enablement of Secure Boot and vTPM for Guacamole VMs ([#4235](https://github.com/microsoft/AzureTRE/issues/4235)) * Surface the server-layout parameter of Guacamole [server-layout](https://guacamole.apache.org/doc/gug/configuring-guacamole.html#session-settings) ([#4234](https://github.com/microsoft/AzureTRE/issues/4234)) * Add encryption at host for VMs ([#4263](https://github.com/microsoft/AzureTRE/pull/4263)) +* Airlock function host storage to use the user-assigned managed identity ([#4276](https://github.com/microsoft/AzureTRE/issues/4276)) BUG FIXES: * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) @@ -54,7 +55,6 @@ BUG FIXES: * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222)) * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)]) * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)]) -* Fix dev container build failure on missing mount directories, add copilot extensions, and CI fixes ([#4290](https://github.com/microsoft/AzureTRE/pull/4290)) COMPONENTS: diff --git a/core/terraform/airlock/airlock_processor.tf b/core/terraform/airlock/airlock_processor.tf index 80a6968e97..a95bf54eaa 100644 --- a/core/terraform/airlock/airlock_processor.tf +++ b/core/terraform/airlock/airlock_processor.tf @@ -21,9 +21,8 @@ resource "azurerm_storage_account" "sa_airlock_processor_func_app" { allow_nested_items_to_be_public = false cross_tenant_replication_enabled = false local_user_enabled = false - # Function Host Storage doesn't seem to be able to use a User Managed ID, which is why we continue to use a key. - shared_access_key_enabled = true - tags = var.tre_core_tags + shared_access_key_enabled = false + tags = var.tre_core_tags dynamic "identity" { for_each = var.enable_cmk_encryption ? [1] : [] @@ -57,9 +56,7 @@ resource "azurerm_linux_function_app" "airlock_function_app" { ftp_publish_basic_authentication_enabled = false webdeploy_publish_basic_authentication_enabled = false storage_account_name = azurerm_storage_account.sa_airlock_processor_func_app.name - - # Function Host Storage doesn't seem to be able to use a User Managed ID, which is why we continue to use a key. - storage_account_access_key = azurerm_storage_account.sa_airlock_processor_func_app.primary_access_key + storage_uses_managed_identity = true tags = var.tre_core_tags @@ -86,6 +83,8 @@ resource "azurerm_linux_function_app" "airlock_function_app" { "TRE_ID" = var.tre_id "WEBSITE_CONTENTOVERVNET" = 1 "STORAGE_ENDPOINT_SUFFIX" = module.terraform_azurerm_environment_configuration.storage_suffix + "AzureWebJobsStorage__clientId" = azurerm_user_assigned_identity.airlock_id.client_id + "AzureWebJobsStorage__credential" = "managedidentity" } site_config { diff --git a/core/terraform/airlock/identity.tf b/core/terraform/airlock/identity.tf index 9711f19ab6..7f452ebdbb 100644 --- a/core/terraform/airlock/identity.tf +++ b/core/terraform/airlock/identity.tf @@ -52,3 +52,11 @@ resource "azurerm_role_assignment" "api_sa_data_contributor" { role_definition_name = "Storage Blob Data Contributor" principal_id = var.api_principal_id } + +# Permissions needed for the Function Host to work correctly. +resource "azurerm_role_assignment" "function_host_storage" { + for_each = toset(["Storage Account Contributor", "Storage Blob Data Owner", "Storage Queue Data Contributor"]) + scope = azurerm_storage_account.sa_airlock_processor_func_app.id + role_definition_name = each.value + principal_id = azurerm_user_assigned_identity.airlock_id.principal_id +} diff --git a/core/version.txt b/core/version.txt index 318bf6c824..a43ff2b5d7 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.19" +__version__ = "0.11.20" From 3838df278693193012fd3c9ef4034362294cf36c Mon Sep 17 00:00:00 2001 From: Jonny Rylands Date: Wed, 5 Feb 2025 10:34:51 +0000 Subject: [PATCH 10/38] Downgrade certs shared service App Gateway to Basic SKU (#4301) Downgrade certs shared service App Gateway to Basic SKU #4300 --- CHANGELOG.md | 1 + templates/shared_services/certs/porter.yaml | 2 +- .../certs/terraform/.terraform.lock.hcl | 28 +++++++++---------- .../certs/terraform/appgateway.tf | 4 +-- .../shared_services/certs/terraform/main.tf | 2 +- 5 files changed, 19 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11b92d2d74..4e4302bfde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ ENHANCEMENTS: * Allow enablement of Secure Boot and vTPM for Guacamole VMs ([#4235](https://github.com/microsoft/AzureTRE/issues/4235)) * Surface the server-layout parameter of Guacamole [server-layout](https://guacamole.apache.org/doc/gug/configuring-guacamole.html#session-settings) ([#4234](https://github.com/microsoft/AzureTRE/issues/4234)) * Add encryption at host for VMs ([#4263](https://github.com/microsoft/AzureTRE/pull/4263)) +* Downgrade certs shared service App Gateway to Basic SKU ([#4300](https://github.com/microsoft/AzureTRE/issues/4300)) * Airlock function host storage to use the user-assigned managed identity ([#4276](https://github.com/microsoft/AzureTRE/issues/4276)) BUG FIXES: diff --git a/templates/shared_services/certs/porter.yaml b/templates/shared_services/certs/porter.yaml index 92de8ee40d..c334eda592 100755 --- a/templates/shared_services/certs/porter.yaml +++ b/templates/shared_services/certs/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-certs -version: 0.7.2 +version: 0.7.3 description: "An Azure TRE shared service to generate certificates for a specified internal domain using Letsencrypt" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/shared_services/certs/terraform/.terraform.lock.hcl b/templates/shared_services/certs/terraform/.terraform.lock.hcl index 6e8b87ea6b..41771bac4d 100644 --- a/templates/shared_services/certs/terraform/.terraform.lock.hcl +++ b/templates/shared_services/certs/terraform/.terraform.lock.hcl @@ -2,21 +2,21 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.117.0" - constraints = "3.117.0" + version = "4.17.0" + constraints = "4.17.0" hashes = [ - "h1:Ynfg+Iy7x6K8M6W1AhqXCe3wkoiqIQhROlca7C3KC3w=", - "zh:2e25f47492366821a786762369f0e0921cc9452d64bfd5075f6fdfcf1a9c6d70", - "zh:41eb34f2f7469bf3eb1019dfb0e7fc28256f809824016f4f8b9d691bf473b2ac", - "zh:48bb9c87b3d928da1abc1d3db75453c9725de4674c612daf3800160cc7145d30", - "zh:5d6b0de0bbd78943fcc65c53944ef4496329e247f434c6eab86ed051c5cea67b", - "zh:78c9f6fdb1206a89cf0e6706b4f46178169a93b6c964a4cad8a321058ccbd9b4", - "zh:793b702c352589d4360b580d4a1cf654a7439d2ad6bdb7bfea91de07bc4b0fac", - "zh:7ed687ff0a5509463a592f97431863574fe5cc80a34e395be06766215b8c6285", - "zh:955ba18789bd15592824eb426a8d0f38595bd09fffc6939c1c58933489c1a71e", - "zh:bf5949a55be0714cd9c8815d472eae4baa48ba06d0f6bf2b96775869acda8a54", - "zh:da5d31f635abd2c645ffc76d6176d73f646128e73720cc368247cc424975c127", - "zh:eed5a66d59883c9c56729b0a964a2b60d758ea7489ef3e920a6fbd48518ce5f5", + "h1:gpFgaBSkRTxhavgPAuqQcElHJqmRJ1RpQGr1K0dvVW8=", + "zh:163b81a3bf29c8f161a1c100a48164b1bd1af434cd564b44596cb71a6c33f03d", + "zh:2996b107d3c05a9db14458b32b6f22f8cde0adb96263196d82d3dc302907a257", + "zh:361abd84b6e73016ebebb9ef9cd14c237d8b1e4500ea75f73243ff0534e5e4fb", + "zh:4872445dcb109fe8bbaba439d3dffaaef849a92645df3f8a854d3a40ac962f68", + "zh:61974eb7379acadbceb47b001ae1de2cdefe8cf078a15fff3a6fcc753cd24273", + "zh:75c60ca6e7851fe1d52fe9f5a0ae3d219e300ee5aa63bc8f807e3e0cab569ff0", + "zh:7c79305cff7849e6c5d9d60fe570510f95fb2e2bd5ae801da0281702f21dd779", + "zh:964b7da03f2dc55583cda3c277fef3511824b183a3a88344ae4ff9823af79109", + "zh:cad1593d364eb22b68578a1da4fd4d84749dc81f20e6591b27c6cb1eed9d2072", + "zh:db1a2ca17aae78813e8e0676bb9ef941e1a1e32d9fc6e1b239c24661605a8425", + "zh:e3a65d2f6f5a63cd1beeeb60a23e7e6b7328ebbd46ffe994792aaac6738186c3", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } diff --git a/templates/shared_services/certs/terraform/appgateway.tf b/templates/shared_services/certs/terraform/appgateway.tf index 909b3cb784..69d2b08db3 100644 --- a/templates/shared_services/certs/terraform/appgateway.tf +++ b/templates/shared_services/certs/terraform/appgateway.tf @@ -26,8 +26,8 @@ resource "azurerm_application_gateway" "agw" { tags = local.tre_shared_service_tags sku { - name = "Standard_v2" - tier = "Standard_v2" + name = "Basic" + tier = "Basic" capacity = 1 } diff --git a/templates/shared_services/certs/terraform/main.tf b/templates/shared_services/certs/terraform/main.tf index 001ccd9042..9f483e7251 100644 --- a/templates/shared_services/certs/terraform/main.tf +++ b/templates/shared_services/certs/terraform/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "=3.117.0" + version = "=4.17.0" } } From 7ecd1841b4281b446daff922eee49419cad8bc7f Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Wed, 5 Feb 2025 11:40:38 +0000 Subject: [PATCH 11/38] Fix letsencrypt missing parameters (#4292) * Fix letsencrypt missing parameters Fixes #3638 Update `core/terraform/outputs.sh` to handle empty `tre_output.json` file * Add a check to see if `tre_output.json` is empty * If `tre_output.json` is empty, run `terraform init` to recreate it --- For more details, open the [Copilot Workspace session](https://copilot-workspace.githubnext.com/microsoft/AzureTRE/issues/3638?shareId=XXXX-XXXX-XXXX-XXXX). * Update version and changelog. --- CHANGELOG.md | 1 + core/terraform/outputs.sh | 2 +- core/version.txt | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) mode change 100755 => 100644 core/terraform/outputs.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e4302bfde..53171c7c1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ BUG FIXES: * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222)) * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)]) * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)]) +* Recreate tre_output.json if empty. ([[#4292](https://github.com/microsoft/AzureTRE/issues/4292)]) COMPONENTS: diff --git a/core/terraform/outputs.sh b/core/terraform/outputs.sh old mode 100755 new mode 100644 index e00090bbc1..078b20be53 --- a/core/terraform/outputs.sh +++ b/core/terraform/outputs.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -if [ ! -f ../tre_output.json ]; then +if [ ! -f ../tre_output.json ] || [ ! -s ../tre_output.json ]; then # Connect to the remote backend of Terraform export TF_LOG="" # shellcheck disable=SC2154 diff --git a/core/version.txt b/core/version.txt index a43ff2b5d7..3c85494aac 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.20" +__version__ = "0.11.21" From 8103f9634cb7eec0c5bc6509c44ee5f2db31ee86 Mon Sep 17 00:00:00 2001 From: Jonny Rylands Date: Wed, 5 Feb 2025 14:06:09 +0000 Subject: [PATCH 12/38] Genericise Windows VM post install script to deal with different R versions (#4289) Windows R version must be 4.1.2 otherwise post install script doesn't update package mirror URL #4288 --- CHANGELOG.md | 1 + .../guacamole-azure-windowsvm/porter.yaml | 2 +- .../guacamole-azure-windowsvm/terraform/vm_config.ps1 | 11 +++++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53171c7c1c..4b9113b73a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ BUG FIXES: * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222)) * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)]) * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)]) +* Windows R version must be 4.1.2 otherwise post install script doesn't update package mirror URL ([#4288](https://github.com/microsoft/AzureTRE/issues/4288)) * Recreate tre_output.json if empty. ([[#4292](https://github.com/microsoft/AzureTRE/issues/4292)]) COMPONENTS: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index 8587988668..0f727c688b 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-windowsvm -version: 1.2.4 +version: 1.2.5 description: "An Azure TRE User Resource Template for Guacamole (Windows 10)" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 index f2604f7a3b..343ac0e33c 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 @@ -44,9 +44,6 @@ $DaemonConfig = @" $DaemonConfig | Out-File -Encoding Ascii ( New-Item -Path $env:ProgramData\docker\config\daemon.json -Force ) # R config -# $RconfigFilePathWindows = C:\Progra~1\R\4.1.2\etc\Rprofile.site -#Add-Content $RconfigFilePathWindows "local({`n r <- getOption(`"repos`")`n r[`"Nexus`"] <- `"${nexus_proxy_url}/repository/r-proxy/`"`n options(repos = r)`n})" -# echo "local({`n r <- getOption(`"repos`")`n r[`"Nexus`"] <- `"${nexus_proxy_url}/repository/r-proxy/`"`n options(repos = r)`n})" > $RconfigFilePathWindows $RConfig = @" local({ r <- getOption("repos") @@ -54,5 +51,11 @@ local({ options(repos = r) }) "@ -$RConfig | Out-File -Encoding Ascii ( New-Item -Path $Env:ProgramFiles\R\R-4.1.2\etc\Rprofile.site -Force ) +$RBasePath = "$Env:ProgramFiles\R" +$RVersions = Get-ChildItem -Path $RBasePath -Directory | Where-Object { $_.Name -like "R-*" } + +foreach ($RVersion in $RVersions) { + $ConfigPath = Join-Path -Path $RVersion.FullName -ChildPath "etc\Rprofile.site" + $RConfig | Out-File -Encoding Ascii (New-Item -Path $ConfigPath -Force) +} From 18bf38594f190365a2caf7d28a7ca23ea5674d49 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Wed, 5 Feb 2025 19:10:06 +0200 Subject: [PATCH 13/38] Disable local auth in EventGrid (#4254) * Disable local authentication in EventGrid * fix publish test results * fix python unit test procedure --- .github/workflows/deploy_tre_reusable.yml | 1 + CHANGELOG.md | 1 + .../BlobCreatedTrigger/function.json | 6 +-- .../ScanResultTrigger/function.json | 3 +- .../StatusChangedQueueTrigger/function.json | 6 +-- airlock_processor/_version.py | 2 +- .../run_tests_and_exit_succesfully.sh | 2 +- api_app/_version.py | 2 +- api_app/run_tests_and_exit_succesfully.sh | 2 +- core/terraform/airlock/airlock_processor.tf | 44 +++++++++++-------- core/terraform/airlock/eventgrid_topics.tf | 5 +++ core/terraform/airlock/identity.tf | 14 +++++- core/terraform/airlock/locals.tf | 3 ++ core/version.txt | 2 +- 14 files changed, 58 insertions(+), 35 deletions(-) diff --git a/.github/workflows/deploy_tre_reusable.yml b/.github/workflows/deploy_tre_reusable.yml index e775bed80b..99b9c7e4d0 100644 --- a/.github/workflows/deploy_tre_reusable.yml +++ b/.github/workflows/deploy_tre_reusable.yml @@ -863,3 +863,4 @@ jobs: with: junit_files: "artifacts/**/*.xml" check_name: "E2E Test Results" + comment_mode: off diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b9113b73a..f6ea6de28e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ ENHANCEMENTS: * Add encryption at host for VMs ([#4263](https://github.com/microsoft/AzureTRE/pull/4263)) * Downgrade certs shared service App Gateway to Basic SKU ([#4300](https://github.com/microsoft/AzureTRE/issues/4300)) * Airlock function host storage to use the user-assigned managed identity ([#4276](https://github.com/microsoft/AzureTRE/issues/4276)) +* Disable local authentication in EventGrid ([#4254](https://github.com/microsoft/AzureTRE/issues/4254)) BUG FIXES: * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) diff --git a/airlock_processor/BlobCreatedTrigger/function.json b/airlock_processor/BlobCreatedTrigger/function.json index 5bde252c39..5a652a8eff 100644 --- a/airlock_processor/BlobCreatedTrigger/function.json +++ b/airlock_processor/BlobCreatedTrigger/function.json @@ -13,15 +13,13 @@ { "type": "eventGrid", "name": "stepResultEvent", - "topicEndpointUri": "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING", - "topicKeySetting": "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING", + "connection": "EVENT_GRID_STEP_RESULT_CONNECTION", "direction": "out" }, { "type": "eventGrid", "name": "dataDeletionEvent", - "topicEndpointUri": "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING", - "topicKeySetting": "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING", + "connection": "EVENT_GRID_DATA_DELETION_CONNECTION", "direction": "out" } ] diff --git a/airlock_processor/ScanResultTrigger/function.json b/airlock_processor/ScanResultTrigger/function.json index 4dee63e389..32758cea1c 100644 --- a/airlock_processor/ScanResultTrigger/function.json +++ b/airlock_processor/ScanResultTrigger/function.json @@ -12,8 +12,7 @@ { "type": "eventGrid", "name": "outputEvent", - "topicEndpointUri": "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING", - "topicKeySetting": "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING", + "connection": "EVENT_GRID_STEP_RESULT_CONNECTION", "direction": "out" } ] diff --git a/airlock_processor/StatusChangedQueueTrigger/function.json b/airlock_processor/StatusChangedQueueTrigger/function.json index c5e7be3356..f686eca80a 100644 --- a/airlock_processor/StatusChangedQueueTrigger/function.json +++ b/airlock_processor/StatusChangedQueueTrigger/function.json @@ -11,15 +11,13 @@ { "type": "eventGrid", "name": "stepResultEvent", - "topicEndpointUri": "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING", - "topicKeySetting": "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING", + "connection": "EVENT_GRID_STEP_RESULT_CONNECTION", "direction": "out" }, { "type": "eventGrid", "name": "dataDeletionEvent", - "topicEndpointUri": "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING", - "topicKeySetting": "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING", + "connection": "EVENT_GRID_DATA_DELETION_CONNECTION", "direction": "out" } ] diff --git a/airlock_processor/_version.py b/airlock_processor/_version.py index 777f190df0..8088f75131 100644 --- a/airlock_processor/_version.py +++ b/airlock_processor/_version.py @@ -1 +1 @@ -__version__ = "0.8.0" +__version__ = "0.8.1" diff --git a/airlock_processor/run_tests_and_exit_succesfully.sh b/airlock_processor/run_tests_and_exit_succesfully.sh index 0b50ba6067..12884a743d 100755 --- a/airlock_processor/run_tests_and_exit_succesfully.sh +++ b/airlock_processor/run_tests_and_exit_succesfully.sh @@ -6,6 +6,6 @@ rm -f ../test-results/pytest_airlock_processor* mkdir -p ../test-results -if ! pytest --junit-xml ../test-results/pytest_airlock_processor_unit.xml --ignore e2e_tests; then +if ! python -m pytest --junit-xml ../test-results/pytest_airlock_processor_unit.xml --ignore e2e_tests; then touch ../test-results/pytest_airlock_processor_unit_failed fi diff --git a/api_app/_version.py b/api_app/_version.py index 8815fb52f3..8b8252f484 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.20.3" +__version__ = "0.20.4" diff --git a/api_app/run_tests_and_exit_succesfully.sh b/api_app/run_tests_and_exit_succesfully.sh index 34873d4c16..311a59f2d1 100755 --- a/api_app/run_tests_and_exit_succesfully.sh +++ b/api_app/run_tests_and_exit_succesfully.sh @@ -6,6 +6,6 @@ rm -f ../test-results/pytest_api* mkdir -p ../test-results -if ! pytest --junit-xml ../test-results/pytest_api_unit.xml --ignore e2e_tests -W ignore::pytest.PytestUnraisableExceptionWarning -W ignore::DeprecationWarning; then +if ! python -m pytest --junit-xml ../test-results/pytest_api_unit.xml --ignore e2e_tests -W ignore::pytest.PytestUnraisableExceptionWarning -W ignore::DeprecationWarning; then touch ../test-results/pytest_api_unit_failed fi diff --git a/core/terraform/airlock/airlock_processor.tf b/core/terraform/airlock/airlock_processor.tf index a95bf54eaa..ccb36b81bb 100644 --- a/core/terraform/airlock/airlock_processor.tf +++ b/core/terraform/airlock/airlock_processor.tf @@ -66,25 +66,31 @@ resource "azurerm_linux_function_app" "airlock_function_app" { } app_settings = { - "SB_CONNECTION_STRING" = var.airlock_servicebus.default_primary_connection_string - "BLOB_CREATED_TOPIC_NAME" = azurerm_servicebus_topic.blob_created.name - "TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name - "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.step_result.endpoint - "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.step_result.primary_access_key - "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.data_deletion.endpoint - "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.data_deletion.primary_access_key - "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false - "AIRLOCK_STATUS_CHANGED_QUEUE_NAME" = local.status_changed_queue_name - "AIRLOCK_SCAN_RESULT_QUEUE_NAME" = local.scan_result_queue_name - "AIRLOCK_DATA_DELETION_QUEUE_NAME" = local.data_deletion_queue_name - "ENABLE_MALWARE_SCANNING" = var.enable_malware_scanning - "ARM_ENVIRONMENT" = var.arm_environment - "MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.airlock_id.client_id - "TRE_ID" = var.tre_id - "WEBSITE_CONTENTOVERVNET" = 1 - "STORAGE_ENDPOINT_SUFFIX" = module.terraform_azurerm_environment_configuration.storage_suffix - "AzureWebJobsStorage__clientId" = azurerm_user_assigned_identity.airlock_id.client_id - "AzureWebJobsStorage__credential" = "managedidentity" + "SB_CONNECTION_STRING" = var.airlock_servicebus.default_primary_connection_string + "BLOB_CREATED_TOPIC_NAME" = azurerm_servicebus_topic.blob_created.name + "TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name + "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false + "AIRLOCK_STATUS_CHANGED_QUEUE_NAME" = local.status_changed_queue_name + "AIRLOCK_SCAN_RESULT_QUEUE_NAME" = local.scan_result_queue_name + "AIRLOCK_DATA_DELETION_QUEUE_NAME" = local.data_deletion_queue_name + "ENABLE_MALWARE_SCANNING" = var.enable_malware_scanning + "ARM_ENVIRONMENT" = var.arm_environment + "MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.airlock_id.client_id + "TRE_ID" = var.tre_id + "WEBSITE_CONTENTOVERVNET" = 1 + "STORAGE_ENDPOINT_SUFFIX" = module.terraform_azurerm_environment_configuration.storage_suffix + "AzureWebJobsStorage__clientId" = azurerm_user_assigned_identity.airlock_id.client_id + "AzureWebJobsStorage__credential" = "managedidentity" + + "EVENT_GRID_STEP_RESULT_CONNECTION" = local.step_result_eventgrid_connection + "${local.step_result_eventgrid_connection}__topicEndpointUri" = azurerm_eventgrid_topic.step_result.endpoint + "${local.step_result_eventgrid_connection}__credential" = "managedidentity" + "${local.step_result_eventgrid_connection}__clientId" = azurerm_user_assigned_identity.airlock_id.client_id + + "EVENT_GRID_DATA_DELETION_CONNECTION" = local.data_deletion_eventgrid_connection + "${local.data_deletion_eventgrid_connection}__topicEndpointUri" = azurerm_eventgrid_topic.data_deletion.endpoint + "${local.data_deletion_eventgrid_connection}__credential" = "managedidentity" + "${local.data_deletion_eventgrid_connection}__clientId" = azurerm_user_assigned_identity.airlock_id.client_id } site_config { diff --git a/core/terraform/airlock/eventgrid_topics.tf b/core/terraform/airlock/eventgrid_topics.tf index 2b967a6b79..d9faaef013 100644 --- a/core/terraform/airlock/eventgrid_topics.tf +++ b/core/terraform/airlock/eventgrid_topics.tf @@ -6,6 +6,7 @@ resource "azurerm_eventgrid_topic" "step_result" { location = var.location resource_group_name = var.resource_group_name public_network_access_enabled = var.enable_local_debugging + local_auth_enabled = false identity { type = "SystemAssigned" @@ -60,6 +61,7 @@ resource "azurerm_eventgrid_topic" "status_changed" { location = var.location resource_group_name = var.resource_group_name public_network_access_enabled = var.enable_local_debugging + local_auth_enabled = false identity { type = "SystemAssigned" @@ -113,6 +115,7 @@ resource "azurerm_eventgrid_topic" "data_deletion" { location = var.location resource_group_name = var.resource_group_name public_network_access_enabled = var.enable_local_debugging + local_auth_enabled = false identity { type = "SystemAssigned" @@ -163,6 +166,7 @@ resource "azurerm_eventgrid_topic" "scan_result" { resource_group_name = var.resource_group_name # This is mandatory for the scan result to be published since private networks are not supported yet public_network_access_enabled = true + local_auth_enabled = false identity { type = "SystemAssigned" @@ -323,6 +327,7 @@ resource "azurerm_eventgrid_topic" "airlock_notification" { location = var.location resource_group_name = var.resource_group_name public_network_access_enabled = var.enable_local_debugging + local_auth_enabled = false identity { type = "SystemAssigned" diff --git a/core/terraform/airlock/identity.tf b/core/terraform/airlock/identity.tf index 7f452ebdbb..a21a26f562 100644 --- a/core/terraform/airlock/identity.tf +++ b/core/terraform/airlock/identity.tf @@ -25,7 +25,7 @@ resource "azurerm_role_assignment" "servicebus_receiver" { principal_id = azurerm_user_assigned_identity.airlock_id.principal_id } -resource "azurerm_role_assignment" "eventgrid_data_sender" { +resource "azurerm_role_assignment" "eventgrid_data_sender_status_changed" { scope = azurerm_eventgrid_topic.status_changed.id role_definition_name = "EventGrid Data Sender" principal_id = var.api_principal_id @@ -37,6 +37,18 @@ resource "azurerm_role_assignment" "eventgrid_data_sender_notification" { principal_id = var.api_principal_id } +resource "azurerm_role_assignment" "eventgrid_data_sender_step_result" { + scope = azurerm_eventgrid_topic.step_result.id + role_definition_name = "EventGrid Data Sender" + principal_id = azurerm_user_assigned_identity.airlock_id.principal_id +} + +resource "azurerm_role_assignment" "eventgrid_data_sender_data_deletion" { + scope = azurerm_eventgrid_topic.data_deletion.id + role_definition_name = "EventGrid Data Sender" + principal_id = azurerm_user_assigned_identity.airlock_id.principal_id +} + resource "azurerm_role_assignment" "airlock_blob_data_contributor" { count = length(local.airlock_sa_blob_data_contributor) scope = local.airlock_sa_blob_data_contributor[count.index] diff --git a/core/terraform/airlock/locals.tf b/core/terraform/airlock/locals.tf index 3bc09392b6..8ed6805e0e 100644 --- a/core/terraform/airlock/locals.tf +++ b/core/terraform/airlock/locals.tf @@ -60,4 +60,7 @@ locals { azurerm_storage_account.sa_import_in_progress.id, azurerm_storage_account.sa_export_approved.id ] + + step_result_eventgrid_connection = "EVENT_GRID_STEP_RESULT_CONNECTION" + data_deletion_eventgrid_connection = "EVENT_GRID_DATA_DELETION_CONNECTION" } diff --git a/core/version.txt b/core/version.txt index 3c85494aac..663d6b3572 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.21" +__version__ = "0.11.22" From 595bbfb00444876ef368cf844819d46cc1016956 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Thu, 6 Feb 2025 07:51:33 +0000 Subject: [PATCH 14/38] [Snyk] Upgrade @fluentui/react-file-type-icons from 8.12.5 to 8.12.6 (#4262) fix: upgrade @fluentui/react-file-type-icons from 8.12.5 to 8.12.6 Snyk has created this PR to upgrade @fluentui/react-file-type-icons from 8.12.5 to 8.12.6. See this package in yarn: @fluentui/react-file-type-icons See this project in Snyk: https://app.snyk.io/org/marrobi/project/dd2ad75a-7a6d-4559-9a18-d70318d44707?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot Co-authored-by: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> --- ui/app/package.json | 2 +- ui/app/yarn.lock | 32 +++++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/ui/app/package.json b/ui/app/package.json index c23a15d135..0445e7c341 100644 --- a/ui/app/package.json +++ b/ui/app/package.json @@ -6,7 +6,7 @@ "@azure/msal-browser": "^2.35.0", "@azure/msal-react": "^1.5.12", "@fluentui/react": "^8.120.3", - "@fluentui/react-file-type-icons": "^8.11.20", + "@fluentui/react-file-type-icons": "^8.12.6", "@reduxjs/toolkit": "^1.8.6", "@rjsf/core": "^4.2.3", "@rjsf/fluent-ui": "^4.2.3", diff --git a/ui/app/yarn.lock b/ui/app/yarn.lock index 77b1aa6167..e6215fd6a1 100644 --- a/ui/app/yarn.lock +++ b/ui/app/yarn.lock @@ -1334,13 +1334,13 @@ "@fluentui/set-version" "^8.2.23" tslib "^2.1.0" -"@fluentui/react-file-type-icons@^8.11.20": - version "8.12.5" - resolved "https://registry.yarnpkg.com/@fluentui/react-file-type-icons/-/react-file-type-icons-8.12.5.tgz#7ba1f7f526a6c7454e49ed3bac0f1a8a60ee7656" - integrity sha512-SKidkVj2el1b1s+7cL5lXWtRwiHkWbRy/8iWn9SwwCYvob9jRc4hkVoNW/9ahlSJfVMyUmtG03/TeOdLzV05uA== +"@fluentui/react-file-type-icons@^8.12.6": + version "8.12.7" + resolved "https://registry.yarnpkg.com/@fluentui/react-file-type-icons/-/react-file-type-icons-8.12.7.tgz#6a46dd8f5829b0b901c67ab54d7ffce2f2c3f824" + integrity sha512-Q/J5l93vJySVBZF+PyirB77M5EXiFuv5tDXWqkVY4a8xH4w1OIaC6L4Irv4aa3ORwyaJuvbAh6YR1PYgpCy6Qw== dependencies: "@fluentui/set-version" "^8.2.23" - "@fluentui/style-utilities" "^8.11.4" + "@fluentui/style-utilities" "^8.11.6" tslib "^2.1.0" "@fluentui/react-focus@^8.9.18": @@ -1419,6 +1419,18 @@ "@microsoft/load-themed-styles" "^1.10.26" tslib "^2.1.0" +"@fluentui/style-utilities@^8.11.6": + version "8.11.6" + resolved "https://registry.yarnpkg.com/@fluentui/style-utilities/-/style-utilities-8.11.6.tgz#0a7c505d7b13b9671ca7b28ee7bd562789cf96a1" + integrity sha512-bVFu/ONP2+GZ/JzR6NhN7+1fuMHvi+LjOfgo21HQoDakY/KwFaitLiQBQFlRpbRUVcZXQDqe4Ur6EDFAlb2I7Q== + dependencies: + "@fluentui/merge-styles" "^8.6.13" + "@fluentui/set-version" "^8.2.23" + "@fluentui/theme" "^2.6.64" + "@fluentui/utilities" "^8.15.19" + "@microsoft/load-themed-styles" "^1.10.26" + tslib "^2.1.0" + "@fluentui/theme@^2.6.63": version "2.6.63" resolved "https://registry.yarnpkg.com/@fluentui/theme/-/theme-2.6.63.tgz#dfac29a06c54c3405d58772dc6863ed592b1cda3" @@ -1429,6 +1441,16 @@ "@fluentui/utilities" "^8.15.19" tslib "^2.1.0" +"@fluentui/theme@^2.6.64": + version "2.6.64" + resolved "https://registry.yarnpkg.com/@fluentui/theme/-/theme-2.6.64.tgz#6b8f1620ba1f217461441d9fab9ffd44325deb9c" + integrity sha512-cjzwPgq3Zsw4F6Xy7A7yN8WCeEXKTwk9lfJzEr5b00euJRuPMxkxesBbAWW43+/1l1eWVYmSm4GcEMDVD4BfXQ== + dependencies: + "@fluentui/merge-styles" "^8.6.13" + "@fluentui/set-version" "^8.2.23" + "@fluentui/utilities" "^8.15.19" + tslib "^2.1.0" + "@fluentui/utilities@^8.15.19": version "8.15.19" resolved "https://registry.yarnpkg.com/@fluentui/utilities/-/utilities-8.15.19.tgz#2229d4c294d4d6b1eb618a8ea6d6ec05392c5962" From cdb59e4da56e06fbd9d29342cf7e9ee767058515 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Thu, 6 Feb 2025 07:53:24 +0000 Subject: [PATCH 15/38] [Snyk] Upgrade @types/react from 18.3.12 to 18.3.16 (#4239) fix: upgrade @types/react from 18.3.12 to 18.3.16 Snyk has created this PR to upgrade @types/react from 18.3.12 to 18.3.16. See this package in yarn: @types/react See this project in Snyk: https://app.snyk.io/org/marrobi/project/dd2ad75a-7a6d-4559-9a18-d70318d44707?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot Co-authored-by: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> --- ui/app/package.json | 2 +- ui/app/yarn.lock | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ui/app/package.json b/ui/app/package.json index 0445e7c341..e158265a67 100644 --- a/ui/app/package.json +++ b/ui/app/package.json @@ -16,7 +16,7 @@ "@testing-library/user-event": "^14.4.3", "@types/jest": "^29.5.0", "@types/node": "^20.16.12", - "@types/react": "^18.3.3", + "@types/react": "^18.3.16", "@types/react-dom": "^18.2.6", "moment": "^2.29.4", "node-sass": "^8.0.0", diff --git a/ui/app/yarn.lock b/ui/app/yarn.lock index e6215fd6a1..0144c744de 100644 --- a/ui/app/yarn.lock +++ b/ui/app/yarn.lock @@ -2504,7 +2504,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@^18.3.3": +"@types/react@*": version "18.3.12" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.12.tgz#99419f182ccd69151813b7ee24b792fe08774f60" integrity sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw== @@ -2512,6 +2512,14 @@ "@types/prop-types" "*" csstype "^3.0.2" +"@types/react@^18.3.16": + version "18.3.18" + resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.18.tgz#9b382c4cd32e13e463f97df07c2ee3bbcd26904b" + integrity sha512-t4yC+vtgnkYjNSKlFx1jkAhH8LgTo2N/7Qvi83kdEaUtMDiwpbLAktKDaAMlRcJ5eSxZkH74eEGt1ky31d7kfQ== + dependencies: + "@types/prop-types" "*" + csstype "^3.0.2" + "@types/resolve@1.17.1": version "1.17.1" resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.17.1.tgz#3afd6ad8967c77e4376c598a82ddd58f46ec45d6" From 052aeffc06c9d0cdb5ebc9c53b16629b0c0f4633 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Thu, 6 Feb 2025 08:12:13 +0000 Subject: [PATCH 16/38] [Snyk] Upgrade react-router-dom from 6.27.0 to 6.28.0 (#4191) fix: upgrade react-router-dom from 6.27.0 to 6.28.0 Snyk has created this PR to upgrade react-router-dom from 6.27.0 to 6.28.0. See this package in yarn: react-router-dom See this project in Snyk: https://app.snyk.io/org/marrobi/project/dd2ad75a-7a6d-4559-9a18-d70318d44707?utm_source=github&utm_medium=referral&page=upgrade-pr Co-authored-by: snyk-bot Co-authored-by: Tim Allen Co-authored-by: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> --- ui/app/package.json | 2 +- ui/app/yarn.lock | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ui/app/package.json b/ui/app/package.json index e158265a67..b1a9a66992 100644 --- a/ui/app/package.json +++ b/ui/app/package.json @@ -24,7 +24,7 @@ "react-dom": "^18.3.1", "react-markdown": "^8.0.3", "react-redux": "^8.0.4", - "react-router-dom": "6.27.0", + "react-router-dom": "6.28.0", "remark-gfm": "^3.0.1", "typescript": "^5.6.3", "web-vitals": "^3.3.0" diff --git a/ui/app/yarn.lock b/ui/app/yarn.lock index 0144c744de..1cc293094d 100644 --- a/ui/app/yarn.lock +++ b/ui/app/yarn.lock @@ -1899,10 +1899,10 @@ redux-thunk "^2.4.2" reselect "^4.1.8" -"@remix-run/router@1.20.0": - version "1.20.0" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.20.0.tgz#03554155b45d8b529adf635b2f6ad1165d70d8b4" - integrity sha512-mUnk8rPJBI9loFDZ+YzPGdeniYK+FTmRD1TMCz7ev2SNIozyKKpnGgsxO34u6Z4z/t0ITuu7voi/AshfsGsgFg== +"@remix-run/router@1.21.0": + version "1.21.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.21.0.tgz#c65ae4262bdcfe415dbd4f64ec87676e4a56e2b5" + integrity sha512-xfSkCAchbdG5PnbrKqFWwia4Bi61nH+wm8wLEqfHDyp7Y3dZzgqS2itV8i4gAq9pC2HsTpwyBC6Ds8VHZ96JlA== "@rjsf/core@^4.2.3": version "4.2.3" @@ -9503,20 +9503,20 @@ react-refresh@^0.11.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.11.0.tgz#77198b944733f0f1f1a90e791de4541f9f074046" integrity sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A== -react-router-dom@6.27.0: - version "6.27.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.27.0.tgz#8d7972a425fd75f91c1e1ff67e47240c5752dc3f" - integrity sha512-+bvtFWMC0DgAFrfKXKG9Fc+BcXWRUO1aJIihbB79xaeq0v5UzfvnM5houGUm1Y461WVRcgAQ+Clh5rdb1eCx4g== +react-router-dom@6.28.0: + version "6.28.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.28.0.tgz#f73ebb3490e59ac9f299377062ad1d10a9f579e6" + integrity sha512-kQ7Unsl5YdyOltsPGl31zOjLrDv+m2VcIEcIHqYYD3Lp0UppLjrzcfJqDJwXxFw3TH/yvapbnUvPlAj7Kx5nbg== dependencies: - "@remix-run/router" "1.20.0" - react-router "6.27.0" + "@remix-run/router" "1.21.0" + react-router "6.28.0" -react-router@6.27.0: - version "6.27.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.27.0.tgz#db292474926c814c996c0ff3ef0162d1f9f60ed4" - integrity sha512-YA+HGZXz4jaAkVoYBE98VQl+nVzI+cVI2Oj/06F5ZM+0u3TgedN9Y9kmMRo2mnkSK2nCpNQn0DVob4HCsY/WLw== +react-router@6.28.0: + version "6.28.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.28.0.tgz#29247c86d7ba901d7e5a13aa79a96723c3e59d0d" + integrity sha512-HrYdIFqdrnhDw0PqG/AKjAqEqM7AvxCz0DQ4h2W8k6nqmc5uRBYDag0SBxx9iYz5G8gnuNVLzUe13wl9eAsXXg== dependencies: - "@remix-run/router" "1.20.0" + "@remix-run/router" "1.21.0" react-scripts@5.0.1: version "5.0.1" From 7ae3d18afb9af29ee5d6031ddd69d01cbf879546 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Thu, 6 Feb 2025 10:32:09 +0000 Subject: [PATCH 17/38] Fix dev container initialization on Windows (#4327) --- .devcontainer/devcontainer.json | 2 +- .devcontainer/scripts/initialize | 3 +++ .devcontainer/scripts/initialize.cmd | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100755 .devcontainer/scripts/initialize create mode 100644 .devcontainer/scripts/initialize.cmd diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8d4eefdebf..90b080ddb1 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -294,5 +294,5 @@ ], // Run commands after the container is created. "postCreateCommand": "./.devcontainer/scripts/post-create.sh", - "initializeCommand": "mkdir -p $HOME/.azure $HOME/.config || true" + "initializeCommand": ["./.devcontainer/scripts/initialize"] } diff --git a/.devcontainer/scripts/initialize b/.devcontainer/scripts/initialize new file mode 100755 index 0000000000..8cfb7eb8df --- /dev/null +++ b/.devcontainer/scripts/initialize @@ -0,0 +1,3 @@ +#!/bin/bash + +mkdir -p "$HOME/.azure" "$HOME/.config" || true diff --git a/.devcontainer/scripts/initialize.cmd b/.devcontainer/scripts/initialize.cmd new file mode 100644 index 0000000000..d8d6060e09 --- /dev/null +++ b/.devcontainer/scripts/initialize.cmd @@ -0,0 +1,2 @@ +@echo off +mkdir %USERPROFILE%\.azure %USERPROFILE%\.config || exit /b 0 From 8153f8e4d59128c351e485e0ea9becae063046b9 Mon Sep 17 00:00:00 2001 From: David Goon <19471397+david-goon@users.noreply.github.com> Date: Thu, 6 Feb 2025 14:21:57 +0000 Subject: [PATCH 18/38] Expanding industry coverage in Azure TRE Overview documentation (#4319) --- README.md | 12 +++++++----- docs/index.md | 12 ++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 2fbbb0b2b5..a0f4abcac9 100644 --- a/README.md +++ b/README.md @@ -5,21 +5,23 @@ ## Background -Across the health industry, be it a pharmaceutical company interrogating clinical trial results, or a public health provider analyzing electronic health records, there is the need to enable researchers, analysts, and developers to work with sensitive data sets. +Across multiple industries, be it a pharmaceutical company interrogating clinical trial results, a public health provider analyzing electronic health records, or an engineering organization working on next-generation systems, there is a need to enable researchers and solution developers to collaborate on sensitive data. -Trusted Research Environments (TREs) enable organisations to provide research teams secure access to these data sets alongside appropriate tooling to ensure researchers can remain efficient and productive despite the security controls in place. +Trusted Research Environments (TREs) enable organisations to provide research and development (R&D) teams secure access to data alongside tooling to ensure productivity while keeping security controls in place. -Further information on TREs in general can be found in many places, one good resource is [HDR UK's website](https://www.hdruk.ac.uk/access-to-health-data/trusted-research-environments/). +Further information on TREs in general can be found in many places, one good resource specifically for healthcare is [HDR UK's website](https://www.hdruk.ac.uk/access-to-health-data/trusted-research-environments/). + +Azure TRE is finding new applications and to date, has been used as the basis for exploring future secure collaboration environments in engineering. Workloads investigated include systems and software development. In truth, any workload in any industry could theoretically be supported with extensions to Azure TRE. The Azure Trusted Research Environment project is an accelerator to assist Microsoft customers and partners who want to build out Trusted Research environments on Azure. This project enables authorized users to deploy and configure secure workspaces and researcher tooling without a dependency on IT teams. -This project is typically implemented alongside a data platform that provides research ready datasets to TRE workspaces. +This project is typically implemented alongside a data platform that provides research ready datasets to TRE workspaces. Azure TRE has also been proven to handle unstructured data scenarios. For example, collaborative Computer Aided Design (CAD), with Product Lifecycle Management integration. TREs are not “one size fits all”, hence although the Azure TRE has a number of out of the box features, the project has been built be extensible, and hence tooling and data platform agnostic. Core features include: - Self-service workspace management for TRE administrators -- Self-service provisioning of research tooling for research teams +- Self-service provisioning of R&D tooling for R&D teams - Package and repository mirroring - PyPi, R-CRAN, Apt and more. - Extensible architecture - build your own service templates as required - Microsoft Entra ID integration diff --git a/docs/index.md b/docs/index.md index daa36b31fb..9499aba262 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,22 +2,26 @@ ## What is Azure TRE? -Across the health industry, be it a pharmaceutical company interrogating clinical trial results, or a public health provider analyzing electronic health records, there is the need to enable researchers, analysts, and developers to work with sensitive data sets. +Across multiple industries, be it a pharmaceutical company interrogating clinical trial results, a public health provider analyzing electronic health records, or an engineering organization working on next-generation systems, there is a need to enable researchers and solution developers to collaborate on sensitive data. -Trusted Research Environments (TREs) enable organisations to provide research teams secure access to these data sets along side tooling to ensure a researchers can be productive. Further information on TREs in general can be found in many places, one good resource is [HDR UK](https://www.hdruk.ac.uk/access-to-health-data/trusted-research-environments/). +Trusted Research Environments (TREs) enable organisations to provide research and development (R&D) teams secure access to data alongside tooling to ensure productivity. Further information on TREs in general can be found in many places, one good resource specifically for healthcare is [HDR UK](https://www.hdruk.ac.uk/access-to-health-data/trusted-research-environments/). -The Azure Trusted Research Environment project is an accelerator to assist Microsoft customers and partners who want to build out Trusted Research environments on Azure. This project enables authorized users to deploy and configure secure workspaces and researcher tooling without a dependency on IT teams. +Azure TRE is finding new applications and to date, has been used as the basis for exploring future secure collaboration environments in engineering. Workloads investigated include systems and software development. In truth, any workload in any industry could theoretically be supported with extensions to Azure TRE. + +The Azure Trusted Research Environment project is an accelerator to assist Microsoft customers and partners who want to build out Trusted Research environments on Azure. This project enables authorized users to deploy and configure secure workspaces and R&D tooling without a dependency on IT teams. This project is typically implemented alongside a data platform that provides research ready datasets to TRE workspaces: ![Concepts](assets/TRE_Overview.png) +Azure TRE has also been proven to handle unstructured data scenarios. For example, collaborative Computer Aided Design (CAD), with Product Lifecycle Management integration. + TREs are not “one size fits all”, hence although the Azure TRE has a number of out of the box features, the project has been built be extensible, and hence tooling and data platform agnostic. Core features include: - Self-service for administrators – workspace creation and administration -- Self-service for research teams – research tooling creation and administration +- Self-service for R&D teams – R&D tooling creation and administration - Package and repository mirroring - Extensible architecture - build your own service templates as required - Microsoft Entra ID integration From a73cf2f32b62fce506b91b2b7f37ec2b2439de9f Mon Sep 17 00:00:00 2001 From: Jonny Rylands Date: Thu, 6 Feb 2025 17:21:59 +0000 Subject: [PATCH 19/38] Ensure R directory is present before attempting to update package mirror config (#4332) --- CHANGELOG.md | 1 + .../guacamole-azure-windowsvm/porter.yaml | 2 +- .../guacamole-azure-windowsvm/terraform/vm_config.ps1 | 11 +++++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f6ea6de28e..2c41d6f0ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ BUG FIXES: * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)]) * Windows R version must be 4.1.2 otherwise post install script doesn't update package mirror URL ([#4288](https://github.com/microsoft/AzureTRE/issues/4288)) * Recreate tre_output.json if empty. ([[#4292](https://github.com/microsoft/AzureTRE/issues/4292)]) +* Ensure R directory is present before attempting to update package mirror URL ([#4332](https://github.com/microsoft/AzureTRE/pull/4332)) COMPONENTS: diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml index 0f727c688b..f9056f0f0b 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-guacamole-windowsvm -version: 1.2.5 +version: 1.2.6 description: "An Azure TRE User Resource Template for Guacamole (Windows 10)" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 index 343ac0e33c..6664c652e8 100644 --- a/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 +++ b/templates/workspace_services/guacamole/user_resources/guacamole-azure-windowsvm/terraform/vm_config.ps1 @@ -53,9 +53,12 @@ local({ "@ $RBasePath = "$Env:ProgramFiles\R" -$RVersions = Get-ChildItem -Path $RBasePath -Directory | Where-Object { $_.Name -like "R-*" } -foreach ($RVersion in $RVersions) { - $ConfigPath = Join-Path -Path $RVersion.FullName -ChildPath "etc\Rprofile.site" - $RConfig | Out-File -Encoding Ascii (New-Item -Path $ConfigPath -Force) +if (Test-Path $RBasePath) { + $RVersions = Get-ChildItem -Path $RBasePath -Directory | Where-Object { $_.Name -like "R-*" } + + foreach ($RVersion in $RVersions) { + $ConfigPath = Join-Path -Path $RVersion.FullName -ChildPath "etc\Rprofile.site" + $RConfig | Out-File -Encoding Ascii (New-Item -Path $ConfigPath -Force) + } } From 133746a40046cd72369437d3521252d8202d55a8 Mon Sep 17 00:00:00 2001 From: Guy Bertental Date: Sun, 9 Feb 2025 00:06:17 +0200 Subject: [PATCH 20/38] Disable local authentication between Airlock Processor and Azure Service Bus in Function Binding (#4277) * Support managed identity authentication in azure function binding between airlock and service bus queue * Disable service bus local authentication --- CHANGELOG.md | 2 + .../BlobCreatedTrigger/function.json | 4 +- .../DataDeletionTrigger/function.json | 4 +- .../ScanResultTrigger/function.json | 4 +- .../StatusChangedQueueTrigger/function.json | 4 +- airlock_processor/_version.py | 2 +- airlock_processor/host.json | 2 +- core/terraform/airlock/airlock_processor.tf | 41 ++++++++++++------- core/terraform/airlock/locals.tf | 1 + core/terraform/airlock/variables.tf | 3 ++ core/terraform/main.tf | 1 + core/terraform/servicebus.tf | 1 + core/version.txt | 2 +- 13 files changed, 49 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c41d6f0ae..a9f3dcaaad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ ENHANCEMENTS: * Update Guacamole dependencies ([[#4232](https://github.com/microsoft/AzureTRE/issues/4232)]) * Add option to force tunnel TRE's Firewall ([#4237](https://github.com/microsoft/AzureTRE/issues/4237)) * Add EventGrid diagnostics to identify airlock issues ([#4258](https://github.com/microsoft/AzureTRE/issues/4258)) +* Disable local authentication in ServiceBus ([#4259](https://github.com/microsoft/AzureTRE/issues/4259)) * Allow enablement of Secure Boot and vTPM for Guacamole VMs ([#4235](https://github.com/microsoft/AzureTRE/issues/4235)) * Surface the server-layout parameter of Guacamole [server-layout](https://guacamole.apache.org/doc/gug/configuring-guacamole.html#session-settings) ([#4234](https://github.com/microsoft/AzureTRE/issues/4234)) * Add encryption at host for VMs ([#4263](https://github.com/microsoft/AzureTRE/pull/4263)) @@ -41,6 +42,7 @@ ENHANCEMENTS: * Airlock function host storage to use the user-assigned managed identity ([#4276](https://github.com/microsoft/AzureTRE/issues/4276)) * Disable local authentication in EventGrid ([#4254](https://github.com/microsoft/AzureTRE/issues/4254)) + BUG FIXES: * Update KeyVault references in API to use the version so Terraform cascades the update ([#4112](https://github.com/microsoft/AzureTRE/pull/4112)) * Template images are showing CVEs ([#4153](https://github.com/microsoft/AzureTRE/issues/4153)) diff --git a/airlock_processor/BlobCreatedTrigger/function.json b/airlock_processor/BlobCreatedTrigger/function.json index 5a652a8eff..c34edbeeb7 100644 --- a/airlock_processor/BlobCreatedTrigger/function.json +++ b/airlock_processor/BlobCreatedTrigger/function.json @@ -8,7 +8,9 @@ "direction": "in", "topicName": "%BLOB_CREATED_TOPIC_NAME%", "subscriptionName": "%TOPIC_SUBSCRIPTION_NAME%", - "connection": "SB_CONNECTION_STRING" + "connection": "%SERVICEBUS_CONNECTION_NAME%", + "accessRights": "listen", + "autoComplete": true }, { "type": "eventGrid", diff --git a/airlock_processor/DataDeletionTrigger/function.json b/airlock_processor/DataDeletionTrigger/function.json index 2b2bb580da..0cb7f66eab 100644 --- a/airlock_processor/DataDeletionTrigger/function.json +++ b/airlock_processor/DataDeletionTrigger/function.json @@ -7,7 +7,9 @@ "type": "serviceBusTrigger", "direction": "in", "queueName": "%AIRLOCK_DATA_DELETION_QUEUE_NAME%", - "connection": "SB_CONNECTION_STRING" + "connection": "%SERVICEBUS_CONNECTION_NAME%", + "accessRights": "listen", + "autoComplete": true } ] } diff --git a/airlock_processor/ScanResultTrigger/function.json b/airlock_processor/ScanResultTrigger/function.json index 32758cea1c..266bd059fe 100644 --- a/airlock_processor/ScanResultTrigger/function.json +++ b/airlock_processor/ScanResultTrigger/function.json @@ -7,7 +7,9 @@ "type": "serviceBusTrigger", "direction": "in", "queueName": "%AIRLOCK_SCAN_RESULT_QUEUE_NAME%", - "connection": "SB_CONNECTION_STRING" + "connection": "%SERVICEBUS_CONNECTION_NAME%", + "accessRights": "listen", + "autoComplete": true }, { "type": "eventGrid", diff --git a/airlock_processor/StatusChangedQueueTrigger/function.json b/airlock_processor/StatusChangedQueueTrigger/function.json index f686eca80a..b96de6710c 100644 --- a/airlock_processor/StatusChangedQueueTrigger/function.json +++ b/airlock_processor/StatusChangedQueueTrigger/function.json @@ -6,7 +6,9 @@ "type": "serviceBusTrigger", "direction": "in", "queueName": "%AIRLOCK_STATUS_CHANGED_QUEUE_NAME%", - "connection": "SB_CONNECTION_STRING" + "connection": "%SERVICEBUS_CONNECTION_NAME%", + "accessRights": "listen", + "autoComplete": true }, { "type": "eventGrid", diff --git a/airlock_processor/_version.py b/airlock_processor/_version.py index 8088f75131..deded3247f 100644 --- a/airlock_processor/_version.py +++ b/airlock_processor/_version.py @@ -1 +1 @@ -__version__ = "0.8.1" +__version__ = "0.8.2" diff --git a/airlock_processor/host.json b/airlock_processor/host.json index 95b6b4b7d6..f9667b1f23 100644 --- a/airlock_processor/host.json +++ b/airlock_processor/host.json @@ -8,7 +8,7 @@ } } }, - "extensionBundle": { +"extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", "version": "[4.0.0, 5.0.0)" } diff --git a/core/terraform/airlock/airlock_processor.tf b/core/terraform/airlock/airlock_processor.tf index ccb36b81bb..f6a0f98ed4 100644 --- a/core/terraform/airlock/airlock_processor.tf +++ b/core/terraform/airlock/airlock_processor.tf @@ -66,21 +66,32 @@ resource "azurerm_linux_function_app" "airlock_function_app" { } app_settings = { - "SB_CONNECTION_STRING" = var.airlock_servicebus.default_primary_connection_string - "BLOB_CREATED_TOPIC_NAME" = azurerm_servicebus_topic.blob_created.name - "TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name - "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false - "AIRLOCK_STATUS_CHANGED_QUEUE_NAME" = local.status_changed_queue_name - "AIRLOCK_SCAN_RESULT_QUEUE_NAME" = local.scan_result_queue_name - "AIRLOCK_DATA_DELETION_QUEUE_NAME" = local.data_deletion_queue_name - "ENABLE_MALWARE_SCANNING" = var.enable_malware_scanning - "ARM_ENVIRONMENT" = var.arm_environment - "MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.airlock_id.client_id - "TRE_ID" = var.tre_id - "WEBSITE_CONTENTOVERVNET" = 1 - "STORAGE_ENDPOINT_SUFFIX" = module.terraform_azurerm_environment_configuration.storage_suffix - "AzureWebJobsStorage__clientId" = azurerm_user_assigned_identity.airlock_id.client_id - "AzureWebJobsStorage__credential" = "managedidentity" + "SERVICEBUS_CONNECTION_NAME" = local.servicebus_connection + "${local.servicebus_connection}__tenantId" = azurerm_user_assigned_identity.airlock_id.tenant_id + "${local.servicebus_connection}__clientId" = azurerm_user_assigned_identity.airlock_id.client_id + "${local.servicebus_connection}__credential" = "managedidentity" + "${local.servicebus_connection}__fullyQualifiedNamespace" = var.airlock_servicebus_fqdn + + "BLOB_CREATED_TOPIC_NAME" = azurerm_servicebus_topic.blob_created.name + "TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name + "EVENT_GRID_STEP_RESULT_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.step_result.endpoint + "EVENT_GRID_STEP_RESULT_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.step_result.primary_access_key + "EVENT_GRID_DATA_DELETION_TOPIC_URI_SETTING" = azurerm_eventgrid_topic.data_deletion.endpoint + "EVENT_GRID_DATA_DELETION_TOPIC_KEY_SETTING" = azurerm_eventgrid_topic.data_deletion.primary_access_key + "WEBSITES_ENABLE_APP_SERVICE_STORAGE" = false + "AIRLOCK_STATUS_CHANGED_QUEUE_NAME" = local.status_changed_queue_name + "AIRLOCK_SCAN_RESULT_QUEUE_NAME" = local.scan_result_queue_name + "AIRLOCK_DATA_DELETION_QUEUE_NAME" = local.data_deletion_queue_name + "ENABLE_MALWARE_SCANNING" = var.enable_malware_scanning + "ARM_ENVIRONMENT" = var.arm_environment + "MANAGED_IDENTITY_CLIENT_ID" = azurerm_user_assigned_identity.airlock_id.client_id + "TRE_ID" = var.tre_id + "WEBSITE_CONTENTOVERVNET" = 1 + "STORAGE_ENDPOINT_SUFFIX" = module.terraform_azurerm_environment_configuration.storage_suffix + + "TOPIC_SUBSCRIPTION_NAME" = azurerm_servicebus_subscription.airlock_processor.name + "AzureWebJobsStorage__clientId" = azurerm_user_assigned_identity.airlock_id.client_id + "AzureWebJobsStorage__credential" = "managedidentity" "EVENT_GRID_STEP_RESULT_CONNECTION" = local.step_result_eventgrid_connection "${local.step_result_eventgrid_connection}__topicEndpointUri" = azurerm_eventgrid_topic.step_result.endpoint diff --git a/core/terraform/airlock/locals.tf b/core/terraform/airlock/locals.tf index 8ed6805e0e..838ddf091a 100644 --- a/core/terraform/airlock/locals.tf +++ b/core/terraform/airlock/locals.tf @@ -61,6 +61,7 @@ locals { azurerm_storage_account.sa_export_approved.id ] + servicebus_connection = "SERVICEBUS_CONNECTION" step_result_eventgrid_connection = "EVENT_GRID_STEP_RESULT_CONNECTION" data_deletion_eventgrid_connection = "EVENT_GRID_DATA_DELETION_CONNECTION" } diff --git a/core/terraform/airlock/variables.tf b/core/terraform/airlock/variables.tf index 95e03b4ba4..bb0fad04df 100644 --- a/core/terraform/airlock/variables.tf +++ b/core/terraform/airlock/variables.tf @@ -62,6 +62,9 @@ variable "airlock_servicebus" { default_primary_connection_string = string }) } +variable "airlock_servicebus_fqdn" { + type = string +} variable "tre_core_tags" { type = map(string) } diff --git a/core/terraform/main.tf b/core/terraform/main.tf index 49693884c1..4d6d910257 100644 --- a/core/terraform/main.tf +++ b/core/terraform/main.tf @@ -132,6 +132,7 @@ module "airlock_resources" { airlock_app_service_plan_sku = var.core_app_service_plan_sku airlock_processor_subnet_id = module.network.airlock_processor_subnet_id airlock_servicebus = azurerm_servicebus_namespace.sb + airlock_servicebus_fqdn = azurerm_servicebus_namespace.sb.endpoint applicationinsights_connection_string = module.azure_monitor.app_insights_connection_string enable_malware_scanning = var.enable_airlock_malware_scanning arm_environment = var.arm_environment diff --git a/core/terraform/servicebus.tf b/core/terraform/servicebus.tf index f686a8e08e..7c03d661c0 100644 --- a/core/terraform/servicebus.tf +++ b/core/terraform/servicebus.tf @@ -5,6 +5,7 @@ resource "azurerm_servicebus_namespace" "sb" { sku = "Premium" premium_messaging_partitions = "1" capacity = "1" + local_auth_enabled = false tags = local.tre_core_tags # Block public access diff --git a/core/version.txt b/core/version.txt index 663d6b3572..836582489b 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.22" +__version__ = "0.11.23" From 9327874584b1045b40a58a40512e37bc4c6e9751 Mon Sep 17 00:00:00 2001 From: Tamir Kamara <26870601+tamirkamara@users.noreply.github.com> Date: Sun, 9 Feb 2025 12:14:24 +0200 Subject: [PATCH 21/38] Release 0.20.0 (#4345) * update release doc formatting * update change log * fix typos --- CHANGELOG.md | 65 +++++++++++++++++++++++++++++++--- docs/tre-developers/release.md | 1 + 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9f3dcaaad..6e919c37e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,19 @@ -## 0.20.0 (Unreleased) +## 0.21.0 (Unreleased) **BREAKING CHANGES & MIGRATIONS**: -* InnerEye and MLFlow bundles depreciated and removed from main. If you wish to update and deploy these worksapce services they can be retrieved from release 0.19.1. ([#4127](https://github.com/microsoft/AzureTRE/issues/4127)) -* This released removed support for Porter v0.*. If you're upgrading from a much earlier verion you can't go directly to this one. ([#4228](https://github.com/microsoft/AzureTRE/issues/4228)) + +ENHANCEMENTS: + +BUG FIXES: + +COMPONENTS: + +## 0.20.0 (Feburary 9, 2025) + +**BREAKING CHANGES & MIGRATIONS**: +* InnerEye and MLFlow bundles depreciated and removed from main. If you wish to update and deploy these workspace services they can be retrieved from release 0.19.1. ([#4127](https://github.com/microsoft/AzureTRE/issues/4127)) +* This release removed support for Porter v0.*. If you're upgrading from a much earlier version you can't go directly to this one. ([#4228](https://github.com/microsoft/AzureTRE/issues/4228)) FEATURES: * Add support for customer-managed keys encryption. Core support ([#4141](https://github.com/microsoft/AzureTRE/issues/4142), [#4144](https://github.com/microsoft/AzureTRE/issues/4144)), Base workspace ([#4161](https://github.com/microsoft/AzureTRE/pull/4161)), other templates ([#4145](https://github.com/microsoft/AzureTRE/issues/4145)) @@ -55,7 +65,7 @@ BUG FIXES: * Fix failing tests, .env missing and storage logs ([#4207](https://github.com/microsoft/AzureTRE/issues/4207)) * Unable to delete virtual machines, add skip_shutdown_and_force_delete = true ([#4135](https://github.com/microsoft/AzureTRE/issues/4135)) * Bump terraform version in windows VM template ([#4212](https://github.com/microsoft/AzureTRE/issues/4212)) -* Upgrade azurerm terraform provider from v3.112.0 to v3.117.0 to mitiagte storage account deployment issue ([#4004](https://github.com/microsoft/AzureTRE/issues/4004)) +* Upgrade azurerm terraform provider from v3.112.0 to v3.117.0 to mitigate storage account deployment issue ([#4004](https://github.com/microsoft/AzureTRE/issues/4004)) * Fix VM actions where Workspace shared storage doesn't allow shared key access ([#4222](https://github.com/microsoft/AzureTRE/issues/4222)) * Fix public exposure in Guacamole service ([[#4199](https://github.com/microsoft/AzureTRE/issues/4199)]) * Fix Azure ML network tags to use name rather than ID ([[#4151](https://github.com/microsoft/AzureTRE/issues/4151)]) @@ -65,6 +75,37 @@ BUG FIXES: COMPONENTS: +| name | version | +| ----- | ----- | +| devops | 0.5.5 | +| core | 0.11.23 | +| ui | 0.6.3 | +| tre-shared-service-databricks-private-auth | 0.1.11 | +| tre-shared-service-gitea | 1.1.4 | +| tre-shared-service-sonatype-nexus | 3.3.2 | +| tre-shared-service-firewall | 1.3.0 | +| tre-shared-service-admin-vm | 0.5.2 | +| tre-shared-service-certs | 0.7.3 | +| tre-shared-service-airlock-notifier | 1.0.8 | +| tre-shared-service-cyclecloud | 0.7.2 | +| tre-workspace-airlock-import-review | 0.14.2 | +| tre-workspace-base | 1.9.2 | +| tre-workspace-unrestricted | 0.13.2 | +| tre-workspace-service-gitea | 1.2.2 | +| tre-workspace-service-mysql | 1.0.9 | +| tre-workspace-service-health | 0.2.11 | +| tre-workspace-service-openai | 1.0.6 | +| tre-service-azureml | 0.9.2 | +| tre-user-resource-aml-compute-instance | 0.5.11 | +| tre-service-databricks | 1.0.10 | +| tre-workspace-service-azuresql | 1.0.15 | +| tre-service-guacamole | 0.12.7 | +| tre-service-guacamole-export-reviewvm | 0.2.2 | +| tre-service-guacamole-linuxvm | 1.2.4 | +| tre-service-guacamole-import-reviewvm | 0.3.2 | +| tre-service-guacamole-windowsvm | 1.2.6 | +| tre-workspace-service-ohdsi | 0.3.2 | + ## 0.19.1 **BREAKING CHANGES & MIGRATIONS**: @@ -80,6 +121,7 @@ BUG FIXES: * Workspace creation blocked due to Azure API depreciation ([#4095](https://github.com/microsoft/AzureTRE/issues/4095)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.2 | @@ -138,6 +180,7 @@ BUG FIXES: * Update .NET version on Linux VMs ([#4067](https://github.com/microsoft/AzureTRE/issues/4067)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -201,6 +244,7 @@ BUG FIXES: * Add lifecycle rule to the Gitea Shared Service template for the MySQL resource to stop it recreating on `update` ([#4006](https://github.com/microsoft/AzureTRE/issues/4006)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -255,6 +299,7 @@ BUG FIXES: * Fix issue with firewall failing to deploy on a new TRE deploy ([#3775](https://github.com/microsoft/AzureTRE/issues/3775)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -304,6 +349,7 @@ BUG FIXES: * Airlock Import Review workspace uses dedicated DNS zone to prevent conflict with core ([#3767](https://github.com/microsoft/AzureTRE/pull/3767)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -346,6 +392,7 @@ BUG FIXES: * Fix workspace not loading fails if operation or history roles are not loaded ([#3755](https://github.com/microsoft/AzureTRE/issues/3755)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -384,6 +431,7 @@ BUG FIXES: * SecuredByRole failing if roles are null ([#3740](https://github.com/microsoft/AzureTRE/issues/3740 )) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -433,6 +481,7 @@ BUG FIXES: * Fix issue with cost tags not displaying correctly for some user roles ([#3721](https://github.com/microsoft/AzureTRE/issues/3721)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -469,6 +518,7 @@ BUG FIXES: * Fix firewall config related to Nexus so that `pypi.org` is added to the allow-list ([#3694](https://github.com/microsoft/AzureTRE/issues/3694)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -518,6 +568,7 @@ BUG FIXES: * Added missing region entries in `databricks-udr.json` ([[#3688](https://github.com/microsoft/AzureTRE/pull/3688)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -557,6 +608,7 @@ BUG FIXES: * Upgrade airlock and unrestricted workspaces to base workspace version 0.12.0 ([#3659](https://github.com/microsoft/AzureTRE/pull/3659)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -616,6 +668,7 @@ BUG FIXES: COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -657,6 +710,7 @@ BUG FIXES: * Nexus fails to install due to `az login` and firewall rules ([#3453](https://github.com/microsoft/AzureTRE/issues/3453)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.5.1 | @@ -859,6 +913,7 @@ BUG FIXES: * Fix KeyVault purge error on MLFlow uninstall ([#3082](https://github.com/microsoft/AzureTRE/pull/3082)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.4.4 | @@ -935,6 +990,7 @@ BUG FIXES: * Handle 429 TooManyRequests and 503 ServiceUnavailable which might return from Azure Cost Management in TRE Cost API ([#2835](https://github.com/microsoft/AzureTRE/issues/2835)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.4.2 | @@ -982,6 +1038,7 @@ BUG FIXES: * Fix issues with AML workspace service deployment ([#2768](https://github.com/microsoft/AzureTRE/pull/2768)) COMPONENTS: + | name | version | | ----- | ----- | | devops | 0.4.2 | diff --git a/docs/tre-developers/release.md b/docs/tre-developers/release.md index 19aeb12866..40a431eea5 100644 --- a/docs/tre-developers/release.md +++ b/docs/tre-developers/release.md @@ -21,6 +21,7 @@ The process follows these steps: 5. Include a final line with a link to the full changelog similar to this: **Full Changelog**: https://github.com/microsoft/AzureTRE/compare/v0.9.1...v0.9.2 + 7. Update [AzureTRE-Deployment](https://github.com/microsoft/AzureTRE-Deployment). The procedure may vary depending on the level of changes introduced in the new version but should include the following steps: 1. Update the tag used in [devcontainer.json](https://github.com/microsoft/AzureTRE-Deployment/blob/main/.devcontainer/devcontainer.json). 2. Rebuild the container. From 0bcf5b3b20d030e321f86dd8139f0b30ca728097 Mon Sep 17 00:00:00 2001 From: Jonny Rylands Date: Sun, 9 Feb 2025 18:05:39 +0000 Subject: [PATCH 22/38] Allow workspace App Service Plan SKU to be updated (#4339) * Allow workspace App Service Plan SKU to be updated #4331 --- CHANGELOG.md | 1 + templates/workspaces/airlock-import-review/porter.yaml | 2 +- templates/workspaces/airlock-import-review/template_schema.json | 1 + templates/workspaces/base/porter.yaml | 2 +- templates/workspaces/base/template_schema.json | 1 + templates/workspaces/unrestricted/porter.yaml | 2 +- templates/workspaces/unrestricted/template_schema.json | 1 + 7 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e919c37e7..a3ea27a0ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ **BREAKING CHANGES & MIGRATIONS**: ENHANCEMENTS: +* Allow workspace App Service Plan SKU to be updated ([#4331](https://github.com/microsoft/AzureTRE/issues/4331)) BUG FIXES: diff --git a/templates/workspaces/airlock-import-review/porter.yaml b/templates/workspaces/airlock-import-review/porter.yaml index 56f90dbc70..a651294ea9 100644 --- a/templates/workspaces/airlock-import-review/porter.yaml +++ b/templates/workspaces/airlock-import-review/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-airlock-import-review -version: 0.14.2 +version: 0.14.3 description: "A workspace to do Airlock Data Import Reviews for Azure TRE" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/airlock-import-review/template_schema.json b/templates/workspaces/airlock-import-review/template_schema.json index 180e360abc..09d7d6be87 100644 --- a/templates/workspaces/airlock-import-review/template_schema.json +++ b/templates/workspaces/airlock-import-review/template_schema.json @@ -15,6 +15,7 @@ "title": "App Service Plan SKU", "description": "The SKU that will be used when deploying an Azure App Service Plan.", "default": "P1v3", + "updateable": true, "enum": [ "P0v3", "P1v3", diff --git a/templates/workspaces/base/porter.yaml b/templates/workspaces/base/porter.yaml index a7e09fa692..ed48af1a9b 100644 --- a/templates/workspaces/base/porter.yaml +++ b/templates/workspaces/base/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-base -version: 1.9.2 +version: 1.9.3 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/base/template_schema.json b/templates/workspaces/base/template_schema.json index b456b5f044..24cec47f34 100644 --- a/templates/workspaces/base/template_schema.json +++ b/templates/workspaces/base/template_schema.json @@ -27,6 +27,7 @@ "title": "App Service Plan SKU", "description": "The SKU that will be used when deploying an Azure App Service Plan.", "default": "P1v3", + "updateable": true, "enum": [ "P0v3", "P1v3", diff --git a/templates/workspaces/unrestricted/porter.yaml b/templates/workspaces/unrestricted/porter.yaml index b8bd2becae..6c9cb1e43b 100644 --- a/templates/workspaces/unrestricted/porter.yaml +++ b/templates/workspaces/unrestricted/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-workspace-unrestricted -version: 0.13.2 +version: 0.13.3 description: "A base Azure TRE workspace" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/workspaces/unrestricted/template_schema.json b/templates/workspaces/unrestricted/template_schema.json index f9c8a807f1..6ebbb1c159 100644 --- a/templates/workspaces/unrestricted/template_schema.json +++ b/templates/workspaces/unrestricted/template_schema.json @@ -27,6 +27,7 @@ "title": "App Service Plan SKU", "description": "The SKU that will be used when deploying an Azure App Service Plan.", "default": "P1v3", + "updateable": true, "enum": [ "P0v3", "P1v3", From 998c3d4eb97ae9daed35e9c12d022029101b0862 Mon Sep 17 00:00:00 2001 From: Yuval Yaron <43217306+yuvalyaron@users.noreply.github.com> Date: Tue, 11 Feb 2025 00:46:57 +0200 Subject: [PATCH 23/38] Remove public IP from TRE's firewall when forced tunneling is configured (#4346) * remove public IP from TRE's firewall when forced tunneling is configured * update changelog * update version as this change is not backward compatible * update template version --- CHANGELOG.md | 1 + templates/shared_services/firewall/porter.yaml | 2 +- templates/shared_services/firewall/terraform/firewall.tf | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3ea27a0ac..9979b55d2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ENHANCEMENTS: * Allow workspace App Service Plan SKU to be updated ([#4331](https://github.com/microsoft/AzureTRE/issues/4331)) +* Remove public IP from TRE's firewall when forced tunneling is configured ([#4346](https://github.com/microsoft/AzureTRE/pull/4346)) BUG FIXES: diff --git a/templates/shared_services/firewall/porter.yaml b/templates/shared_services/firewall/porter.yaml index ffba80504b..d5759218a7 100644 --- a/templates/shared_services/firewall/porter.yaml +++ b/templates/shared_services/firewall/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-firewall -version: 1.3.0 +version: 1.3.1 description: "An Azure TRE Firewall shared service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/shared_services/firewall/terraform/firewall.tf b/templates/shared_services/firewall/terraform/firewall.tf index 6697a359b6..3f42237a0f 100644 --- a/templates/shared_services/firewall/terraform/firewall.tf +++ b/templates/shared_services/firewall/terraform/firewall.tf @@ -1,4 +1,5 @@ resource "azurerm_public_ip" "fwtransit" { + count = var.firewall_force_tunnel_ip != "" ? 0 : 1 name = "pip-fw-${var.tre_id}" resource_group_name = local.core_resource_group_name location = data.azurerm_resource_group.rg.location @@ -11,7 +12,7 @@ resource "azurerm_public_ip" "fwtransit" { moved { from = azurerm_public_ip.fwpip - to = azurerm_public_ip.fwtransit + to = azurerm_public_ip.fwtransit[0] } resource "azurerm_public_ip" "fwmanagement" { @@ -38,7 +39,7 @@ resource "azurerm_firewall" "fw" { ip_configuration { name = "fw-ip-configuration" subnet_id = data.azurerm_subnet.firewall.id - public_ip_address_id = azurerm_public_ip.fwtransit.id + public_ip_address_id = var.firewall_force_tunnel_ip != "" ? null : azurerm_public_ip.fwtransit[0].id } dynamic "management_ip_configuration" { From a6d85e2600ffac2a3dcc9576607a637911e330ec Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Mon, 10 Feb 2025 23:34:44 +0000 Subject: [PATCH 24/38] Fix upgrade if porter install has failed and add tests to resource processor (#4338) --- .devcontainer/devcontainer.json | 1 + CHANGELOG.md | 3 + airlock_processor/_version.py | 2 +- .../tests/shared_code}/__init__.py | 0 resource_processor/_version.py | 2 +- resource_processor/helpers/__init__.py | 0 .../{resources => helpers}/commands.py | 29 +- .../{resources => helpers}/httpserver.py | 0 .../{resources => helpers}/statuses.py | 2 +- .../{resources => helpers}/strings.py | 0 resource_processor/resources/helpers.py | 5 - resource_processor/tests_rp/test_commands.py | 94 ++++++ resource_processor/tests_rp/test_runner.py | 284 ++++++++++++++++++ resource_processor/vmss_porter/runner.py | 37 ++- 14 files changed, 427 insertions(+), 32 deletions(-) rename {resource_processor/resources => airlock_processor/tests/shared_code}/__init__.py (100%) create mode 100644 resource_processor/helpers/__init__.py rename resource_processor/{resources => helpers}/commands.py (89%) rename resource_processor/{resources => helpers}/httpserver.py (100%) rename resource_processor/{resources => helpers}/statuses.py (96%) rename resource_processor/{resources => helpers}/strings.py (100%) delete mode 100644 resource_processor/resources/helpers.py create mode 100644 resource_processor/tests_rp/test_commands.py create mode 100644 resource_processor/tests_rp/test_runner.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 90b080ddb1..e8966910e5 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -275,6 +275,7 @@ "extensions": [ "ms-python.python", "ms-python.pylance", + "ms-python.flake8", "hashicorp.terraform", "github.vscode-pull-request-github", "gitHub.copilot", diff --git a/CHANGELOG.md b/CHANGELOG.md index 9979b55d2c..9cf9862269 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ ENHANCEMENTS: * Remove public IP from TRE's firewall when forced tunneling is configured ([#4346](https://github.com/microsoft/AzureTRE/pull/4346)) BUG FIXES: +* Fix upgrade when porter install has failed ([#4338](https://github.com/microsoft/AzureTRE/pull/4338)) + + COMPONENTS: diff --git a/airlock_processor/_version.py b/airlock_processor/_version.py index deded3247f..732155f8df 100644 --- a/airlock_processor/_version.py +++ b/airlock_processor/_version.py @@ -1 +1 @@ -__version__ = "0.8.2" +__version__ = "0.8.3" diff --git a/resource_processor/resources/__init__.py b/airlock_processor/tests/shared_code/__init__.py similarity index 100% rename from resource_processor/resources/__init__.py rename to airlock_processor/tests/shared_code/__init__.py diff --git a/resource_processor/_version.py b/resource_processor/_version.py index fee46bd8ce..def467e071 100644 --- a/resource_processor/_version.py +++ b/resource_processor/_version.py @@ -1 +1 @@ -__version__ = "0.11.1" +__version__ = "0.12.1" diff --git a/resource_processor/helpers/__init__.py b/resource_processor/helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/resource_processor/resources/commands.py b/resource_processor/helpers/commands.py similarity index 89% rename from resource_processor/resources/commands.py rename to resource_processor/helpers/commands.py index 0111b52358..59c0eaea82 100644 --- a/resource_processor/resources/commands.py +++ b/resource_processor/helpers/commands.py @@ -4,14 +4,13 @@ import logging from urllib.parse import urlparse -from resources.helpers import get_installation_id from shared.logging import logger, shell_output_logger def azure_login_command(config): set_cloud_command = f"az cloud set --name {config['azure_environment']} >/dev/null " - if config["vmss_msi_id"]: + if config.get("vmss_msi_id"): # Use the Managed Identity when in VMSS context login_command = f"az login --identity -u {config['vmss_msi_id']} >/dev/null " @@ -23,7 +22,7 @@ def azure_login_command(config): def apply_porter_credentials_sets_command(config): - if config["vmss_msi_id"]: + if config.get("vmss_msi_id"): # Use the Managed Identity when in VMSS context porter_credential_sets = "porter credentials apply vmss_porter/arm_auth_local_debugging.json >/dev/null 2>&1 && porter credentials apply vmss_porter/aad_auth.json >/dev/null 2>&1" @@ -80,25 +79,31 @@ async def build_porter_command(config, msg_body, custom_action=False): val_base64_bytes = base64.b64encode(val_bytes) parameter_value = val_base64_bytes.decode("ascii") - porter_parameters = porter_parameters + f" --param {parameter_name}=\"{parameter_value}\"" + porter_parameters += f" --param {parameter_name}=\"{parameter_value}\"" - installation_id = get_installation_id(msg_body) + installation_id = msg_body['id'] command_line = [f"porter" # If a custom action (i.e. not install, uninstall, upgrade) we need to use 'invoke' - f"{' invoke --action' if custom_action else ''}" - f" {msg_body['action']} \"{installation_id}\"" - f" --reference {config['registry_server']}/{msg_body['name']}:v{msg_body['version']}" - f" {porter_parameters} --force" - f" --credential-set arm_auth" - f" --credential-set aad_auth" + f"{' invoke --action' if custom_action else ''} " + f"{msg_body['action']} \"{installation_id}\" " + f"--reference {config['registry_server']}/{msg_body['name']}:v{msg_body['version']}" + f"{porter_parameters} " + f"--force " + f"--credential-set arm_auth " + f"--credential-set aad_auth " ] + if msg_body['action'] == 'upgrade': + command_line[0] = command_line[0] + "--force-upgrade " + + command_line[0] = command_line[0].strip() + return command_line async def build_porter_command_for_outputs(msg_body): - installation_id = get_installation_id(msg_body) + installation_id = msg_body['id'] command_line = [f"porter installations output list --installation {installation_id} --output json"] return command_line diff --git a/resource_processor/resources/httpserver.py b/resource_processor/helpers/httpserver.py similarity index 100% rename from resource_processor/resources/httpserver.py rename to resource_processor/helpers/httpserver.py diff --git a/resource_processor/resources/statuses.py b/resource_processor/helpers/statuses.py similarity index 96% rename from resource_processor/resources/statuses.py rename to resource_processor/helpers/statuses.py index 952dcef24b..2e1941f482 100644 --- a/resource_processor/resources/statuses.py +++ b/resource_processor/helpers/statuses.py @@ -1,5 +1,5 @@ from collections import defaultdict -from resources import strings +from helpers import strings # Specify pass and fail status strings so we can return the right statuses to the api depending on the action type (with a default of custom action) diff --git a/resource_processor/resources/strings.py b/resource_processor/helpers/strings.py similarity index 100% rename from resource_processor/resources/strings.py rename to resource_processor/helpers/strings.py diff --git a/resource_processor/resources/helpers.py b/resource_processor/resources/helpers.py deleted file mode 100644 index 98ef4d2e0d..0000000000 --- a/resource_processor/resources/helpers.py +++ /dev/null @@ -1,5 +0,0 @@ -def get_installation_id(msg_body): - """ - This is used to identify each bundle install within the porter state store. - """ - return msg_body['id'] diff --git a/resource_processor/tests_rp/test_commands.py b/resource_processor/tests_rp/test_commands.py new file mode 100644 index 0000000000..bb2f9f20e0 --- /dev/null +++ b/resource_processor/tests_rp/test_commands.py @@ -0,0 +1,94 @@ +import json +import pytest +from unittest.mock import patch, AsyncMock +from helpers.commands import azure_login_command, apply_porter_credentials_sets_command, azure_acr_login_command, build_porter_command, build_porter_command_for_outputs, get_porter_parameter_keys + + +@pytest.fixture +def mock_get_porter_parameter_keys(): + with patch("helpers.commands.get_porter_parameter_keys", new_callable=AsyncMock) as mock: + yield mock + + +@pytest.mark.parametrize("config, expected_command", [ + ({"azure_environment": "AzureCloud", "vmss_msi_id": "msi_id"}, "az cloud set --name AzureCloud >/dev/null && az login --identity -u msi_id >/dev/null "), + ({"azure_environment": "AzureCloud", "arm_client_id": "client_id", "arm_client_secret": "client_secret", "arm_tenant_id": "tenant_id"}, "az cloud set --name AzureCloud >/dev/null && az login --service-principal --username client_id --password client_secret --tenant tenant_id >/dev/null") +]) +def test_azure_login_command(config, expected_command): + """Test azure_login_command function.""" + assert azure_login_command(config) == expected_command + + +@pytest.mark.parametrize("config, expected_command", [ + ({"vmss_msi_id": "msi_id"}, "porter credentials apply vmss_porter/arm_auth_local_debugging.json >/dev/null 2>&1 && porter credentials apply vmss_porter/aad_auth.json >/dev/null 2>&1"), + ({}, "porter credentials apply vmss_porter/arm_auth_local_debugging.json >/dev/null 2>&1 && porter credentials apply vmss_porter/aad_auth_local_debugging.json >/dev/null 2>&1") +]) +def test_apply_porter_credentials_sets_command(config, expected_command): + """Test apply_porter_credentials_sets_command function.""" + assert apply_porter_credentials_sets_command(config) == expected_command + + +@pytest.mark.parametrize("config, expected_command", [ + ({"registry_server": "myregistry.azurecr.io"}, "az acr login --name myregistry >/dev/null ") +]) +def test_azure_acr_login_command(config, expected_command): + """Test azure_acr_login_command function.""" + assert azure_acr_login_command(config) == expected_command + + +@pytest.mark.asyncio +async def test_build_porter_command(mock_get_porter_parameter_keys): + """Test build_porter_command function.""" + config = {"registry_server": "myregistry.azurecr.io"} + msg_body = {"id": "guid", "action": "install", "name": "mybundle", "version": "1.0.0", "parameters": {"param1": "value1"}} + mock_get_porter_parameter_keys.return_value = ["param1"] + + expected_command = [ + "porter install \"guid\" --reference myregistry.azurecr.io/mybundle:v1.0.0 --param param1=\"value1\" --force --credential-set arm_auth --credential-set aad_auth" + ] + + command = await build_porter_command(config, msg_body) + assert command == expected_command + + +@pytest.mark.asyncio +async def test_build_porter_command_for_upgrade(mock_get_porter_parameter_keys): + """Test build_porter_command function for upgrade action.""" + config = {"registry_server": "myregistry.azurecr.io"} + msg_body = {"id": "guid", "action": "upgrade", "name": "mybundle", "version": "1.0.0", "parameters": {"param1": "value1"}} + mock_get_porter_parameter_keys.return_value = ["param1"] + + expected_command = [ + "porter upgrade \"guid\" --reference myregistry.azurecr.io/mybundle:v1.0.0 --param param1=\"value1\" --force --credential-set arm_auth --credential-set aad_auth --force-upgrade" + ] + + command = await build_porter_command(config, msg_body) + assert command == expected_command + + +@pytest.mark.asyncio +async def test_build_porter_command_for_outputs(): + """Test build_porter_command_for_outputs function.""" + msg_body = {"id": "guid", "action": "install", "name": "mybundle", "version": "1.0.0"} + expected_command = ["porter installations output list --installation guid --output json"] + + command = await build_porter_command_for_outputs(msg_body) + assert command == expected_command + + +@pytest.mark.asyncio +@patch("helpers.commands.azure_login_command", return_value="az login command") +@patch("helpers.commands.azure_acr_login_command", return_value="az acr login command") +@patch("asyncio.create_subprocess_shell") +async def test_get_porter_parameter_keys(mock_create_subprocess_shell, mock_azure_acr_login_command, mock_azure_login_command): + """Test get_porter_parameter_keys function.""" + config = {"registry_server": "myregistry.azurecr.io", "porter_env": {}} + msg_body = {"name": "mybundle", "version": "1.0.0"} + mock_proc = AsyncMock() + mock_proc.communicate.return_value = (json.dumps({"parameters": [{"name": "param1"}]}).encode(), b"") + mock_create_subprocess_shell.return_value = mock_proc + + expected_keys = ["param1"] + + keys = await get_porter_parameter_keys(config, msg_body) + assert keys == expected_keys diff --git a/resource_processor/tests_rp/test_runner.py b/resource_processor/tests_rp/test_runner.py new file mode 100644 index 0000000000..7c1dc6d2e4 --- /dev/null +++ b/resource_processor/tests_rp/test_runner.py @@ -0,0 +1,284 @@ +import json +from unittest.mock import patch, AsyncMock, Mock +import pytest +from resource_processor.vmss_porter.runner import ( + set_up_config, receive_message, invoke_porter_action, get_porter_outputs, check_runners, runner +) +from azure.servicebus.aio import ServiceBusClient +from azure.servicebus import ServiceBusSessionFilter + + +@pytest.fixture +def mock_service_bus_client(): + with patch("resource_processor.vmss_porter.runner.ServiceBusClient") as mock: + yield mock + + +@pytest.fixture +def mock_default_credential(): + with patch("resource_processor.vmss_porter.runner.default_credentials") as mock: + yield mock + + +@pytest.fixture +def mock_auto_lock_renewer(): + with patch("resource_processor.vmss_porter.runner.AutoLockRenewer") as mock: + yield mock + + +@pytest.fixture +def mock_logger(): + with patch("resource_processor.vmss_porter.runner.logger") as mock: + yield mock + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.get_config", return_value={"resource_request_queue": "test_queue", "service_bus_namespace": "test_namespace", "vmss_msi_id": "test_msi_id", "porter_env": {}}) +async def test_set_up_config(mock_get_config): + """Test setting up configuration.""" + config = set_up_config() + assert config == {"resource_request_queue": "test_queue", "service_bus_namespace": "test_namespace", "vmss_msi_id": "test_msi_id", "porter_env": {}} + + +async def setup_service_bus_client_and_credential(mock_service_bus_client, mock_default_credential, msi_id): + mock_credential = AsyncMock() + mock_default_credential.return_value.__aenter__.return_value = mock_credential + mock_service_bus_client_instance = mock_service_bus_client.return_value + return mock_service_bus_client_instance, mock_credential + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.receive_message") +async def test_runner(mock_receive_message, mock_service_bus_client, mock_default_credential): + """Test runner with valid MSI ID.""" + mock_service_bus_client_instance, mock_credential = await setup_service_bus_client_and_credential(mock_service_bus_client, mock_default_credential, 'test_msi_id') + + config = {"vmss_msi_id": "test_msi_id", "service_bus_namespace": "test_namespace"} + + await runner(0, config) + + mock_default_credential.assert_called_once_with('test_msi_id') + mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, config) + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.receive_message") +async def test_runner_no_msi_id(mock_receive_message, mock_service_bus_client, mock_default_credential): + """Test runner with no MSI ID.""" + mock_service_bus_client_instance, mock_credential = await setup_service_bus_client_and_credential(mock_service_bus_client, mock_default_credential, None) + + config = {"vmss_msi_id": None, "service_bus_namespace": "test_namespace"} + + await runner(0, config) + + mock_default_credential.assert_called_once_with(None) + mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, config) + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.receive_message") +async def test_runner_exception(mock_receive_message, mock_service_bus_client, mock_default_credential): + """Test runner with an exception.""" + mock_service_bus_client_instance, mock_credential = await setup_service_bus_client_and_credential(mock_service_bus_client, mock_default_credential, 'test_msi_id') + mock_receive_message.side_effect = Exception("Test Exception") + + config = {"vmss_msi_id": "test_msi_id", "service_bus_namespace": "test_namespace"} + + with pytest.raises(Exception, match="Test Exception"): + await runner(0, config) + + mock_default_credential.assert_called_once_with('test_msi_id') + mock_service_bus_client.assert_called_once_with("test_namespace", mock_credential) + mock_receive_message.assert_called_once_with(mock_service_bus_client_instance, config) + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.invoke_porter_action", return_value=True) +async def test_receive_message(mock_invoke_porter_action, mock_service_bus_client, mock_auto_lock_renewer): + mock_service_bus_client_instance = mock_service_bus_client.return_value + mock_auto_lock_renewer.return_value = AsyncMock() + + mock_receiver = AsyncMock() + mock_receiver.__aenter__.return_value = mock_receiver + mock_receiver.__aexit__.return_value = None + mock_receiver.session.session_id = "test_session_id" + mock_receiver.__aiter__.return_value = [AsyncMock()] + mock_receiver.__aiter__.return_value[0] = json.dumps({"id": "test_id", "action": "install", "stepId": "test_step_id", "operationId": "test_operation_id"}) + + mock_service_bus_client_instance.get_queue_receiver.return_value.__aenter__.return_value = mock_receiver + + run_once = Mock(side_effect=[True, False]) + + config = {"resource_request_queue": "test_queue"} + + await receive_message(mock_service_bus_client_instance, config, keep_running=run_once) + mock_receiver.complete_message.assert_called_once() + mock_service_bus_client_instance.get_queue_receiver.assert_called_once_with(queue_name="test_queue", max_wait_time=1, session_id=ServiceBusSessionFilter.NEXT_AVAILABLE) + + +@pytest.mark.asyncio +async def test_receive_message_unknown_exception(mock_auto_lock_renewer, mock_service_bus_client, mock_logger): + """Test receiving a message with an unknown exception.""" + mock_service_bus_client_instance = mock_service_bus_client.return_value + mock_auto_lock_renewer.return_value = AsyncMock() + + mock_receiver = AsyncMock() + mock_receiver.__aenter__.return_value = mock_receiver + mock_receiver.__aexit__.return_value = None + mock_receiver.session.session_id = "test_session_id" + mock_receiver.__aiter__.return_value = [AsyncMock()] + mock_receiver.__aiter__.return_value[0] = json.dumps({"id": "test_id", "action": "install", "stepId": "test_step_id", "operationId": "test_operation_id"}) + + mock_service_bus_client_instance.get_queue_receiver.return_value.__aenter__.return_value = mock_receiver + + run_once = Mock(side_effect=[True, False]) + + config = {"resource_request_queue": "test_queue"} + + with patch("resource_processor.vmss_porter.runner.receive_message", side_effect=Exception("Test Exception")): + await receive_message(mock_service_bus_client_instance, config, keep_running=run_once) + mock_logger.exception.assert_any_call("Unknown exception. Will retry...") + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.build_porter_command", return_value=["porter install"]) +@patch("resource_processor.vmss_porter.runner.run_porter", return_value=(0, "stdout", "stderr")) +@patch("resource_processor.vmss_porter.runner.service_bus_message_generator", return_value="test_message") +async def test_invoke_porter_action(mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client): + """Test invoking a porter action.""" + mock_sb_sender = AsyncMock() + mock_service_bus_client.get_queue_sender.return_value = mock_sb_sender + + config = {"deployment_status_queue": "test_queue"} + msg_body = {"id": "test_id", "action": "install", "stepId": "test_step_id", "operationId": "test_operation_id"} + + result = await invoke_porter_action(msg_body, mock_service_bus_client, config) + + assert result is True + mock_sb_sender.send_messages.assert_called() + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.build_porter_command", return_value=["porter install"]) +@patch("resource_processor.vmss_porter.runner.run_porter", return_value=(1, "", "error")) +@patch("resource_processor.vmss_porter.runner.service_bus_message_generator", return_value="test_message") +async def test_invoke_porter_action_failure(mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client): + """Test invoking a porter action with failure.""" + mock_sb_client = AsyncMock(spec=ServiceBusClient) + mock_sb_sender = AsyncMock() + mock_sb_client.get_queue_sender.return_value = mock_sb_sender + + config = {"deployment_status_queue": "test_queue"} + msg_body = {"id": "test_id", "action": "install", "stepId": "test_step_id", "operationId": "test_operation_id"} + + result = await invoke_porter_action(msg_body, mock_sb_client, config) + + assert result is False + mock_sb_sender.send_messages.assert_called() + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.build_porter_command", return_value=["porter install"]) +@patch("resource_processor.vmss_porter.runner.run_porter", side_effect=[(1, "", "could not find installation"), (0, "", "")]) +@patch("resource_processor.vmss_porter.runner.service_bus_message_generator", return_value="test_message") +async def test_invoke_porter_action_upgrade_failure_install_success(mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client): + """Test invoking a porter action with upgrade failure and install success.""" + mock_sb_client = AsyncMock(spec=ServiceBusClient) + mock_sb_sender = AsyncMock() + mock_sb_client.get_queue_sender.return_value = mock_sb_sender + + config = {"deployment_status_queue": "test_queue"} + msg_body = {"id": "test_id", "action": "upgrade", "stepId": "test_step_id", "operationId": "test_operation_id"} + + result = await invoke_porter_action(msg_body, mock_sb_client, config) + + assert result is True + mock_sb_sender.send_messages.assert_called() + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.build_porter_command", return_value=["porter install"]) +@patch("resource_processor.vmss_porter.runner.run_porter", side_effect=[(1, "", "could not find installation"), (1, "", "installation failed")]) +@patch("resource_processor.vmss_porter.runner.service_bus_message_generator", return_value="test_message") +async def test_invoke_porter_action_upgrade_failure_install_failure(mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client): + """Test invoking a porter action with upgrade and install failure.""" + mock_sb_client = AsyncMock(spec=ServiceBusClient) + mock_sb_sender = AsyncMock() + mock_sb_client.get_queue_sender.return_value = mock_sb_sender + + config = {"deployment_status_queue": "test_queue"} + msg_body = {"id": "test_id", "action": "upgrade", "stepId": "test_step_id", "operationId": "test_operation_id"} + + result = await invoke_porter_action(msg_body, mock_sb_client, config) + + assert result is False + mock_sb_sender.send_messages.assert_called() + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.build_porter_command", return_value=["porter install"]) +@patch("resource_processor.vmss_porter.runner.run_porter", return_value=(1, "", "could not find installation")) +@patch("resource_processor.vmss_porter.runner.service_bus_message_generator", return_value="test_message") +async def test_invoke_porter_action_uninstall_failure(mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client): + """Test invoking a porter action with uninstall failure.""" + mock_sb_client = AsyncMock(spec=ServiceBusClient) + mock_sb_sender = AsyncMock() + mock_sb_client.get_queue_sender.return_value = mock_sb_sender + + config = {"deployment_status_queue": "test_queue"} + msg_body = {"id": "test_id", "action": "uninstall", "stepId": "test_step_id", "operationId": "test_operation_id"} + + result = await invoke_porter_action(msg_body, mock_sb_client, config) + + assert result is True + mock_sb_sender.send_messages.assert_called() + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.build_porter_command", return_value=["porter custom-action"]) +@patch("resource_processor.vmss_porter.runner.run_porter", return_value=(0, "stdout", "stderr")) +@patch("resource_processor.vmss_porter.runner.service_bus_message_generator", return_value="test_message") +async def test_invoke_porter_action_custom_action(mock_service_bus_message_generator, mock_run_porter, mock_build_porter_command, mock_service_bus_client): + """Test invoking a porter custom action.""" + mock_sb_client = AsyncMock(spec=ServiceBusClient) + mock_sb_sender = AsyncMock() + mock_sb_client.get_queue_sender.return_value = mock_sb_sender + + config = {"deployment_status_queue": "test_queue"} + msg_body = {"id": "test_id", "action": "custom-action", "stepId": "test_step_id", "operationId": "test_operation_id"} + + result = await invoke_porter_action(msg_body, mock_sb_client, config) + + assert result is True + mock_sb_sender.send_messages.assert_called() + + +@pytest.mark.asyncio +@patch("resource_processor.vmss_porter.runner.build_porter_command_for_outputs", return_value=["porter installations output list"]) +@patch("resource_processor.vmss_porter.runner.run_porter", return_value=(0, json.dumps([{"name": "output1", "value": "value1"}]), "stderr")) +async def test_get_porter_outputs(mock_run_porter, mock_build_porter_command_for_outputs): + """Test getting porter outputs.""" + config = {} + msg_body = {"id": "test_id", "action": "install"} + + success, outputs = await get_porter_outputs(msg_body, config) + + assert success is True + assert outputs == [{"name": "output1", "value": "value1"}] + + +@pytest.mark.asyncio +@patch("asyncio.sleep", new_callable=AsyncMock) +async def test_check_runners(_): + """Test checking runners.""" + mock_process = Mock() + mock_process.is_alive.return_value = False + processes = [mock_process] + mock_httpserver = AsyncMock() + + run_once = Mock(side_effect=[True, False]) + + await check_runners(processes, mock_httpserver, keep_running=run_once) + mock_httpserver.kill.assert_called_once() diff --git a/resource_processor/vmss_porter/runner.py b/resource_processor/vmss_porter/runner.py index 3de4ac06fe..6f6d6e21e1 100644 --- a/resource_processor/vmss_porter/runner.py +++ b/resource_processor/vmss_porter/runner.py @@ -4,14 +4,13 @@ import asyncio import logging import sys -from resources.commands import azure_acr_login_command, azure_login_command, build_porter_command, build_porter_command_for_outputs, apply_porter_credentials_sets_command +from helpers.commands import azure_acr_login_command, azure_login_command, build_porter_command, build_porter_command_for_outputs, apply_porter_credentials_sets_command from shared.config import get_config -from resources.helpers import get_installation_id -from resources.httpserver import start_server +from helpers.httpserver import start_server from shared.logging import initialize_logging, logger, shell_output_logger, tracer from shared.config import VERSION -from resources import statuses +from helpers import statuses from contextlib import asynccontextmanager from azure.servicebus import ServiceBusMessage, NEXT_AVAILABLE_SESSION from azure.servicebus.exceptions import OperationTimeoutError, ServiceBusConnectionError @@ -38,7 +37,7 @@ async def default_credentials(msi_id): await credential.close() -async def receive_message(service_bus_client, config: dict): +async def receive_message(service_bus_client, config: dict, keep_running=lambda: True): """ This method is run per process. Each process will connect to service bus and try to establish a session. If messages are there, the process will continue to receive all the messages associated with that session. @@ -46,7 +45,7 @@ async def receive_message(service_bus_client, config: dict): """ q_name = config["resource_request_queue"] - while True: + while keep_running(): try: logger.info("Looking for new session...") # max_wait_time=1 -> don't hold the session open after processing of the message has finished @@ -94,6 +93,7 @@ async def receive_message(service_bus_client, config: dict): except Exception: # Catch all other exceptions, log them via .exception to get the stack trace, sleep, and reconnect + logger.exception("Unknown exception. Will retry...") @@ -135,7 +135,7 @@ def service_bus_message_generator(sb_message: dict, status: str, deployment_mess """ Generate a resource request message """ - installation_id = get_installation_id(sb_message) + installation_id = sb_message["id"] message_dict = { "operationId": sb_message["operationId"], "stepId": sb_message["stepId"], @@ -156,7 +156,7 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf Handle resource message by invoking specified porter action (i.e. install, uninstall) """ - installation_id = get_installation_id(msg_body) + installation_id = msg_body["id"] action = msg_body["action"] logger.info(f"{action} action starting for {installation_id}...") sb_sender = sb_client.get_queue_sender(queue_name=config["deployment_status_queue"]) @@ -173,13 +173,25 @@ async def invoke_porter_action(msg_body: dict, sb_client: ServiceBusClient, conf logger.debug("Starting to run porter execution command...") returncode, _, err = await run_porter(porter_command, config) logger.debug("Finished running porter execution command.") - action_completed_without_error = True + + action_completed_without_error = False + + if returncode == 0: + action_completed_without_error = True # Handle command output if returncode != 0 and err is not None: error_message = "Error message: " + " ".join(err.split('\n')) + "; Command executed: " + " ".join(porter_command) action_completed_without_error = False + if "upgrade" == action and ("could not find installation" in err or "The installation cannot be upgraded, because it is not installed." in err): + logger.warning("Upgrade failed, attempting install...") + msg_body['action'] = "install" + porter_command = await build_porter_command(config, msg_body, False) + returncode, _, err = await run_porter(porter_command, config) + if returncode == 0: + action_completed_without_error = True + if "uninstall" == action and "could not find installation" in err: logger.warning("The installation doesn't exist. Treating as a successful action to allow the flow to proceed.") action_completed_without_error = True @@ -227,7 +239,8 @@ async def get_porter_outputs(msg_body: dict, config: dict): if returncode != 0: error_message = "Error context message = " + " ".join(err.split('\n')) - logger.info(f"{get_installation_id(msg_body)}: Failed to get outputs with error = {error_message}") + installation_id = msg_body["id"] + logger.info(f"{installation_id}: Failed to get outputs with error = {error_message}") return False, {} else: outputs_json = {} @@ -253,10 +266,10 @@ async def runner(process_number: int, config: dict): await receive_message(service_bus_client, config) -async def check_runners(processes: list, httpserver: Process): +async def check_runners(processes: list, httpserver: Process, keep_running=lambda: True): logger.info("Starting runners check...") - while True: + while keep_running(): await asyncio.sleep(30) if all(not process.is_alive() for process in processes): logger.error("All runner processes have failed!") From 0715eca9dfc0673a9a948d192233a0e3e0f76df7 Mon Sep 17 00:00:00 2001 From: Yuval Yaron <43217306+yuvalyaron@users.noreply.github.com> Date: Tue, 11 Feb 2025 18:02:24 +0200 Subject: [PATCH 25/38] Add moved block to handle firewall public IP count migration (#4357) Adds moved block to handle resource address change when adding count to the firewall's public IP resource, preventing unwanted recreation of the IP address --- templates/shared_services/firewall/porter.yaml | 2 +- templates/shared_services/firewall/terraform/firewall.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/shared_services/firewall/porter.yaml b/templates/shared_services/firewall/porter.yaml index d5759218a7..19e820c4d7 100644 --- a/templates/shared_services/firewall/porter.yaml +++ b/templates/shared_services/firewall/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-shared-service-firewall -version: 1.3.1 +version: 1.3.2 description: "An Azure TRE Firewall shared service" dockerfile: Dockerfile.tmpl registry: azuretre diff --git a/templates/shared_services/firewall/terraform/firewall.tf b/templates/shared_services/firewall/terraform/firewall.tf index 3f42237a0f..6f4dedc816 100644 --- a/templates/shared_services/firewall/terraform/firewall.tf +++ b/templates/shared_services/firewall/terraform/firewall.tf @@ -11,7 +11,7 @@ resource "azurerm_public_ip" "fwtransit" { } moved { - from = azurerm_public_ip.fwpip + from = azurerm_public_ip.fwtransit to = azurerm_public_ip.fwtransit[0] } From 78f4a1351625ce9f1f6bd4fcc532e396bf989338 Mon Sep 17 00:00:00 2001 From: Ron Shakutai <58519179+ShakutaiGit@users.noreply.github.com> Date: Wed, 12 Feb 2025 01:57:30 +0200 Subject: [PATCH 26/38] Update Core Subnets Creation and upgrade core Terraform to 4.14.0 (#4255) --- CHANGELOG.md | 3 + core/terraform/.terraform.lock.hcl | 29 ++- core/terraform/cosmos_mongo.tf | 2 +- core/terraform/locals.tf | 15 +- core/terraform/main.tf | 2 +- core/terraform/migrate.sh | 110 ++++++--- core/terraform/network/.terraform.lock.hcl | 22 ++ core/terraform/network/locals.tf | 2 + core/terraform/network/main.tf | 2 +- core/terraform/network/network.tf | 210 ++++++++---------- .../network/network_security_groups.tf | 55 ----- core/terraform/network/outputs.tf | 20 +- core/terraform/statestore.tf | 3 +- core/version.txt | 2 +- 14 files changed, 231 insertions(+), 246 deletions(-) create mode 100644 core/terraform/network/.terraform.lock.hcl diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cf9862269..879a3f47e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ ENHANCEMENTS: * Allow workspace App Service Plan SKU to be updated ([#4331](https://github.com/microsoft/AzureTRE/issues/4331)) * Remove public IP from TRE's firewall when forced tunneling is configured ([#4346](https://github.com/microsoft/AzureTRE/pull/4346)) +* Upgrade AzureRM Terraform provider from `3.117.0` to `4.14.0`. ([[#4255](https://github.com/microsoft/AzureTRE/pull/4255/)]) +* Subnet definitions are now inline in the `azurerm_virtual_network` resource, and NSG associations are set using `security_group` in each subnet block (no separate `azurerm_subnet_network_security_group_association` needed). ([[#4255](https://github.com/microsoft/AzureTRE/pull/4255/)]) BUG FIXES: * Fix upgrade when porter install has failed ([#4338](https://github.com/microsoft/AzureTRE/pull/4338)) @@ -78,6 +80,7 @@ BUG FIXES: * Recreate tre_output.json if empty. ([[#4292](https://github.com/microsoft/AzureTRE/issues/4292)]) * Ensure R directory is present before attempting to update package mirror URL ([#4332](https://github.com/microsoft/AzureTRE/pull/4332)) + COMPONENTS: | name | version | diff --git a/core/terraform/.terraform.lock.hcl b/core/terraform/.terraform.lock.hcl index 1c20359910..41d8da1a19 100644 --- a/core/terraform/.terraform.lock.hcl +++ b/core/terraform/.terraform.lock.hcl @@ -6,7 +6,6 @@ provider "registry.terraform.io/azure/azapi" { constraints = ">= 1.15.0, ~> 1.15.0" hashes = [ "h1:Y7ruMuPh8UJRTRl4rm+cdpGtmURx2taqiuqfYaH3o48=", - "h1:gIOgxVmFSxHrR+XOzgUEA+ybOmp8kxZlZH3eYeB/eFI=", "zh:0627a8bc77254debc25dc0c7b62e055138217c97b03221e593c3c56dc7550671", "zh:2fe045f07070ef75d0bec4b0595a74c14394daa838ddb964e2fd23cc98c40c34", "zh:343009f39c957883b2c06145a5954e524c70f93585f943f1ea3d28ef6995d0d0", @@ -23,22 +22,22 @@ provider "registry.terraform.io/azure/azapi" { } provider "registry.terraform.io/hashicorp/azurerm" { - version = "3.117.0" - constraints = ">= 3.117.0, 3.117.0" + version = "4.14.0" + constraints = ">= 3.117.0, 4.14.0" hashes = [ - "h1:Ynfg+Iy7x6K8M6W1AhqXCe3wkoiqIQhROlca7C3KC3w=", - "zh:2e25f47492366821a786762369f0e0921cc9452d64bfd5075f6fdfcf1a9c6d70", - "zh:41eb34f2f7469bf3eb1019dfb0e7fc28256f809824016f4f8b9d691bf473b2ac", - "zh:48bb9c87b3d928da1abc1d3db75453c9725de4674c612daf3800160cc7145d30", - "zh:5d6b0de0bbd78943fcc65c53944ef4496329e247f434c6eab86ed051c5cea67b", - "zh:78c9f6fdb1206a89cf0e6706b4f46178169a93b6c964a4cad8a321058ccbd9b4", - "zh:793b702c352589d4360b580d4a1cf654a7439d2ad6bdb7bfea91de07bc4b0fac", - "zh:7ed687ff0a5509463a592f97431863574fe5cc80a34e395be06766215b8c6285", - "zh:955ba18789bd15592824eb426a8d0f38595bd09fffc6939c1c58933489c1a71e", - "zh:bf5949a55be0714cd9c8815d472eae4baa48ba06d0f6bf2b96775869acda8a54", - "zh:da5d31f635abd2c645ffc76d6176d73f646128e73720cc368247cc424975c127", - "zh:eed5a66d59883c9c56729b0a964a2b60d758ea7489ef3e920a6fbd48518ce5f5", + "h1:FYZ9qh8i3X2gDmUTe1jJ/VzdSyjGjVmhBzv2R8D6CBo=", + "zh:05aaea16fc5f27b14d9fbad81654edf0638949ed3585576b2219c76a2bee095a", + "zh:065ce6ed16ba3fa7efcf77888ea582aead54e6a28f184c6701b73d71edd64bb0", + "zh:3c0cd17c249d18aa2e0120acb5f0c14810725158b379a67fec1331110e7c50df", + "zh:5a3ba3ffb2f1ce519fe3bf84a7296aa5862c437c70c62f0b0a5293bea9f2d01c", + "zh:7a8e9d72fa2714f4d567270b1761d4b4e788de7c15dada7db0cf0e29933185a2", + "zh:a11e190073f31c1238c15af29b9162e0f4564f6b0cd0310a3fa94102738450dc", + "zh:a5c004114410cc6dcb8fed584c9f3b84283b58025b0073a7e88d2bdb27840dfa", + "zh:a674a41db118e244eda7591e455d2ec338626664e0856e4125e909eb038f78db", + "zh:b5139010e4cbb2cb1a27c775610593c1c8063d3a7c82b00a65006509c434df2f", + "zh:cbb031223ccd8b099ac4d19b92641142f330b90f2fc6452843e445bae28f832c", "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f7e7db1b94082a4ac3d4af3dabe7bbd335e1679305bf8e29d011f0ee440724ca", ] } diff --git a/core/terraform/cosmos_mongo.tf b/core/terraform/cosmos_mongo.tf index 65812cc8f1..6bb4ec4594 100644 --- a/core/terraform/cosmos_mongo.tf +++ b/core/terraform/cosmos_mongo.tf @@ -6,7 +6,7 @@ resource "azurerm_cosmosdb_account" "mongo" { kind = "MongoDB" automatic_failover_enabled = false mongo_server_version = 4.2 - ip_range_filter = "${local.azure_portal_cosmos_ips}${var.enable_local_debugging ? ",${local.myip}" : ""}" + ip_range_filter = local.cosmos_ip_filter_set capabilities { name = "EnableServerless" diff --git a/core/terraform/locals.tf b/core/terraform/locals.tf index 22d327f96f..15f066ae7b 100644 --- a/core/terraform/locals.tf +++ b/core/terraform/locals.tf @@ -14,7 +14,20 @@ locals { docker_registry_server = data.azurerm_container_registry.mgmt_acr.login_server # https://learn.microsoft.com/en-us/azure/cosmos-db/how-to-configure-firewall#allow-requests-from-the-azure-portal - azure_portal_cosmos_ips = "104.42.195.92,40.76.54.131,52.176.6.30,52.169.50.45,52.187.184.26" + + azure_portal_cosmos_ips_list = [ + "104.42.195.92", + "40.76.54.131", + "52.176.6.30", + "52.169.50.45", + "52.187.184.26" + ] + + cosmos_ip_filter_set = toset( + var.enable_local_debugging + ? concat(local.azure_portal_cosmos_ips_list, [local.myip]) + : local.azure_portal_cosmos_ips_list + ) # we define some zones in core despite not used by the core infra because # it's the easier way to make them available to other services in the system. diff --git a/core/terraform/main.tf b/core/terraform/main.tf index 4d6d910257..b231621603 100644 --- a/core/terraform/main.tf +++ b/core/terraform/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "=3.117.0" + version = "=4.14.0" } random = { source = "hashicorp/random" diff --git a/core/terraform/migrate.sh b/core/terraform/migrate.sh index 3c88e2ed2d..5f64abeb15 100755 --- a/core/terraform/migrate.sh +++ b/core/terraform/migrate.sh @@ -5,16 +5,25 @@ set -o pipefail set -o nounset # set -o xtrace -# Configure AzureRM provider to user Azure AD to connect to storage accounts +get_resource_id() { + local json_data="$1" + local resource_addr="$2" + echo "$json_data" | jq -r --arg addr "$resource_addr" ' + def walk_resources: + (.resources[]?), + (.child_modules[]? | walk_resources); + .values.root_module | walk_resources | select(.address==$addr) | .values.id + ' +} + +# Configure AzureRM provider to use Azure AD to connect to storage accounts export ARM_STORAGE_USE_AZUREAD=true -# Configure AzureRM backend to user Azure AD to connect to storage accounts +# Configure AzureRM backend to use Azure AD to connect to storage accounts export ARM_USE_AZUREAD=true export ARM_USE_OIDC=true -# terraform_wrapper_path="../../devops/scripts/terraform_wrapper.sh" - -# This variables are loaded in for us +# These variables are loaded in for us # shellcheck disable=SC2154 terraform init -input=false -backend=true -reconfigure \ -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \ @@ -24,41 +33,68 @@ terraform init -input=false -backend=true -reconfigure \ echo "*** Migrating TF Resources... ***" +terraform refresh +# get TF state in JSON terraform_show_json=$(terraform show -json) -# Remove cnab-state legacy state path form state. Needs to be run before refresh, as refresh will fail. -state_store_legacy_path=$(echo "${terraform_show_json}" \ - | jq 'select(.values.root_module.resources != null) | .values.root_module.resources[] | select(.address=="azurerm_storage_share.storage_state_path") | .values.id') - -if [ -n "${state_store_legacy_path}" ]; then - echo -e "\n\e[96mRemoving legacy state path from TF state\e[0m..." - terraform state rm azurerm_storage_share.storage_state_path -fi - -# terraform show might fail if provider schema has changed. Since we don't call apply at this stage a refresh is needed -terraform refresh +# List of resource addresses to remove. +declare -a RESOURCES_TO_REMOVE=( + "module.network.azurerm_subnet_network_security_group_association.bastion" + "module.network.azurerm_subnet_network_security_group_association.app_gw" + "module.network.azurerm_subnet_network_security_group_association.shared" + "module.network.azurerm_subnet_network_security_group_association.web_app" + "module.network.azurerm_subnet_network_security_group_association.resource_processor" + "module.network.azurerm_subnet_network_security_group_association.airlock_processor" + "module.network.azurerm_subnet_network_security_group_association.airlock_notification" + "module.network.azurerm_subnet_network_security_group_association.airlock_storage" + "module.network.azurerm_subnet_network_security_group_association.airlock_events" + "module.network.azurerm_subnet.bastion" + "module.network.azurerm_subnet.azure_firewall" + "module.network.azurerm_subnet.app_gw" + "module.network.azurerm_subnet.web_app" + "module.network.azurerm_subnet.shared" + "module.network.azurerm_subnet.resource_processor" + "module.network.azurerm_subnet.airlock_processor" + "module.network.azurerm_subnet.airlock_notification" + "module.network.azurerm_subnet.airlock_storage" + "module.network.azurerm_subnet.airlock_events" + "module.network.azurerm_subnet.firewall_management" +) +vnet_address="module.network.azurerm_virtual_network.core" -# 1. Check we have a root_module in state -# 2. Grab the Resource ID -# 3. Delete the old resource from state -# 4. Import the new resource type in using the existing Azure Resource ID +# Check if migration is needed +migration_needed=0 +for resource in "${RESOURCES_TO_REMOVE[@]}"; do + resource_id=$(get_resource_id "${terraform_show_json}" "$resource") + if [ -n "$resource_id" ] && [ "$resource_id" != "null" ]; then + migration_needed=1 + break + fi +done -terraform_show_json=$(terraform show -json) +# Remove old resources +if [ "$migration_needed" -eq 1 ]; then + for resource in "${RESOURCES_TO_REMOVE[@]}"; do + resource_id=$(get_resource_id "${terraform_show_json}" "$resource") + if [ -n "$resource_id" ] && [ "$resource_id" != "null" ]; then + terraform state rm "$resource" + else + echo "Resource that was supposed to be removed not found in state: ${resource}" + fi + done -# example migration -# # azurerm_app_service_plan -> azurerm_service_plan -# core_app_service_plan_id=$(echo "${terraform_show_json}" \ -# | jq -r 'select(.values.root_module.resources != null) | .values.root_module.resources[] | select(.address=="azurerm_app_service_plan.core") | .values.id') -# if [ -n "${core_app_service_plan_id}" ]; then -# echo "Migrating ${core_app_service_plan_id}" -# terraform state rm azurerm_app_service_plan.core -# if [[ $(az resource list --query "[?id=='${core_app_service_plan_id}'] | length(@)") == 0 ]]; -# then -# echo "The resource doesn't exist on Azure. Skipping importing it back to state." -# else -# terraform import azurerm_service_plan.core "${core_app_service_plan_id}" -# fi -# fi - -echo "*** Migration is done. ***" + # Remove and re-import the VNet + vnet_address="module.network.azurerm_virtual_network.core" + vnet_id=$(get_resource_id "${terraform_show_json}" "$vnet_address" "vnet") + if [ -n "${vnet_id}" ] && [ "${vnet_id}" != "null" ]; then + terraform state rm "${vnet_address}" + terraform import "${vnet_address}" "${vnet_id}" + else + echo "VNet resource not found in state: ${vnet_address}" + fi + echo "*** Migration Done ***" +else + echo "No old resources found in the state, skipping migration." + echo "*** Migration Skipped ***" +fi diff --git a/core/terraform/network/.terraform.lock.hcl b/core/terraform/network/.terraform.lock.hcl new file mode 100644 index 0000000000..ec690305f6 --- /dev/null +++ b/core/terraform/network/.terraform.lock.hcl @@ -0,0 +1,22 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "4.14.0" + constraints = "4.14.0" + hashes = [ + "h1:FYZ9qh8i3X2gDmUTe1jJ/VzdSyjGjVmhBzv2R8D6CBo=", + "zh:05aaea16fc5f27b14d9fbad81654edf0638949ed3585576b2219c76a2bee095a", + "zh:065ce6ed16ba3fa7efcf77888ea582aead54e6a28f184c6701b73d71edd64bb0", + "zh:3c0cd17c249d18aa2e0120acb5f0c14810725158b379a67fec1331110e7c50df", + "zh:5a3ba3ffb2f1ce519fe3bf84a7296aa5862c437c70c62f0b0a5293bea9f2d01c", + "zh:7a8e9d72fa2714f4d567270b1761d4b4e788de7c15dada7db0cf0e29933185a2", + "zh:a11e190073f31c1238c15af29b9162e0f4564f6b0cd0310a3fa94102738450dc", + "zh:a5c004114410cc6dcb8fed584c9f3b84283b58025b0073a7e88d2bdb27840dfa", + "zh:a674a41db118e244eda7591e455d2ec338626664e0856e4125e909eb038f78db", + "zh:b5139010e4cbb2cb1a27c775610593c1c8063d3a7c82b00a65006509c434df2f", + "zh:cbb031223ccd8b099ac4d19b92641142f330b90f2fc6452843e445bae28f832c", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:f7e7db1b94082a4ac3d4af3dabe7bbd335e1679305bf8e29d011f0ee440724ca", + ] +} diff --git a/core/terraform/network/locals.tf b/core/terraform/network/locals.tf index aaa2aea7d1..82ae26fb2d 100644 --- a/core/terraform/network/locals.tf +++ b/core/terraform/network/locals.tf @@ -32,4 +32,6 @@ locals { "privatelink.queue.core.windows.net", "privatelink.table.core.windows.net" ]) + + subnet_ids_map = { for subnet in azurerm_virtual_network.core.subnet : subnet.name => subnet.id } } diff --git a/core/terraform/network/main.tf b/core/terraform/network/main.tf index a4eb095f9c..5cced47bb0 100644 --- a/core/terraform/network/main.tf +++ b/core/terraform/network/main.tf @@ -3,7 +3,7 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = ">= 3.117" + version = ">= 4.14.0" } } } diff --git a/core/terraform/network/network.tf b/core/terraform/network/network.tf index db71fe554f..a511365326 100644 --- a/core/terraform/network/network.tf +++ b/core/terraform/network/network.tf @@ -5,146 +5,112 @@ resource "azurerm_virtual_network" "core" { address_space = [var.core_address_space] tags = local.tre_core_tags lifecycle { ignore_changes = [tags] } -} -resource "azurerm_subnet" "bastion" { - name = "AzureBastionSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.bastion_subnet_address_prefix] -} + subnet { + name = "AzureBastionSubnet" + address_prefixes = [local.bastion_subnet_address_prefix] + security_group = azurerm_network_security_group.bastion.id + } -resource "azurerm_subnet" "azure_firewall" { - name = "AzureFirewallSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.firewall_subnet_address_space] - depends_on = [azurerm_subnet.bastion] -} + subnet { + name = "AzureFirewallSubnet" + address_prefixes = [local.firewall_subnet_address_space] + } -resource "azurerm_subnet" "app_gw" { - name = "AppGwSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.app_gw_subnet_address_prefix] - private_endpoint_network_policies = "Disabled" - private_link_service_network_policies_enabled = true - depends_on = [azurerm_subnet.azure_firewall] -} + subnet { + name = "AppGwSubnet" + address_prefixes = [local.app_gw_subnet_address_prefix] + private_endpoint_network_policies = "Disabled" + private_link_service_network_policies_enabled = true + security_group = azurerm_network_security_group.app_gw.id + } -resource "azurerm_subnet" "web_app" { - name = "WebAppSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.web_app_subnet_address_prefix] - private_endpoint_network_policies = "Disabled" - private_link_service_network_policies_enabled = true - depends_on = [azurerm_subnet.app_gw] - - delegation { - name = "delegation" - - service_delegation { - name = "Microsoft.Web/serverFarms" - actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + subnet { + name = "WebAppSubnet" + address_prefixes = [local.web_app_subnet_address_prefix] + private_endpoint_network_policies = "Disabled" + private_link_service_network_policies_enabled = true + security_group = azurerm_network_security_group.default_rules.id + + delegation { + name = "delegation" + + service_delegation { + name = "Microsoft.Web/serverFarms" + actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + } } } -} -resource "azurerm_subnet" "shared" { - name = "SharedSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.shared_services_subnet_address_prefix] - # notice that private endpoints do not adhere to NSG rules - private_endpoint_network_policies = "Disabled" - depends_on = [azurerm_subnet.web_app] -} + subnet { + name = "SharedSubnet" + address_prefixes = [local.shared_services_subnet_address_prefix] + private_endpoint_network_policies = "Disabled" + security_group = azurerm_network_security_group.default_rules.id + } -resource "azurerm_subnet" "resource_processor" { - name = "ResourceProcessorSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.resource_processor_subnet_address_prefix] - # notice that private endpoints do not adhere to NSG rules - private_endpoint_network_policies = "Disabled" - depends_on = [azurerm_subnet.shared] -} + subnet { + name = "ResourceProcessorSubnet" + address_prefixes = [local.resource_processor_subnet_address_prefix] + private_endpoint_network_policies = "Disabled" + security_group = azurerm_network_security_group.default_rules.id + } + + subnet { + name = "AirlockProcessorSubnet" + address_prefixes = [local.airlock_processor_subnet_address_prefix] + private_endpoint_network_policies = "Disabled" + security_group = azurerm_network_security_group.default_rules.id -resource "azurerm_subnet" "airlock_processor" { - name = "AirlockProcessorSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.airlock_processor_subnet_address_prefix] - # notice that private endpoints do not adhere to NSG rules - private_endpoint_network_policies = "Disabled" - depends_on = [azurerm_subnet.resource_processor] - - delegation { - name = "delegation" - - service_delegation { - name = "Microsoft.Web/serverFarms" - actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + delegation { + name = "delegation" + + service_delegation { + name = "Microsoft.Web/serverFarms" + actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + } } + + service_endpoints = ["Microsoft.Storage"] } - # Todo: needed as we want to open the fw for this subnet in some of the airlock storages (export inprogress) - # https://github.com/microsoft/AzureTRE/issues/2098 - service_endpoints = ["Microsoft.Storage"] -} + subnet { + name = "AirlockNotifiactionSubnet" + address_prefixes = [local.airlock_notifications_subnet_address_prefix] + private_endpoint_network_policies = "Disabled" + security_group = azurerm_network_security_group.default_rules.id + + delegation { + name = "delegation" -resource "azurerm_subnet" "airlock_notification" { - name = "AirlockNotifiactionSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.airlock_notifications_subnet_address_prefix] - # notice that private endpoints do not adhere to NSG rules - private_endpoint_network_policies = "Disabled" - depends_on = [azurerm_subnet.airlock_processor] - - delegation { - name = "delegation" - - service_delegation { - name = "Microsoft.Web/serverFarms" - actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + service_delegation { + name = "Microsoft.Web/serverFarms" + actions = ["Microsoft.Network/virtualNetworks/subnets/action"] + } } + service_endpoints = ["Microsoft.ServiceBus"] } - service_endpoints = ["Microsoft.ServiceBus"] -} -resource "azurerm_subnet" "airlock_storage" { - name = "AirlockStorageSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.airlock_storage_subnet_address_prefix] - # notice that private endpoints do not adhere to NSG rules - private_endpoint_network_policies = "Disabled" - depends_on = [azurerm_subnet.airlock_notification] -} + subnet { + name = "AirlockStorageSubnet" + address_prefixes = [local.airlock_storage_subnet_address_prefix] + private_endpoint_network_policies = "Disabled" + security_group = azurerm_network_security_group.default_rules.id + } -resource "azurerm_subnet" "airlock_events" { - name = "AirlockEventsSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.airlock_events_subnet_address_prefix] - # notice that private endpoints do not adhere to NSG rules - private_endpoint_network_policies = "Disabled" - depends_on = [azurerm_subnet.airlock_storage] - - # Eventgrid CAN'T send messages over private endpoints, hence we need to allow service endpoints to the service bus - # We are using service endpoints + managed identity to send these messaages - # https://docs.microsoft.com/en-us/azure/event-grid/consume-private-endpoints - service_endpoints = ["Microsoft.ServiceBus"] -} + subnet { + name = "AirlockEventsSubnet" + address_prefixes = [local.airlock_events_subnet_address_prefix] + private_endpoint_network_policies = "Disabled" + security_group = azurerm_network_security_group.default_rules.id -resource "azurerm_subnet" "firewall_management" { - name = "AzureFirewallManagementSubnet" - virtual_network_name = azurerm_virtual_network.core.name - resource_group_name = var.resource_group_name - address_prefixes = [local.firewall_management_subnet_address_prefix] - depends_on = [azurerm_subnet.airlock_events] + service_endpoints = ["Microsoft.ServiceBus"] + } + + subnet { + name = "AzureFirewallManagementSubnet" + address_prefixes = [local.firewall_management_subnet_address_prefix] + } } resource "azurerm_ip_group" "resource_processor" { diff --git a/core/terraform/network/network_security_groups.tf b/core/terraform/network/network_security_groups.tf index 50accf846b..34371dc145 100644 --- a/core/terraform/network/network_security_groups.tf +++ b/core/terraform/network/network_security_groups.tf @@ -105,13 +105,6 @@ resource "azurerm_network_security_group" "bastion" { lifecycle { ignore_changes = [tags] } } -resource "azurerm_subnet_network_security_group_association" "bastion" { - subnet_id = azurerm_subnet.bastion.id - network_security_group_id = azurerm_network_security_group.bastion.id - # depend on the last subnet we created in the vnet - depends_on = [azurerm_subnet.firewall_management] -} - # Network security group for Application Gateway # See https://docs.microsoft.com/azure/application-gateway/configuration-infrastructure#network-security-groups resource "azurerm_network_security_group" "app_gw" { @@ -147,12 +140,6 @@ resource "azurerm_network_security_group" "app_gw" { lifecycle { ignore_changes = [tags] } } -resource "azurerm_subnet_network_security_group_association" "app_gw" { - subnet_id = azurerm_subnet.app_gw.id - network_security_group_id = azurerm_network_security_group.app_gw.id - depends_on = [azurerm_subnet_network_security_group_association.bastion] -} - # Network security group with only default security rules # See https://docs.microsoft.com/azure/virtual-network/network-security-groups-overview#default-security-rules resource "azurerm_network_security_group" "default_rules" { @@ -163,45 +150,3 @@ resource "azurerm_network_security_group" "default_rules" { lifecycle { ignore_changes = [tags] } } - -resource "azurerm_subnet_network_security_group_association" "shared" { - subnet_id = azurerm_subnet.shared.id - network_security_group_id = azurerm_network_security_group.default_rules.id - depends_on = [azurerm_subnet_network_security_group_association.app_gw] -} - -resource "azurerm_subnet_network_security_group_association" "web_app" { - subnet_id = azurerm_subnet.web_app.id - network_security_group_id = azurerm_network_security_group.default_rules.id - depends_on = [azurerm_subnet_network_security_group_association.shared] -} - -resource "azurerm_subnet_network_security_group_association" "resource_processor" { - subnet_id = azurerm_subnet.resource_processor.id - network_security_group_id = azurerm_network_security_group.default_rules.id - depends_on = [azurerm_subnet_network_security_group_association.web_app] -} - -resource "azurerm_subnet_network_security_group_association" "airlock_processor" { - subnet_id = azurerm_subnet.airlock_processor.id - network_security_group_id = azurerm_network_security_group.default_rules.id - depends_on = [azurerm_subnet_network_security_group_association.resource_processor] -} - -resource "azurerm_subnet_network_security_group_association" "airlock_storage" { - subnet_id = azurerm_subnet.airlock_storage.id - network_security_group_id = azurerm_network_security_group.default_rules.id - depends_on = [azurerm_subnet_network_security_group_association.airlock_processor] -} - -resource "azurerm_subnet_network_security_group_association" "airlock_events" { - subnet_id = azurerm_subnet.airlock_events.id - network_security_group_id = azurerm_network_security_group.default_rules.id - depends_on = [azurerm_subnet_network_security_group_association.airlock_storage] -} - -resource "azurerm_subnet_network_security_group_association" "airlock_notification" { - subnet_id = azurerm_subnet.airlock_notification.id - network_security_group_id = azurerm_network_security_group.default_rules.id - depends_on = [azurerm_subnet_network_security_group_association.airlock_events] -} diff --git a/core/terraform/network/outputs.tf b/core/terraform/network/outputs.tf index 3e0aab407d..e2a7fba134 100644 --- a/core/terraform/network/outputs.tf +++ b/core/terraform/network/outputs.tf @@ -3,43 +3,43 @@ output "core_vnet_id" { } output "bastion_subnet_id" { - value = azurerm_subnet.bastion.id + value = local.subnet_ids_map["AzureBastionSubnet"] } output "azure_firewall_subnet_id" { - value = azurerm_subnet.azure_firewall.id + value = local.subnet_ids_map["AzureFirewallSubnet"] } output "app_gw_subnet_id" { - value = azurerm_subnet.app_gw.id + value = local.subnet_ids_map["AppGwSubnet"] } output "web_app_subnet_id" { - value = azurerm_subnet.web_app.id + value = local.subnet_ids_map["WebAppSubnet"] } output "shared_subnet_id" { - value = azurerm_subnet.shared.id + value = local.subnet_ids_map["SharedSubnet"] } output "airlock_processor_subnet_id" { - value = azurerm_subnet.airlock_processor.id + value = local.subnet_ids_map["AirlockProcessorSubnet"] } output "airlock_storage_subnet_id" { - value = azurerm_subnet.airlock_storage.id + value = local.subnet_ids_map["AirlockStorageSubnet"] } output "airlock_events_subnet_id" { - value = azurerm_subnet.airlock_events.id + value = local.subnet_ids_map["AirlockEventsSubnet"] } output "resource_processor_subnet_id" { - value = azurerm_subnet.resource_processor.id + value = local.subnet_ids_map["ResourceProcessorSubnet"] } output "airlock_notification_subnet_id" { - value = azurerm_subnet.airlock_notification.id + value = local.subnet_ids_map["AirlockNotifiactionSubnet"] } # DNS Zones diff --git a/core/terraform/statestore.tf b/core/terraform/statestore.tf index 66748fda58..ec8950084e 100644 --- a/core/terraform/statestore.tf +++ b/core/terraform/statestore.tf @@ -5,10 +5,9 @@ resource "azurerm_cosmosdb_account" "tre_db_account" { offer_type = "Standard" kind = "GlobalDocumentDB" automatic_failover_enabled = false - ip_range_filter = "${local.azure_portal_cosmos_ips}${var.enable_local_debugging ? ",${local.myip}" : ""}" + ip_range_filter = local.cosmos_ip_filter_set local_authentication_disabled = true tags = local.tre_core_tags - dynamic "capabilities" { # We can't change an existing cosmos for_each = var.is_cosmos_defined_throughput ? [] : [1] diff --git a/core/version.txt b/core/version.txt index 836582489b..ea370a8e55 100644 --- a/core/version.txt +++ b/core/version.txt @@ -1 +1 @@ -__version__ = "0.11.23" +__version__ = "0.12.0" From 72e98c370ee35b0f7c1c599460b2172c578d9e90 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Wed, 12 Feb 2025 15:47:24 +0000 Subject: [PATCH 27/38] Add requests endpoint to core TRE API (#4352) --- CHANGELOG.md | 1 + api_app/_version.py | 2 +- api_app/api/routes/api.py | 5 +- api_app/api/routes/requests.py | 38 ++ api_app/db/repositories/airlock_requests.py | 49 ++- api_app/resources/strings.py | 2 + .../test_api/test_routes/test_requests.py | 42 +++ .../test_airlock_request_repository.py | 196 +++++++++- ui/app/package.json | 2 +- ui/app/src/components/root/LeftNav.tsx | 36 +- ui/app/src/components/root/RootLayout.tsx | 63 ++-- ui/app/src/components/shared/RequestsList.tsx | 339 ++++++++++++++++++ ui/app/src/models/apiEndpoints.ts | 1 + 13 files changed, 728 insertions(+), 48 deletions(-) create mode 100644 api_app/api/routes/requests.py create mode 100644 api_app/tests_ma/test_api/test_routes/test_requests.py create mode 100644 ui/app/src/components/shared/RequestsList.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 879a3f47e5..d451bec8fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ENHANCEMENTS: * Allow workspace App Service Plan SKU to be updated ([#4331](https://github.com/microsoft/AzureTRE/issues/4331)) +* Add core requests endpoint and UI to enable requests to be managed TRE wide. ([[#2510](https://github.com/microsoft/AzureTRE/issues/2510)]) * Remove public IP from TRE's firewall when forced tunneling is configured ([#4346](https://github.com/microsoft/AzureTRE/pull/4346)) * Upgrade AzureRM Terraform provider from `3.117.0` to `4.14.0`. ([[#4255](https://github.com/microsoft/AzureTRE/pull/4255/)]) * Subnet definitions are now inline in the `azurerm_virtual_network` resource, and NSG associations are set using `security_group` in each subnet block (no separate `azurerm_subnet_network_security_group_association` needed). ([[#4255](https://github.com/microsoft/AzureTRE/pull/4255/)]) diff --git a/api_app/_version.py b/api_app/_version.py index 8b8252f484..6a726d853b 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.20.4" +__version__ = "0.21.0" diff --git a/api_app/api/routes/api.py b/api_app/api/routes/api.py index 0dd1a7d2c5..6dabc88275 100644 --- a/api_app/api/routes/api.py +++ b/api_app/api/routes/api.py @@ -8,7 +8,7 @@ from api.helpers import get_repository from db.repositories.workspaces import WorkspaceRepository from api.routes import health, ping, workspaces, workspace_templates, workspace_service_templates, user_resource_templates, \ - shared_services, shared_service_templates, migrations, costs, airlock, operations, metadata + shared_services, shared_service_templates, migrations, costs, airlock, operations, metadata, requests from core import config from resources import strings @@ -49,6 +49,7 @@ core_router.include_router(migrations.migrations_core_router, tags=["migrations"]) core_router.include_router(costs.costs_core_router, tags=["costs"]) core_router.include_router(costs.costs_workspace_router, tags=["costs"]) +core_router.include_router(requests.router, tags=["requests"]) core_swagger_router = APIRouter() swagger_disabled_router = APIRouter() @@ -112,7 +113,7 @@ async def get_disabled_swagger(): def get_scope(workspace) -> str: # Cope with the fact that scope id can have api:// at the front. - return f"api://{workspace.properties['scope_id'].replace('api://','')}/user_impersonation" + return f"api://{workspace.properties['scope_id'].replace('api://', '')}/user_impersonation" @workspace_swagger_router.get("/workspaces/{workspace_id}/openapi.json", include_in_schema=False, name="openapi_definitions") diff --git a/api_app/api/routes/requests.py b/api_app/api/routes/requests.py new file mode 100644 index 0000000000..1fffa6f356 --- /dev/null +++ b/api_app/api/routes/requests.py @@ -0,0 +1,38 @@ +from fastapi import APIRouter, Depends, HTTPException, status as status_code +from typing import List, Optional + +from api.helpers import get_repository +from resources import strings +from db.repositories.airlock_requests import AirlockRequestRepository +from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType +from services.authentication import get_current_tre_user_or_tre_admin + +router = APIRouter(dependencies=[Depends(get_current_tre_user_or_tre_admin)]) + + +@router.get("/requests", response_model=List[AirlockRequest], name=strings.API_LIST_REQUESTS) +async def get_requests( + user=Depends(get_current_tre_user_or_tre_admin), + airlock_request_repo: AirlockRequestRepository = Depends(get_repository(AirlockRequestRepository)), + airlock_manager: bool = False, + creator_user_id: Optional[str] = None, type: Optional[AirlockRequestType] = None, status: Optional[AirlockRequestStatus] = None, + order_by: Optional[str] = None, order_ascending: bool = True +) -> List[AirlockRequest]: + try: + if not airlock_manager: + requests = await airlock_request_repo.get_airlock_requests( + creator_user_id=creator_user_id or user.id, + type=type, + status=status, + order_by=order_by, + order_ascending=order_ascending, + ) + else: + requests = await airlock_request_repo.get_airlock_requests_for_airlock_manager(user) + + return requests + + except ValueError as ve: + raise HTTPException(status_code=status_code.HTTP_400_BAD_REQUEST, detail=str(ve)) + except Exception as e: + raise HTTPException(status_code=status_code.HTTP_500_INTERNAL_SERVER_ERROR, detail=str(e)) diff --git a/api_app/db/repositories/airlock_requests.py b/api_app/db/repositories/airlock_requests.py index f4a1926348..8bb8275aa6 100644 --- a/api_app/db/repositories/airlock_requests.py +++ b/api_app/db/repositories/airlock_requests.py @@ -7,6 +7,8 @@ from azure.cosmos.exceptions import CosmosResourceNotFoundError, CosmosAccessConditionFailedError from fastapi import HTTPException, status from pydantic import parse_obj_as +from db.repositories.workspaces import WorkspaceRepository +from services.authentication import get_access_service from models.domain.authentication import User from db.errors import EntityDoesNotExist from models.domain.airlock_request import AirlockFile, AirlockRequest, AirlockRequestStatus, \ @@ -107,27 +109,33 @@ def create_airlock_request_item(self, airlock_request_input: AirlockRequestInCre return airlock_request - async def get_airlock_requests(self, workspace_id: str, creator_user_id: Optional[str] = None, type: Optional[AirlockRequestType] = None, status: Optional[AirlockRequestStatus] = None, order_by: Optional[str] = None, order_ascending=True) -> List[AirlockRequest]: - query = self.airlock_requests_query() + f' WHERE c.workspaceId = "{workspace_id}"' + async def get_airlock_requests(self, workspace_id: Optional[str] = None, creator_user_id: Optional[str] = None, type: Optional[AirlockRequestType] = None, status: Optional[AirlockRequestStatus] = None, order_by: Optional[str] = None, order_ascending=True) -> List[AirlockRequest]: + query = self.airlock_requests_query() # optional filters + conditions = [] + parameters = [] + if workspace_id: + conditions.append('c.workspaceId=@workspace_id') + parameters.append({"name": "@workspace_id", "value": workspace_id}) if creator_user_id: - query += ' AND c.createdBy.id=@user_id' + conditions.append('c.createdBy.id=@user_id') + parameters.append({"name": "@user_id", "value": creator_user_id}) if status: - query += ' AND c.status=@status' + conditions.append('c.status=@status') + parameters.append({"name": "@status", "value": status}) if type: - query += ' AND c.type=@type' + conditions.append('c.type=@type') + parameters.append({"name": "@type", "value": type}) + + if conditions: + query += ' WHERE ' + ' AND '.join(conditions) # optional sorting if order_by: query += ' ORDER BY c.' + order_by query += ' ASC' if order_ascending else ' DESC' - parameters = [ - {"name": "@user_id", "value": creator_user_id}, - {"name": "@status", "value": status}, - {"name": "@type", "value": type}, - ] airlock_requests = await self.query(query=query, parameters=parameters) return parse_obj_as(List[AirlockRequest], airlock_requests) @@ -138,6 +146,27 @@ async def get_airlock_request_by_id(self, airlock_request_id: UUID4) -> AirlockR raise EntityDoesNotExist return parse_obj_as(AirlockRequest, airlock_requests) + async def get_airlock_requests_for_airlock_manager(self, user: User, type: Optional[AirlockRequestType] = None, status: Optional[AirlockRequestStatus] = None, order_by: Optional[str] = None, order_ascending=True) -> List[AirlockRequest]: + workspace_repo = await WorkspaceRepository.create() + access_service = get_access_service() + + workspaces = await workspace_repo.get_active_workspaces() + user_role_assignments = access_service.get_identity_role_assignments(user.id) + + valid_roles = {ra.role_id for ra in user_role_assignments} + + workspace_ids = [ + workspace.id + for workspace in workspaces + if workspace.properties["app_role_id_workspace_airlock_manager"] in valid_roles + ] + requests = [] + + for workspace_id in workspace_ids: + requests += await self.get_airlock_requests(workspace_id=workspace_id, type=type, status=status, order_by=order_by, order_ascending=order_ascending) + + return requests + async def update_airlock_request( self, original_request: AirlockRequest, diff --git a/api_app/resources/strings.py b/api_app/resources/strings.py index a9184cf289..896b385f85 100644 --- a/api_app/resources/strings.py +++ b/api_app/resources/strings.py @@ -34,6 +34,8 @@ API_UPDATE_USER_RESOURCE = "Update an existing user resource" API_INVOKE_ACTION_ON_USER_RESOURCE = "Invoke action on a user resource" +API_LIST_REQUESTS = "Get requests" + API_CREATE_AIRLOCK_REQUEST = "Create an airlock request" API_GET_AIRLOCK_REQUEST = "Get an airlock request" API_LIST_AIRLOCK_REQUESTS = "Get all airlock requests for a workspace" diff --git a/api_app/tests_ma/test_api/test_routes/test_requests.py b/api_app/tests_ma/test_api/test_routes/test_requests.py new file mode 100644 index 0000000000..4a10544ca1 --- /dev/null +++ b/api_app/tests_ma/test_api/test_routes/test_requests.py @@ -0,0 +1,42 @@ +import pytest +from fastapi import status +from mock import patch + +from resources import strings +from services.authentication import get_current_tre_user_or_tre_admin + + +pytestmark = pytest.mark.asyncio + + +class TestRequestsThatDontRequireAdminRigths: + @pytest.fixture(autouse=True, scope='class') + def log_in_with_non_admin_user(self, app, non_admin_user): + with patch('services.aad_authentication.AzureADAuthorization._get_user_from_token', return_value=non_admin_user()): + app.dependency_overrides[get_current_tre_user_or_tre_admin] = non_admin_user + yield + app.dependency_overrides = {} + + # [GET] /requests/ - get_requests + @patch("api.routes.airlock.AirlockRequestRepository.get_airlock_requests", return_value=[]) + async def test_get_all_requests_returns_200(self, _, app, client): + response = await client.get(app.url_path_for(strings.API_LIST_REQUESTS)) + assert response.status_code == status.HTTP_200_OK + + @patch("api.routes.airlock.AirlockRequestRepository.get_airlock_requests_for_airlock_manager") + async def test_get_airlock_manager_requests_returns_200(self, mock_get_airlock_requests_for_airlock_manager, app, client): + mock_get_airlock_requests_for_airlock_manager.return_value = [] + response = await client.get(app.url_path_for(strings.API_LIST_REQUESTS), params={"airlock_manager": True}) + + assert response.status_code == status.HTTP_200_OK + mock_get_airlock_requests_for_airlock_manager.assert_called_once() + + @patch("api.routes.airlock.AirlockRequestRepository.get_airlock_requests", side_effect=Exception("Internal Server Error")) + async def test_get_all_requests_returns_500(self, _, app, client): + response = await client.get(app.url_path_for(strings.API_LIST_REQUESTS)) + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + + @patch("api.routes.airlock.AirlockRequestRepository.get_airlock_requests_for_airlock_manager", side_effect=Exception("Internal Server Error")) + async def test_get_airlock_manager_requests_returns_500(self, _, app, client): + response = await client.get(app.url_path_for(strings.API_LIST_REQUESTS), params={"airlock_manager": True}) + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR diff --git a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py index 4c773db327..92ae8b7ae7 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_airlock_request_repository.py @@ -3,6 +3,8 @@ from mock import patch import pytest import pytest_asyncio +from models.domain.authentication import RoleAssignment, User +from models.domain.workspace import Workspace from tests_ma.test_api.conftest import create_test_user from models.schemas.airlock_request import AirlockRequestInCreate from models.domain.airlock_request import AirlockRequest, AirlockRequestStatus, AirlockRequestType @@ -63,6 +65,18 @@ def verify_dictionary_contains_all_enum_values(): raise Exception(f"Status '{status}' was not added to the ALLOWED_STATUS_CHANGES dictionary") +def sample_workspace(workspace_id=WORKSPACE_ID, workspace_properties: dict = {}) -> Workspace: + workspace = Workspace( + id=workspace_id, + templateName="tre-workspace-base", + templateVersion="0.1.0", + etag="", + properties=workspace_properties, + resourcePath=f'/workspaces/{workspace_id}' + ) + return workspace + + def airlock_request_mock(status=AirlockRequestStatus.Draft): airlock_request = AirlockRequest( id=AIRLOCK_REQUEST_ID, @@ -143,12 +157,186 @@ async def test_update_airlock_request_should_retry_update_when_etag_is_not_up_to async def test_get_airlock_requests_queries_db(airlock_request_repo): airlock_request_repo.container.query_items = MagicMock() - expected_query = airlock_request_repo.airlock_requests_query() + f' WHERE c.workspaceId = "{WORKSPACE_ID}"' + expected_query = airlock_request_repo.airlock_requests_query() + ' WHERE c.workspaceId=@workspace_id' expected_parameters = [ - {"name": "@user_id", "value": None}, - {"name": "@status", "value": None}, - {"name": "@type", "value": None}, + {"name": "@workspace_id", "value": WORKSPACE_ID}, ] await airlock_request_repo.get_airlock_requests(WORKSPACE_ID) airlock_request_repo.container.query_items.assert_called_once_with(query=expected_query, parameters=expected_parameters) + + +async def test_get_airlock_requests_with_user_id(airlock_request_repo): + airlock_request_repo.container.query_items = MagicMock() + user_id = "test_user_id" + expected_query = airlock_request_repo.airlock_requests_query() + ' WHERE c.createdBy.id=@user_id' + expected_parameters = [ + {"name": "@user_id", "value": user_id}, + ] + + await airlock_request_repo.get_airlock_requests(creator_user_id=user_id) + airlock_request_repo.container.query_items.assert_called_once_with(query=expected_query, parameters=expected_parameters) + + +async def test_get_airlock_requests_with_status(airlock_request_repo): + airlock_request_repo.container.query_items = MagicMock() + status = AirlockRequestStatus.Submitted + expected_query = airlock_request_repo.airlock_requests_query() + ' WHERE c.status=@status' + expected_parameters = [ + {"name": "@status", "value": status} + ] + + await airlock_request_repo.get_airlock_requests(status=status) + airlock_request_repo.container.query_items.assert_called_once_with(query=expected_query, parameters=expected_parameters) + + +async def test_get_airlock_requests_with_type(airlock_request_repo): + airlock_request_repo.container.query_items = MagicMock() + request_type = AirlockRequestType.Import + expected_query = airlock_request_repo.airlock_requests_query() + ' WHERE c.type=@type' + expected_parameters = [ + {"name": "@type", "value": request_type}, + ] + + await airlock_request_repo.get_airlock_requests(type=request_type) + airlock_request_repo.container.query_items.assert_called_once_with(query=expected_query, parameters=expected_parameters) + + +async def test_get_airlock_requests_with_multiple_filters(airlock_request_repo): + airlock_request_repo.container.query_items = MagicMock() + user_id = "test_user_id" + status = AirlockRequestStatus.Submitted + request_type = AirlockRequestType.Import + expected_query = airlock_request_repo.airlock_requests_query() + ' WHERE c.createdBy.id=@user_id AND c.status=@status AND c.type=@type' + expected_parameters = [ + {"name": "@user_id", "value": user_id}, + {"name": "@status", "value": status}, + {"name": "@type", "value": request_type}, + ] + + await airlock_request_repo.get_airlock_requests(creator_user_id=user_id, status=status, type=request_type) + airlock_request_repo.container.query_items.assert_called_once_with(query=expected_query, parameters=expected_parameters) + + +@pytest.mark.asyncio +@patch.object(AirlockRequestRepository, 'get_airlock_requests', new_callable=AsyncMock) +@patch('db.repositories.airlock_requests.get_access_service', autospec=True) +@patch('db.repositories.airlock_requests.WorkspaceRepository', autospec=True) +async def test_get_airlock_requests_for_airlock_manager_no_roles( + mock_workspace_repo, + mock_access_service, + mock_get_requests, + airlock_request_repo +): + # Mock no user roles + mock_access_service.return_value.get_identity_role_assignments.return_value = [] + + # Mock active workspaces + mock_workspace_instance = MagicMock() + mock_workspace_instance.get_active_workspaces = AsyncMock(return_value=[]) + mock_workspace_repo.create = AsyncMock(return_value=mock_workspace_instance) + + # Call function + user = User(id="user1", name="TestUser") + result = await airlock_request_repo.get_airlock_requests_for_airlock_manager(user) + + # validate + assert result == [] + mock_get_requests.assert_not_called() + + +@pytest.mark.asyncio +@patch.object(AirlockRequestRepository, 'get_airlock_requests', new_callable=AsyncMock) +@patch('db.repositories.airlock_requests.get_access_service', autospec=True) +@patch('db.repositories.airlock_requests.WorkspaceRepository', autospec=True) +async def test_get_airlock_requests_for_airlock_manager_single_workspace( + mock_workspace_repo, + mock_access_service, + mock_get_requests, + airlock_request_repo +): + # Setup workspace and manager role + workspace = sample_workspace(workspace_properties={"app_role_id_workspace_airlock_manager": "manager-role-1"}) + mock_workspace_instance = MagicMock() + mock_workspace_instance.get_active_workspaces = AsyncMock(return_value=[workspace]) + mock_workspace_repo.create = AsyncMock(return_value=mock_workspace_instance) + + # Setup user roles + role_assignment = RoleAssignment(resource_id="resource_id", role_id="manager-role-1") + mock_access_service.return_value.get_identity_role_assignments.return_value = [role_assignment] + + # Setup corresponding requests from that workspace + request_mock = AirlockRequest(id="request-1", workspaceId=WORKSPACE_ID, type=AirlockRequestType.Import, reviews=[]) + mock_get_requests.return_value = [request_mock] + + user = User(id="user1", name="TestUser") + result = await airlock_request_repo.get_airlock_requests_for_airlock_manager(user) + + assert len(result) == 1 + assert result[0].id == "request-1" + mock_get_requests.assert_called_once_with(workspace_id=WORKSPACE_ID, type=None, status=None, order_by=None, order_ascending=True) + + +@pytest.mark.asyncio +@patch.object(AirlockRequestRepository, 'get_airlock_requests', new_callable=AsyncMock) +@patch('db.repositories.airlock_requests.get_access_service', autospec=True) +@patch('db.repositories.airlock_requests.WorkspaceRepository', autospec=True) +async def test_get_airlock_requests_for_airlock_manager_multiple_workspaces( + mock_workspace_repo, + mock_access_service, + mock_get_requests, + airlock_request_repo +): + # Setup multiple workspaces + workspace1 = sample_workspace(workspace_properties={"app_role_id_workspace_airlock_manager": "manager-role-1"}) + workspace2 = sample_workspace(workspace_properties={"app_role_id_workspace_airlock_manager": "manager-role-2"}) + mock_workspace_instance = MagicMock() + mock_workspace_instance.get_active_workspaces = AsyncMock(return_value=[workspace1, workspace2]) + mock_workspace_repo.create = AsyncMock(return_value=mock_workspace_instance) + + # Setup user roles + role_assignment_1 = RoleAssignment(resource_id="resource_id", role_id="manager-role-1") + role_assignment_2 = RoleAssignment(resource_id="resource_id", role_id="manager-role-2") + mock_access_service.return_value.get_identity_role_assignments.return_value = [role_assignment_1, role_assignment_2] + + # Setup requests for each workspace + first_ws_requests = [AirlockRequest(id="request-1", workspaceId="workspace-1", type=AirlockRequestType.Import, reviews=[])] + second_ws_requests = [AirlockRequest(id="request-2", workspaceId="workspace-2", type=AirlockRequestType.Import, reviews=[])] + mock_get_requests.side_effect = [first_ws_requests, second_ws_requests] + + user = User(id="user1", name="TestUser") + result = await airlock_request_repo.get_airlock_requests_for_airlock_manager(user) + + # combined requests from both + assert len(result) == 2 + assert result[0].id == "request-1" + assert result[1].id == "request-2" + assert mock_get_requests.call_count == 2 + + +@pytest.mark.asyncio +@patch.object(AirlockRequestRepository, 'get_airlock_requests', new_callable=AsyncMock) +@patch('db.repositories.airlock_requests.get_access_service', autospec=True) +@patch('db.repositories.airlock_requests.WorkspaceRepository', autospec=True) +async def test_get_airlock_requests_for_airlock_manager_active_workspaces_but_no_manager_role( + mock_workspace_repo, + mock_access_service, + mock_get_requests, + airlock_request_repo +): + # Setup multiple workspaces, but user doesn't have manager roles + workspace1 = sample_workspace(workspace_properties={"app_role_id_workspace_airlock_manager": "manager-role-1"}) + workspace2 = sample_workspace(workspace_properties={"app_role_id_workspace_airlock_manager": "manager-role-2"}) + mock_workspace_instance = MagicMock() + mock_workspace_instance.get_active_workspaces = AsyncMock(return_value=[workspace1, workspace2]) + mock_workspace_repo.create = AsyncMock(return_value=mock_workspace_instance) + + # No matching roles for these workspaces + mock_access_service.return_value.get_identity_role_assignments.return_value = [ + RoleAssignment(resource_id="resource_id", role_id="some-other-role") + ] + + user = User(id="user1", name="TestUser") + result = await airlock_request_repo.get_airlock_requests_for_airlock_manager(user) + assert result == [] + mock_get_requests.assert_not_called() diff --git a/ui/app/package.json b/ui/app/package.json index b1a9a66992..b3a14712be 100644 --- a/ui/app/package.json +++ b/ui/app/package.json @@ -1,6 +1,6 @@ { "name": "tre-ui", - "version": "0.6.3", + "version": "0.7.0", "private": true, "dependencies": { "@azure/msal-browser": "^2.35.0", diff --git a/ui/app/src/components/root/LeftNav.tsx b/ui/app/src/components/root/LeftNav.tsx index 568d326151..706cc8c8b1 100644 --- a/ui/app/src/components/root/LeftNav.tsx +++ b/ui/app/src/components/root/LeftNav.tsx @@ -1,13 +1,16 @@ import React, { useContext } from 'react'; import { Nav, INavLinkGroup } from '@fluentui/react/lib/Nav'; -import { useNavigate } from 'react-router-dom'; +import { useLocation, useNavigate } from 'react-router-dom'; import { AppRolesContext } from '../../contexts/AppRolesContext'; import { RoleName } from '../../models/roleNames'; export const LeftNav: React.FunctionComponent = () => { const navigate = useNavigate(); + const location = useLocation(); const appRolesCtx = useContext(AppRolesContext); + const isRequestsRoute = location.pathname.startsWith('/requests'); // ← True if URL starts with /requests + const navLinkGroups: INavLinkGroup[] = [ { links: [ @@ -32,9 +35,38 @@ export const LeftNav: React.FunctionComponent = () => { }); } + const requestsLinkArray: { name: string; url: string; key: string; icon: string }[] = []; + + requestsLinkArray.push( + { + name: 'Airlock', + url: '/requests/airlock', + key: 'airlock', + icon: 'Lock', + + }); + + // add Requests link + navLinkGroups[0].links.push( + { + name: 'Requests', + url: '/requests', + key: 'requests', + icon: '', + links: requestsLinkArray, + isExpanded: isRequestsRoute + }); + return (