Consolidate deploy.sh scripts

Fixes #4379

CHANGELOG.md

@@ -1,7 +1,17 @@
 <!-- markdownlint-disable MD041 -->
-## 0.21.0 (Unreleased)
+## 0.22.0 (Unreleased)
+**BREAKING CHANGES & MIGRATIONS**:
+
+ENHANCEMENTS:
+* Deny public access to TRE management storage account, and add private endpoint for TRE core [#4353](https://github.com/microsoft/AzureTRE/issues/4353)
+
+BUG FIXES:
+
+
+## 0.21.0
 
 **BREAKING CHANGES & MIGRATIONS**:
+* Workspace bundle uses infrastructure encryption on shared storage which will recreate storage share. Major verison increase will prevent upgrade, do not force the upgrade unless you are fully aware of the consequences.
 
 ENHANCEMENTS:
 * Core key vault firewall should not be set to "Allow public access from all networks" ([#4250](https://github.com/microsoft/AzureTRE/issues/4250))
@@ -14,16 +24,49 @@ ENHANCEMENTS:
 * Add bundle target to Makefile for handling different bundle types in single command ([#4372](https://github.com/microsoft/AzureTRE/issues/4372))
 * Migrate UI to Vite build engine and update dependencies ([#4368](https://github.com/microsoft/AzureTRE/pull/4368))
 * Add Windows image field to the Admin VM template ([#4274](https://github.com/microsoft/AzureTRE/pull/4274))
-* Update TLS to the latest version for web apps / function apps (([#4351](https://github.com/microsoft/AzureTRE/issues/4351))
+* Update TLS to the latest version for web apps / function apps ([#4351](https://github.com/microsoft/AzureTRE/issues/4351))
+* Set `stairlockp` Airlock Processor storage account firewall to "Enabled from selected virtual networks and IP addresses" ([#4386](https://github.com/microsoft/AzureTRE/issues/4386))
 
 BUG FIXES:
 * Fix upgrade when porter install has failed ([#4338](https://github.com/microsoft/AzureTRE/pull/4338))
-* Certs shared service: Secret nexus-ssl-password is currently in a deleted but recoverable state ([#4294](https://github.com/microsoft/AzureTRE/issues/4294)])
+* Certs shared service: Secret nexus-ssl-password is currently in a deleted but recoverable state ([#4294](https://github.com/microsoft/AzureTRE/issues/4294))
 * Fix Cosmos DB local debugging configuration ([#4340](https://github.com/microsoft/AzureTRE/pull/4340))
 * Add firewall rules to upgrade steps for Guacamole service ([#4343](https://github.com/microsoft/AzureTRE/pull/4343))
 
 COMPONENTS:
 
+| name | version |
+| ----- | ----- |
+| devops | 0.5.5 |
+| core | 0.12.3 |
+| ui | 0.7.0 |
+| tre-workspace-airlock-import-review | 0.14.3 |
+| tre-workspace-base | 2.0.0 |
+| tre-workspace-unrestricted | 0.13.3 |
+| tre-shared-service-airlock-notifier | 1.0.8 |
+| tre-shared-service-certs | 0.7.4 |
+| tre-shared-service-firewall | 1.3.2 |
+| tre-shared-service-gitea | 1.1.5 |
+| tre-shared-service-cyclecloud | 0.7.2 |
+| tre-shared-service-databricks-private-auth | 0.1.11 |
+| tre-shared-service-admin-vm | 0.5.3 |
+| tre-shared-service-sonatype-nexus | 3.3.2 |
+| tre-workspace-service-mysql | 1.0.9 |
+| tre-workspace-service-ohdsi | 0.3.3 |
+| tre-user-resource-aml-compute-instance | 0.5.11 |
+| tre-service-azureml | 0.9.2 |
+| tre-service-guacamole-linuxvm | 1.2.8 |
+| tre-service-guacamole-windowsvm | 1.2.10 |
+| tre-service-guacamole-import-reviewvm | 0.3.2 |
+| tre-service-guacamole-export-reviewvm | 0.2.2 |
+| tre-service-guacamole | 0.12.9 |
+| tre-workspace-service-health | 0.2.11 |
+| tre-workspace-service-gitea | 1.2.3 |
+| tre-service-databricks | 1.0.10 |
+| tre-workspace-service-openai | 1.0.6 |
+| tre-workspace-service-azuresql | 1.0.15 |
+
+
 ## 0.20.0 (Feburary 9, 2025)
 
 **BREAKING CHANGES & MIGRATIONS**:

core/terraform/airlock/airlock_processor.tf

@@ -22,8 +22,14 @@ resource "azurerm_storage_account" "sa_airlock_processor_func_app" {
   cross_tenant_replication_enabled = false
   local_user_enabled               = false
   shared_access_key_enabled        = false
+  public_network_access_enabled    = true
   tags                             = var.tre_core_tags
 
+  network_rules {
+    default_action = var.enable_local_debugging ? "Allow" : "Deny"
+    bypass         = ["AzureServices"]
+  }
+
   dynamic "identity" {
     for_each = var.enable_cmk_encryption ? [1] : []
     content {

core/terraform/data.tf

@@ -37,4 +37,4 @@ data "azurerm_monitor_diagnostic_categories" "sb" {
   depends_on = [
     azurerm_servicebus_namespace.sb
   ]
-}
+}

core/terraform/main.tf

@@ -165,6 +165,7 @@ module "resource_processor_vmss_porter" {
   acr_id                                           = data.azurerm_container_registry.mgmt_acr.id
   app_insights_connection_string                   = module.azure_monitor.app_insights_connection_string
   resource_processor_subnet_id                     = module.network.resource_processor_subnet_id
+  blob_core_dns_zone_id                            = module.network.blob_core_dns_zone_id
   docker_registry_server                           = local.docker_registry_server
   resource_processor_vmss_porter_image_repository  = var.resource_processor_vmss_porter_image_repository
   service_bus_namespace_id                         = azurerm_servicebus_namespace.sb.id

core/terraform/migrate.sh

@@ -5,6 +5,9 @@ set -o pipefail
 set -o nounset
 # set -o xtrace
 
+# shellcheck disable=SC1091
+source ../../devops/scripts/mgmtstorage_enable_public_access.sh
+
 get_resource_id() {
   local json_data="$1"
   local resource_addr="$2"

core/terraform/outputs.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e
 
+# shellcheck disable=SC1091
+source ../../devops/scripts/mgmtstorage_enable_public_access.sh
+
 if [ ! -f ../tre_output.json ] || [ ! -s ../tre_output.json ]; then
   # Connect to the remote backend of Terraform
   export TF_LOG=""

core/terraform/resource_processor/vmss_porter/main.tf

@@ -224,6 +224,27 @@ resource "azurerm_role_assignment" "vmss_kv_encryption_key_user" {
   principal_id         = azurerm_user_assigned_identity.vmss_msi.principal_id
 }
 
+resource "azurerm_private_endpoint" "mgmtblobpe" {
+  name                = "pe-mgmt-blob-${var.tre_id}"
+  location            = var.location
+  resource_group_name = var.mgmt_resource_group_name
+  subnet_id           = var.resource_processor_subnet_id
+  tags                = local.tre_core_tags
+  lifecycle { ignore_changes = [tags] }
+
+  private_dns_zone_group {
+    name                 = "private-dns-zone-group-blobcore"
+    private_dns_zone_ids = [var.blob_core_dns_zone_id]
+  }
+
+  private_service_connection {
+    name                           = "psc-mgmt-${var.tre_id}"
+    private_connection_resource_id = data.azurerm_storage_account.mgmt_storage.id
+    is_manual_connection           = false
+    subresource_names              = ["Blob"]
+  }
+}
+
 module "terraform_azurerm_environment_configuration" {
   source          = "git::https://github.com/microsoft/terraform-azurerm-environment-configuration.git?ref=0.2.0"
   arm_environment = var.arm_environment

core/terraform/resource_processor/vmss_porter/variables.tf

@@ -13,6 +13,9 @@ variable "resource_group_name" {
 variable "resource_processor_subnet_id" {
   type = string
 }
+variable "blob_core_dns_zone_id" {
+  type = string
+}
 variable "resource_processor_vmss_porter_image_repository" {
   type = string
 }

core/terraform/update_tags.sh

@@ -5,6 +5,9 @@ set -o pipefail
 set -o nounset
 # set -o xtrace
 
+# shellcheck disable=SC1091
+source ../../devops/scripts/mgmtstorage_enable_public_access.sh
+
 script_dir=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 core_rg_rid=$(terraform show -json | jq -r '.values.root_module.resources[] | select(.address=="azurerm_resource_group.core") | .values.id')

core/version.txt

@@ -1 +1 @@
-__version__ = "0.12.4"
+__version__ = "0.12.6"

devops/scripts/mgmtstorage_enable_public_access.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+
+#
+# Add an exception to the TRE management storage account by making it public for deployment, and remove it on script exit.
+#
+# Note: Ensure you "source" this script, or else the EXIT trap won't fire at the right time.
+#
+
+function mgmtstorage_enable_public_access() {
+  local RESOURCE_GROUP
+  RESOURCE_GROUP=$(get_resource_group_name)
+
+  local SA_NAME
+  SA_NAME=$(get_storage_account_name)
+
+  # Check that the storage account exists before making changes
+  if ! does_storage_account_exist "$SA_NAME"; then
+    echo -e "Error: Storage account $SA_NAME does not exist.\n" >&2
+    exit 1
+  fi
+
+  # Pre-check: if public access is already enabled, no need to update.
+  if is_public_access_enabled "$RESOURCE_GROUP" "$SA_NAME"; then
+    echo -e " Storage account $SA_NAME is already publicly accessible\n"
+    return
+  fi
+
+  echo -e "\nEnabling public access on storage account $SA_NAME"
+
+  # Enable public network access with explicit default action allow
+  az storage account update --resource-group "$RESOURCE_GROUP" --name "$SA_NAME" --public-network-access Enabled --default-action Allow --output none
+
+  for ATTEMPT in {1..10}; do
+    if is_public_access_enabled "$RESOURCE_GROUP" "$SA_NAME"; then
+      echo -e " Storage account $SA_NAME is now publicly accessible\n"
+      return
+    fi
+
+    echo " Unable to confirm public access on storage account $SA_NAME after $ATTEMPT/10. Waiting for update to take effect..."
+    sleep 10
+  done
+
+  echo -e "Error: Could not enable public access for $SA_NAME after 10 attempts.\n"
+  exit 1
+}
+
+function mgmtstorage_disable_public_access() {
+  local RESOURCE_GROUP
+  RESOURCE_GROUP=$(get_resource_group_name)
+
+  local SA_NAME
+  SA_NAME=$(get_storage_account_name)
+
+  # Check that the storage account exists before making changes
+  if ! does_storage_account_exist "$SA_NAME"; then
+    echo -e "Error: Storage account $SA_NAME does not exist.\n" >&2
+    exit 1
+  fi
+
+  # Pre-check: if public access is already disabled, no need to update.
+  if ! is_public_access_enabled "$RESOURCE_GROUP" "$SA_NAME"; then
+    echo -e " Storage account $SA_NAME is already not publicly accessible\n"
+    return
+  fi
+
+  echo -e "\nDisabling public access on storage account $SA_NAME"
+
+  # Disable public network access with explicit default action deny
+  az storage account update --resource-group "$RESOURCE_GROUP" --name "$SA_NAME" --public-network-access Disabled --default-action Deny --output none
+
+  for ATTEMPT in {1..10}; do
+    if ! is_public_access_enabled "$RESOURCE_GROUP" "$SA_NAME"; then
+      echo -e " Public access has been disabled successfully\n"
+      return
+    fi
+
+    echo " Unable to confirm public access is disabled on storage account $SA_NAME after $ATTEMPT/10. Waiting for update to take effect..."
+    sleep 10
+  done
+
+  echo -e "Error: Could not disable public access for $SA_NAME after 10 attempts.\n"
+  exit 1
+}
+
+function get_resource_group_name() {
+  if [[ -z "${TF_VAR_mgmt_resource_group_name:-}" ]]; then
+    echo -e "Error: TF_VAR_mgmt_resource_group_name is not set\nExiting...\n" >&2
+    exit 1
+  fi
+  echo "$TF_VAR_mgmt_resource_group_name"
+}
+
+function get_storage_account_name() {
+  if [[ -z "${TF_VAR_mgmt_storage_account_name:-}" ]]; then
+    echo -e "Error: TF_VAR_mgmt_storage_account_name is not set\nExiting...\n" >&2
+    exit 1
+  fi
+  echo "$TF_VAR_mgmt_storage_account_name"
+}
+
+function does_storage_account_exist() {
+  [[ -n "$(az storage account show --name "$1" --query "id" --output tsv)" ]]
+}
+
+function is_public_access_enabled() {
+  local RESOURCE_GROUP="$1"
+  local SA_NAME="$2"
+
+  # Try listing containers
+  local containers
+  if ! containers=$(az storage container list --account-name "$SA_NAME" --auth-mode login --query "[].name" --output tsv); then
+    return 1
+  fi
+
+  # For each container found, check blob listing
+  for container in $containers; do
+    if ! az storage blob list --container-name "$container" --account-name "$SA_NAME" --auth-mode login --output none; then
+      return 1
+    fi
+  done
+
+  # If container list succeeded (even if empty) and blob list (if any) succeeded, public access is enabled
+  return 0
+}
+
+# Setup the trap to disable public access on exit
+trap mgmtstorage_disable_public_access EXIT
+
+# Enable public access for deployment
+mgmtstorage_enable_public_access "$@"

devops/scripts/terraform_upgrade_provider.sh

@@ -53,6 +53,9 @@ else
   KEY="${TRE_ID?}_${PARENT_DIR}"
 fi
 
+# shellcheck disable=SC1091
+source "$(dirname "$0")/mgmtstorage_enable_public_access.sh"
+
 # Run terraform init with upgrade and reconfigure options
 terraform -chdir="$DIR/terraform" init -upgrade -reconfigure -input=false -backend=true \
     -backend-config="resource_group_name=${TF_VAR_mgmt_resource_group_name}" \

devops/scripts/terraform_wrapper.sh

@@ -99,6 +99,10 @@ if [[ -z ${tf_logfile+x} ]]; then
     echo -e "No logfile provided, using ${tf_logfile}\n"
 fi
 
+
+# shellcheck disable=SC1091
+source "$(dirname "$0")/mgmtstorage_enable_public_access.sh"
+
 # Change directory to $DIR
 pushd "$DIR" > /dev/null
 

devops/terraform/bootstrap.sh

@@ -28,6 +28,9 @@ else
   az storage account show --resource-group "$TF_VAR_mgmt_resource_group_name" --name "$TF_VAR_mgmt_storage_account_name" --output table
 fi
 
+# shellcheck disable=SC1091
+source ../scripts/mgmtstorage_enable_public_access.sh
+
 # Grant user blob data contributor permissions
 echo -e "\n\e[34m»»» 🔑 \e[96mGranting Storage Blob Data Contributor role to the current user\e[0m..."
 if [ -n "${ARM_CLIENT_ID:-}" ]; then

devops/terraform/deploy.sh

@@ -5,6 +5,9 @@ set -o pipefail
 set -o nounset
 # set -o xtrace
 
+# shellcheck disable=SC1091
+source ../scripts/mgmtstorage_enable_public_access.sh
+
 PLAN_FILE="devops.tfplan"
 
 terraform init -input=false -backend=true -reconfigure

devops/terraform/destroy.sh

@@ -1 +1,11 @@
-terraform destroy -auto-approve
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+# set -o xtrace
+
+# shellcheck disable=SC1091
+source ../scripts/mgmtstorage_enable_public_access.sh
+
+terraform destroy -auto-approve

devops/version.txt

@@ -1 +1 @@
-__version__ = "0.5.5"
+__version__ = "0.5.6"

e2e_tests/test_performance.py

@@ -106,7 +106,8 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(verify) ->
         "properties": {
             "display_name": "Perf test VM",
             "description": "",
-            "os_image": "Ubuntu 22.04 LTS"
+            "os_image": "Ubuntu 22.04 LTS",
+            "admin_username": "researcher"
         }
     }
 

e2e_tests/test_workspace_services.py

@@ -31,7 +31,8 @@ async def test_create_guacamole_service_into_base_workspace(setup_test_workspace
         "properties": {
             "display_name": "My VM",
             "description": "Will be using this VM for my research",
-            "os_image": "Windows 10"
+            "os_image": "Windows 10",
+            "admin_username": "researcher"
         }
     }