From 85f7b74b3498d3022da1abf0c1318c01a30dc10b Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Thu, 9 Jan 2025 14:53:19 +0000 Subject: [PATCH 1/4] No SAS. --- .../workspace_services/azureml/porter.yaml | 2 +- .../azureml/terraform/.terraform.lock.hcl | 31 ++++---- .../azureml/terraform/compute.tf | 23 ++---- .../azureml/terraform/main.tf | 78 +++++++++++++------ .../azureml/terraform/network.tf | 4 +- .../azureml/terraform/outputs.tf | 4 +- .../azureml/terraform/providers.tf | 2 +- .../azureml/terraform/roles.tf | 35 ++++++++- 8 files changed, 117 insertions(+), 62 deletions(-) diff --git a/templates/workspace_services/azureml/porter.yaml b/templates/workspace_services/azureml/porter.yaml index 24f5488f33..6a75726b98 100644 --- a/templates/workspace_services/azureml/porter.yaml +++ b/templates/workspace_services/azureml/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-azureml -version: 0.9.2 +version: 0.9.111 description: "An Azure TRE service for Azure Machine Learning" registry: azuretre dockerfile: Dockerfile.tmpl diff --git a/templates/workspace_services/azureml/terraform/.terraform.lock.hcl b/templates/workspace_services/azureml/terraform/.terraform.lock.hcl index d18c22de62..8724075450 100644 --- a/templates/workspace_services/azureml/terraform/.terraform.lock.hcl +++ b/templates/workspace_services/azureml/terraform/.terraform.lock.hcl @@ -2,23 +2,22 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/azure/azapi" { - version = "1.15.0" - constraints = "1.15.0" + version = "2.2.0" + constraints = "2.2.0" hashes = [ - "h1:Y7ruMuPh8UJRTRl4rm+cdpGtmURx2taqiuqfYaH3o48=", - "h1:gIOgxVmFSxHrR+XOzgUEA+ybOmp8kxZlZH3eYeB/eFI=", - "zh:0627a8bc77254debc25dc0c7b62e055138217c97b03221e593c3c56dc7550671", - "zh:2fe045f07070ef75d0bec4b0595a74c14394daa838ddb964e2fd23cc98c40c34", - "zh:343009f39c957883b2c06145a5954e524c70f93585f943f1ea3d28ef6995d0d0", - "zh:53fe9ab54485aaebc9b91e27a10bce2729a1c95b1399079e631dc6bb9e3f27dc", - "zh:63c407e7dc04d178d4798c17ad489d9cc92f7d1941d7f4a3f560b95908b6107b", - "zh:7d6fc2b432b264f036bb80ab2b2ba67f80a5d98da8a8c322aa097833dad598c9", - "zh:7ec49c0a8799d469eb6e2a1f856693f9862f1b73f5ed70adc1b346e5a4c6458d", - "zh:889704f10319d301d677539d788fc82a7c73608ab78cb93e1280ac2be39e6e00", - "zh:90b4b07405b7cde9ebae3b034cb5bb5dd18484d1b95bd250f905451f1e86ac3f", - "zh:92aa9c241a8cb2a6d81ad47bc007c119f8b818464a960ebaf39008766c361e6b", - "zh:f28fbd0a2c59e239b53067bc1adc691be444876bcb2d4f78d310f549724da6e0", - "zh:ffb15e0ddfa505d0e9b75341570199076ae574887124f398162b1ead9376b25f", + "h1:yckm1jqUMUGSeS57a3uR0gG/V7scwvjpkRVXnQIUAo4=", + "zh:062be5d8272cac297a88c2057449f449ea6906c4121ba3dfdeb5cecb3ff91178", + "zh:1fd9abec3ffcbf8d0244408334e9bfc8f49ada50978cd73ee0ed5f8560987267", + "zh:48e84b0302af99d7e7f4248a724088fb1c34aeee78c9ca63ec5a9464ec5054a0", + "zh:4e7302883fd9dd83bfbbcd72ebd55f83d8b16ccc6d12d1573d578058e604d5cf", + "zh:5b6e181e32cbf62f5d2ce34f9d6d9ffe17192e24943450bbe335e1baf0494e66", + "zh:62d525d426c6d5f10109ab04a9abc231b204ea413238f5690f69b420a8b8583a", + "zh:90aab23497ec9c7af44ad9ea1a1d6063dc3331334915e1c549527a73c2c6948d", + "zh:91ecf30a01df5e832191e0c55c87f8403a1f584796fd70f9c9c913d35c2e2a37", + "zh:bc3a5db5e4b9695a69dff47cf1e7184eaf5564d3dc50f231cbcbf535dd140d19", + "zh:cb566bec2676511bf4722e24d0dfc9bf58aff78af38b8e0864970f20d263118f", + "zh:d4fa0c1462b389cee313e1c152e00f5dfc175a1be3615d3b23b526a8581e39a5", + "zh:f8136b0f41045a1e5a6dedc6b6fb055faee3d825f84a3192312e3ac5d057ff72", ] } diff --git a/templates/workspace_services/azureml/terraform/compute.tf b/templates/workspace_services/azureml/terraform/compute.tf index 549995e6ab..e66e6017ad 100644 --- a/templates/workspace_services/azureml/terraform/compute.tf +++ b/templates/workspace_services/azureml/terraform/compute.tf @@ -25,7 +25,7 @@ resource "azapi_resource" "compute_cluster" { type = "Microsoft.MachineLearningServices/workspaces/computes@2022-10-01" name = "cp-${local.short_service_id}" location = data.azurerm_resource_group.ws.location - parent_id = azurerm_machine_learning_workspace.aml_workspace.id + parent_id = azapi_resource.aml_workspace.output.id tags = local.tre_workspace_service_tags lifecycle { ignore_changes = [tags] } @@ -34,7 +34,7 @@ resource "azapi_resource" "compute_cluster" { type = "SystemAssigned" } - body = jsonencode({ + body = { properties = { computeLocation = data.azurerm_resource_group.ws.location description = "Default Compute Cluster" @@ -57,7 +57,7 @@ resource "azapi_resource" "compute_cluster" { vmSize = "Standard_DS2_v2" } } - }) + } depends_on = [ azurerm_private_endpoint.mlpe, @@ -69,27 +69,18 @@ resource "azapi_resource" "compute_cluster" { } -# This seems to be added automatically -# resource "azurerm_role_assignment" "compute_cluster_acr_pull" { -# scope = azurerm_container_registry.acr.id -# role_definition_name = "AcrPull" -# principal_id = jsondecode(azapi_resource.compute_cluster.output).identity.principalId -# } - resource "azapi_update_resource" "set_image_build_compute" { type = "Microsoft.MachineLearningServices/workspaces@2022-10-01" - name = azurerm_machine_learning_workspace.aml_workspace.name + name = azapi_resource.aml_workspace.output.name parent_id = data.azurerm_resource_group.ws.id - body = jsonencode({ + body = { properties = { - imageBuildCompute = jsondecode(azapi_resource.compute_cluster.output).name + imageBuildCompute = azapi_resource.compute_cluster.output.name } - }) + } depends_on = [ azapi_resource.compute_cluster - #, - #azurerm_role_assignment.compute_cluster_acr_pull ] } diff --git a/templates/workspace_services/azureml/terraform/main.tf b/templates/workspace_services/azureml/terraform/main.tf index da6996243f..37a964b2f4 100644 --- a/templates/workspace_services/azureml/terraform/main.tf +++ b/templates/workspace_services/azureml/terraform/main.tf @@ -1,29 +1,61 @@ -resource "azurerm_machine_learning_workspace" "aml_workspace" { - name = local.workspace_name - resource_group_name = data.azurerm_resource_group.ws.name - location = data.azurerm_resource_group.ws.location - application_insights_id = azurerm_application_insights.ai.id - container_registry_id = azurerm_container_registry.acr.id - friendly_name = var.display_name - description = var.description - high_business_impact = true - key_vault_id = data.azurerm_key_vault.ws.id - public_network_access_enabled = var.is_exposed_externally ? true : false - storage_account_id = azurerm_storage_account.aml.id - tags = local.tre_workspace_service_tags - - identity { - type = "SystemAssigned" +resource "azapi_resource" "aml_workspace" { + type = "Microsoft.MachineLearningServices/workspaces@2024-10-01-preview" + name = local.workspace_name + location = data.azurerm_resource_group.ws.location + parent_id = data.azurerm_resource_group.ws.id + tags = local.tre_workspace_service_tags + + lifecycle { ignore_changes = [tags] } + + + dynamic "identity" { + for_each = var.enable_cmk_encryption ? [] : [1] + content { + type = "SystemAssigned" + } + } + + + dynamic "identity" { + for_each = var.enable_cmk_encryption ? [1] : [] + content { + type = "SystemAssigned, UserAssigned" + identity_ids = [data.azurerm_user_assigned_identity.ws_encryption_identity[0].id] + } } - lifecycle { - ignore_changes = [ - tags, - image_build_compute_name, - public_network_access_enabled - ] + body = { + properties = { + applicationInsights = azurerm_application_insights.ai.id + containerRegistry = azurerm_container_registry.acr.id + description = var.description + friendlyName = var.display_name + hbiWorkspace = true + keyVault = data.azurerm_key_vault.ws.id + publicNetworkAccess = var.is_exposed_externally ? "Enabled" : "Disabled" + storageAccount = azurerm_storage_account.aml.id + systemDatastoresAuthMode = "identity" + + workspaceHubConfig = { + additionalWorkspaceStorageAccounts = [] + } + + + encryption = { + status = var.enable_cmk_encryption ? "Enabled" : "Disabled" + identity = { + userAssignedIdentity = var.enable_cmk_encryption ? data.azurerm_user_assigned_identity.ws_encryption_identity[0].id : null + } + keyVaultProperties = { + keyIdentifier = var.enable_cmk_encryption ? data.azurerm_key_vault_key.ws_encryption_key[0].versionless_id : null + keyVaultArmId = var.enable_cmk_encryption ? var.key_store_id : null + identityClientId = var.enable_cmk_encryption ? data.azurerm_user_assigned_identity.ws_encryption_identity[0].client_id : null + } + } + } } + response_export_values = ["id", "name", "identity.principalId"] } resource "azurerm_private_endpoint" "mlpe" { @@ -42,7 +74,7 @@ resource "azurerm_private_endpoint" "mlpe" { private_service_connection { name = "mlpesc-${local.service_resource_name_suffix}" - private_connection_resource_id = azurerm_machine_learning_workspace.aml_workspace.id + private_connection_resource_id = azapi_resource.aml_workspace.output.id is_manual_connection = false subresource_names = ["amlworkspace"] } diff --git a/templates/workspace_services/azureml/terraform/network.tf b/templates/workspace_services/azureml/terraform/network.tf index b82ed52cc4..10b01aec26 100644 --- a/templates/workspace_services/azureml/terraform/network.tf +++ b/templates/workspace_services/azureml/terraform/network.tf @@ -30,7 +30,7 @@ resource "azapi_resource" "aml_service_endpoint_policy" { location = data.azurerm_virtual_network.ws.location parent_id = data.azurerm_resource_group.ws.id tags = local.tre_workspace_service_tags - body = jsonencode({ + body = { properties = { serviceEndpointPolicyDefinitions = [ { @@ -55,7 +55,7 @@ resource "azapi_resource" "aml_service_endpoint_policy" { } ] } - }) + } lifecycle { ignore_changes = [tags] } } diff --git a/templates/workspace_services/azureml/terraform/outputs.tf b/templates/workspace_services/azureml/terraform/outputs.tf index 6f75c6547d..f889ab5f17 100644 --- a/templates/workspace_services/azureml/terraform/outputs.tf +++ b/templates/workspace_services/azureml/terraform/outputs.tf @@ -1,5 +1,5 @@ output "azureml_workspace_name" { - value = azurerm_machine_learning_workspace.aml_workspace.name + value = azapi_resource.aml_workspace.output.name } output "azureml_acr_id" { @@ -15,7 +15,7 @@ output "aml_fqdn" { } output "connection_uri" { - value = format("%s/?wsid=%s&tid=%s", module.terraform_azurerm_environment_configuration.aml_studio_endpoint, azurerm_machine_learning_workspace.aml_workspace.id, var.arm_tenant_id) + value = format("%s/?wsid=%s&tid=%s", module.terraform_azurerm_environment_configuration.aml_studio_endpoint, azapi_resource.aml_workspace.output.id, var.arm_tenant_id) } output "workspace_address_spaces" { diff --git a/templates/workspace_services/azureml/terraform/providers.tf b/templates/workspace_services/azureml/terraform/providers.tf index f243be5378..6049cb2610 100644 --- a/templates/workspace_services/azureml/terraform/providers.tf +++ b/templates/workspace_services/azureml/terraform/providers.tf @@ -6,7 +6,7 @@ terraform { } azapi = { source = "Azure/azapi" - version = "=1.15.0" + version = "=2.2.0" } external = { source = "hashicorp/external" diff --git a/templates/workspace_services/azureml/terraform/roles.tf b/templates/workspace_services/azureml/terraform/roles.tf index af965cae2d..d94448d011 100644 --- a/templates/workspace_services/azureml/terraform/roles.tf +++ b/templates/workspace_services/azureml/terraform/roles.tf @@ -22,7 +22,40 @@ data "azurerm_role_definition" "azure_ml_data_scientist" { resource "azurerm_role_assignment" "app_role_members_aml_data_scientist" { for_each = (data.external.app_role_members.result.principals == "") ? [] : toset(split("\n", data.external.app_role_members.result.principals)) - scope = azurerm_machine_learning_workspace.aml_workspace.id + scope = azapi_resource.aml_workspace.output.id role_definition_id = data.azurerm_role_definition.azure_ml_data_scientist.id principal_id = each.value } + +data "azurerm_role_definition" "reader" { + name = "Reader" +} + +resource "azurerm_role_assignment" "app_role_members_reader" { + for_each = (data.external.app_role_members.result.principals == "") ? [] : toset(split("\n", data.external.app_role_members.result.principals)) + scope = azapi_resource.aml_workspace.output.id + role_definition_id = data.azurerm_role_definition.reader.id + principal_id = each.value +} + +data "azurerm_role_definition" "storage_blob_data_contributor" { + name = "Storage Blob Data Contributor" +} + +resource "azurerm_role_assignment" "app_role_members_storage_blob_data_contributor" { + for_each = (data.external.app_role_members.result.principals == "") ? [] : toset(split("\n", data.external.app_role_members.result.principals)) + scope = azurerm_storage_account.aml.id + role_definition_id = data.azurerm_role_definition.storage_blob_data_contributor.id + principal_id = each.value +} + +data "azurerm_role_definition" "storage_file_data_contributor" { + name = "Storage File Data Privileged Contributor" +} + +resource "azurerm_role_assignment" "app_role_members_storage_file_data_contributor" { + for_each = (data.external.app_role_members.result.principals == "") ? [] : toset(split("\n", data.external.app_role_members.result.principals)) + scope = azurerm_storage_account.aml.id + role_definition_id = data.azurerm_role_definition.storage_file_data_contributor.id + principal_id = each.value +} From 7e14a3e56055c2fefc016637b5c344e7c43709ee Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Wed, 19 Feb 2025 12:22:06 +0000 Subject: [PATCH 2/4] Update bundle version and changelog --- CHANGELOG.md | 1 + templates/workspace_services/azureml/porter.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c596b48c02..eabe3713bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ ENHANCEMENTS: * Migrate UI to Vite build engine and update dependencies ([#4368](https://github.com/microsoft/AzureTRE/pull/4368)) * Add Windows image field to the Admin VM template ([#4274](https://github.com/microsoft/AzureTRE/pull/4274)) * Update TLS to the latest version for web apps / function apps (([#4351](https://github.com/microsoft/AzureTRE/issues/4351)) +* Modify the AML workspace service so does not use local authentication keys for storage access ([#4341](https://github.com/microsoft/AzureTRE/issues/4341)) BUG FIXES: * Fix upgrade when porter install has failed ([#4338](https://github.com/microsoft/AzureTRE/pull/4338)) diff --git a/templates/workspace_services/azureml/porter.yaml b/templates/workspace_services/azureml/porter.yaml index 6a75726b98..4f8ef1adae 100644 --- a/templates/workspace_services/azureml/porter.yaml +++ b/templates/workspace_services/azureml/porter.yaml @@ -1,7 +1,7 @@ --- schemaVersion: 1.0.0 name: tre-service-azureml -version: 0.9.111 +version: 0.10.0 description: "An Azure TRE service for Azure Machine Learning" registry: azuretre dockerfile: Dockerfile.tmpl From 8d0231d60266cb21e2103cf7e161d1a796cd80b3 Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Wed, 19 Feb 2025 12:24:16 +0000 Subject: [PATCH 3/4] Update CHANGELOG.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 248528f63a..d05e5e6069 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,8 +15,7 @@ ENHANCEMENTS: * Migrate UI to Vite build engine and update dependencies ([#4368](https://github.com/microsoft/AzureTRE/pull/4368)) * Add Windows image field to the Admin VM template ([#4274](https://github.com/microsoft/AzureTRE/pull/4274)) * Update TLS to the latest version for web apps / function apps (([#4351](https://github.com/microsoft/AzureTRE/issues/4351)) -* Modify the AML workspace service so does not use local authentication keys for storage access ([#4341](https://github.com/microsoft/AzureTRE/issues/4341)) - +* Modify the AML workspace service so it does not use local authentication keys for storage access ([#4341](https://github.com/microsoft/AzureTRE/issues/4341)) BUG FIXES: * Fix upgrade when porter install has failed ([#4338](https://github.com/microsoft/AzureTRE/pull/4338)) * Certs shared service: Secret nexus-ssl-password is currently in a deleted but recoverable state ([#4294](https://github.com/microsoft/AzureTRE/issues/4294)]) From c8d6cc6a1d7457b2be581f77fd85e666cd1da16c Mon Sep 17 00:00:00 2001 From: Marcus Robinson Date: Mon, 24 Feb 2025 13:45:13 +0000 Subject: [PATCH 4/4] Review comments. --- templates/workspace_services/azureml/terraform/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/workspace_services/azureml/terraform/main.tf b/templates/workspace_services/azureml/terraform/main.tf index 37a964b2f4..719b6bdd8d 100644 --- a/templates/workspace_services/azureml/terraform/main.tf +++ b/templates/workspace_services/azureml/terraform/main.tf @@ -1,11 +1,11 @@ resource "azapi_resource" "aml_workspace" { - type = "Microsoft.MachineLearningServices/workspaces@2024-10-01-preview" + type = "Microsoft.MachineLearningServices/workspaces@2025-01-01-preview" name = local.workspace_name location = data.azurerm_resource_group.ws.location parent_id = data.azurerm_resource_group.ws.id tags = local.tre_workspace_service_tags - lifecycle { ignore_changes = [tags] } + lifecycle { ignore_changes = [tags, imageBuildCompute] } dynamic "identity" {