Merge pull request #16 from monte-carlo-data/mrostan/private-storage-azapi

Use azapi for private_storage example to avoid public IP workaround

Author: mrostan

examples/private_storage/README.md

@@ -3,13 +3,20 @@
 This example deploys an Agent using a VNet and storage accounts using private access.
 Private endpoints are created for the storage accounts and the agent is configured to use them.
 
-Please note that Storage Accounts are configured with public access enabled only from the IP
-address running Terraform. 
-This is due to a limitation in the way the Azure provider for TF works, in order to create
-shares or containers it needs access to the storage account.
-You can remove this rule if you run TF from a subnet with access to the Storage Account or you 
-can update this module to remove the creation of the storage accounts (and the corresponding 
-share and container) and receive them as input variables.
+The Storage Accounts in this example are kept fully private (`public_network_access_enabled = false`):
+they are only reachable through the private endpoints. This is possible because the file share and
+blob container are created with the [`azapi`](https://registry.terraform.io/providers/Azure/azapi/latest)
+provider, which manages them through the Azure Resource Manager (ARM) management plane rather than the
+storage data plane. Management-plane calls are not subject to the storage account firewall, so the
+machine running Terraform does not need any network path to the storage accounts.
+
+> The `azurerm_storage_share` / `azurerm_storage_container` resources operate over the storage data
+> plane (`*.file.core.windows.net` / `*.blob.core.windows.net`), so with a private account they
+> would require Terraform to run from a network with access to the account (e.g. a subnet/private
+> endpoint), or a temporary public firewall rule allowing the runner's IP. Using `azapi` avoids that
+> entirely. If you prefer the `azurerm`-only approach, you can instead run Terraform from a subnet
+> with access to the Storage Account, or remove the storage account creation from this example and
+> pass them in as input variables.
 
 Note that there are additional requirements and limitations for using private storage accounts 
 with Monte Carlo. Please review the docs [here](https://mc-d.io/LgZYUfJ) for further details.

examples/private_storage/main.tf

@@ -2,16 +2,6 @@ resource "random_id" "unique_id" {
   byte_length = 4
 }
 
-# get the current IP address, used to allow access to the storage account from TF
-# to create the share
-data "http" "ip" {
-  url = "https://checkip.amazonaws.com"
-}
-
-locals {
-  my_ip = chomp(data.http.ip.response_body)
-}
-
 locals {
   suffix              = random_id.unique_id.hex
   resource_group_name = "mcd-agent-private-storage"
@@ -36,10 +26,10 @@ module "apollo" {
   existing_storage_accounts = {
     agent_durable_function_storage_account_name       = azurerm_storage_account.durable_function_storage.name
     agent_durable_function_storage_account_access_key = azurerm_storage_account.durable_function_storage.primary_access_key
-    agent_durable_function_storage_account_share_name = azurerm_storage_share.durable_function_storage.name
+    agent_durable_function_storage_account_share_name = azapi_resource.durable_function_storage_share.name
 
     agent_data_storage_account_name   = azurerm_storage_account.mcd_agent_storage.name
-    agent_data_storage_container_name = azurerm_storage_container.mcd_agent_storage_container.name
+    agent_data_storage_container_name = azapi_resource.mcd_agent_storage_container.name
 
     private_access = true
   }

examples/private_storage/provider.tf

@@ -1,7 +1,21 @@
+terraform {
+  required_providers {
+    # azapi is used to create the blob container and file share via the ARM
+    # management plane, which is not subject to the storage account firewall.
+    # This lets us keep the storage accounts fully private (no public access).
+    azapi = {
+      source  = "Azure/azapi"
+      version = "~> 2.0"
+    }
+  }
+}
+
 provider "azurerm" {
   features {
     resource_group {
       prevent_deletion_if_contains_resources = false
     }
   }
-}
+}
+
+provider "azapi" {}

examples/private_storage/storage.tf

@@ -10,23 +10,28 @@ resource "azurerm_storage_account" "durable_function_storage" {
   allow_nested_items_to_be_public   = false
   infrastructure_encryption_enabled = true
 
-  # Using this approach to allow access from the IP address running Terraform,
-  # this is required for the creation of the share below and run TF plan/apply
-  # in the future.
-  # You can manually disable public access completely after deploying this module,
-  # just remember to restore this rule before executing TF again.
-  public_network_access_enabled = true
+  # The account is kept fully private: access is only possible through the
+  # private endpoints below. The share is created via the azapi provider
+  # (ARM management plane), so no public/data-plane access from the Terraform
+  # runner is required.
+  public_network_access_enabled = false
   network_rules {
     default_action = "Deny"
-    ip_rules       = [local.my_ip]
   }
 }
 
-# Share to use for the Azure Function, for the WEBSITE_CONTENTSHARE setting
-resource "azurerm_storage_share" "durable_function_storage" {
-  name                 = azurerm_storage_account.durable_function_storage.name
-  storage_account_name = azurerm_storage_account.durable_function_storage.name
-  quota                = 50
+# Share to use for the Azure Function, for the WEBSITE_CONTENTSHARE setting.
+# Created through the ARM management plane (azapi) rather than the storage data
+# plane, so it works even with public_network_access_enabled = false.
+resource "azapi_resource" "durable_function_storage_share" {
+  type      = "Microsoft.Storage/storageAccounts/fileServices/shares@2023-01-01"
+  name      = azurerm_storage_account.durable_function_storage.name
+  parent_id = "${azurerm_storage_account.durable_function_storage.id}/fileServices/default"
+  body = {
+    properties = {
+      shareQuota = 50
+    }
+  }
 }
 
 # Private endpoints for the Durable function storage account
@@ -114,23 +119,28 @@ resource "azurerm_storage_account" "mcd_agent_storage" {
   allow_nested_items_to_be_public   = false
   infrastructure_encryption_enabled = true
 
-  # Using this approach to allow access from the IP address running Terraform,
-  # this is required for the creation of the container below and run TF plan/apply
-  # in the future.
-  # You can manually disable public access completely after deploying this module,
-  # just remember to restore this rule before executing TF again.
-  public_network_access_enabled = true
+  # The account is kept fully private: access is only possible through the
+  # private endpoint below. The container is created via the azapi provider
+  # (ARM management plane), so no public/data-plane access from the Terraform
+  # runner is required.
+  public_network_access_enabled = false
   network_rules {
     default_action = "Deny"
-    ip_rules       = [local.my_ip]
   }
 }
 
-# Container used by the MC agent
-resource "azurerm_storage_container" "mcd_agent_storage_container" {
-  name                  = local.agent_data_storage_container_name
-  storage_account_name  = azurerm_storage_account.mcd_agent_storage.name
-  container_access_type = "private"
+# Container used by the MC agent.
+# Created through the ARM management plane (azapi) rather than the storage data
+# plane, so it works even with public_network_access_enabled = false.
+resource "azapi_resource" "mcd_agent_storage_container" {
+  type      = "Microsoft.Storage/storageAccounts/blobServices/containers@2023-01-01"
+  name      = local.agent_data_storage_container_name
+  parent_id = "${azurerm_storage_account.mcd_agent_storage.id}/blobServices/default"
+  body = {
+    properties = {
+      publicAccess = "None"
+    }
+  }
 }
 
 # Private endpoint for the MC agent storage account
@@ -160,7 +170,7 @@ resource "azurerm_storage_management_policy" "mcd_agent_storage_lifecycle" {
     enabled = true
     filters {
       blob_types   = ["blockBlob", "appendBlob"]
-      prefix_match = ["${azurerm_storage_container.mcd_agent_storage_container.name}/${local.agent_data_store_data_prefix}"]
+      prefix_match = ["${azapi_resource.mcd_agent_storage_container.name}/${local.agent_data_store_data_prefix}"]
     }
     actions {
       base_blob {
@@ -173,7 +183,7 @@ resource "azurerm_storage_management_policy" "mcd_agent_storage_lifecycle" {
     enabled = true
     filters {
       blob_types   = ["blockBlob", "appendBlob"]
-      prefix_match = ["${azurerm_storage_container.mcd_agent_storage_container.name}/${local.agent_data_store_data_prefix}/tmp"]
+      prefix_match = ["${azapi_resource.mcd_agent_storage_container.name}/${local.agent_data_store_data_prefix}/tmp"]
     }
     actions {
       base_blob {

main.tf

@@ -1,6 +1,6 @@
 locals {
   # Wrapper metadata
-  mcd_wrapper_version       = "1.0.3"
+  mcd_wrapper_version       = "1.0.5"
   mcd_agent_platform        = "AZURE"
   mcd_agent_deployment_type = "TERRAFORM"