refactor(gcp): workspace_deployment module

gcp/examples/byo_gcp_workspace_deployment/init.tf

@@ -1,5 +1,32 @@
+terraform {
+  # Remote state backend.
+  # Uncomment the `backend "gcs" {}` line below to store state in GCS instead
+  # of the local filesystem. Provide bucket and prefix at init time:
+  #   terraform init \
+  #     -backend-config="bucket=my-tfstate-bucket" \
+  #     -backend-config="prefix=databricks/byo_gcp_workspace_deployment"
+  # If left commented, Terraform uses the default local backend
+  # (terraform.tfstate in this directory).
+  # backend "gcs" {}
+
+  required_providers {
+    databricks = {
+      source  = "databricks/databricks"
+      version = ">=1.113.0"
+    }
+    google = {
+      source  = "hashicorp/google"
+      version = ">=5.43.1"
+    }
+  }
+}
+
 provider "google" {
   project = var.project
 }
 
-# data "google_client_config" "current" {}
+provider "databricks" {
+  alias      = "accounts"
+  host       = var.account_console_url
+  account_id = var.databricks_account_id
+}

gcp/examples/byo_gcp_workspace_deployment/main.tf

@@ -1,49 +1,50 @@
 
 module "customer_managed_vpc" {
-  source                      = "../../modules/workspace_deployment/"
-  google_project = var.google_project #Google Cloud project id (GCP)
-  google_region = var.google_region  #Google Cloud region (GCP)
-  databricks_account_id = var.databricks_account_id #Databricks account id (Databricks)
+  source                            = "../../modules/workspace_deployment/"
+  google_project                    = var.google_project        #Google Cloud project id (GCP)
+  google_region                     = var.google_region         #Google Cloud region (GCP)
+  databricks_account_id             = var.databricks_account_id #Databricks account id (Databricks)
   databricks_google_service_account = var.databricks_google_service_account
-  workspace_name = var.workspace_name #Name of the Databricks workspace to be created (Databricks)
-  account_console_url = var.account_console_url #Databricks Account console url (Databricks)
+  workspace_name                    = var.workspace_name      #Name of the Databricks workspace to be created (Databricks)
+  account_console_url               = var.account_console_url #Databricks Account console url (Databricks)
+
+
 
-
-
   #IP ACCESS LIST
-  ip_addresses=var.ip_addresses #Databricks Workspace IP Access List (Databricks)
- 
+  ip_addresses = var.ip_addresses #Databricks Workspace IP Access List (Databricks)
+
   #VPC RESOURCES
-  use_existing_vpc = var.use_existing_vpc #Flag to use existing vpc or create a new one
-  existing_vpc_name = var.existing_vpc_name
+  use_existing_vpc     = var.use_existing_vpc #Flag to use existing vpc or create a new one
+  existing_vpc_name    = var.existing_vpc_name
   existing_subnet_name = var.existing_subnet_name
 
   #PSC RESOURCES
   google_pe_subnet = var.google_pe_subnet #Name of the subnet to be used for the PSC endpoints (GCP)
 
-  use_psc = var.use_psc #Flag to enable Private Service Connect (PSC) for the workspace
-  use_existing_pas = var.use_existing_pas #Flag to use existing private access settings or create a new one
-  existing_pas_id = var.existing_pas_id #Required if use_existing_pas is true
-  use_existing_PSC_EP = var.use_existing_psc_eps #Flag to use existing PSC endpoints or create a new ones
+  use_psc                         = var.use_psc                         #Flag to enable Private Service Connect (PSC) for the workspace
+  use_existing_pas                = var.use_existing_pas                #Flag to use existing private access settings or create a new one
+  existing_pas_id                 = var.existing_pas_id                 #Required if use_existing_pas is true
+  use_existing_PSC_EP             = var.use_existing_psc_eps            #Flag to use existing PSC endpoints or create a new ones
   use_existing_databricks_vpc_eps = var.use_existing_databricks_vpc_eps #Flag to use existing Databricks VPC Endpoints for PSC or create a new ones
 
-  workspace_service_attachment = var.workspace_service_attachment #Workspace service attachment. Regional values - https://docs.databricks.com/gcp/en/resources/ip-domain-region#private-service-connect-psc-attachment-uris-and-project-numbers
-  workspace_pe = var.workspace_pe #Name of the PSC endpoint (found in GCP console) used for the workspace communication (GCP)
-  workspace_pe_ip_name = var.workspace_pe_ip_name #Workspace private endpoint IP name if not using an existing one (GCP)
+  workspace_service_attachment         = var.workspace_service_attachment         #Workspace service attachment. Regional values - https://docs.databricks.com/gcp/en/resources/ip-domain-region#private-service-connect-psc-attachment-uris-and-project-numbers
+  workspace_pe                         = var.workspace_pe                         #Name of the PSC endpoint (found in GCP console) used for the workspace communication (GCP)
+  workspace_pe_ip_name                 = var.workspace_pe_ip_name                 #Workspace private endpoint IP name if not using an existing one (GCP)
   existing_databricks_vpc_ep_workspace = var.existing_databricks_vpc_ep_workspace #Required if use_existing_databricks_vpc_eps is true. 
-  existing_databricks_vpc_ep_relay = var.existing_databricks_vpc_ep_relay #Required if use_existing_databricks_vpc_eps is true.
+  existing_databricks_vpc_ep_relay     = var.existing_databricks_vpc_ep_relay     #Required if use_existing_databricks_vpc_eps is true.
 
   relay_service_attachment = var.relay_service_attachment #Relay service attachment. Regional values - https://docs.databricks.com/gcp/en/resources/ip-domain-region#private-service-connect-psc-attachment-uris-and-project-numbers
-  relay_pe =  var.relay_pe #Name of the PSC endpoint (found in GCP console) used for the relay communication (GCP)
-  relay_pe_ip_name = var.relay_pe_ip_name #Relay private endpoint IP name if not using an existing one (GCP)
-  
+  relay_pe                 = var.relay_pe                 #Name of the PSC endpoint (found in GCP console) used for the relay communication (GCP)
+  relay_pe_ip_name         = var.relay_pe_ip_name         #Relay private endpoint IP name if not using an existing one (GCP)
+
   #CMEK RESOURCES
+  use_cmek          = var.use_cmek          #Master flag to enable CMEK
   use_existing_cmek = var.use_existing_cmek #Flag to use existing Cloud KMS Key or create a new one
-  key_name = var.key_name #Name of the key to be created if not using an existing one (GCP)
-  keyring_name = var.keyring_name #Name of the keyring to be created if not using an existing one (GCP)
-  cmek_resource_id = var.cmek_resource_id #Resource ID for the existing Cloud KMS Key (GCP)
+  key_name          = var.key_name          #Name of the key to be created if not using an existing one (GCP)
+  keyring_name      = var.keyring_name      #Name of the keyring to be created if not using an existing one (GCP)
+  cmek_resource_id  = var.cmek_resource_id  #Resource ID for the existing Cloud KMS Key (GCP)
 
-    # Flags
+  # Flags
   harden_network               = var.harden_network
-  provision_regional_metastore  = false 
+  provision_regional_metastore = false
 }

gcp/examples/byo_gcp_workspace_deployment/variables.tf

@@ -34,6 +34,12 @@ variable "use_existing_vpc" {
 }
 variable "harden_network" {}
 
+variable "use_cmek" {
+  description = "Enable Customer-Managed Encryption Keys"
+  type        = bool
+  default     = false
+}
+
 variable "use_existing_cmek" {
   description = "Use existing cmek"
   type        = bool

gcp/examples/end-to-end provisionning/init.tf

@@ -1,8 +1,18 @@
 terraform {
+  # Remote state backend.
+  # Uncomment the `backend "gcs" {}` line below to store state in GCS instead
+  # of the local filesystem. Provide bucket and prefix at init time:
+  #   terraform init \
+  #     -backend-config="bucket=my-tfstate-bucket" \
+  #     -backend-config="prefix=databricks/end-to-end"
+  # If left commented, Terraform uses the default local backend
+  # (terraform.tfstate in this directory).
+  # backend "gcs" {}
+
   required_providers {
     databricks = {
-      source = "databricks/databricks"
-      version = ">=1.51.0"
+      source  = "databricks/databricks"
+      version = ">=1.113.0"
     }
     google = {
       source  = "hashicorp/google"
@@ -11,15 +21,15 @@ terraform {
   }
 }
 
-# # Default provider for service account creation (uses your current auth)
+# Default Google provider (uses your current auth).
 provider "google" {
   project = var.google_project
   region  = var.google_region
 }
 
-# # Databricks provider for account operations
+# Databricks provider for account operations.
 provider "databricks" {
-  alias                  = "accounts"
-  host                   = "https://accounts.gcp.databricks.com"
-  account_id             = var.databricks_account_id
-}
+  alias      = "accounts"
+  host       = "https://accounts.gcp.databricks.com"
+  account_id = var.databricks_account_id
+}

gcp/examples/end-to-end provisionning/main.tf

@@ -1,45 +1,57 @@
 
 module "service_account" {
   source = "../../modules/service_account/"
-  # Bare Minimum Variables
-  project = var.google_project
-  sa_name = var.sa_name
+
+  project                    = var.google_project
+  sa_name                    = var.sa_name
   create_service_account_key = true
-  delegate_from = var.delegate_from
+  delegate_from              = var.delegate_from
 }
 
-
 module "make_sa_dbx_admin" {
   source = "../../modules/make_sa_dbx_admin/"
-  
-  databricks_account_id = var.databricks_account_id
-  new_admin_account = module.service_account.workspace_creator_email  # Use output from service_account module
-  dbx_existing_admin_account = data.google_client_openid_userinfo.me.email  # Use the current user's email as the existing admin account
+
+  databricks_account_id      = var.databricks_account_id
+  new_admin_account          = module.service_account.workspace_creator_email
+  dbx_existing_admin_account = data.google_client_openid_userinfo.me.email
 }
 
 module "customer_managed_vpc" {
-  source                      = "../../modules/workspace_deployment/"
-
-  # Bare Minimum Variables
-  google_project               = var.google_project
-  google_region                = var.google_region
-  databricks_account_id        = var.databricks_account_id
-  databricks_google_service_account = module.service_account.workspace_creator_email  # Use output from service_account module
-  workspace_name               = var.workspace_name
-  databricks_google_service_account_key = module.service_account.workspace_creator_key  # Use output from service_account module
-  regional_metastore_id = var.regional_metastore_id
-  can_create_workspaces = module.service_account.workspace_creator_role_applied 
-  admin_user_email            = module.make_sa_dbx_admin.original_admin_account
-  cmek_resource_id           = var.cmek_resource_id
-
-  # Flags
-  use_existing_vpc             = false
-  use_existing_pas             = false
-  use_existing_PSC_EP          = false
-  use_existing_cmek            = true
-  use_psc                      = false
-  harden_network               = true
-  provision_regional_metastore  = false 
-
-}
+  source = "../../modules/workspace_deployment/"
+
+  google_project                        = var.google_project
+  google_region                         = var.google_region
+  databricks_account_id                 = var.databricks_account_id
+  databricks_google_service_account     = module.service_account.workspace_creator_email
+  workspace_name                        = var.workspace_name
+  databricks_google_service_account_key = module.service_account.workspace_creator_key
+  regional_metastore_id                 = var.regional_metastore_id
+  can_create_workspaces                 = module.service_account.workspace_creator_role_applied
+  admin_user_email                      = module.make_sa_dbx_admin.original_admin_account
 
+  # Networking — module creates VPC, subnet, PSC endpoints, firewalls
+  use_existing_vpc    = false
+  use_existing_pas    = false
+  use_existing_PSC_EP = false
+  use_psc             = true
+  harden_network      = true
+
+  # PSC service attachments and endpoint names
+  workspace_service_attachment = var.workspace_service_attachment
+  relay_service_attachment     = var.relay_service_attachment
+  workspace_pe                 = "sra-workspace-pe"
+  workspace_pe_ip_name         = "sra-workspace-pe-ip"
+  relay_pe                     = "sra-relay-pe"
+  relay_pe_ip_name             = "sra-relay-pe-ip"
+
+  # CMEK — module creates KMS keyring + key and registers with Databricks
+  use_cmek          = true
+  use_existing_cmek = false
+  keyring_name      = var.keyring_name
+  key_name          = var.key_name
+
+  # DNS — create a private zone for PSC resolution
+  create_dns_zone = true
+
+  provision_regional_metastore = false
+}

gcp/examples/end-to-end provisionning/outputs.tf

@@ -2,6 +2,10 @@ output "service_account_email" {
   value = module.service_account.workspace_creator_email
 }
 
-output "databricks_host" {
-  value = module.customer_managed_vpc.databricks_host
-}
+output "workspace_url" {
+  value = module.customer_managed_vpc.workspace_url
+}
+
+output "workspace_id" {
+  value = module.customer_managed_vpc.workspace_id
+}

gcp/examples/end-to-end provisionning/variables.tf

@@ -1,49 +1,65 @@
 
 ##### GENERAL VARIABLES #####
 variable "databricks_account_id" {
-    # Databricks account ID (found in the account console)
-} 
+  type        = string
+  description = "Databricks account ID (found in the account console)"
+}
+
 variable "sa_name" {
-    # Google service account for Databricks
-    default = "databricks-workspace-creator"
-} 
+  type        = string
+  default     = "databricks-workspace-creator"
+  description = "Google service account name for Databricks provisioning"
+}
+
 variable "google_project" {
-    # Name of the Google Cloud project
-} 
-variable "cmek_resource_id" {
-    # Customer-managed encryption key resource ID
-    default = ""
+  type        = string
+  description = "Google Cloud project ID"
 }
 
 variable "google_region" {
-    # Google Cloud region
-} 
-variable "workspace_name" { 
-    # Name you want to give to the Databricks workspace you are creating
-    default = "sra-deployed-ws"
+  type        = string
+  description = "Google Cloud region"
+}
+
+variable "workspace_name" {
+  type        = string
+  default     = "sra-deployed-ws"
+  description = "Name of the Databricks workspace to create"
 }
 
 variable "delegate_from" {
-    # List of users or service accounts to delegate permissions from
-    type    = list(string)
-    default = []
+  type        = list(string)
+  default     = []
+  description = "List of users or service accounts to delegate impersonation from (e.g. [\"user:you@example.com\"])"
+}
+
+##### PSC #####
+variable "workspace_service_attachment" {
+  type        = string
+  description = "PSC service attachment URI for the workspace endpoint (plproxy). See https://docs.databricks.com/gcp/en/resources/ip-domain-region"
 }
 
 variable "relay_service_attachment" {
-    # Relay service attachment. regional values - https://docs.gcp.databricks.com/resources/supported-regions.html#psc
-    default = "projects/prod-gcp-europe-west1/regions/europe-west1/serviceAttachments/ngrok-psc-endpoint"
-} 
-variable "workspace_service_attachment" { 
-    # Workspace service attachment. Regional values - https://docs.gcp.databricks.com/resources/supported-regions.html#psc
-    default = "projects/general-prod-europewest1-01/regions/europe-west1/serviceAttachments/plproxy-psc-endpoint-all-ports"
+  type        = string
+  description = "PSC service attachment URI for the SCC relay endpoint (ngrok). See https://docs.databricks.com/gcp/en/resources/ip-domain-region"
+}
+
+##### CMEK #####
+variable "keyring_name" {
+  type        = string
+  default     = "databricks-keyring"
+  description = "Name of the KMS keyring to create"
 }
 
+variable "key_name" {
+  type        = string
+  default     = "databricks-key"
+  description = "Name of the KMS crypto key to create"
+}
 
+##### METASTORE #####
 variable "regional_metastore_id" {
-    # ID of the regional Hive Metastore
-    default = "regional-metastore"
+  type        = string
+  default     = ""
+  description = "ID of the regional Unity Catalog metastore to assign (leave empty to skip)"
 }
-
-variable "provision_regional_metastore"{
-    default = false
-}

gcp/examples/service_account/main.tf

@@ -1,7 +1,7 @@
 module "service_account" {
-  source                      = "../../modules/service_account/"
+  source        = "../../modules/service_account/"
   project       = var.project
-  prefix = var.prefix
+  prefix        = var.prefix
   delegate_from = var.delegate_from
 }