From 05bc2fb961e2aadcaf68232dff139f56cb0af88c Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Mon, 30 Mar 2026 19:28:47 +0200
Subject: [PATCH 01/62] Initial TF module refactoring

---
 .../charms/azure-storage-integrator/README.md |  44 ++
 .../azure-storage-integrator/applications.tf  |  21 +
 .../azure-storage-integrator/integrations.tf  |  30 ++
 .../charms/azure-storage-integrator/locals.tf |   2 +
 .../charms/azure-storage-integrator/main.tf   |   2 +
 .../azure-storage-integrator/outputs.tf       |  21 +
 .../azure-storage-integrator/providers.tf     |   6 +
 .../azure-storage-integrator/resources.tf     |  30 ++
 .../azure-storage-integrator/terraform.tf     |  10 +
 .../azure-storage-integrator/variables.tf     |  37 ++
 .../terraform/charms/manual-tls/README.md     |  41 ++
 .../terraform/charms/manual-tls/locals.tf     |   2 +
 releases/terraform/charms/manual-tls/main.tf  |  91 ++++
 .../terraform/charms/manual-tls/outputs.tf    |  16 +
 .../terraform/charms/manual-tls/providers.tf  |   6 +
 .../terraform/charms/manual-tls/terraform.tf  |  10 +
 .../terraform/charms/manual-tls/variables.tf  |  50 +++
 .../terraform/charms/s3-integrator/README.md  |  42 ++
 .../charms/s3-integrator/applications.tf      |  20 +
 .../charms/s3-integrator/integrations.tf      |  30 ++
 .../terraform/charms/s3-integrator/locals.tf  |   2 +
 .../terraform/charms/s3-integrator/main.tf    |   2 +
 .../terraform/charms/s3-integrator/outputs.tf |  21 +
 .../charms/s3-integrator/providers.tf         |   6 +
 .../charms/s3-integrator/resources.tf         |  12 +
 .../charms/s3-integrator/terraform.tf         |  10 +
 .../charms/s3-integrator/variables.tf         |  39 ++
 .../components/observability/README.md        |  67 +++
 .../components/observability/applications.tf  |  64 +++
 .../components/observability/integrations.tf  | 236 +++++++++++
 .../components/observability/locals.tf        |   2 +
 .../components/observability/main.tf          |   2 +
 .../components/observability/outputs.tf       |  21 +
 .../components/observability/providers.tf     |   6 +
 .../components/observability/resources.tf     |  12 +
 .../components/observability/terraform.tf     |  10 +
 .../components/observability/variables.tf     |  81 ++++
 .../terraform/components/spark-3.4/README.md  |  89 ++++
 .../components/spark-3.4/applications.tf      | 190 +++++++++
 .../components/spark-3.4/integrations.tf      |  86 ++++
 .../terraform/components/spark-3.4/locals.tf  |  43 ++
 .../terraform/components/spark-3.4/main.tf    |   2 +
 .../terraform/components/spark-3.4/offers.tf  |  11 +
 .../terraform/components/spark-3.4/outputs.tf |  34 ++
 .../components/spark-3.4/providers.tf         |   6 +
 .../components/spark-3.4/resources.tf         |  36 ++
 .../components/spark-3.4/terraform.tf         |   8 +
 .../components/spark-3.4/variables.tf         | 244 +++++++++++
 .../terraform/components/spark-3.5/README.md  |  90 ++++
 .../components/spark-3.5/applications.tf      | 190 +++++++++
 .../components/spark-3.5/integrations.tf      |  86 ++++
 .../terraform/components/spark-3.5/locals.tf  |  43 ++
 .../terraform/components/spark-3.5/main.tf    |   2 +
 .../terraform/components/spark-3.5/offers.tf  |  11 +
 .../terraform/components/spark-3.5/outputs.tf |  34 ++
 .../components/spark-3.5/providers.tf         |   6 +
 .../components/spark-3.5/resources.tf         |  36 ++
 .../components/spark-3.5/terraform.tf         |  10 +
 .../components/spark-3.5/variables.tf         | 244 +++++++++++
 .../products/charmed-spark-3.4/README.md      |  99 +++++
 .../products/charmed-spark-3.4/locals.tf      |  43 ++
 .../products/charmed-spark-3.4/main.tf        | 146 +++++++
 .../products/charmed-spark-3.4/outputs.tf     |  51 +++
 .../products/charmed-spark-3.4/providers.tf   |   6 +
 .../products/charmed-spark-3.4/terraform.tf   |  11 +
 .../products/charmed-spark-3.4/variables.tf   | 395 ++++++++++++++++++
 .../products/charmed-spark-3.5/README.md      | 100 +++++
 .../products/charmed-spark-3.5/locals.tf      |  43 ++
 .../products/charmed-spark-3.5/main.tf        | 146 +++++++
 .../products/charmed-spark-3.5/outputs.tf     |  51 +++
 .../products/charmed-spark-3.5/providers.tf   |   6 +
 .../products/charmed-spark-3.5/terraform.tf   |  13 +
 .../products/charmed-spark-3.5/variables.tf   | 395 ++++++++++++++++++
 releases/terraform/products/cos/README.md     |  77 ++++
 .../terraform/products/cos/applications.tf    | 103 +++++
 .../terraform/products/cos/integrations.tf    | 300 +++++++++++++
 releases/terraform/products/cos/locals.tf     |   2 +
 releases/terraform/products/cos/main.tf       |   2 +
 releases/terraform/products/cos/outputs.tf    |  30 ++
 releases/terraform/products/cos/providers.tf  |   6 +
 releases/terraform/products/cos/resources.tf  |  27 ++
 releases/terraform/products/cos/terraform.tf  |  10 +
 releases/terraform/products/cos/variables.tf  |  69 +++
 83 files changed, 4736 insertions(+)
 create mode 100644 releases/terraform/charms/azure-storage-integrator/README.md
 create mode 100644 releases/terraform/charms/azure-storage-integrator/applications.tf
 create mode 100644 releases/terraform/charms/azure-storage-integrator/integrations.tf
 create mode 100644 releases/terraform/charms/azure-storage-integrator/locals.tf
 create mode 100644 releases/terraform/charms/azure-storage-integrator/main.tf
 create mode 100644 releases/terraform/charms/azure-storage-integrator/outputs.tf
 create mode 100644 releases/terraform/charms/azure-storage-integrator/providers.tf
 create mode 100644 releases/terraform/charms/azure-storage-integrator/resources.tf
 create mode 100644 releases/terraform/charms/azure-storage-integrator/terraform.tf
 create mode 100644 releases/terraform/charms/azure-storage-integrator/variables.tf
 create mode 100644 releases/terraform/charms/manual-tls/README.md
 create mode 100644 releases/terraform/charms/manual-tls/locals.tf
 create mode 100644 releases/terraform/charms/manual-tls/main.tf
 create mode 100644 releases/terraform/charms/manual-tls/outputs.tf
 create mode 100644 releases/terraform/charms/manual-tls/providers.tf
 create mode 100644 releases/terraform/charms/manual-tls/terraform.tf
 create mode 100644 releases/terraform/charms/manual-tls/variables.tf
 create mode 100644 releases/terraform/charms/s3-integrator/README.md
 create mode 100644 releases/terraform/charms/s3-integrator/applications.tf
 create mode 100644 releases/terraform/charms/s3-integrator/integrations.tf
 create mode 100644 releases/terraform/charms/s3-integrator/locals.tf
 create mode 100644 releases/terraform/charms/s3-integrator/main.tf
 create mode 100644 releases/terraform/charms/s3-integrator/outputs.tf
 create mode 100644 releases/terraform/charms/s3-integrator/providers.tf
 create mode 100644 releases/terraform/charms/s3-integrator/resources.tf
 create mode 100644 releases/terraform/charms/s3-integrator/terraform.tf
 create mode 100644 releases/terraform/charms/s3-integrator/variables.tf
 create mode 100644 releases/terraform/components/observability/README.md
 create mode 100644 releases/terraform/components/observability/applications.tf
 create mode 100644 releases/terraform/components/observability/integrations.tf
 create mode 100644 releases/terraform/components/observability/locals.tf
 create mode 100644 releases/terraform/components/observability/main.tf
 create mode 100644 releases/terraform/components/observability/outputs.tf
 create mode 100644 releases/terraform/components/observability/providers.tf
 create mode 100644 releases/terraform/components/observability/resources.tf
 create mode 100644 releases/terraform/components/observability/terraform.tf
 create mode 100644 releases/terraform/components/observability/variables.tf
 create mode 100644 releases/terraform/components/spark-3.4/README.md
 create mode 100644 releases/terraform/components/spark-3.4/applications.tf
 create mode 100644 releases/terraform/components/spark-3.4/integrations.tf
 create mode 100644 releases/terraform/components/spark-3.4/locals.tf
 create mode 100644 releases/terraform/components/spark-3.4/main.tf
 create mode 100644 releases/terraform/components/spark-3.4/offers.tf
 create mode 100644 releases/terraform/components/spark-3.4/outputs.tf
 create mode 100644 releases/terraform/components/spark-3.4/providers.tf
 create mode 100644 releases/terraform/components/spark-3.4/resources.tf
 create mode 100644 releases/terraform/components/spark-3.4/terraform.tf
 create mode 100644 releases/terraform/components/spark-3.4/variables.tf
 create mode 100644 releases/terraform/components/spark-3.5/README.md
 create mode 100644 releases/terraform/components/spark-3.5/applications.tf
 create mode 100644 releases/terraform/components/spark-3.5/integrations.tf
 create mode 100644 releases/terraform/components/spark-3.5/locals.tf
 create mode 100644 releases/terraform/components/spark-3.5/main.tf
 create mode 100644 releases/terraform/components/spark-3.5/offers.tf
 create mode 100644 releases/terraform/components/spark-3.5/outputs.tf
 create mode 100644 releases/terraform/components/spark-3.5/providers.tf
 create mode 100644 releases/terraform/components/spark-3.5/resources.tf
 create mode 100644 releases/terraform/components/spark-3.5/terraform.tf
 create mode 100644 releases/terraform/components/spark-3.5/variables.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.4/README.md
 create mode 100644 releases/terraform/products/charmed-spark-3.4/locals.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.4/main.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.4/outputs.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.4/providers.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.4/terraform.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.4/variables.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.5/README.md
 create mode 100644 releases/terraform/products/charmed-spark-3.5/locals.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.5/main.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.5/outputs.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.5/providers.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.5/terraform.tf
 create mode 100644 releases/terraform/products/charmed-spark-3.5/variables.tf
 create mode 100644 releases/terraform/products/cos/README.md
 create mode 100644 releases/terraform/products/cos/applications.tf
 create mode 100644 releases/terraform/products/cos/integrations.tf
 create mode 100644 releases/terraform/products/cos/locals.tf
 create mode 100644 releases/terraform/products/cos/main.tf
 create mode 100644 releases/terraform/products/cos/outputs.tf
 create mode 100644 releases/terraform/products/cos/providers.tf
 create mode 100644 releases/terraform/products/cos/resources.tf
 create mode 100644 releases/terraform/products/cos/terraform.tf
 create mode 100644 releases/terraform/products/cos/variables.tf

diff --git a/releases/terraform/charms/azure-storage-integrator/README.md b/releases/terraform/charms/azure-storage-integrator/README.md
new file mode 100644
index 00000000..d8926087
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/README.md
@@ -0,0 +1,44 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_access_secret.azure_blob_storage_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
+| [juju_application.azure_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.azure_storage_integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.history_server_azure_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_secret.azure_blob_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
+| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_azure_storage"></a> [azure\_storage](#input\_azure\_storage) | Azure Object storage information | <pre>object({<br/>    container       = optional(string, "azurecontainer")<br/>    storage_account = optional(string, "azurestorageaccount")<br/>    protocol        = optional(string, "abfss")<br/>    secret_key      = optional(string, "secret-key")<br/>    path            = optional(string, "spark-events")<br/>  })</pre> | `{}` | no |
+| <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | n/a | yes |
+| <a name="input_model"></a> [model](#input\_model) | Name of the Spark Juju Model to deploy to | `string` | n/a | yes |
+| <a name="input_spark_charms"></a> [spark\_charms](#input\_spark\_charms) | Names of the Spark applications in the Spark Juju model. | `map(string)` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_application"></a> [application](#output\_application) | The deployed application name. |
+| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Relation endpoint provided by this module. |
diff --git a/releases/terraform/charms/azure-storage-integrator/applications.tf b/releases/terraform/charms/azure-storage-integrator/applications.tf
new file mode 100644
index 00000000..64d1485a
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/applications.tf
@@ -0,0 +1,21 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "azure_storage" {
+  name  = "azure-storage"
+  model_uuid = data.juju_model.spark.id
+  charm {
+    name     = "azure-storage-integrator"
+    channel  = "latest/edge"
+    revision = var.azure_storage_revision
+  }
+  config = {
+    container           = var.azure_storage.container
+    storage-account     = var.azure_storage.storage_account
+    path                = var.azure_storage.path
+    connection-protocol = var.azure_storage.protocol
+    credentials         = "secret:${juju_secret.azure_blob_storage_secret.secret_id}"
+  }
+  units       = 1
+  constraints = "arch=amd64"
+}
diff --git a/releases/terraform/charms/azure-storage-integrator/integrations.tf b/releases/terraform/charms/azure-storage-integrator/integrations.tf
new file mode 100644
index 00000000..b008ad80
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/integrations.tf
@@ -0,0 +1,30 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_integration" "history_server_azure_storage" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.azure_storage.name
+    endpoint = "azure-storage-credentials"
+  }
+
+  application {
+    name     = var.spark_charms.history_server
+    endpoint = "azure-storage-credentials"
+  }
+}
+
+resource "juju_integration" "azure_storage_integration_hub" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.azure_storage.name
+    endpoint = "azure-storage-credentials"
+  }
+
+  application {
+    name     = var.spark_charms.integration_hub
+    endpoint = "azure-storage-credentials"
+  }
+}
diff --git a/releases/terraform/charms/azure-storage-integrator/locals.tf b/releases/terraform/charms/azure-storage-integrator/locals.tf
new file mode 100644
index 00000000..44f3a0f6
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/locals.tf
@@ -0,0 +1,2 @@
+# Local values for $(basename "$moduleDir").
+# Define computed values used in this module.
diff --git a/releases/terraform/charms/azure-storage-integrator/main.tf b/releases/terraform/charms/azure-storage-integrator/main.tf
new file mode 100644
index 00000000..46aca146
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/main.tf
@@ -0,0 +1,2 @@
+# Main Terraform code for $(basename "$moduleDir").
+# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/charms/azure-storage-integrator/outputs.tf b/releases/terraform/charms/azure-storage-integrator/outputs.tf
new file mode 100644
index 00000000..0a4eec05
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/outputs.tf
@@ -0,0 +1,21 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "charms" {
+  description = "The name of the charms which are part of the deployment."
+  value = {
+    azure_storage = juju_application.azure_storage.name
+  }
+}
+
+output "application" {
+  description = "The deployed application name."
+  value       = juju_application.azure_storage.name
+}
+
+output "provides" {
+  description = "Relation endpoint provided by this module."
+  value = {
+    "azure-storage-credentials" = juju_application.azure_storage.name
+  }
+}
diff --git a/releases/terraform/charms/azure-storage-integrator/providers.tf b/releases/terraform/charms/azure-storage-integrator/providers.tf
new file mode 100644
index 00000000..0693f27b
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/providers.tf
@@ -0,0 +1,6 @@
+# Provider configuration for module $(basename "$moduleDir").
+# Terraform block moved to terraform.tf and should not be in providers.tf.
+
+# provider "juju" {
+#   # Configure provider here if required by the module.
+# }
diff --git a/releases/terraform/charms/azure-storage-integrator/resources.tf b/releases/terraform/charms/azure-storage-integrator/resources.tf
new file mode 100644
index 00000000..e342298a
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/resources.tf
@@ -0,0 +1,30 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define juju resources (model, secrets, storage)
+
+locals {
+  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
+}
+
+data "juju_model" "spark" {
+  name = var.model
+}
+
+resource "juju_secret" "azure_blob_storage_secret" {
+  model_uuid = local.target_model_uuid
+  name  = "azure_blob_storage_secret"
+  value = {
+    secret-key = var.azure_storage.secret_key
+  }
+  info = "This is the secret key for the Azure storage account"
+}
+
+resource "juju_access_secret" "azure_blob_storage_secret_access" {
+  model_uuid = local.target_model_uuid
+  applications = [
+    juju_application.azure_storage.name
+  ]
+  # Use the secret_id from your secret resource or data source.
+  secret_id = juju_secret.azure_blob_storage_secret.secret_id
+}
diff --git a/releases/terraform/charms/azure-storage-integrator/terraform.tf b/releases/terraform/charms/azure-storage-integrator/terraform.tf
new file mode 100644
index 00000000..6412125a
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/charms/azure-storage-integrator/variables.tf b/releases/terraform/charms/azure-storage-integrator/variables.tf
new file mode 100644
index 00000000..04072b59
--- /dev/null
+++ b/releases/terraform/charms/azure-storage-integrator/variables.tf
@@ -0,0 +1,37 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "azure_storage" {
+  description = "Azure Object storage information"
+  type = object({
+    container       = optional(string, "azurecontainer")
+    storage_account = optional(string, "azurestorageaccount")
+    protocol        = optional(string, "abfss")
+    secret_key      = optional(string, "secret-key")
+    path            = optional(string, "spark-events")
+  })
+  default = {}
+}
+
+variable "azure_storage_revision" {
+  description = "Charm revision for azure-storage-integrator"
+  type        = number
+  nullable    = false
+}
+
+variable "model" {
+  description = "Name of the Spark Juju Model to deploy to (fallback if model_uuid is not provided)."
+  type        = string
+  nullable    = false
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy to."
+  type        = string
+  default     = null
+}
+
+variable "spark_charms" {
+  description = "Names of the Spark applications in the Spark Juju model."
+  type        = map(string)
+}
diff --git a/releases/terraform/charms/manual-tls/README.md b/releases/terraform/charms/manual-tls/README.md
new file mode 100644
index 00000000..bf0eb31e
--- /dev/null
+++ b/releases/terraform/charms/manual-tls/README.md
@@ -0,0 +1,41 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 >=1.0.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_application.certificates](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_app_name"></a> [app\_name](#input\_app\_name) | n/a | `string` | `"certificates"` | no |
+| <a name="input_channel"></a> [channel](#input\_channel) | n/a | `string` | `"latest/stable"` | no |
+| <a name="input_example"></a> [example](#input\_example) | Example variable | `string` | `""` | no |
+| <a name="input_model"></a> [model](#input\_model) | n/a | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_app_name"></a> [app\_name](#output\_app\_name) | Name of the deployed application. |
+| <a name="output_example"></a> [example](#output\_example) | Example output |
+| <a name="output_provides"></a> [provides](#output\_provides) | n/a |
diff --git a/releases/terraform/charms/manual-tls/locals.tf b/releases/terraform/charms/manual-tls/locals.tf
new file mode 100644
index 00000000..44f3a0f6
--- /dev/null
+++ b/releases/terraform/charms/manual-tls/locals.tf
@@ -0,0 +1,2 @@
+# Local values for $(basename "$moduleDir").
+# Define computed values used in this module.
diff --git a/releases/terraform/charms/manual-tls/main.tf b/releases/terraform/charms/manual-tls/main.tf
new file mode 100644
index 00000000..b6dff313
--- /dev/null
+++ b/releases/terraform/charms/manual-tls/main.tf
@@ -0,0 +1,91 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# This module is but a placeholder until we get an official TF module for the manual TLS operator
+# For testing purpose.
+
+variable "model" {
+  description = "Target Juju model name (fallback if model_uuid is not provided)."
+  type        = string
+  default     = "spark"
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy to."
+  type        = string
+  default     = null
+}
+
+variable "channel" {
+  description = "Charm channel."
+  type        = string
+  default     = "latest/stable"
+}
+
+variable "app_name" {
+  description = "Name of the deployed application."
+  type        = string
+  default     = "certificates"
+}
+
+variable "config" {
+  description = "Charm configuration map."
+  type        = map(string)
+  default     = {}
+}
+
+variable "constraints" {
+  description = "Juju constraints."
+  type        = string
+  default     = "arch=amd64"
+}
+
+variable "revision" {
+  description = "Charm revision number."
+  type        = number
+  default     = null
+}
+
+variable "units" {
+  description = "Number of units."
+  type        = number
+  default     = 1
+}
+
+locals {
+  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.target[0].id
+}
+
+data "juju_model" "target" {
+  count = var.model_uuid == null ? 1 : 0
+  name  = var.model
+}
+
+resource "juju_application" "certificates" {
+  name       = var.app_name
+  model_uuid = local.target_model_uuid
+
+  charm {
+    name     = "manual-tls-certificates"
+    channel  = var.channel
+    revision = var.revision
+  }
+
+  config      = var.config
+  constraints = var.constraints
+  units       = var.units
+  trust       = true
+}
+
+output "application" {
+  description = "Object representing the deployed application."
+  value = {
+    name = juju_application.certificates.name
+  }
+}
+
+output "provides" {
+  value = {
+    certificates = juju_application.certificates.name
+  }
+}
diff --git a/releases/terraform/charms/manual-tls/outputs.tf b/releases/terraform/charms/manual-tls/outputs.tf
new file mode 100644
index 00000000..8af3a313
--- /dev/null
+++ b/releases/terraform/charms/manual-tls/outputs.tf
@@ -0,0 +1,16 @@
+# Outputs for $(basename "$moduleDir").
+
+output "application" {
+  description = "Object representing the deployed application."
+  value = {
+    name = juju_application.certificates.name
+  }
+}
+
+output "provides" {
+  description = "Relation endpoints provided by the manual-tls charm."
+  value = {
+    certificates = juju_application.certificates.name
+  }
+}
+
diff --git a/releases/terraform/charms/manual-tls/providers.tf b/releases/terraform/charms/manual-tls/providers.tf
new file mode 100644
index 00000000..2ae9c8a3
--- /dev/null
+++ b/releases/terraform/charms/manual-tls/providers.tf
@@ -0,0 +1,6 @@
+# Provider configuration for module $(basename "$moduleDir").
+# Root module should configure provider credentials and connection.
+
+# provider "juju" {
+#   # Configure provider here if required by the module.
+# }
diff --git a/releases/terraform/charms/manual-tls/terraform.tf b/releases/terraform/charms/manual-tls/terraform.tf
new file mode 100644
index 00000000..6412125a
--- /dev/null
+++ b/releases/terraform/charms/manual-tls/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/charms/manual-tls/variables.tf b/releases/terraform/charms/manual-tls/variables.tf
new file mode 100644
index 00000000..a9d09057
--- /dev/null
+++ b/releases/terraform/charms/manual-tls/variables.tf
@@ -0,0 +1,50 @@
+# Variables for $(basename "$moduleDir").
+
+variable "app_name" {
+  description = "Name to give the deployed application."
+  type        = string
+  default     = "certificates"
+}
+
+variable "channel" {
+  description = "Channel for the charm."
+  type        = string
+  default     = "latest/stable"
+}
+
+variable "config" {
+  description = "Application config for the charm."
+  type        = map(string)
+  default     = {}
+}
+
+variable "constraints" {
+  description = "Juju constraints for the charm deployment."
+  type        = string
+  default     = "arch=amd64"
+}
+
+variable "model" {
+  description = "Target Juju model name (fallback if model_uuid is not provided)."
+  type        = string
+  default     = "spark"
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy the charm into."
+  type        = string
+  default     = null
+}
+
+variable "revision" {
+  description = "Charm revision number (null means latest)."
+  type        = number
+  default     = null
+}
+
+variable "units" {
+  description = "Number of units for the application."
+  type        = number
+  default     = 1
+}
+
diff --git a/releases/terraform/charms/s3-integrator/README.md b/releases/terraform/charms/s3-integrator/README.md
new file mode 100644
index 00000000..dbd80b99
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/README.md
@@ -0,0 +1,42 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_application.s3](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.s3_history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.s3_integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_model"></a> [model](#input\_model) | Name of the Spark Juju Model to deploy to | `string` | n/a | yes |
+| <a name="input_s3"></a> [s3](#input\_s3) | S3 Bucket information | <pre>object({<br/>    bucket   = optional(string, "spark-bucket")<br/>    endpoint = optional(string, "https://s3.amazonaws.com")<br/>    region   = optional(string, "us-east-1")<br/>  })</pre> | <pre>{<br/>  "bucket": "spark-bucket",<br/>  "endpoint": "https://s3.amazonaws.com",<br/>  "region": "us-east-1"<br/>}</pre> | no |
+| <a name="input_s3_revision"></a> [s3\_revision](#input\_s3\_revision) | Charm revision for s3-integrator | `number` | n/a | yes |
+| <a name="input_spark_charms"></a> [spark\_charms](#input\_spark\_charms) | Names of the Spark applications in the Spark Juju model. | `map(string)` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_application"></a> [application](#output\_application) | The deployed application name. |
+| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Relation endpoint provided by this module. |
diff --git a/releases/terraform/charms/s3-integrator/applications.tf b/releases/terraform/charms/s3-integrator/applications.tf
new file mode 100644
index 00000000..e56ec9eb
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/applications.tf
@@ -0,0 +1,20 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "s3" {
+  name  = "s3"
+  model_uuid = data.juju_model.spark.id
+  charm {
+    name     = "s3-integrator"
+    channel  = "1/stable"
+    revision = var.s3_revision
+  }
+  config = {
+    path     = "spark-events"
+    bucket   = var.s3.bucket
+    endpoint = var.s3.endpoint
+    region   = var.s3.region
+  }
+  units       = 1
+  constraints = "arch=amd64"
+}
diff --git a/releases/terraform/charms/s3-integrator/integrations.tf b/releases/terraform/charms/s3-integrator/integrations.tf
new file mode 100644
index 00000000..c4bda8b4
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/integrations.tf
@@ -0,0 +1,30 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_integration" "s3_history_server" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.s3.name
+    endpoint = "s3-credentials"
+  }
+
+  application {
+    name     = var.spark_charms.history_server
+    endpoint = "s3-credentials"
+  }
+}
+
+resource "juju_integration" "s3_integration_hub" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.s3.name
+    endpoint = "s3-credentials"
+  }
+
+  application {
+    name     = var.spark_charms.integration_hub
+    endpoint = "s3-credentials"
+  }
+}
diff --git a/releases/terraform/charms/s3-integrator/locals.tf b/releases/terraform/charms/s3-integrator/locals.tf
new file mode 100644
index 00000000..44f3a0f6
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/locals.tf
@@ -0,0 +1,2 @@
+# Local values for $(basename "$moduleDir").
+# Define computed values used in this module.
diff --git a/releases/terraform/charms/s3-integrator/main.tf b/releases/terraform/charms/s3-integrator/main.tf
new file mode 100644
index 00000000..46aca146
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/main.tf
@@ -0,0 +1,2 @@
+# Main Terraform code for $(basename "$moduleDir").
+# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/charms/s3-integrator/outputs.tf b/releases/terraform/charms/s3-integrator/outputs.tf
new file mode 100644
index 00000000..05253e4f
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/outputs.tf
@@ -0,0 +1,21 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "charms" {
+  description = "The name of the charms which are part of the deployment."
+  value = {
+    s3 = juju_application.s3.name
+  }
+}
+
+output "application" {
+  description = "The deployed application name."
+  value       = juju_application.s3.name
+}
+
+output "provides" {
+  description = "Relation endpoint provided by this module."
+  value = {
+    "s3-credentials" = juju_application.s3.name
+  }
+}
diff --git a/releases/terraform/charms/s3-integrator/providers.tf b/releases/terraform/charms/s3-integrator/providers.tf
new file mode 100644
index 00000000..0693f27b
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/providers.tf
@@ -0,0 +1,6 @@
+# Provider configuration for module $(basename "$moduleDir").
+# Terraform block moved to terraform.tf and should not be in providers.tf.
+
+# provider "juju" {
+#   # Configure provider here if required by the module.
+# }
diff --git a/releases/terraform/charms/s3-integrator/resources.tf b/releases/terraform/charms/s3-integrator/resources.tf
new file mode 100644
index 00000000..0ddad034
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/resources.tf
@@ -0,0 +1,12 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define juju resources (model, secrets, storage)
+
+locals {
+  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
+}
+
+data "juju_model" "spark" {
+  name = var.model
+}
diff --git a/releases/terraform/charms/s3-integrator/terraform.tf b/releases/terraform/charms/s3-integrator/terraform.tf
new file mode 100644
index 00000000..6412125a
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/charms/s3-integrator/variables.tf b/releases/terraform/charms/s3-integrator/variables.tf
new file mode 100644
index 00000000..31f13080
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator/variables.tf
@@ -0,0 +1,39 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "model" {
+  description = "Name of the Spark Juju Model to deploy to (fallback if model_uuid is not provided)."
+  type        = string
+  nullable    = false
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy to."
+  type        = string
+  default     = null
+}
+
+variable "s3" {
+  description = "S3 Bucket information"
+  type = object({
+    bucket   = optional(string, "spark-bucket")
+    endpoint = optional(string, "https://s3.amazonaws.com")
+    region   = optional(string, "us-east-1")
+  })
+  default = {
+    bucket   = "spark-bucket"
+    endpoint = "https://s3.amazonaws.com"
+    region   = "us-east-1"
+  }
+}
+
+variable "s3_revision" {
+  description = "Charm revision for s3-integrator"
+  type        = number
+  nullable    = false
+}
+
+variable "spark_charms" {
+  description = "Names of the Spark applications in the Spark Juju model."
+  type        = map(string)
+}
diff --git a/releases/terraform/components/observability/README.md b/releases/terraform/components/observability/README.md
new file mode 100644
index 00000000..ac296091
--- /dev/null
+++ b/releases/terraform/components/observability/README.md
@@ -0,0 +1,67 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_application.cos_configuration](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.pushgateway](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.scrape_config](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.cos_configuration_grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana_agent_grafana_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana_agent_loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana_agent_prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.history_server_grafana_agent_dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.history_server_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.history_server_grafana_agent_metrics](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.integration_hub_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_grafana_agent_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_grafana_agent_metrics](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.pushgateway_integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.pushgateway_scrape_config](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.scrape_config_grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
+| [juju_offer.grafana_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
+| [juju_offer.loki_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
+| [juju_offer.prometheus_receive_remote_write](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_config_grafana_path"></a> [config\_grafana\_path](#input\_config\_grafana\_path) | Grafana dashboard path from configuration repo root. | `string` | `"releases/3.4/resources/grafana/"` | no |
+| <a name="input_config_repo"></a> [config\_repo](#input\_config\_repo) | COS configuration repo URL. | `string` | `"https://github.com/canonical/spark-k8s-bundle"` | no |
+| <a name="input_cos_configuration_revision"></a> [cos\_configuration\_revision](#input\_cos\_configuration\_revision) | Charm revision for cos-configuration-k8s | `number` | n/a | yes |
+| <a name="input_dashboards_offer"></a> [dashboards\_offer](#input\_dashboards\_offer) | URL of the `grafana_dashboard` interface offer. | `string` | n/a | yes |
+| <a name="input_grafana_agent_revision"></a> [grafana\_agent\_revision](#input\_grafana\_agent\_revision) | Charm revision for grafana-agent-k8s | `number` | n/a | yes |
+| <a name="input_logging_offer"></a> [logging\_offer](#input\_logging\_offer) | URL of the `loki_push_api` interface offer. | `string` | n/a | yes |
+| <a name="input_metrics_offer"></a> [metrics\_offer](#input\_metrics\_offer) | URL of the `prometheus_remote_write` interface offer. | `string` | n/a | yes |
+| <a name="input_pushgateway_revision"></a> [pushgateway\_revision](#input\_pushgateway\_revision) | Charm revision for prometheus-pushgateway-k8s | `number` | n/a | yes |
+| <a name="input_pushgateway_size"></a> [pushgateway\_size](#input\_pushgateway\_size) | Storage size for the pushgateway database | `string` | `"10G"` | no |
+| <a name="input_scrape_config_revision"></a> [scrape\_config\_revision](#input\_scrape\_config\_revision) | Charm revision for prometheus-scrape-config-k8s | `number` | n/a | yes |
+| <a name="input_spark_charms"></a> [spark\_charms](#input\_spark\_charms) | Names of the Spark applications in the Spark Juju model. | `map(string)` | n/a | yes |
+| <a name="input_spark_model"></a> [spark\_model](#input\_spark\_model) | Name of the Spark Juju model. | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
+| <a name="output_components"></a> [components](#output\_components) | List of the deployed applications for this component module. |
diff --git a/releases/terraform/components/observability/applications.tf b/releases/terraform/components/observability/applications.tf
new file mode 100644
index 00000000..f3188753
--- /dev/null
+++ b/releases/terraform/components/observability/applications.tf
@@ -0,0 +1,64 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "grafana_agent" {
+  name  = "grafana-agent"
+  model_uuid = local.target_model_uuid
+  charm {
+    name     = "grafana-agent-k8s"
+    channel  = "1/stable"
+    revision = var.grafana_agent_revision
+  }
+  units       = 1
+  trust       = true
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "cos_configuration" {
+  name  = "cos-configuration"
+  model_uuid = local.target_model_uuid
+  charm {
+    name     = "cos-configuration-k8s"
+    channel  = "1/stable"
+    revision = var.cos_configuration_revision
+  }
+  config = {
+    git_branch              = "main"
+    git_depth               = "1"
+    git_repo                = var.config_repo
+    grafana_dashboards_path = var.config_grafana_path
+  }
+  units       = 1
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "pushgateway" {
+  name  = "pushgateway"
+  model_uuid = local.target_model_uuid
+  charm {
+    name     = "prometheus-pushgateway-k8s"
+    channel  = "1/stable"
+    revision = var.pushgateway_revision
+  }
+  units       = 1
+  constraints = "arch=amd64"
+  storage_directives = {
+    pushgateway-store = var.pushgateway_size
+  }
+}
+
+resource "juju_application" "scrape_config" {
+  name  = "scrape-config"
+  model_uuid = local.target_model_uuid
+  charm {
+    name     = "prometheus-scrape-config-k8s"
+    channel  = "1/stable"
+    revision = var.scrape_config_revision
+  }
+  config = {
+    scrape_interval = "10s"
+  }
+  units       = 1
+  constraints = "arch=amd64"
+}
+
diff --git a/releases/terraform/components/observability/integrations.tf b/releases/terraform/components/observability/integrations.tf
new file mode 100644
index 00000000..ea071768
--- /dev/null
+++ b/releases/terraform/components/observability/integrations.tf
@@ -0,0 +1,236 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+data "juju_offer" "grafana_dashboards" {
+  url = var.dashboards_offer
+}
+
+data "juju_offer" "prometheus_receive_remote_write" {
+  url = var.metrics_offer
+}
+
+data "juju_offer" "loki_logging" {
+  url = var.logging_offer
+}
+
+resource "juju_integration" "cos_configuration_grafana_agent" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.cos_configuration.name
+    endpoint = "grafana-dashboards"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "grafana-dashboards-consumer"
+  }
+}
+
+resource "juju_integration" "pushgateway_scrape_config" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.pushgateway.name
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.scrape_config.name
+    endpoint = "configurable-scrape-jobs"
+  }
+}
+
+resource "juju_integration" "scrape_config_grafana_agent" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.scrape_config.name
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+resource "juju_integration" "grafana_agent_grafana_dashboards" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "grafana-dashboards-provider"
+  }
+
+  application {
+    offer_url = data.juju_offer.grafana_dashboards.url
+  }
+
+  lifecycle {
+    replace_triggered_by = [
+      juju_application.grafana_agent.name,
+      juju_application.grafana_agent.model,
+      juju_application.grafana_agent.constraints,
+      juju_application.grafana_agent.placement,
+      juju_application.grafana_agent.charm.name,
+    ]
+  }
+}
+
+resource "juju_integration" "grafana_agent_prometheus" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "send-remote-write"
+  }
+
+  application {
+    offer_url = data.juju_offer.prometheus_receive_remote_write.url
+  }
+
+  lifecycle {
+    replace_triggered_by = [
+      juju_application.grafana_agent.name,
+      juju_application.grafana_agent.model,
+      juju_application.grafana_agent.constraints,
+      juju_application.grafana_agent.placement,
+      juju_application.grafana_agent.charm.name,
+    ]
+  }
+}
+
+resource "juju_integration" "grafana_agent_loki" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "logging-consumer"
+  }
+
+  application {
+    offer_url = data.juju_offer.loki_logging.url
+  }
+
+  lifecycle {
+    replace_triggered_by = [
+      juju_application.grafana_agent.name,
+      juju_application.grafana_agent.model,
+      juju_application.grafana_agent.constraints,
+      juju_application.grafana_agent.placement,
+      juju_application.grafana_agent.charm.name,
+    ]
+  }
+}
+resource "juju_integration" "pushgateway_integration_hub" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.pushgateway.name
+    endpoint = "push-endpoint"
+  }
+
+  application {
+    name     = var.spark_charms.integration_hub
+    endpoint = "cos"
+  }
+}
+
+resource "juju_integration" "history_server_grafana_agent_dashboard" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = var.spark_charms.history_server
+    endpoint = "grafana-dashboard"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "grafana-dashboards-consumer"
+  }
+}
+
+resource "juju_integration" "history_server_grafana_agent_logging" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = var.spark_charms.history_server
+    endpoint = "logging"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "logging-provider"
+  }
+}
+
+resource "juju_integration" "history_server_grafana_agent_metrics" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = var.spark_charms.history_server
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+resource "juju_integration" "kyuubi_grafana_agent_metrics" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = var.spark_charms.kyuubi
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+resource "juju_integration" "kyuubi_grafana_agent_dashboards" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = var.spark_charms.kyuubi
+    endpoint = "grafana-dashboard"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "grafana-dashboards-consumer"
+  }
+}
+
+resource "juju_integration" "kyuubi_grafana_agent_logging" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = var.spark_charms.kyuubi
+    endpoint = "logging"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "logging-provider"
+  }
+}
+
+resource "juju_integration" "integration_hub_grafana_agent_logging" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = var.spark_charms.integration_hub
+    endpoint = "logging"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "logging-provider"
+  }
+}
diff --git a/releases/terraform/components/observability/locals.tf b/releases/terraform/components/observability/locals.tf
new file mode 100644
index 00000000..44f3a0f6
--- /dev/null
+++ b/releases/terraform/components/observability/locals.tf
@@ -0,0 +1,2 @@
+# Local values for $(basename "$moduleDir").
+# Define computed values used in this module.
diff --git a/releases/terraform/components/observability/main.tf b/releases/terraform/components/observability/main.tf
new file mode 100644
index 00000000..46aca146
--- /dev/null
+++ b/releases/terraform/components/observability/main.tf
@@ -0,0 +1,2 @@
+# Main Terraform code for $(basename "$moduleDir").
+# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/components/observability/outputs.tf b/releases/terraform/components/observability/outputs.tf
new file mode 100644
index 00000000..f98fc3a0
--- /dev/null
+++ b/releases/terraform/components/observability/outputs.tf
@@ -0,0 +1,21 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "charms" {
+  description = "The name of the charms which are part of the deployment."
+  value = {
+    grafana_agent     = juju_application.grafana_agent.name
+    cos_configuration = juju_application.cos_configuration.name
+    pushgateway       = juju_application.pushgateway.name
+    scrape_config     = juju_application.scrape_config.name
+  }
+}
+output "components" {
+  description = "List of the deployed applications for this component module."
+  value = [
+    { name = juju_application.grafana_agent.name, type = "grafana_agent" },
+    { name = juju_application.cos_configuration.name, type = "cos_configuration" },
+    { name = juju_application.pushgateway.name, type = "pushgateway" },
+    { name = juju_application.scrape_config.name, type = "scrape_config" },
+  ]
+}
diff --git a/releases/terraform/components/observability/providers.tf b/releases/terraform/components/observability/providers.tf
new file mode 100644
index 00000000..0693f27b
--- /dev/null
+++ b/releases/terraform/components/observability/providers.tf
@@ -0,0 +1,6 @@
+# Provider configuration for module $(basename "$moduleDir").
+# Terraform block moved to terraform.tf and should not be in providers.tf.
+
+# provider "juju" {
+#   # Configure provider here if required by the module.
+# }
diff --git a/releases/terraform/components/observability/resources.tf b/releases/terraform/components/observability/resources.tf
new file mode 100644
index 00000000..765d64b8
--- /dev/null
+++ b/releases/terraform/components/observability/resources.tf
@@ -0,0 +1,12 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define juju resources (model, secrets, storage)
+
+locals {
+  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
+}
+
+data "juju_model" "spark" {
+  name = var.spark_model
+}
diff --git a/releases/terraform/components/observability/terraform.tf b/releases/terraform/components/observability/terraform.tf
new file mode 100644
index 00000000..6412125a
--- /dev/null
+++ b/releases/terraform/components/observability/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/components/observability/variables.tf b/releases/terraform/components/observability/variables.tf
new file mode 100644
index 00000000..94aaef5b
--- /dev/null
+++ b/releases/terraform/components/observability/variables.tf
@@ -0,0 +1,81 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "config_grafana_path" {
+  description = "Grafana dashboard path from configuration repo root."
+  type        = string
+  default     = "releases/3.4/resources/grafana/"
+  nullable    = false
+}
+
+variable "config_repo" {
+  description = "COS configuration repo URL."
+  type        = string
+  default     = "https://github.com/canonical/spark-k8s-bundle"
+  nullable    = false
+}
+
+variable "cos_configuration_revision" {
+  description = "Charm revision for cos-configuration-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "dashboards_offer" {
+  description = "URL of the `grafana_dashboard` interface offer."
+  type        = string
+  nullable    = false
+}
+
+variable "grafana_agent_revision" {
+  description = "Charm revision for grafana-agent-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "logging_offer" {
+  description = "URL of the `loki_push_api` interface offer."
+  type        = string
+  nullable    = false
+}
+
+variable "metrics_offer" {
+  description = "URL of the `prometheus_remote_write` interface offer."
+  type        = string
+  nullable    = false
+}
+
+variable "pushgateway_revision" {
+  description = "Charm revision for prometheus-pushgateway-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "pushgateway_size" {
+  description = "Storage size for the pushgateway database"
+  type        = string
+  default     = "10G"
+}
+
+variable "scrape_config_revision" {
+  description = "Charm revision for prometheus-scrape-config-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "spark_charms" {
+  description = "Names of the Spark applications in the Spark Juju model."
+  type        = map(string)
+}
+
+variable "spark_model" {
+  description = "Name of the Spark Juju model."
+  type        = string
+  nullable    = false
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy to."
+  type        = string
+  default     = null
+}
diff --git a/releases/terraform/components/spark-3.4/README.md b/releases/terraform/components/spark-3.4/README.md
new file mode 100644
index 00000000..6de53b81
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/README.md
@@ -0,0 +1,89 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=0.20.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | >=0.20.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
+| [juju_application.data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.kyuubi](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.kyuubi_data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_tls](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_offer.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_offer.metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
+| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
+| <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | n/a | yes |
+| <a name="input_driver_pod_template"></a> [driver\_pod\_template](#input\_driver\_pod\_template) | Define K8s driver pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
+| <a name="input_enable_dynamic_allocation"></a> [enable\_dynamic\_allocation](#input\_enable\_dynamic\_allocation) | Enable dynamic allocation of pods for Spark jobs. | `bool` | `false` | no |
+| <a name="input_executor_pod_template"></a> [executor\_pod\_template](#input\_executor\_pod\_template) | Define K8s executor pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
+| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `map(string)` | n/a | yes |
+| <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | n/a | yes |
+| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | n/a | yes |
+| <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
+| <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | n/a | yes |
+| <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
+| <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
+| <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
+| <a name="input_kyuubi_executor_pod_template"></a> [kyuubi\_executor\_pod\_template](#input\_kyuubi\_executor\_pod\_template) | Define K8s executor pod from a file accessible in the object storage. | `string` | `null` | no |
+| <a name="input_kyuubi_gpu_enable"></a> [kyuubi\_gpu\_enable](#input\_kyuubi\_gpu\_enable) | Enable GPU acceleration for SparkSQLEngine. | `bool` | n/a | yes |
+| <a name="input_kyuubi_gpu_engine_executors_limit"></a> [kyuubi\_gpu\_engine\_executors\_limit](#input\_kyuubi\_gpu\_engine\_executors\_limit) | Limit the number of GPUs an engine can schedule executor pods on. | `number` | n/a | yes |
+| <a name="input_kyuubi_gpu_pinned_memory"></a> [kyuubi\_gpu\_pinned\_memory](#input\_kyuubi\_gpu\_pinned\_memory) | Set the host memory (in GB) per executor reserved for fast data transfer on GPUs. | `number` | n/a | yes |
+| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `map(string)` | n/a | yes |
+| <a name="input_kyuubi_k8s_node_selectors"></a> [kyuubi\_k8s\_node\_selectors](#input\_kyuubi\_k8s\_node\_selectors) | Comma separated label:value selectors for K8s pods in Kyuubi. | `string` | `null` | no |
+| <a name="input_kyuubi_loadbalancer_extra_annotations"></a> [kyuubi\_loadbalancer\_extra\_annotations](#input\_kyuubi\_loadbalancer\_extra\_annotations) | Optional extra annotations to be supplied to the load balancer service in Kyuubi. | `string` | `null` | no |
+| <a name="input_kyuubi_profile"></a> [kyuubi\_profile](#input\_kyuubi\_profile) | The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'. | `string` | `"production"` | no |
+| <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | n/a | yes |
+| <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
+| <a name="input_kyuubi_user"></a> [kyuubi\_user](#input\_kyuubi\_user) | User name to be used for running Kyuubi engines. | `string` | `"kyuubi-spark-engine"` | no |
+| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | n/a | yes |
+| <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | n/a | yes |
+| <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
+| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | n/a | yes |
+| <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | n/a | yes |
+| <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
+| <a name="input_model"></a> [model](#input\_model) | Name of the Juju Model to deploy to. | `string` | n/a | yes |
+| <a name="input_tls_app_name"></a> [tls\_app\_name](#input\_tls\_app\_name) | Name of the application providing the `tls-certificates` interface. | `string` | n/a | yes |
+| <a name="input_tls_certificates_endpoint"></a> [tls\_certificates\_endpoint](#input\_tls\_certificates\_endpoint) | Name of the endpoint providing the `tls-certificates` interface. | `string` | n/a | yes |
+| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
+| <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | n/a | yes |
+| <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | n/a | yes |
+| <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the zookeeper unit | `string` | `"10G"` | no |
+| <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
+| <a name="output_components"></a> [components](#output\_components) | List of the deployed applications for this component module. |
+| <a name="output_offers"></a> [offers](#output\_offers) | n/a |
diff --git a/releases/terraform/components/spark-3.4/applications.tf b/releases/terraform/components/spark-3.4/applications.tf
new file mode 100644
index 00000000..3943ad1c
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/applications.tf
@@ -0,0 +1,190 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "history_server" {
+  name  = "history-server"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "spark-history-server-k8s"
+    channel  = "3/stable"
+    revision = var.history_server_revision
+  }
+
+  resources = var.history_server_image
+  units     = 1
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "kyuubi" {
+  name  = "kyuubi"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "kyuubi-k8s"
+    channel  = "3.5/stable"
+    revision = var.kyuubi_revision
+  }
+
+  resources = var.kyuubi_image
+
+  config = merge(
+    {
+      enable-dynamic-allocation  = var.enable_dynamic_allocation
+      expose-external            = "loadbalancer"
+      gpu-enable                 = var.kyuubi_gpu_enable
+      gpu-engine-executors-limit = var.kyuubi_gpu_engine_executors_limit
+      gpu-pinned-memory          = var.kyuubi_gpu_pinned_memory
+      namespace                  = data.juju_model.spark.name
+      profile                    = var.kyuubi_profile
+      service-account            = var.kyuubi_user
+    },
+    var.kyuubi_k8s_node_selectors == null ? {} : {
+      k8s-node-selectors = var.kyuubi_k8s_node_selectors
+    },
+    var.kyuubi_loadbalancer_extra_annotations == null ? {} : {
+      loadbalancer-extra-annotations = var.kyuubi_loadbalancer_extra_annotations
+    },
+    var.tls_private_key == null ? {} : {
+      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+    },
+    var.admin_password == null ? {} : {
+      system-users = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+    },
+    var.kyuubi_executor_cores == null ? {} : {
+      executor-cores = var.kyuubi_executor_cores
+    },
+    var.kyuubi_executor_memory == null ? {} : {
+      executor-memory = var.kyuubi_executor_memory
+    },
+    var.kyuubi_driver_pod_template == null ? {} : {
+      driver-pod-template = var.kyuubi_driver_pod_template
+    },
+    var.kyuubi_executor_pod_template == null ? {} : {
+      executor-pod-template = var.kyuubi_executor_pod_template
+    }
+  )
+
+  units = var.kyuubi_units
+  trust = true
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "kyuubi_users" {
+  name  = "kyuubi-users"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "postgresql-k8s"
+    channel  = "14/stable"
+    revision = var.kyuubi_users_revision
+  }
+
+  storage_directives = {
+    pgdata = var.kyuubi_users_size
+  }
+
+  resources = var.kyuubi_users_image
+
+  units = 1
+  trust = true
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "metastore" {
+  name  = "metastore"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "postgresql-k8s"
+    channel  = "14/stable"
+    revision = var.metastore_revision
+  }
+
+  storage_directives = {
+    pgdata = var.kyuubi_users_size
+  }
+
+
+  resources = var.metastore_image
+
+  units = 1
+  trust = true
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "integration_hub" {
+  name  = "integration-hub"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "spark-integration-hub-k8s"
+    channel  = "3/stable"
+    revision = var.integration_hub_revision
+  }
+
+  config = merge(
+    {
+      enable-dynamic-allocation = var.enable_dynamic_allocation
+    },
+    var.driver_pod_template == null ? {} : {
+      driver-pod-template = var.driver_pod_template
+    },
+    var.executor_pod_template == null ? {} : {
+      executor-pod-template = var.executor_pod_template
+    },
+    var.integration_hub_monitored_service_accounts == null ? {} : {
+      monitored-service-accounts = var.integration_hub_monitored_service_accounts
+    }
+  )
+
+  resources = var.integration_hub_image
+
+  units = 1
+  trust = true
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "zookeeper" {
+  name  = "zookeeper"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "zookeeper-k8s"
+    channel  = "3/stable"
+    revision = var.zookeeper_revision
+  }
+
+  storage_directives = {
+    zookeeper = var.zookeeper_size
+  }
+
+
+  resources = var.zookeeper_image
+
+  units       = var.zookeeper_units
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "data_integrator" {
+  name  = "data-integrator"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "data-integrator"
+    channel  = "latest/stable"
+    revision = var.data_integrator_revision
+  }
+
+  config = {
+    database-name = "integrator"
+  }
+
+  units       = 1
+  constraints = "arch=amd64"
+}
diff --git a/releases/terraform/components/spark-3.4/integrations.tf b/releases/terraform/components/spark-3.4/integrations.tf
new file mode 100644
index 00000000..8031e0d5
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/integrations.tf
@@ -0,0 +1,86 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_integration" "kyuubi_metastore" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.metastore.name
+    endpoint = "database"
+  }
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "metastore-db"
+  }
+}
+
+resource "juju_integration" "kyuubi_users" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.kyuubi_users.name
+    endpoint = "database"
+  }
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "auth-db"
+  }
+}
+
+resource "juju_integration" "kyuubi_service_account" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "spark-service-account"
+  }
+
+  application {
+    name     = juju_application.integration_hub.name
+    endpoint = "spark-service-account"
+  }
+}
+
+resource "juju_integration" "kyuubi_zookeeper" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.zookeeper.name
+    endpoint = "zookeeper"
+  }
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "zookeeper"
+  }
+}
+
+resource "juju_integration" "kyuubi_tls" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "certificates"
+  }
+
+  application {
+    name     = var.tls_app_name
+    endpoint = var.tls_certificates_endpoint
+  }
+}
+
+resource "juju_integration" "kyuubi_data_integrator" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "jdbc"
+  }
+
+  application {
+    name     = juju_application.data_integrator.name
+    endpoint = "kyuubi"
+  }
+}
diff --git a/releases/terraform/components/spark-3.4/locals.tf b/releases/terraform/components/spark-3.4/locals.tf
new file mode 100644
index 00000000..22eaae95
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/locals.tf
@@ -0,0 +1,43 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+locals {
+  revisions = {
+    history_server  = 94
+    integration_hub = 113
+    kyuubi          = 140
+    kyuubi_users    = 495
+    metastore       = 495
+    zookeeper       = 78
+    data_integrator = 179
+    s3              = 145
+    azure_storage   = 15
+    grafana_agent   = 121
+    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
+    cos_configuration = 65
+    pushgateway       = 27
+    scrape_config     = 67
+  }
+  images = {
+    history_server = {
+      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
+    } # rev23, spark-version: 3.5.7, release date 2026-02-27
+    integration_hub = {
+      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
+    } # rev11, release date 2026-03-02
+    kyuubi = {
+      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e"
+    } # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
+    kyuubi_users = {
+      postgresql-image = 165
+    }
+    metastore = {
+      postgresql-image = 165
+    }
+    zookeeper = {
+      zookeeper-image = 34
+    }
+  }
+}
diff --git a/releases/terraform/components/spark-3.4/main.tf b/releases/terraform/components/spark-3.4/main.tf
new file mode 100644
index 00000000..46aca146
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/main.tf
@@ -0,0 +1,2 @@
+# Main Terraform code for $(basename "$moduleDir").
+# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/components/spark-3.4/offers.tf b/releases/terraform/components/spark-3.4/offers.tf
new file mode 100644
index 00000000..46f9b8dc
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/offers.tf
@@ -0,0 +1,11 @@
+resource "juju_offer" "integration_hub" {
+  model_uuid = data.juju_model.spark.id
+  application_name = juju_application.integration_hub.name
+  endpoints        = ["spark-service-account"]
+}
+
+resource "juju_offer" "metastore" {
+  model_uuid = data.juju_model.spark.id
+  application_name = juju_application.metastore.name
+  endpoints        = ["database"]
+}
\ No newline at end of file
diff --git a/releases/terraform/components/spark-3.4/outputs.tf b/releases/terraform/components/spark-3.4/outputs.tf
new file mode 100644
index 00000000..01714e58
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/outputs.tf
@@ -0,0 +1,34 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "charms" {
+  description = "The name of the charms which are part of the deployment."
+  value = {
+    history_server  = juju_application.history_server.name
+    kyuubi          = juju_application.kyuubi.name
+    kyuubi_users    = juju_application.kyuubi_users.name
+    metastore       = juju_application.metastore.name
+    integration_hub = juju_application.integration_hub.name
+    zookeeper       = juju_application.zookeeper.name
+    certificates    = var.tls_app_name
+  }
+}
+
+output "components" {
+  description = "List of the deployed applications for this component module."
+  value = [
+    { name = juju_application.history_server.name, type = "history_server" },
+    { name = juju_application.kyuubi.name, type = "kyuubi" },
+    { name = juju_application.kyuubi_users.name, type = "kyuubi_users" },
+    { name = juju_application.metastore.name, type = "metastore" },
+    { name = juju_application.integration_hub.name, type = "integration_hub" },
+    { name = juju_application.zookeeper.name, type = "zookeeper" },
+  ]
+}
+
+output "offers" {
+  value = {
+    hub_service_account = juju_offer.integration_hub
+    metastore_database  = juju_offer.metastore
+  }
+}
diff --git a/releases/terraform/components/spark-3.4/providers.tf b/releases/terraform/components/spark-3.4/providers.tf
new file mode 100644
index 00000000..0693f27b
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/providers.tf
@@ -0,0 +1,6 @@
+# Provider configuration for module $(basename "$moduleDir").
+# Terraform block moved to terraform.tf and should not be in providers.tf.
+
+# provider "juju" {
+#   # Configure provider here if required by the module.
+# }
diff --git a/releases/terraform/components/spark-3.4/resources.tf b/releases/terraform/components/spark-3.4/resources.tf
new file mode 100644
index 00000000..e25d743f
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/resources.tf
@@ -0,0 +1,36 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define juju resources (model, secrets, storage)
+
+locals {
+  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
+}
+
+data "juju_model" "spark" {
+  name = var.model
+}
+
+resource "juju_secret" "system_users_and_private_key_secret" {
+  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = local.target_model_uuid
+  name  = "system_users_and_private_key_secret"
+  value = merge(
+    var.admin_password == null ? {} : {
+      admin = var.admin_password
+    },
+    var.tls_private_key == null ? {} : {
+      private-key = var.tls_private_key
+    }
+  )
+  info = "This secret contains password for admin user and the TLS private key."
+}
+
+resource "juju_access_secret" "system_users_and_private_key_secret_access" {
+  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = local.target_model_uuid
+  applications = [
+    juju_application.kyuubi.name
+  ]
+  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
+}
diff --git a/releases/terraform/components/spark-3.4/terraform.tf b/releases/terraform/components/spark-3.4/terraform.tf
new file mode 100644
index 00000000..9a82f31b
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/terraform.tf
@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=0.20.0"
+    }
+  }
+}
diff --git a/releases/terraform/components/spark-3.4/variables.tf b/releases/terraform/components/spark-3.4/variables.tf
new file mode 100644
index 00000000..673a1de8
--- /dev/null
+++ b/releases/terraform/components/spark-3.4/variables.tf
@@ -0,0 +1,244 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define spark-specific variables
+
+variable "admin_password" {
+  description = "The password for the admin user."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "data_integrator_revision" {
+  description = "Charm revision for data-integrator"
+  type        = number
+  nullable    = false
+}
+
+variable "driver_pod_template" {
+  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
+  type        = string
+  default     = null
+}
+
+variable "enable_dynamic_allocation" {
+  description = "Enable dynamic allocation of pods for Spark jobs."
+  type        = bool
+  default     = false
+}
+
+variable "executor_pod_template" {
+  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
+  type        = string
+  default     = null
+}
+
+variable "history_server_image" {
+  description = "Image for spark-history-server-k8s"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "history_server_revision" {
+  description = "Charm revision for spark-history-server-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "integration_hub_image" {
+  description = "Image for spark-integration-hub-k8s"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "integration_hub_monitored_service_accounts" {
+  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
+  type        = string
+  default     = null
+}
+
+variable "integration_hub_revision" {
+  description = "Charm revision for spark-integration-hub-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_driver_pod_template" {
+  description = "Define K8s driver pod from a file accessible in the object storage."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_executor_cores" {
+  description = "Set kyuubi executor pods cpu cores."
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_executor_memory" {
+  description = "Set kyuubi executor pods memory (in GB)."
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_executor_pod_template" {
+  description = "Define K8s executor pod from a file accessible in the object storage."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_gpu_enable" {
+  description = "Enable GPU acceleration for SparkSQLEngine."
+  type        = bool
+  nullable    = false
+}
+
+variable "kyuubi_gpu_engine_executors_limit" {
+  description = "Limit the number of GPUs an engine can schedule executor pods on."
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_gpu_pinned_memory" {
+  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_image" {
+  description = "Image for kyuubi-k8s"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "kyuubi_k8s_node_selectors" {
+  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_loadbalancer_extra_annotations" {
+  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_profile" {
+  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
+  type        = string
+  nullable    = false
+  default     = "production"
+}
+
+variable "kyuubi_revision" {
+  description = "Charm revision for kyuubi-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_units" {
+  description = "Number of Kyuubi units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
+
+variable "kyuubi_user" {
+  description = "User name to be used for running Kyuubi engines."
+  type        = string
+  nullable    = false
+  default     = "kyuubi-spark-engine"
+}
+
+variable "kyuubi_users_image" {
+  description = "Image for postgresql-k8s (auth-db)"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "kyuubi_users_revision" {
+  description = "Charm revision for postgresql-k8s (auth-db)"
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_users_size" {
+  description = "Storage size for the Kyuubi users database"
+  type        = string
+  default     = "1G"
+}
+
+variable "metastore_image" {
+  description = "Image for postgresql-k8s (metastore)"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "metastore_revision" {
+  description = "Charm revision for postgresql-k8s (metastore)"
+  type        = number
+  nullable    = false
+}
+
+variable "metastore_size" {
+  description = "Storage size for the metastore database"
+  type        = string
+  default     = "10G"
+}
+
+variable "model" {
+  description = "Name of the Juju Model to deploy to (fallback if model_uuid is not provided)."
+  type        = string
+  nullable    = false
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy to."
+  type        = string
+  default     = null
+}
+
+variable "tls_app_name" {
+  description = "Name of the application providing the `tls-certificates` interface."
+  type        = string
+  nullable    = false
+}
+
+variable "tls_certificates_endpoint" {
+  description = "Name of the endpoint providing the `tls-certificates` interface."
+  type        = string
+  nullable    = false
+}
+
+variable "tls_private_key" {
+  description = "The private key to be used for TLS certificates."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "zookeeper_image" {
+  description = "Image for zookeeper-k8s"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "zookeeper_revision" {
+  description = "Charm revision for zookeeper-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "zookeeper_size" {
+  description = "Storage size for the zookeeper unit"
+  type        = string
+  default     = "10G"
+}
+
+variable "zookeeper_units" {
+  description = "Number of zookeeper units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
diff --git a/releases/terraform/components/spark-3.5/README.md b/releases/terraform/components/spark-3.5/README.md
new file mode 100644
index 00000000..fe9f167f
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/README.md
@@ -0,0 +1,90 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
+| [juju_application.data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.kyuubi](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.kyuubi_data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_tls](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_offer.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_offer.metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
+| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
+| <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | n/a | yes |
+| <a name="input_driver_pod_template"></a> [driver\_pod\_template](#input\_driver\_pod\_template) | Define K8s driver pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
+| <a name="input_enable_dynamic_allocation"></a> [enable\_dynamic\_allocation](#input\_enable\_dynamic\_allocation) | Enable dynamic allocation of pods for Spark jobs. | `bool` | `false` | no |
+| <a name="input_executor_pod_template"></a> [executor\_pod\_template](#input\_executor\_pod\_template) | Define K8s executor pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
+| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `map(string)` | n/a | yes |
+| <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | n/a | yes |
+| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | n/a | yes |
+| <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
+| <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | n/a | yes |
+| <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
+| <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
+| <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
+| <a name="input_kyuubi_executor_pod_template"></a> [kyuubi\_executor\_pod\_template](#input\_kyuubi\_executor\_pod\_template) | Define K8s executor pod from a file accessible in the object storage. | `string` | `null` | no |
+| <a name="input_kyuubi_gpu_enable"></a> [kyuubi\_gpu\_enable](#input\_kyuubi\_gpu\_enable) | Enable GPU acceleration for SparkSQLEngine. | `bool` | n/a | yes |
+| <a name="input_kyuubi_gpu_engine_executors_limit"></a> [kyuubi\_gpu\_engine\_executors\_limit](#input\_kyuubi\_gpu\_engine\_executors\_limit) | Limit the number of GPUs an engine can schedule executor pods on. | `number` | n/a | yes |
+| <a name="input_kyuubi_gpu_pinned_memory"></a> [kyuubi\_gpu\_pinned\_memory](#input\_kyuubi\_gpu\_pinned\_memory) | Set the host memory (in GB) per executor reserved for fast data transfer on GPUs. | `number` | n/a | yes |
+| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `map(string)` | n/a | yes |
+| <a name="input_kyuubi_k8s_node_selectors"></a> [kyuubi\_k8s\_node\_selectors](#input\_kyuubi\_k8s\_node\_selectors) | Comma separated label:value selectors for K8s pods in Kyuubi. | `string` | `null` | no |
+| <a name="input_kyuubi_loadbalancer_extra_annotations"></a> [kyuubi\_loadbalancer\_extra\_annotations](#input\_kyuubi\_loadbalancer\_extra\_annotations) | Optional extra annotations to be supplied to the load balancer service in Kyuubi. | `string` | `null` | no |
+| <a name="input_kyuubi_profile"></a> [kyuubi\_profile](#input\_kyuubi\_profile) | The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'. | `string` | `"production"` | no |
+| <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | n/a | yes |
+| <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
+| <a name="input_kyuubi_user"></a> [kyuubi\_user](#input\_kyuubi\_user) | User name to be used for running Kyuubi engines. | `string` | `"kyuubi-spark-engine"` | no |
+| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | n/a | yes |
+| <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | n/a | yes |
+| <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
+| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | n/a | yes |
+| <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | n/a | yes |
+| <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
+| <a name="input_model"></a> [model](#input\_model) | Name of the Juju Model to deploy to. | `string` | n/a | yes |
+| <a name="input_tls_app_name"></a> [tls\_app\_name](#input\_tls\_app\_name) | Name of the application providing the `tls-certificates` interface. | `string` | n/a | yes |
+| <a name="input_tls_certificates_endpoint"></a> [tls\_certificates\_endpoint](#input\_tls\_certificates\_endpoint) | Name of the endpoint providing the `tls-certificates` interface. | `string` | n/a | yes |
+| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
+| <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | n/a | yes |
+| <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | n/a | yes |
+| <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the zookeeper unit | `string` | `"10G"` | no |
+| <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
+| <a name="output_components"></a> [components](#output\_components) | List of the deployed applications for this component module. |
+| <a name="output_offers"></a> [offers](#output\_offers) | n/a |
diff --git a/releases/terraform/components/spark-3.5/applications.tf b/releases/terraform/components/spark-3.5/applications.tf
new file mode 100644
index 00000000..3943ad1c
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/applications.tf
@@ -0,0 +1,190 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "history_server" {
+  name  = "history-server"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "spark-history-server-k8s"
+    channel  = "3/stable"
+    revision = var.history_server_revision
+  }
+
+  resources = var.history_server_image
+  units     = 1
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "kyuubi" {
+  name  = "kyuubi"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "kyuubi-k8s"
+    channel  = "3.5/stable"
+    revision = var.kyuubi_revision
+  }
+
+  resources = var.kyuubi_image
+
+  config = merge(
+    {
+      enable-dynamic-allocation  = var.enable_dynamic_allocation
+      expose-external            = "loadbalancer"
+      gpu-enable                 = var.kyuubi_gpu_enable
+      gpu-engine-executors-limit = var.kyuubi_gpu_engine_executors_limit
+      gpu-pinned-memory          = var.kyuubi_gpu_pinned_memory
+      namespace                  = data.juju_model.spark.name
+      profile                    = var.kyuubi_profile
+      service-account            = var.kyuubi_user
+    },
+    var.kyuubi_k8s_node_selectors == null ? {} : {
+      k8s-node-selectors = var.kyuubi_k8s_node_selectors
+    },
+    var.kyuubi_loadbalancer_extra_annotations == null ? {} : {
+      loadbalancer-extra-annotations = var.kyuubi_loadbalancer_extra_annotations
+    },
+    var.tls_private_key == null ? {} : {
+      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+    },
+    var.admin_password == null ? {} : {
+      system-users = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+    },
+    var.kyuubi_executor_cores == null ? {} : {
+      executor-cores = var.kyuubi_executor_cores
+    },
+    var.kyuubi_executor_memory == null ? {} : {
+      executor-memory = var.kyuubi_executor_memory
+    },
+    var.kyuubi_driver_pod_template == null ? {} : {
+      driver-pod-template = var.kyuubi_driver_pod_template
+    },
+    var.kyuubi_executor_pod_template == null ? {} : {
+      executor-pod-template = var.kyuubi_executor_pod_template
+    }
+  )
+
+  units = var.kyuubi_units
+  trust = true
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "kyuubi_users" {
+  name  = "kyuubi-users"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "postgresql-k8s"
+    channel  = "14/stable"
+    revision = var.kyuubi_users_revision
+  }
+
+  storage_directives = {
+    pgdata = var.kyuubi_users_size
+  }
+
+  resources = var.kyuubi_users_image
+
+  units = 1
+  trust = true
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "metastore" {
+  name  = "metastore"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "postgresql-k8s"
+    channel  = "14/stable"
+    revision = var.metastore_revision
+  }
+
+  storage_directives = {
+    pgdata = var.kyuubi_users_size
+  }
+
+
+  resources = var.metastore_image
+
+  units = 1
+  trust = true
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "integration_hub" {
+  name  = "integration-hub"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "spark-integration-hub-k8s"
+    channel  = "3/stable"
+    revision = var.integration_hub_revision
+  }
+
+  config = merge(
+    {
+      enable-dynamic-allocation = var.enable_dynamic_allocation
+    },
+    var.driver_pod_template == null ? {} : {
+      driver-pod-template = var.driver_pod_template
+    },
+    var.executor_pod_template == null ? {} : {
+      executor-pod-template = var.executor_pod_template
+    },
+    var.integration_hub_monitored_service_accounts == null ? {} : {
+      monitored-service-accounts = var.integration_hub_monitored_service_accounts
+    }
+  )
+
+  resources = var.integration_hub_image
+
+  units = 1
+  trust = true
+
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "zookeeper" {
+  name  = "zookeeper"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "zookeeper-k8s"
+    channel  = "3/stable"
+    revision = var.zookeeper_revision
+  }
+
+  storage_directives = {
+    zookeeper = var.zookeeper_size
+  }
+
+
+  resources = var.zookeeper_image
+
+  units       = var.zookeeper_units
+  constraints = "arch=amd64"
+}
+
+resource "juju_application" "data_integrator" {
+  name  = "data-integrator"
+  model_uuid = data.juju_model.spark.id
+
+  charm {
+    name     = "data-integrator"
+    channel  = "latest/stable"
+    revision = var.data_integrator_revision
+  }
+
+  config = {
+    database-name = "integrator"
+  }
+
+  units       = 1
+  constraints = "arch=amd64"
+}
diff --git a/releases/terraform/components/spark-3.5/integrations.tf b/releases/terraform/components/spark-3.5/integrations.tf
new file mode 100644
index 00000000..8031e0d5
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/integrations.tf
@@ -0,0 +1,86 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_integration" "kyuubi_metastore" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.metastore.name
+    endpoint = "database"
+  }
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "metastore-db"
+  }
+}
+
+resource "juju_integration" "kyuubi_users" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.kyuubi_users.name
+    endpoint = "database"
+  }
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "auth-db"
+  }
+}
+
+resource "juju_integration" "kyuubi_service_account" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "spark-service-account"
+  }
+
+  application {
+    name     = juju_application.integration_hub.name
+    endpoint = "spark-service-account"
+  }
+}
+
+resource "juju_integration" "kyuubi_zookeeper" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.zookeeper.name
+    endpoint = "zookeeper"
+  }
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "zookeeper"
+  }
+}
+
+resource "juju_integration" "kyuubi_tls" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "certificates"
+  }
+
+  application {
+    name     = var.tls_app_name
+    endpoint = var.tls_certificates_endpoint
+  }
+}
+
+resource "juju_integration" "kyuubi_data_integrator" {
+  model_uuid = data.juju_model.spark.id
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "jdbc"
+  }
+
+  application {
+    name     = juju_application.data_integrator.name
+    endpoint = "kyuubi"
+  }
+}
diff --git a/releases/terraform/components/spark-3.5/locals.tf b/releases/terraform/components/spark-3.5/locals.tf
new file mode 100644
index 00000000..10b756b2
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/locals.tf
@@ -0,0 +1,43 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+locals {
+  revisions = {
+    history_server  = 94
+    integration_hub = 113
+    kyuubi          = 142
+    kyuubi_users    = 495
+    metastore       = 495
+    zookeeper       = 78
+    data_integrator = 179
+    s3              = 145
+    azure_storage   = 15
+    grafana_agent   = 121
+    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
+    cos_configuration = 65
+    pushgateway       = 27
+    scrape_config     = 67
+  }
+  images = {
+    history_server = {
+      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
+    }
+    integration_hub = {
+      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
+    }
+    kyuubi = {
+      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:cdde52a4f72112ea09de11c6d21f901323619cf41015db89cb9d5742c57da303"
+    }
+    kyuubi_users = {
+      postgresql-image = 165
+    }
+    metastore = {
+      postgresql-image = 165
+    }
+    zookeeper = {
+      zookeeper-image = 34
+    }
+  }
+}
diff --git a/releases/terraform/components/spark-3.5/main.tf b/releases/terraform/components/spark-3.5/main.tf
new file mode 100644
index 00000000..46aca146
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/main.tf
@@ -0,0 +1,2 @@
+# Main Terraform code for $(basename "$moduleDir").
+# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/components/spark-3.5/offers.tf b/releases/terraform/components/spark-3.5/offers.tf
new file mode 100644
index 00000000..46f9b8dc
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/offers.tf
@@ -0,0 +1,11 @@
+resource "juju_offer" "integration_hub" {
+  model_uuid = data.juju_model.spark.id
+  application_name = juju_application.integration_hub.name
+  endpoints        = ["spark-service-account"]
+}
+
+resource "juju_offer" "metastore" {
+  model_uuid = data.juju_model.spark.id
+  application_name = juju_application.metastore.name
+  endpoints        = ["database"]
+}
\ No newline at end of file
diff --git a/releases/terraform/components/spark-3.5/outputs.tf b/releases/terraform/components/spark-3.5/outputs.tf
new file mode 100644
index 00000000..01714e58
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/outputs.tf
@@ -0,0 +1,34 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "charms" {
+  description = "The name of the charms which are part of the deployment."
+  value = {
+    history_server  = juju_application.history_server.name
+    kyuubi          = juju_application.kyuubi.name
+    kyuubi_users    = juju_application.kyuubi_users.name
+    metastore       = juju_application.metastore.name
+    integration_hub = juju_application.integration_hub.name
+    zookeeper       = juju_application.zookeeper.name
+    certificates    = var.tls_app_name
+  }
+}
+
+output "components" {
+  description = "List of the deployed applications for this component module."
+  value = [
+    { name = juju_application.history_server.name, type = "history_server" },
+    { name = juju_application.kyuubi.name, type = "kyuubi" },
+    { name = juju_application.kyuubi_users.name, type = "kyuubi_users" },
+    { name = juju_application.metastore.name, type = "metastore" },
+    { name = juju_application.integration_hub.name, type = "integration_hub" },
+    { name = juju_application.zookeeper.name, type = "zookeeper" },
+  ]
+}
+
+output "offers" {
+  value = {
+    hub_service_account = juju_offer.integration_hub
+    metastore_database  = juju_offer.metastore
+  }
+}
diff --git a/releases/terraform/components/spark-3.5/providers.tf b/releases/terraform/components/spark-3.5/providers.tf
new file mode 100644
index 00000000..0693f27b
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/providers.tf
@@ -0,0 +1,6 @@
+# Provider configuration for module $(basename "$moduleDir").
+# Terraform block moved to terraform.tf and should not be in providers.tf.
+
+# provider "juju" {
+#   # Configure provider here if required by the module.
+# }
diff --git a/releases/terraform/components/spark-3.5/resources.tf b/releases/terraform/components/spark-3.5/resources.tf
new file mode 100644
index 00000000..e25d743f
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/resources.tf
@@ -0,0 +1,36 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define juju resources (model, secrets, storage)
+
+locals {
+  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
+}
+
+data "juju_model" "spark" {
+  name = var.model
+}
+
+resource "juju_secret" "system_users_and_private_key_secret" {
+  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = local.target_model_uuid
+  name  = "system_users_and_private_key_secret"
+  value = merge(
+    var.admin_password == null ? {} : {
+      admin = var.admin_password
+    },
+    var.tls_private_key == null ? {} : {
+      private-key = var.tls_private_key
+    }
+  )
+  info = "This secret contains password for admin user and the TLS private key."
+}
+
+resource "juju_access_secret" "system_users_and_private_key_secret_access" {
+  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = local.target_model_uuid
+  applications = [
+    juju_application.kyuubi.name
+  ]
+  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
+}
diff --git a/releases/terraform/components/spark-3.5/terraform.tf b/releases/terraform/components/spark-3.5/terraform.tf
new file mode 100644
index 00000000..6412125a
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/components/spark-3.5/variables.tf b/releases/terraform/components/spark-3.5/variables.tf
new file mode 100644
index 00000000..673a1de8
--- /dev/null
+++ b/releases/terraform/components/spark-3.5/variables.tf
@@ -0,0 +1,244 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define spark-specific variables
+
+variable "admin_password" {
+  description = "The password for the admin user."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "data_integrator_revision" {
+  description = "Charm revision for data-integrator"
+  type        = number
+  nullable    = false
+}
+
+variable "driver_pod_template" {
+  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
+  type        = string
+  default     = null
+}
+
+variable "enable_dynamic_allocation" {
+  description = "Enable dynamic allocation of pods for Spark jobs."
+  type        = bool
+  default     = false
+}
+
+variable "executor_pod_template" {
+  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
+  type        = string
+  default     = null
+}
+
+variable "history_server_image" {
+  description = "Image for spark-history-server-k8s"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "history_server_revision" {
+  description = "Charm revision for spark-history-server-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "integration_hub_image" {
+  description = "Image for spark-integration-hub-k8s"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "integration_hub_monitored_service_accounts" {
+  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
+  type        = string
+  default     = null
+}
+
+variable "integration_hub_revision" {
+  description = "Charm revision for spark-integration-hub-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_driver_pod_template" {
+  description = "Define K8s driver pod from a file accessible in the object storage."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_executor_cores" {
+  description = "Set kyuubi executor pods cpu cores."
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_executor_memory" {
+  description = "Set kyuubi executor pods memory (in GB)."
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_executor_pod_template" {
+  description = "Define K8s executor pod from a file accessible in the object storage."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_gpu_enable" {
+  description = "Enable GPU acceleration for SparkSQLEngine."
+  type        = bool
+  nullable    = false
+}
+
+variable "kyuubi_gpu_engine_executors_limit" {
+  description = "Limit the number of GPUs an engine can schedule executor pods on."
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_gpu_pinned_memory" {
+  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_image" {
+  description = "Image for kyuubi-k8s"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "kyuubi_k8s_node_selectors" {
+  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_loadbalancer_extra_annotations" {
+  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_profile" {
+  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
+  type        = string
+  nullable    = false
+  default     = "production"
+}
+
+variable "kyuubi_revision" {
+  description = "Charm revision for kyuubi-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_units" {
+  description = "Number of Kyuubi units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
+
+variable "kyuubi_user" {
+  description = "User name to be used for running Kyuubi engines."
+  type        = string
+  nullable    = false
+  default     = "kyuubi-spark-engine"
+}
+
+variable "kyuubi_users_image" {
+  description = "Image for postgresql-k8s (auth-db)"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "kyuubi_users_revision" {
+  description = "Charm revision for postgresql-k8s (auth-db)"
+  type        = number
+  nullable    = false
+}
+
+variable "kyuubi_users_size" {
+  description = "Storage size for the Kyuubi users database"
+  type        = string
+  default     = "1G"
+}
+
+variable "metastore_image" {
+  description = "Image for postgresql-k8s (metastore)"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "metastore_revision" {
+  description = "Charm revision for postgresql-k8s (metastore)"
+  type        = number
+  nullable    = false
+}
+
+variable "metastore_size" {
+  description = "Storage size for the metastore database"
+  type        = string
+  default     = "10G"
+}
+
+variable "model" {
+  description = "Name of the Juju Model to deploy to (fallback if model_uuid is not provided)."
+  type        = string
+  nullable    = false
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy to."
+  type        = string
+  default     = null
+}
+
+variable "tls_app_name" {
+  description = "Name of the application providing the `tls-certificates` interface."
+  type        = string
+  nullable    = false
+}
+
+variable "tls_certificates_endpoint" {
+  description = "Name of the endpoint providing the `tls-certificates` interface."
+  type        = string
+  nullable    = false
+}
+
+variable "tls_private_key" {
+  description = "The private key to be used for TLS certificates."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "zookeeper_image" {
+  description = "Image for zookeeper-k8s"
+  type        = map(string)
+  nullable    = false
+}
+
+variable "zookeeper_revision" {
+  description = "Charm revision for zookeeper-k8s"
+  type        = number
+  nullable    = false
+}
+
+variable "zookeeper_size" {
+  description = "Storage size for the zookeeper unit"
+  type        = string
+  default     = "10G"
+}
+
+variable "zookeeper_units" {
+  description = "Number of zookeeper units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
diff --git a/releases/terraform/products/charmed-spark-3.4/README.md b/releases/terraform/products/charmed-spark-3.4/README.md
new file mode 100644
index 00000000..40f903f3
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.4/README.md
@@ -0,0 +1,99 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=0.20.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | >=0.20.0 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_azure_storage"></a> [azure\_storage](#module\_azure\_storage) | ./charms/azure-storage-integrator | n/a |
+| <a name="module_bundled_cos"></a> [bundled\_cos](#module\_bundled\_cos) | ./products/cos | n/a |
+| <a name="module_observability"></a> [observability](#module\_observability) | ./components/observability | n/a |
+| <a name="module_s3"></a> [s3](#module\_s3) | ./charms/s3-integrator | n/a |
+| <a name="module_spark"></a> [spark](#module\_spark) | ./components/spark-3.4 | n/a |
+| <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev326 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_model.cos](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
+| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_K8S_CLOUD"></a> [K8S\_CLOUD](#input\_K8S\_CLOUD) | The kubernetes juju cloud name. | `string` | `"microk8s"` | no |
+| <a name="input_K8S_CREDENTIAL"></a> [K8S\_CREDENTIAL](#input\_K8S\_CREDENTIAL) | The name of the kubernetes juju credential. | `string` | `"microk8s"` | no |
+| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
+| <a name="input_alertmanager_size"></a> [alertmanager\_size](#input\_alertmanager\_size) | Storage size for the alertmanager database | `string` | `"10G"` | no |
+| <a name="input_azure_storage"></a> [azure\_storage](#input\_azure\_storage) | Azure Object storage information | <pre>object({<br/>    container       = optional(string, "azurecontainer")<br/>    storage_account = optional(string, "azurestorageaccount")<br/>    secret_key      = optional(string, "azurestoragesecret")<br/>    protocol        = optional(string, "abfss")<br/>  })</pre> | `{}` | no |
+| <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | `null` | no |
+| <a name="input_certificate_common_name"></a> [certificate\_common\_name](#input\_certificate\_common\_name) | Common name for the certificate to be used in self-signed | `string` | `"charmed-spark"` | no |
+| <a name="input_cos"></a> [cos](#input\_cos) | Observability settings | <pre>object({<br/>    model    = optional(string, "cos")<br/>    deployed = optional(string, "bundled")<br/>    offers = optional(object({<br/>      dashboard = optional(string, null),<br/>      metrics   = optional(string, null),<br/>      logging   = optional(string, null)<br/>    }), {}),<br/>    tls = optional(object({<br/>      cert = optional(string, "")<br/>      key  = optional(string, "")<br/>      ca   = optional(string, "")<br/>    }), {})<br/>  })</pre> | <pre>{<br/>  "deployed": "bundled",<br/>  "model": "cos",<br/>  "offers": {},<br/>  "tls": {}<br/>}</pre> | no |
+| <a name="input_cos_configuration_revision"></a> [cos\_configuration\_revision](#input\_cos\_configuration\_revision) | Charm revision for cos-configuration-k8s | `number` | `null` | no |
+| <a name="input_create_model"></a> [create\_model](#input\_create\_model) | Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism. | `bool` | `true` | no |
+| <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | `null` | no |
+| <a name="input_driver_pod_template"></a> [driver\_pod\_template](#input\_driver\_pod\_template) | Define K8s driver pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
+| <a name="input_enable_dynamic_allocation"></a> [enable\_dynamic\_allocation](#input\_enable\_dynamic\_allocation) | Enable dynamic allocation of pods for Spark jobs. | `bool` | `false` | no |
+| <a name="input_executor_pod_template"></a> [executor\_pod\_template](#input\_executor\_pod\_template) | Define K8s executor pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
+| <a name="input_grafana_agent_revision"></a> [grafana\_agent\_revision](#input\_grafana\_agent\_revision) | Charm revision for grafana-agent-k8s | `number` | `null` | no |
+| <a name="input_grafana_size"></a> [grafana\_size](#input\_grafana\_size) | Storage size for the grafana database | `string` | `"10G"` | no |
+| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `map(string)` | `null` | no |
+| <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | `null` | no |
+| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | `null` | no |
+| <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
+| <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | `null` | no |
+| <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
+| <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
+| <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
+| <a name="input_kyuubi_executor_pod_template"></a> [kyuubi\_executor\_pod\_template](#input\_kyuubi\_executor\_pod\_template) | Define K8s executor pod from a file accessible in the object storage. | `string` | `null` | no |
+| <a name="input_kyuubi_gpu_enable"></a> [kyuubi\_gpu\_enable](#input\_kyuubi\_gpu\_enable) | Enable GPU acceleration for SparkSQLEngine. | `bool` | `false` | no |
+| <a name="input_kyuubi_gpu_engine_executors_limit"></a> [kyuubi\_gpu\_engine\_executors\_limit](#input\_kyuubi\_gpu\_engine\_executors\_limit) | Limit the number of GPUs an engine can schedule executor pods on. | `number` | `1` | no |
+| <a name="input_kyuubi_gpu_pinned_memory"></a> [kyuubi\_gpu\_pinned\_memory](#input\_kyuubi\_gpu\_pinned\_memory) | Set the host memory (in GB) per executor reserved for fast data transfer on GPUs. | `number` | `1` | no |
+| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `map(string)` | `null` | no |
+| <a name="input_kyuubi_k8s_node_selectors"></a> [kyuubi\_k8s\_node\_selectors](#input\_kyuubi\_k8s\_node\_selectors) | Comma separated label:value selectors for K8s pods in Kyuubi. | `string` | `null` | no |
+| <a name="input_kyuubi_loadbalancer_extra_annotations"></a> [kyuubi\_loadbalancer\_extra\_annotations](#input\_kyuubi\_loadbalancer\_extra\_annotations) | Optional extra annotations to be supplied to the load balancer service in Kyuubi. | `string` | `null` | no |
+| <a name="input_kyuubi_profile"></a> [kyuubi\_profile](#input\_kyuubi\_profile) | The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'. | `string` | `"production"` | no |
+| <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | `null` | no |
+| <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
+| <a name="input_kyuubi_user"></a> [kyuubi\_user](#input\_kyuubi\_user) | Define the user to be used for running Kyuubi enginers | `string` | `"kyuubi-spark-engine"` | no |
+| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | `null` | no |
+| <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | `null` | no |
+| <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
+| <a name="input_loki_active_index_directory_size"></a> [loki\_active\_index\_directory\_size](#input\_loki\_active\_index\_directory\_size) | Storage size for the active index directory for Loki | `string` | `"10G"` | no |
+| <a name="input_loki_chunks_size"></a> [loki\_chunks\_size](#input\_loki\_chunks\_size) | Storage size for the Loki chucks storage | `string` | `"500G"` | no |
+| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | `null` | no |
+| <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | `null` | no |
+| <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
+| <a name="input_model"></a> [model](#input\_model) | The name of the juju model to deploy Spark to | `string` | `"spark"` | no |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules. | `string` | `null` | no |
+| <a name="input_prometheus_size"></a> [prometheus\_size](#input\_prometheus\_size) | Storage size for the Prometheus database | `string` | `"500G"` | no |
+| <a name="input_pushgateway_revision"></a> [pushgateway\_revision](#input\_pushgateway\_revision) | Charm revision for prometheus-pushgateway-k8s | `number` | `null` | no |
+| <a name="input_s3"></a> [s3](#input\_s3) | S3 Bucket information | <pre>object({<br/>    bucket   = optional(string, "spark-test")<br/>    endpoint = optional(string, "https://s3.amazonaws.com")<br/>    region   = optional(string, "us-east-1")<br/>  })</pre> | `{}` | no |
+| <a name="input_s3_revision"></a> [s3\_revision](#input\_s3\_revision) | Charm revision for s3-integrator | `number` | `null` | no |
+| <a name="input_scrape_config_revision"></a> [scrape\_config\_revision](#input\_scrape\_config\_revision) | Charm revision for prometheus-scrape-config-k8s | `number` | `null` | no |
+| <a name="input_storage_backend"></a> [storage\_backend](#input\_storage\_backend) | Storage backend to be used | `string` | `"s3"` | no |
+| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The file path of the private key to use for TLS certificates. | `string` | `null` | no |
+| <a name="input_traefik_size"></a> [traefik\_size](#input\_traefik\_size) | Storage size for the Traefik storage | `string` | `"10G"` | no |
+| <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | `null` | no |
+| <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | `null` | no |
+| <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
+| <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Define the number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
+| <a name="output_components"></a> [components](#output\_components) | List of component modules (spark and optional addons) and their deployed charms/offers. |
+| <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
diff --git a/releases/terraform/products/charmed-spark-3.4/locals.tf b/releases/terraform/products/charmed-spark-3.4/locals.tf
new file mode 100644
index 00000000..034c5a66
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.4/locals.tf
@@ -0,0 +1,43 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+locals {
+  revisions = {
+    history_server  = 94
+    integration_hub = 113
+    kyuubi          = 140
+    kyuubi_users    = 495
+    metastore       = 495
+    zookeeper       = 78
+    data_integrator = 179
+    s3              = 145
+    azure_storage   = 15
+    grafana_agent   = 121
+    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
+    cos_configuration = 65
+    pushgateway       = 27
+    scrape_config     = 67
+  }
+  images = {
+    history_server = {
+      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
+    } # rev23, spark-version: 3.5.7, release date 2026-02-27
+    integration_hub = {
+      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
+    } # rev11, release date 2026-03-02
+    kyuubi = {
+      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e"
+    } # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
+    kyuubi_users = {
+      postgresql-image = 165
+    }
+    metastore = {
+      postgresql-image = 165
+    }
+    zookeeper = {
+      zookeeper-image = 34
+    }
+  }
+}
diff --git a/releases/terraform/products/charmed-spark-3.4/main.tf b/releases/terraform/products/charmed-spark-3.4/main.tf
new file mode 100644
index 00000000..d5c4f24b
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.4/main.tf
@@ -0,0 +1,146 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_model" "spark" {
+  count      = (var.model_uuid == null && var.create_model == true) ? 1 : 0
+  name       = var.model
+  credential = var.K8S_CREDENTIAL
+  cloud {
+    name = var.K8S_CLOUD
+  }
+}
+
+resource "juju_model" "cos" {
+  count      = var.cos.deployed == "bundled" && var.model_uuid == null && var.create_model == true ? 1 : 0
+  name       = var.cos.model
+  credential = var.K8S_CREDENTIAL
+  cloud {
+    name = var.K8S_CLOUD
+  }
+}
+
+locals {
+  active_model = coalesce(var.model_uuid, var.model)
+}
+
+data "juju_model" "spark" {
+  name = var.model
+}
+
+module "ssc" {
+  depends_on  = [juju_model.spark]
+  source      = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev586"
+  model_uuid  = coalesce(var.model_uuid, data.juju_model.spark.id)
+  app_name    = "certificates"
+  channel     = "1/stable"
+  revision    = 317
+  constraints = "arch=amd64"
+  base        = "ubuntu@24.04"
+  units       = 1
+  config = {
+    ca-common-name = var.certificate_common_name
+  }
+}
+
+module "spark" {
+  depends_on                                 = [juju_model.spark, module.ssc]
+  source                                     = "../../components/spark-3.4"
+  model                                      = local.active_model
+  kyuubi_user                                = var.kyuubi_user
+  kyuubi_profile                             = var.kyuubi_profile
+  admin_password                             = var.admin_password
+  tls_private_key                            = var.tls_private_key
+  enable_dynamic_allocation                  = var.enable_dynamic_allocation
+  kyuubi_k8s_node_selectors                  = var.kyuubi_k8s_node_selectors
+  kyuubi_loadbalancer_extra_annotations      = var.kyuubi_loadbalancer_extra_annotations
+  kyuubi_gpu_enable                          = var.kyuubi_gpu_enable
+  kyuubi_gpu_engine_executors_limit          = var.kyuubi_gpu_engine_executors_limit
+  kyuubi_gpu_pinned_memory                   = var.kyuubi_gpu_pinned_memory
+  kyuubi_driver_pod_template                 = var.kyuubi_driver_pod_template
+  kyuubi_executor_pod_template               = var.kyuubi_executor_pod_template
+  kyuubi_executor_cores                      = var.kyuubi_executor_cores
+  kyuubi_executor_memory                     = var.kyuubi_executor_memory
+  driver_pod_template                        = var.driver_pod_template
+  executor_pod_template                      = var.executor_pod_template
+  integration_hub_monitored_service_accounts = var.integration_hub_monitored_service_accounts
+
+  tls_app_name              = module.ssc.app_name
+  tls_certificates_endpoint = module.ssc.provides.certificates
+  zookeeper_units           = var.zookeeper_units
+  kyuubi_units              = var.kyuubi_units
+
+  history_server_revision  = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server
+  history_server_image     = var.history_server_image != null ? var.history_server_image : local.images.history_server
+  integration_hub_revision = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub
+  integration_hub_image    = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub
+  kyuubi_revision          = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi
+  kyuubi_image             = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi
+  kyuubi_users_revision    = var.kyuubi_users_revision != null ? var.kyuubi_users_revision : local.revisions.kyuubi_users
+  kyuubi_users_image       = var.kyuubi_users_image != null ? var.kyuubi_users_image : local.images.kyuubi_users
+  metastore_revision       = var.metastore_revision != null ? var.metastore_revision : local.revisions.metastore
+  metastore_image          = var.metastore_image != null ? var.metastore_image : local.images.metastore
+  zookeeper_revision       = var.zookeeper_revision != null ? var.zookeeper_revision : local.revisions.zookeeper
+  zookeeper_image          = var.zookeeper_image != null ? var.zookeeper_image : local.images.zookeeper
+  data_integrator_revision = var.data_integrator_revision != null ? var.data_integrator_revision : local.revisions.data_integrator
+
+  kyuubi_users_size = var.kyuubi_users_size
+  metastore_size    = var.metastore_size
+  zookeeper_size    = var.zookeeper_size
+}
+
+module "azure_storage" {
+  depends_on    = [module.spark]
+  count         = var.storage_backend == "azure_storage" ? 1 : 0
+  source = "../../charms/azure-storage-integrator"
+  model_uuid    = local.active_model
+  spark_charms  = module.spark.charms
+  azure_storage = var.azure_storage
+
+  azure_storage_revision = var.azure_storage_revision != null ? var.azure_storage_revision : local.revisions.azure_storage
+}
+
+module "s3" {
+  depends_on   = [module.spark]
+  count        = var.storage_backend == "s3" ? 1 : 0
+  source = "../../charms/s3-integrator"
+  model_uuid   = local.active_model
+  spark_charms = module.spark.charms
+  s3           = var.s3
+
+  s3_revision = var.s3_revision != null ? var.s3_revision : local.revisions.s3
+}
+
+module "bundled_cos" {
+  depends_on   = [juju_model.cos]
+  count        = var.cos.deployed == "bundled" ? 1 : 0
+  source = "../cos"
+  model        = var.cos.model
+  cos_tls_ca   = var.cos.tls.ca
+  cos_tls_cert = var.cos.tls.cert
+  cos_tls_key  = var.cos.tls.key
+
+  alertmanager_size                = var.alertmanager_size
+  grafana_size                     = var.grafana_size
+  loki_active_index_directory_size = var.loki_active_index_directory_size
+  loki_chunks_size                 = var.loki_chunks_size
+  prometheus_size                  = var.prometheus_size
+  traefik_size                     = var.traefik_size
+}
+
+
+module "observability" {
+  depends_on       = [module.spark, module.bundled_cos]
+  count            = var.cos.deployed == "no" ? 0 : 1
+  source = "../../components/observability"
+  dashboards_offer = var.cos.deployed == "external" ? var.cos.offers.dashboard : one(module.bundled_cos[*].dashboards_offer)
+  logging_offer    = var.cos.deployed == "external" ? var.cos.offers.logging : one(module.bundled_cos[*].logging_offer)
+  metrics_offer    = var.cos.deployed == "external" ? var.cos.offers.metrics : one(module.bundled_cos[*].metrics_offer)
+  spark_model      = var.model
+  model_uuid       = local.active_model
+  spark_charms     = module.spark.charms
+
+  grafana_agent_revision     = var.grafana_agent_revision != null ? var.grafana_agent_revision : local.revisions.grafana_agent
+  cos_configuration_revision = var.cos_configuration_revision != null ? var.cos_configuration_revision : local.revisions.cos_configuration
+  pushgateway_revision       = var.pushgateway_revision != null ? var.pushgateway_revision : local.revisions.pushgateway
+  scrape_config_revision     = var.scrape_config_revision != null ? var.scrape_config_revision : local.revisions.scrape_config
+}
diff --git a/releases/terraform/products/charmed-spark-3.4/outputs.tf b/releases/terraform/products/charmed-spark-3.4/outputs.tf
new file mode 100644
index 00000000..f1a2b99f
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.4/outputs.tf
@@ -0,0 +1,51 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "charms" {
+  description = "The name of the charms which are part of the deployment."
+  value = merge(
+    concat(
+      module.spark[*].charms, module.azure_storage[*].charms, module.s3[*].charms, module.bundled_cos[*].charms, module.observability[*].charms
+    )...
+  )
+}
+
+output "components" {
+  description = "List of component modules (spark and optional addons) and their deployed charms/offers."
+  value = [
+    {
+      name   = "spark"
+      charms = module.spark.charms
+      offers = module.spark.offers
+    },
+    {
+      name   = "azure_storage"
+      charms = try(module.azure_storage.charms, [])
+      offers = try(module.azure_storage.offers, {})
+    },
+    {
+      name   = "s3"
+      charms = try(module.s3.charms, [])
+      offers = try(module.s3.offers, {})
+    },
+    {
+      name   = "bundled_cos"
+      charms = try(module.bundled_cos.charms, [])
+      offers = try(module.bundled_cos.offers, {})
+    },
+    {
+      name   = "observability"
+      charms = try(module.observability.charms, [])
+      offers = try(module.observability.offers, {})
+    }
+  ]
+}
+
+output "offers" {
+  description = "The name and url of the various offers being exposed"
+  value = merge(
+    concat(
+      module.spark[*].offers
+    )...
+  )
+}
diff --git a/releases/terraform/products/charmed-spark-3.4/providers.tf b/releases/terraform/products/charmed-spark-3.4/providers.tf
new file mode 100644
index 00000000..b60772fa
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.4/providers.tf
@@ -0,0 +1,6 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+provider "juju" {
+  # Configure the Juju provider if needed (e.g., `controller`, `model`, `username`, etc.).
+}
diff --git a/releases/terraform/products/charmed-spark-3.4/terraform.tf b/releases/terraform/products/charmed-spark-3.4/terraform.tf
new file mode 100644
index 00000000..6ae144c8
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.4/terraform.tf
@@ -0,0 +1,11 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+terraform {
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=0.20.0"
+    }
+  }
+}
diff --git a/releases/terraform/products/charmed-spark-3.4/variables.tf b/releases/terraform/products/charmed-spark-3.4/variables.tf
new file mode 100644
index 00000000..57c95ffe
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.4/variables.tf
@@ -0,0 +1,395 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Juju
+
+variable "admin_password" {
+  description = "The password for the admin user."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "alertmanager_size" {
+  description = "Storage size for the alertmanager database"
+  type        = string
+  default     = "10G"
+}
+
+variable "azure_storage" {
+  description = "Azure Object storage information"
+  type = object({
+    container       = optional(string, "azurecontainer")
+    storage_account = optional(string, "azurestorageaccount")
+    secret_key      = optional(string, "azurestoragesecret")
+    protocol        = optional(string, "abfss")
+  })
+  default = {}
+}
+
+variable "azure_storage_revision" {
+  description = "Charm revision for azure-storage-integrator"
+  type        = number
+  default     = null
+}
+
+variable "certificate_common_name" {
+  description = "Common name for the certificate to be used in self-signed"
+  type        = string
+  default     = "charmed-spark"
+}
+
+variable "cos" {
+  description = "Observability settings"
+  type = object({
+    model    = optional(string, "cos")
+    deployed = optional(string, "bundled")
+    offers = optional(object({
+      dashboard = optional(string, null),
+      metrics   = optional(string, null),
+      logging   = optional(string, null)
+    }), {}),
+    tls = optional(object({
+      cert = optional(string, "")
+      key  = optional(string, "")
+      ca   = optional(string, "")
+    }), {})
+  })
+  default = { model = "cos", deployed = "bundled", offers = {}, tls = {} }
+
+  validation {
+    condition     = contains(["external", "bundled", "no"], var.cos.deployed)
+    error_message = "Valid values for var: cos.deployed are (external, bundled, no)"
+  }
+
+  validation {
+    condition = var.cos.deployed != "external" || alltrue([
+      var.cos.offers.dashboard != null,
+      var.cos.offers.metrics != null,
+      var.cos.offers.logging != null,
+    ])
+    error_message = "When using external cos, please define all offers variables"
+  }
+
+}
+
+variable "cos_configuration_revision" {
+  description = "Charm revision for cos-configuration-k8s"
+  type        = number
+  default     = null
+}
+
+variable "create_model" {
+  description = "Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism."
+  type        = bool
+  default     = true
+  nullable    = false
+}
+
+variable "data_integrator_revision" {
+  description = "Charm revision for data-integrator"
+  type        = number
+  default     = null
+}
+
+variable "driver_pod_template" {
+  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
+  type        = string
+  default     = null
+}
+
+variable "enable_dynamic_allocation" {
+  description = "Enable dynamic allocation of pods for Spark jobs."
+  type        = bool
+  default     = false
+}
+
+variable "executor_pod_template" {
+  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
+  type        = string
+  default     = null
+}
+
+variable "grafana_agent_revision" {
+  description = "Charm revision for grafana-agent-k8s"
+  type        = number
+  default     = null
+}
+
+variable "grafana_size" {
+  description = "Storage size for the grafana database"
+  type        = string
+  default     = "10G"
+}
+
+variable "history_server_image" {
+  description = "Image for spark-history-server-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "history_server_revision" {
+  description = "Charm revision for spark-history-server-k8s"
+  type        = number
+  default     = null
+}
+
+variable "integration_hub_image" {
+  description = "Image for spark-integration-hub-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "integration_hub_monitored_service_accounts" {
+  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
+  type        = string
+  default     = null
+}
+
+variable "integration_hub_revision" {
+  description = "Charm revision for spark-integration-hub-k8s"
+  type        = number
+  default     = null
+}
+
+variable "K8S_CLOUD" {
+  type        = string
+  description = "The kubernetes juju cloud name."
+  default     = "microk8s"
+}
+
+variable "K8S_CREDENTIAL" {
+  type        = string
+  description = "The name of the kubernetes juju credential."
+  default     = "microk8s"
+}
+
+variable "kyuubi_driver_pod_template" {
+  description = "Define K8s driver pod from a file accessible in the object storage."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_executor_cores" {
+  description = "Set kyuubi executor pods cpu cores."
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_executor_memory" {
+  description = "Set kyuubi executor pods memory (in GB)."
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_executor_pod_template" {
+  description = "Define K8s executor pod from a file accessible in the object storage."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_gpu_enable" {
+  description = "Enable GPU acceleration for SparkSQLEngine."
+  type        = bool
+  default     = false
+}
+
+variable "kyuubi_gpu_engine_executors_limit" {
+  description = "Limit the number of GPUs an engine can schedule executor pods on."
+  type        = number
+  default     = 1
+}
+
+variable "kyuubi_gpu_pinned_memory" {
+  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
+  type        = number
+  default     = 1
+}
+
+variable "kyuubi_image" {
+  description = "Image for kyuubi-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "kyuubi_k8s_node_selectors" {
+  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_loadbalancer_extra_annotations" {
+  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_profile" {
+  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
+  type        = string
+  nullable    = false
+  default     = "production"
+}
+
+variable "kyuubi_revision" {
+  description = "Charm revision for kyuubi-k8s"
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_units" {
+  description = "Number of Kyuubi units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
+
+variable "kyuubi_user" {
+  description = "Define the user to be used for running Kyuubi enginers"
+  type        = string
+  default     = "kyuubi-spark-engine"
+}
+
+variable "kyuubi_users_image" {
+  description = "Image for postgresql-k8s (auth-db)"
+  type        = map(string)
+  default     = null
+}
+
+variable "kyuubi_users_revision" {
+  description = "Charm revision for postgresql-k8s (auth-db)"
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_users_size" {
+  description = "Storage size for the Kyuubi users database"
+  type        = string
+  default     = "1G"
+}
+
+variable "loki_active_index_directory_size" {
+  description = "Storage size for the active index directory for Loki"
+  type        = string
+  default     = "10G"
+}
+
+variable "loki_chunks_size" {
+  description = "Storage size for the Loki chucks storage"
+  type        = string
+  default     = "500G"
+}
+
+variable "metastore_image" {
+  description = "Image for postgresql-k8s (metastore)"
+  type        = map(string)
+  default     = null
+}
+
+variable "metastore_revision" {
+  description = "Charm revision for postgresql-k8s (metastore)"
+  type        = number
+  default     = null
+}
+
+variable "metastore_size" {
+  description = "Storage size for the metastore database"
+  type        = string
+  default     = "10G"
+}
+
+variable "model" {
+  description = "The name of the juju model to deploy Spark to"
+  type        = string
+  default     = "spark"
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules."
+  type        = string
+  default     = null
+}
+
+variable "prometheus_size" {
+  description = "Storage size for the Prometheus database"
+  type        = string
+  default     = "500G"
+}
+
+variable "pushgateway_revision" {
+  description = "Charm revision for prometheus-pushgateway-k8s"
+  type        = number
+  default     = null
+}
+
+variable "s3" {
+  description = "S3 Bucket information"
+  type = object({
+    bucket   = optional(string, "spark-test")
+    endpoint = optional(string, "https://s3.amazonaws.com")
+    region   = optional(string, "us-east-1")
+  })
+  default = {}
+}
+
+variable "s3_revision" {
+  description = "Charm revision for s3-integrator"
+  type        = number
+  default     = null
+}
+
+variable "scrape_config_revision" {
+  description = "Charm revision for prometheus-scrape-config-k8s"
+  type        = number
+  default     = null
+}
+
+variable "storage_backend" {
+  type        = string
+  description = "Storage backend to be used"
+
+  validation {
+    condition     = contains(["azure_storage", "s3"], var.storage_backend)
+    error_message = "Valid values for var: test_variable are (s3, azure_storage)."
+  }
+
+  default = "s3"
+}
+
+variable "tls_private_key" {
+  description = "The file path of the private key to use for TLS certificates."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "traefik_size" {
+  description = "Storage size for the Traefik storage"
+  type        = string
+  default     = "10G"
+}
+
+variable "zookeeper_image" {
+  description = "Image for zookeeper-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "zookeeper_revision" {
+  description = "Charm revision for zookeeper-k8s"
+  type        = number
+  default     = null
+}
+
+variable "zookeeper_size" {
+  description = "Storage size for the metastore database"
+  type        = string
+  default     = "10G"
+}
+
+variable "zookeeper_units" {
+  description = "Define the number of zookeeper units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
diff --git a/releases/terraform/products/charmed-spark-3.5/README.md b/releases/terraform/products/charmed-spark-3.5/README.md
new file mode 100644
index 00000000..b58335b5
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.5/README.md
@@ -0,0 +1,100 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_azure_storage"></a> [azure\_storage](#module\_azure\_storage) | ./charms/azure-storage-integrator | n/a |
+| <a name="module_bundled_cos"></a> [bundled\_cos](#module\_bundled\_cos) | ./products/cos | n/a |
+| <a name="module_observability"></a> [observability](#module\_observability) | ./components/observability | n/a |
+| <a name="module_s3"></a> [s3](#module\_s3) | ./charms/s3-integrator | n/a |
+| <a name="module_spark"></a> [spark](#module\_spark) | ./components/spark | n/a |
+| <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev326 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_model.cos](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
+| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_K8S_CLOUD"></a> [K8S\_CLOUD](#input\_K8S\_CLOUD) | The kubernetes juju cloud name. | `string` | `"microk8s"` | no |
+| <a name="input_K8S_CREDENTIAL"></a> [K8S\_CREDENTIAL](#input\_K8S\_CREDENTIAL) | The name of the kubernetes juju credential. | `string` | `"microk8s"` | no |
+| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
+| <a name="input_alertmanager_size"></a> [alertmanager\_size](#input\_alertmanager\_size) | Storage size for the alertmanager database | `string` | `"10G"` | no |
+| <a name="input_azure_storage"></a> [azure\_storage](#input\_azure\_storage) | Azure Object storage information | <pre>object({<br/>    container       = optional(string, "azurecontainer")<br/>    storage_account = optional(string, "azurestorageaccount")<br/>    secret_key      = optional(string, "azurestoragesecret")<br/>    protocol        = optional(string, "abfss")<br/>  })</pre> | `{}` | no |
+| <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | `null` | no |
+| <a name="input_certificate_common_name"></a> [certificate\_common\_name](#input\_certificate\_common\_name) | Common name for the certificate to be used in self-signed | `string` | `"charmed-spark"` | no |
+| <a name="input_cos"></a> [cos](#input\_cos) | Observability settings | <pre>object({<br/>    model    = optional(string, "cos")<br/>    deployed = optional(string, "bundled")<br/>    offers = optional(object({<br/>      dashboard = optional(string, null),<br/>      metrics   = optional(string, null),<br/>      logging   = optional(string, null)<br/>    }), {}),<br/>    tls = optional(object({<br/>      cert = optional(string, "")<br/>      key  = optional(string, "")<br/>      ca   = optional(string, "")<br/>    }), {})<br/>  })</pre> | <pre>{<br/>  "deployed": "bundled",<br/>  "model": "cos",<br/>  "offers": {},<br/>  "tls": {}<br/>}</pre> | no |
+| <a name="input_cos_configuration_revision"></a> [cos\_configuration\_revision](#input\_cos\_configuration\_revision) | Charm revision for cos-configuration-k8s | `number` | `null` | no |
+| <a name="input_create_model"></a> [create\_model](#input\_create\_model) | Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism. | `bool` | `true` | no |
+| <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | `null` | no |
+| <a name="input_driver_pod_template"></a> [driver\_pod\_template](#input\_driver\_pod\_template) | Define K8s driver pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
+| <a name="input_enable_dynamic_allocation"></a> [enable\_dynamic\_allocation](#input\_enable\_dynamic\_allocation) | Enable dynamic allocation of pods for Spark jobs. | `bool` | `false` | no |
+| <a name="input_executor_pod_template"></a> [executor\_pod\_template](#input\_executor\_pod\_template) | Define K8s executor pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
+| <a name="input_grafana_agent_revision"></a> [grafana\_agent\_revision](#input\_grafana\_agent\_revision) | Charm revision for grafana-agent-k8s | `number` | `null` | no |
+| <a name="input_grafana_size"></a> [grafana\_size](#input\_grafana\_size) | Storage size for the grafana database | `string` | `"10G"` | no |
+| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `map(string)` | `null` | no |
+| <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | `null` | no |
+| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | `null` | no |
+| <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
+| <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | `null` | no |
+| <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
+| <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
+| <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
+| <a name="input_kyuubi_executor_pod_template"></a> [kyuubi\_executor\_pod\_template](#input\_kyuubi\_executor\_pod\_template) | Define K8s executor pod from a file accessible in the object storage. | `string` | `null` | no |
+| <a name="input_kyuubi_gpu_enable"></a> [kyuubi\_gpu\_enable](#input\_kyuubi\_gpu\_enable) | Enable GPU acceleration for SparkSQLEngine. | `bool` | `false` | no |
+| <a name="input_kyuubi_gpu_engine_executors_limit"></a> [kyuubi\_gpu\_engine\_executors\_limit](#input\_kyuubi\_gpu\_engine\_executors\_limit) | Limit the number of GPUs an engine can schedule executor pods on. | `number` | `1` | no |
+| <a name="input_kyuubi_gpu_pinned_memory"></a> [kyuubi\_gpu\_pinned\_memory](#input\_kyuubi\_gpu\_pinned\_memory) | Set the host memory (in GB) per executor reserved for fast data transfer on GPUs. | `number` | `1` | no |
+| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `map(string)` | `null` | no |
+| <a name="input_kyuubi_k8s_node_selectors"></a> [kyuubi\_k8s\_node\_selectors](#input\_kyuubi\_k8s\_node\_selectors) | Comma separated label:value selectors for K8s pods in Kyuubi. | `string` | `null` | no |
+| <a name="input_kyuubi_loadbalancer_extra_annotations"></a> [kyuubi\_loadbalancer\_extra\_annotations](#input\_kyuubi\_loadbalancer\_extra\_annotations) | Optional extra annotations to be supplied to the load balancer service in Kyuubi. | `string` | `null` | no |
+| <a name="input_kyuubi_profile"></a> [kyuubi\_profile](#input\_kyuubi\_profile) | The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'. | `string` | `"production"` | no |
+| <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | `null` | no |
+| <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
+| <a name="input_kyuubi_user"></a> [kyuubi\_user](#input\_kyuubi\_user) | Define the user to be used for running Kyuubi enginers | `string` | `"kyuubi-spark-engine"` | no |
+| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | `null` | no |
+| <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | `null` | no |
+| <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
+| <a name="input_loki_active_index_directory_size"></a> [loki\_active\_index\_directory\_size](#input\_loki\_active\_index\_directory\_size) | Storage size for the active index directory for Loki | `string` | `"10G"` | no |
+| <a name="input_loki_chunks_size"></a> [loki\_chunks\_size](#input\_loki\_chunks\_size) | Storage size for the Loki chucks storage | `string` | `"500G"` | no |
+| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | `null` | no |
+| <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | `null` | no |
+| <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
+| <a name="input_model"></a> [model](#input\_model) | The name of the juju model to deploy Spark to | `string` | `"spark"` | no |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules. | `string` | `null` | no |
+| <a name="input_prometheus_size"></a> [prometheus\_size](#input\_prometheus\_size) | Storage size for the Prometheus database | `string` | `"500G"` | no |
+| <a name="input_pushgateway_revision"></a> [pushgateway\_revision](#input\_pushgateway\_revision) | Charm revision for prometheus-pushgateway-k8s | `number` | `null` | no |
+| <a name="input_s3"></a> [s3](#input\_s3) | S3 Bucket information | <pre>object({<br/>    bucket   = optional(string, "spark-test")<br/>    endpoint = optional(string, "https://s3.amazonaws.com")<br/>    region   = optional(string, "us-east-1")<br/>  })</pre> | `{}` | no |
+| <a name="input_s3_revision"></a> [s3\_revision](#input\_s3\_revision) | Charm revision for s3-integrator | `number` | `null` | no |
+| <a name="input_scrape_config_revision"></a> [scrape\_config\_revision](#input\_scrape\_config\_revision) | Charm revision for prometheus-scrape-config-k8s | `number` | `null` | no |
+| <a name="input_storage_backend"></a> [storage\_backend](#input\_storage\_backend) | Storage backend to be used | `string` | `"s3"` | no |
+| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The file path of the private key to use for TLS certificates. | `string` | `null` | no |
+| <a name="input_traefik_size"></a> [traefik\_size](#input\_traefik\_size) | Storage size for the Traefik storage | `string` | `"10G"` | no |
+| <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | `null` | no |
+| <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | `null` | no |
+| <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
+| <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Define the number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
+| <a name="output_components"></a> [components](#output\_components) | List of component modules (spark and optional addons) and their deployed charms/offers. |
+| <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
diff --git a/releases/terraform/products/charmed-spark-3.5/locals.tf b/releases/terraform/products/charmed-spark-3.5/locals.tf
new file mode 100644
index 00000000..10b756b2
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.5/locals.tf
@@ -0,0 +1,43 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+locals {
+  revisions = {
+    history_server  = 94
+    integration_hub = 113
+    kyuubi          = 142
+    kyuubi_users    = 495
+    metastore       = 495
+    zookeeper       = 78
+    data_integrator = 179
+    s3              = 145
+    azure_storage   = 15
+    grafana_agent   = 121
+    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
+    cos_configuration = 65
+    pushgateway       = 27
+    scrape_config     = 67
+  }
+  images = {
+    history_server = {
+      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
+    }
+    integration_hub = {
+      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
+    }
+    kyuubi = {
+      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:cdde52a4f72112ea09de11c6d21f901323619cf41015db89cb9d5742c57da303"
+    }
+    kyuubi_users = {
+      postgresql-image = 165
+    }
+    metastore = {
+      postgresql-image = 165
+    }
+    zookeeper = {
+      zookeeper-image = 34
+    }
+  }
+}
diff --git a/releases/terraform/products/charmed-spark-3.5/main.tf b/releases/terraform/products/charmed-spark-3.5/main.tf
new file mode 100644
index 00000000..c7c79671
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.5/main.tf
@@ -0,0 +1,146 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_model" "spark" {
+  count      = (var.model_uuid == null && var.create_model == true) ? 1 : 0
+  name       = var.model
+  credential = var.K8S_CREDENTIAL
+  cloud {
+    name = var.K8S_CLOUD
+  }
+}
+
+resource "juju_model" "cos" {
+  count      = var.cos.deployed == "bundled" && var.model_uuid == null && var.create_model == true ? 1 : 0
+  name       = var.cos.model
+  credential = var.K8S_CREDENTIAL
+  cloud {
+    name = var.K8S_CLOUD
+  }
+}
+
+locals {
+  active_model = coalesce(var.model_uuid, var.model)
+}
+
+data "juju_model" "spark" {
+  name = var.model
+}
+
+module "ssc" {
+  depends_on  = [juju_model.spark]
+  source      = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev586"
+  model_uuid  = coalesce(var.model_uuid, data.juju_model.spark.id)
+  app_name    = "certificates"
+  channel     = "1/stable"
+  revision    = 317
+  constraints = "arch=amd64"
+  base        = "ubuntu@24.04"
+  units       = 1
+  config = {
+    ca-common-name = var.certificate_common_name
+  }
+}
+
+module "spark" {
+  depends_on                                 = [juju_model.spark, module.ssc]
+  source                                     = "../../components/spark-3.5"
+  model                                      = local.active_model
+  kyuubi_user                                = var.kyuubi_user
+  kyuubi_profile                             = var.kyuubi_profile
+  admin_password                             = var.admin_password
+  tls_private_key                            = var.tls_private_key
+  enable_dynamic_allocation                  = var.enable_dynamic_allocation
+  kyuubi_k8s_node_selectors                  = var.kyuubi_k8s_node_selectors
+  kyuubi_loadbalancer_extra_annotations      = var.kyuubi_loadbalancer_extra_annotations
+  kyuubi_gpu_enable                          = var.kyuubi_gpu_enable
+  kyuubi_gpu_engine_executors_limit          = var.kyuubi_gpu_engine_executors_limit
+  kyuubi_gpu_pinned_memory                   = var.kyuubi_gpu_pinned_memory
+  kyuubi_driver_pod_template                 = var.kyuubi_driver_pod_template
+  kyuubi_executor_pod_template               = var.kyuubi_executor_pod_template
+  kyuubi_executor_cores                      = var.kyuubi_executor_cores
+  kyuubi_executor_memory                     = var.kyuubi_executor_memory
+  driver_pod_template                        = var.driver_pod_template
+  executor_pod_template                      = var.executor_pod_template
+  integration_hub_monitored_service_accounts = var.integration_hub_monitored_service_accounts
+
+  tls_app_name              = module.ssc.app_name
+  tls_certificates_endpoint = module.ssc.provides.certificates
+  zookeeper_units           = var.zookeeper_units
+  kyuubi_units              = var.kyuubi_units
+
+  history_server_revision  = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server
+  history_server_image     = var.history_server_image != null ? var.history_server_image : local.images.history_server
+  integration_hub_revision = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub
+  integration_hub_image    = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub
+  kyuubi_revision          = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi
+  kyuubi_image             = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi
+  kyuubi_users_revision    = var.kyuubi_users_revision != null ? var.kyuubi_users_revision : local.revisions.kyuubi_users
+  kyuubi_users_image       = var.kyuubi_users_image != null ? var.kyuubi_users_image : local.images.kyuubi_users
+  metastore_revision       = var.metastore_revision != null ? var.metastore_revision : local.revisions.metastore
+  metastore_image          = var.metastore_image != null ? var.metastore_image : local.images.metastore
+  zookeeper_revision       = var.zookeeper_revision != null ? var.zookeeper_revision : local.revisions.zookeeper
+  zookeeper_image          = var.zookeeper_image != null ? var.zookeeper_image : local.images.zookeeper
+  data_integrator_revision = var.data_integrator_revision != null ? var.data_integrator_revision : local.revisions.data_integrator
+
+  kyuubi_users_size = var.kyuubi_users_size
+  metastore_size    = var.metastore_size
+  zookeeper_size    = var.zookeeper_size
+}
+
+module "azure_storage" {
+  depends_on    = [module.spark]
+  count         = var.storage_backend == "azure_storage" ? 1 : 0
+  source = "../../charms/azure-storage-integrator"
+  model_uuid    = local.active_model
+  spark_charms  = module.spark.charms
+  azure_storage = var.azure_storage
+
+  azure_storage_revision = var.azure_storage_revision != null ? var.azure_storage_revision : local.revisions.azure_storage
+}
+
+module "s3" {
+  depends_on   = [module.spark]
+  count        = var.storage_backend == "s3" ? 1 : 0
+  source = "../../charms/s3-integrator"
+  model_uuid   = local.active_model
+  spark_charms = module.spark.charms
+  s3           = var.s3
+
+  s3_revision = var.s3_revision != null ? var.s3_revision : local.revisions.s3
+}
+
+module "bundled_cos" {
+  depends_on   = [juju_model.cos]
+  count        = var.cos.deployed == "bundled" ? 1 : 0
+  source = "../cos"
+  model        = var.cos.model
+  cos_tls_ca   = var.cos.tls.ca
+  cos_tls_cert = var.cos.tls.cert
+  cos_tls_key  = var.cos.tls.key
+
+  alertmanager_size                = var.alertmanager_size
+  grafana_size                     = var.grafana_size
+  loki_active_index_directory_size = var.loki_active_index_directory_size
+  loki_chunks_size                 = var.loki_chunks_size
+  prometheus_size                  = var.prometheus_size
+  traefik_size                     = var.traefik_size
+}
+
+
+module "observability" {
+  depends_on       = [module.spark, module.bundled_cos]
+  count            = var.cos.deployed == "no" ? 0 : 1
+  source = "../../components/observability"
+  dashboards_offer = var.cos.deployed == "external" ? var.cos.offers.dashboard : one(module.bundled_cos[*].dashboards_offer)
+  logging_offer    = var.cos.deployed == "external" ? var.cos.offers.logging : one(module.bundled_cos[*].logging_offer)
+  metrics_offer    = var.cos.deployed == "external" ? var.cos.offers.metrics : one(module.bundled_cos[*].metrics_offer)
+  spark_model      = var.model
+  model_uuid       = local.active_model
+  spark_charms     = module.spark.charms
+
+  grafana_agent_revision     = var.grafana_agent_revision != null ? var.grafana_agent_revision : local.revisions.grafana_agent
+  cos_configuration_revision = var.cos_configuration_revision != null ? var.cos_configuration_revision : local.revisions.cos_configuration
+  pushgateway_revision       = var.pushgateway_revision != null ? var.pushgateway_revision : local.revisions.pushgateway
+  scrape_config_revision     = var.scrape_config_revision != null ? var.scrape_config_revision : local.revisions.scrape_config
+}
diff --git a/releases/terraform/products/charmed-spark-3.5/outputs.tf b/releases/terraform/products/charmed-spark-3.5/outputs.tf
new file mode 100644
index 00000000..f1a2b99f
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.5/outputs.tf
@@ -0,0 +1,51 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "charms" {
+  description = "The name of the charms which are part of the deployment."
+  value = merge(
+    concat(
+      module.spark[*].charms, module.azure_storage[*].charms, module.s3[*].charms, module.bundled_cos[*].charms, module.observability[*].charms
+    )...
+  )
+}
+
+output "components" {
+  description = "List of component modules (spark and optional addons) and their deployed charms/offers."
+  value = [
+    {
+      name   = "spark"
+      charms = module.spark.charms
+      offers = module.spark.offers
+    },
+    {
+      name   = "azure_storage"
+      charms = try(module.azure_storage.charms, [])
+      offers = try(module.azure_storage.offers, {})
+    },
+    {
+      name   = "s3"
+      charms = try(module.s3.charms, [])
+      offers = try(module.s3.offers, {})
+    },
+    {
+      name   = "bundled_cos"
+      charms = try(module.bundled_cos.charms, [])
+      offers = try(module.bundled_cos.offers, {})
+    },
+    {
+      name   = "observability"
+      charms = try(module.observability.charms, [])
+      offers = try(module.observability.offers, {})
+    }
+  ]
+}
+
+output "offers" {
+  description = "The name and url of the various offers being exposed"
+  value = merge(
+    concat(
+      module.spark[*].offers
+    )...
+  )
+}
diff --git a/releases/terraform/products/charmed-spark-3.5/providers.tf b/releases/terraform/products/charmed-spark-3.5/providers.tf
new file mode 100644
index 00000000..b60772fa
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.5/providers.tf
@@ -0,0 +1,6 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+provider "juju" {
+  # Configure the Juju provider if needed (e.g., `controller`, `model`, `username`, etc.).
+}
diff --git a/releases/terraform/products/charmed-spark-3.5/terraform.tf b/releases/terraform/products/charmed-spark-3.5/terraform.tf
new file mode 100644
index 00000000..975cf66e
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.5/terraform.tf
@@ -0,0 +1,13 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/products/charmed-spark-3.5/variables.tf b/releases/terraform/products/charmed-spark-3.5/variables.tf
new file mode 100644
index 00000000..57c95ffe
--- /dev/null
+++ b/releases/terraform/products/charmed-spark-3.5/variables.tf
@@ -0,0 +1,395 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Juju
+
+variable "admin_password" {
+  description = "The password for the admin user."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "alertmanager_size" {
+  description = "Storage size for the alertmanager database"
+  type        = string
+  default     = "10G"
+}
+
+variable "azure_storage" {
+  description = "Azure Object storage information"
+  type = object({
+    container       = optional(string, "azurecontainer")
+    storage_account = optional(string, "azurestorageaccount")
+    secret_key      = optional(string, "azurestoragesecret")
+    protocol        = optional(string, "abfss")
+  })
+  default = {}
+}
+
+variable "azure_storage_revision" {
+  description = "Charm revision for azure-storage-integrator"
+  type        = number
+  default     = null
+}
+
+variable "certificate_common_name" {
+  description = "Common name for the certificate to be used in self-signed"
+  type        = string
+  default     = "charmed-spark"
+}
+
+variable "cos" {
+  description = "Observability settings"
+  type = object({
+    model    = optional(string, "cos")
+    deployed = optional(string, "bundled")
+    offers = optional(object({
+      dashboard = optional(string, null),
+      metrics   = optional(string, null),
+      logging   = optional(string, null)
+    }), {}),
+    tls = optional(object({
+      cert = optional(string, "")
+      key  = optional(string, "")
+      ca   = optional(string, "")
+    }), {})
+  })
+  default = { model = "cos", deployed = "bundled", offers = {}, tls = {} }
+
+  validation {
+    condition     = contains(["external", "bundled", "no"], var.cos.deployed)
+    error_message = "Valid values for var: cos.deployed are (external, bundled, no)"
+  }
+
+  validation {
+    condition = var.cos.deployed != "external" || alltrue([
+      var.cos.offers.dashboard != null,
+      var.cos.offers.metrics != null,
+      var.cos.offers.logging != null,
+    ])
+    error_message = "When using external cos, please define all offers variables"
+  }
+
+}
+
+variable "cos_configuration_revision" {
+  description = "Charm revision for cos-configuration-k8s"
+  type        = number
+  default     = null
+}
+
+variable "create_model" {
+  description = "Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism."
+  type        = bool
+  default     = true
+  nullable    = false
+}
+
+variable "data_integrator_revision" {
+  description = "Charm revision for data-integrator"
+  type        = number
+  default     = null
+}
+
+variable "driver_pod_template" {
+  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
+  type        = string
+  default     = null
+}
+
+variable "enable_dynamic_allocation" {
+  description = "Enable dynamic allocation of pods for Spark jobs."
+  type        = bool
+  default     = false
+}
+
+variable "executor_pod_template" {
+  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
+  type        = string
+  default     = null
+}
+
+variable "grafana_agent_revision" {
+  description = "Charm revision for grafana-agent-k8s"
+  type        = number
+  default     = null
+}
+
+variable "grafana_size" {
+  description = "Storage size for the grafana database"
+  type        = string
+  default     = "10G"
+}
+
+variable "history_server_image" {
+  description = "Image for spark-history-server-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "history_server_revision" {
+  description = "Charm revision for spark-history-server-k8s"
+  type        = number
+  default     = null
+}
+
+variable "integration_hub_image" {
+  description = "Image for spark-integration-hub-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "integration_hub_monitored_service_accounts" {
+  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
+  type        = string
+  default     = null
+}
+
+variable "integration_hub_revision" {
+  description = "Charm revision for spark-integration-hub-k8s"
+  type        = number
+  default     = null
+}
+
+variable "K8S_CLOUD" {
+  type        = string
+  description = "The kubernetes juju cloud name."
+  default     = "microk8s"
+}
+
+variable "K8S_CREDENTIAL" {
+  type        = string
+  description = "The name of the kubernetes juju credential."
+  default     = "microk8s"
+}
+
+variable "kyuubi_driver_pod_template" {
+  description = "Define K8s driver pod from a file accessible in the object storage."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_executor_cores" {
+  description = "Set kyuubi executor pods cpu cores."
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_executor_memory" {
+  description = "Set kyuubi executor pods memory (in GB)."
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_executor_pod_template" {
+  description = "Define K8s executor pod from a file accessible in the object storage."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_gpu_enable" {
+  description = "Enable GPU acceleration for SparkSQLEngine."
+  type        = bool
+  default     = false
+}
+
+variable "kyuubi_gpu_engine_executors_limit" {
+  description = "Limit the number of GPUs an engine can schedule executor pods on."
+  type        = number
+  default     = 1
+}
+
+variable "kyuubi_gpu_pinned_memory" {
+  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
+  type        = number
+  default     = 1
+}
+
+variable "kyuubi_image" {
+  description = "Image for kyuubi-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "kyuubi_k8s_node_selectors" {
+  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_loadbalancer_extra_annotations" {
+  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_profile" {
+  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
+  type        = string
+  nullable    = false
+  default     = "production"
+}
+
+variable "kyuubi_revision" {
+  description = "Charm revision for kyuubi-k8s"
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_units" {
+  description = "Number of Kyuubi units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
+
+variable "kyuubi_user" {
+  description = "Define the user to be used for running Kyuubi enginers"
+  type        = string
+  default     = "kyuubi-spark-engine"
+}
+
+variable "kyuubi_users_image" {
+  description = "Image for postgresql-k8s (auth-db)"
+  type        = map(string)
+  default     = null
+}
+
+variable "kyuubi_users_revision" {
+  description = "Charm revision for postgresql-k8s (auth-db)"
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_users_size" {
+  description = "Storage size for the Kyuubi users database"
+  type        = string
+  default     = "1G"
+}
+
+variable "loki_active_index_directory_size" {
+  description = "Storage size for the active index directory for Loki"
+  type        = string
+  default     = "10G"
+}
+
+variable "loki_chunks_size" {
+  description = "Storage size for the Loki chucks storage"
+  type        = string
+  default     = "500G"
+}
+
+variable "metastore_image" {
+  description = "Image for postgresql-k8s (metastore)"
+  type        = map(string)
+  default     = null
+}
+
+variable "metastore_revision" {
+  description = "Charm revision for postgresql-k8s (metastore)"
+  type        = number
+  default     = null
+}
+
+variable "metastore_size" {
+  description = "Storage size for the metastore database"
+  type        = string
+  default     = "10G"
+}
+
+variable "model" {
+  description = "The name of the juju model to deploy Spark to"
+  type        = string
+  default     = "spark"
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules."
+  type        = string
+  default     = null
+}
+
+variable "prometheus_size" {
+  description = "Storage size for the Prometheus database"
+  type        = string
+  default     = "500G"
+}
+
+variable "pushgateway_revision" {
+  description = "Charm revision for prometheus-pushgateway-k8s"
+  type        = number
+  default     = null
+}
+
+variable "s3" {
+  description = "S3 Bucket information"
+  type = object({
+    bucket   = optional(string, "spark-test")
+    endpoint = optional(string, "https://s3.amazonaws.com")
+    region   = optional(string, "us-east-1")
+  })
+  default = {}
+}
+
+variable "s3_revision" {
+  description = "Charm revision for s3-integrator"
+  type        = number
+  default     = null
+}
+
+variable "scrape_config_revision" {
+  description = "Charm revision for prometheus-scrape-config-k8s"
+  type        = number
+  default     = null
+}
+
+variable "storage_backend" {
+  type        = string
+  description = "Storage backend to be used"
+
+  validation {
+    condition     = contains(["azure_storage", "s3"], var.storage_backend)
+    error_message = "Valid values for var: test_variable are (s3, azure_storage)."
+  }
+
+  default = "s3"
+}
+
+variable "tls_private_key" {
+  description = "The file path of the private key to use for TLS certificates."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "traefik_size" {
+  description = "Storage size for the Traefik storage"
+  type        = string
+  default     = "10G"
+}
+
+variable "zookeeper_image" {
+  description = "Image for zookeeper-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "zookeeper_revision" {
+  description = "Charm revision for zookeeper-k8s"
+  type        = number
+  default     = null
+}
+
+variable "zookeeper_size" {
+  description = "Storage size for the metastore database"
+  type        = string
+  default     = "10G"
+}
+
+variable "zookeeper_units" {
+  description = "Define the number of zookeeper units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
diff --git a/releases/terraform/products/cos/README.md b/releases/terraform/products/cos/README.md
new file mode 100644
index 00000000..8c609af3
--- /dev/null
+++ b/releases/terraform/products/cos/README.md
@@ -0,0 +1,77 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_application.alertmanager](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.catalogue](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.grafana](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.traefik](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.catalogue-alertmanager](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.catalogue-grafana](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.catalogue-prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.catalogue-traefik](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana-alertmanager-dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana-alertmanager-source](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana-loki-dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana-loki-source](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana-prometheus-dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana-prometheus-source](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.loki-alertmanager](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.prometheus-alertmanager-alerting](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.prometheus-alertmanager-metrics](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.prometheus-grafana](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.prometheus-loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.prometheus-traefik](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.traefik-alertmanager](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.traefik-grafana](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.traefik-loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.traefik-prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_offer.grafana_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_offer.loki_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_offer.prometheus_receive_remote_write](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_model.cos](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_alertmanager_size"></a> [alertmanager\_size](#input\_alertmanager\_size) | Storage size for the alertmanager database | `string` | `"10G"` | no |
+| <a name="input_cos_tls_ca"></a> [cos\_tls\_ca](#input\_cos\_tls\_ca) | COS CA certificate | `string` | `""` | no |
+| <a name="input_cos_tls_cert"></a> [cos\_tls\_cert](#input\_cos\_tls\_cert) | COS certificate | `string` | `""` | no |
+| <a name="input_cos_tls_key"></a> [cos\_tls\_key](#input\_cos\_tls\_key) | COS certificate key | `string` | `""` | no |
+| <a name="input_cos_user"></a> [cos\_user](#input\_cos\_user) | The name of the Juju user of the COS deployment. | `string` | `"admin"` | no |
+| <a name="input_grafana_size"></a> [grafana\_size](#input\_grafana\_size) | Storage size for the grafana database | `string` | `"10G"` | no |
+| <a name="input_loki_active_index_directory_size"></a> [loki\_active\_index\_directory\_size](#input\_loki\_active\_index\_directory\_size) | Storage size for the active index directory for Loki | `string` | `"10G"` | no |
+| <a name="input_loki_chunks_size"></a> [loki\_chunks\_size](#input\_loki\_chunks\_size) | Storage size for the Loki chucks storage | `string` | `"500G"` | no |
+| <a name="input_model"></a> [model](#input\_model) | The name of the Juju model to deploy to | `string` | n/a | yes |
+| <a name="input_prometheus_size"></a> [prometheus\_size](#input\_prometheus\_size) | Storage size for the Prometheus database | `string` | `"500G"` | no |
+| <a name="input_traefik_size"></a> [traefik\_size](#input\_traefik\_size) | Storage size for the Traefik storage | `string` | `"10G"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
+| <a name="output_dashboards_offer"></a> [dashboards\_offer](#output\_dashboards\_offer) | n/a |
+| <a name="output_logging_offer"></a> [logging\_offer](#output\_logging\_offer) | n/a |
+| <a name="output_metrics_offer"></a> [metrics\_offer](#output\_metrics\_offer) | n/a |
+| <a name="output_user"></a> [user](#output\_user) | The name of the Juju user of the COS deployment. |
diff --git a/releases/terraform/products/cos/applications.tf b/releases/terraform/products/cos/applications.tf
new file mode 100644
index 00000000..bfc8f7dc
--- /dev/null
+++ b/releases/terraform/products/cos/applications.tf
@@ -0,0 +1,103 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "alertmanager" {
+  name  = "alertmanager"
+  trust = true
+  model_uuid = data.juju_model.cos.id
+  charm {
+    name    = "alertmanager-k8s"
+    channel = "1/stable"
+  }
+  units       = 1
+  constraints = "arch=amd64"
+  storage_directives = {
+    data = var.alertmanager_size
+  }
+}
+
+resource "juju_application" "catalogue" {
+  name  = "catalogue"
+  trust = true
+  model_uuid = data.juju_model.cos.id
+  charm {
+    name    = "catalogue-k8s"
+    channel = "1/stable"
+  }
+  units       = 1
+  constraints = "arch=amd64"
+  config = {
+    "description" : "Canonical Observability Stack Lite"
+  }
+}
+
+resource "juju_application" "grafana" {
+  name  = "grafana"
+  trust = true
+  model_uuid = data.juju_model.cos.id
+  charm {
+    name    = "grafana-k8s"
+    channel = "1/stable"
+  }
+  units       = 1
+  constraints = "arch=amd64"
+  storage_directives = {
+    database = var.grafana_size
+  }
+}
+
+resource "juju_application" "loki" {
+  name  = "loki"
+  trust = true
+  model_uuid = data.juju_model.cos.id
+  charm {
+    name    = "loki-k8s"
+    channel = "1/stable"
+  }
+  units       = 1
+  constraints = "arch=amd64"
+  storage_directives = {
+    active-index-directory = var.loki_active_index_directory_size
+    loki-chunks            = var.loki_chunks_size
+  }
+}
+
+resource "juju_application" "prometheus" {
+  name  = "prometheus"
+  trust = true
+  model_uuid = data.juju_model.cos.id
+  charm {
+    name    = "prometheus-k8s"
+    channel = "1/stable"
+  }
+  config = {
+    "metrics_retention_time" : "90d"
+  }
+  units       = 1
+  constraints = "arch=amd64"
+  storage_directives = {
+    database = var.prometheus_size
+  }
+}
+
+resource "juju_application" "traefik" {
+  name  = "traefik"
+  trust = true
+  model_uuid = data.juju_model.cos.id
+  charm {
+    name = "traefik-k8s"
+    # FIXME(stable): Use different track when available
+    channel = "latest/stable"
+  }
+  config = {
+    "tls-cert" : var.cos_tls_cert,
+    "tls-key" : var.cos_tls_key,
+    "tls-ca" : var.cos_tls_ca
+  }
+  units       = 1
+  constraints = "arch=amd64"
+  storage_directives = {
+    configurations = var.traefik_size
+  }
+}
+
diff --git a/releases/terraform/products/cos/integrations.tf b/releases/terraform/products/cos/integrations.tf
new file mode 100644
index 00000000..7ecfd638
--- /dev/null
+++ b/releases/terraform/products/cos/integrations.tf
@@ -0,0 +1,300 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+
+
+
+resource "juju_integration" "traefik-grafana" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.traefik.name
+    endpoint = "traefik-route"
+  }
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "ingress"
+  }
+}
+
+resource "juju_integration" "traefik-prometheus" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.traefik.name
+    endpoint = "ingress-per-unit"
+  }
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "ingress"
+  }
+}
+
+resource "juju_integration" "traefik-loki" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.traefik.name
+    endpoint = "ingress-per-unit"
+  }
+
+  application {
+    name     = juju_application.loki.name
+    endpoint = "ingress"
+  }
+}
+
+resource "juju_integration" "traefik-alertmanager" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.traefik.name
+    endpoint = "ingress"
+  }
+
+  application {
+    name     = juju_application.alertmanager.name
+    endpoint = "ingress"
+  }
+}
+
+resource "juju_integration" "prometheus-alertmanager-alerting" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "alertmanager"
+  }
+
+  application {
+    name     = juju_application.alertmanager.name
+    endpoint = "alerting"
+  }
+}
+
+
+resource "juju_integration" "grafana-prometheus-source" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "grafana-source"
+  }
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "grafana-source"
+  }
+}
+
+
+resource "juju_integration" "grafana-loki-source" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "grafana-source"
+  }
+
+  application {
+    name     = juju_application.loki.name
+    endpoint = "grafana-source"
+  }
+}
+
+
+resource "juju_integration" "grafana-alertmanager-source" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "grafana-source"
+  }
+
+  application {
+    name     = juju_application.alertmanager.name
+    endpoint = "grafana-source"
+  }
+}
+
+
+resource "juju_integration" "loki-alertmanager" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.loki.name
+    endpoint = "alertmanager"
+  }
+
+  application {
+    name     = juju_application.alertmanager.name
+    endpoint = "alerting"
+  }
+}
+
+
+resource "juju_integration" "prometheus-traefik" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.traefik.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+
+resource "juju_integration" "prometheus-alertmanager-metrics" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.alertmanager.name
+    endpoint = "self-metrics-endpoint"
+  }
+}
+
+
+resource "juju_integration" "prometheus-loki" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.loki.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+
+resource "juju_integration" "prometheus-grafana" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+
+resource "juju_integration" "grafana-loki-dashboard" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "grafana-dashboard"
+  }
+
+  application {
+    name     = juju_application.loki.name
+    endpoint = "grafana-dashboard"
+  }
+}
+
+
+resource "juju_integration" "grafana-prometheus-dashboard" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "grafana-dashboard"
+  }
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "grafana-dashboard"
+  }
+}
+
+
+resource "juju_integration" "grafana-alertmanager-dashboard" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "grafana-dashboard"
+  }
+
+  application {
+    name     = juju_application.alertmanager.name
+    endpoint = "grafana-dashboard"
+  }
+}
+
+
+resource "juju_integration" "catalogue-traefik" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.catalogue.name
+    endpoint = "ingress"
+  }
+
+  application {
+    name     = juju_application.traefik.name
+    endpoint = "ingress"
+  }
+}
+
+
+resource "juju_integration" "catalogue-grafana" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.catalogue.name
+    endpoint = "catalogue"
+  }
+
+  application {
+    name     = juju_application.grafana.name
+    endpoint = "catalogue"
+  }
+}
+
+
+resource "juju_integration" "catalogue-prometheus" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.catalogue.name
+    endpoint = "catalogue"
+  }
+
+  application {
+    name     = juju_application.prometheus.name
+    endpoint = "catalogue"
+  }
+}
+
+
+resource "juju_integration" "catalogue-alertmanager" {
+  model_uuid = data.juju_model.cos.id
+
+  application {
+    name     = juju_application.catalogue.name
+    endpoint = "catalogue"
+  }
+
+  application {
+    name     = juju_application.alertmanager.name
+    endpoint = "catalogue"
+  }
+}
diff --git a/releases/terraform/products/cos/locals.tf b/releases/terraform/products/cos/locals.tf
new file mode 100644
index 00000000..44f3a0f6
--- /dev/null
+++ b/releases/terraform/products/cos/locals.tf
@@ -0,0 +1,2 @@
+# Local values for $(basename "$moduleDir").
+# Define computed values used in this module.
diff --git a/releases/terraform/products/cos/main.tf b/releases/terraform/products/cos/main.tf
new file mode 100644
index 00000000..46aca146
--- /dev/null
+++ b/releases/terraform/products/cos/main.tf
@@ -0,0 +1,2 @@
+# Main Terraform code for $(basename "$moduleDir").
+# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/products/cos/outputs.tf b/releases/terraform/products/cos/outputs.tf
new file mode 100644
index 00000000..e206ae26
--- /dev/null
+++ b/releases/terraform/products/cos/outputs.tf
@@ -0,0 +1,30 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "charms" {
+  description = "The name of the charms which are part of the deployment."
+  value = {
+    alertmanager = juju_application.alertmanager.name
+    catalogue    = juju_application.catalogue.name
+    grafana      = juju_application.grafana.name
+    loki         = juju_application.loki.name
+    prometheus   = juju_application.prometheus.name
+  }
+}
+
+output "dashboards_offer" {
+  value = juju_offer.grafana_dashboards.url
+}
+
+output "logging_offer" {
+  value = juju_offer.loki_logging.url
+}
+
+output "metrics_offer" {
+  value = juju_offer.prometheus_receive_remote_write.url
+}
+
+output "user" {
+  description = "The name of the Juju user of the COS deployment."
+  value       = var.cos_user
+}
diff --git a/releases/terraform/products/cos/providers.tf b/releases/terraform/products/cos/providers.tf
new file mode 100644
index 00000000..0693f27b
--- /dev/null
+++ b/releases/terraform/products/cos/providers.tf
@@ -0,0 +1,6 @@
+# Provider configuration for module $(basename "$moduleDir").
+# Terraform block moved to terraform.tf and should not be in providers.tf.
+
+# provider "juju" {
+#   # Configure provider here if required by the module.
+# }
diff --git a/releases/terraform/products/cos/resources.tf b/releases/terraform/products/cos/resources.tf
new file mode 100644
index 00000000..c48001a0
--- /dev/null
+++ b/releases/terraform/products/cos/resources.tf
@@ -0,0 +1,27 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define juju resources (model, secrets, storage)
+
+data "juju_model" "cos" {
+  name = var.model
+}
+
+resource "juju_offer" "prometheus_receive_remote_write" {
+  model_uuid = data.juju_model.cos.id
+  application_name = juju_application.prometheus.name
+  endpoints        = ["receive-remote-write"]
+}
+
+resource "juju_offer" "grafana_dashboards" {
+  model_uuid = data.juju_model.cos.id
+  application_name = juju_application.grafana.name
+  endpoints        = ["grafana-dashboard"]
+}
+
+resource "juju_offer" "loki_logging" {
+  model_uuid = data.juju_model.cos.id
+  application_name = juju_application.loki.name
+  endpoints        = ["logging"]
+}
+
diff --git a/releases/terraform/products/cos/terraform.tf b/releases/terraform/products/cos/terraform.tf
new file mode 100644
index 00000000..6412125a
--- /dev/null
+++ b/releases/terraform/products/cos/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/products/cos/variables.tf b/releases/terraform/products/cos/variables.tf
new file mode 100644
index 00000000..16a0b362
--- /dev/null
+++ b/releases/terraform/products/cos/variables.tf
@@ -0,0 +1,69 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Define cos-specific variables
+
+variable "alertmanager_size" {
+  description = "Storage size for the alertmanager database"
+  type        = string
+  default     = "10G"
+}
+
+variable "cos_tls_ca" {
+  type        = string
+  default     = ""
+  description = "COS CA certificate"
+}
+
+variable "cos_tls_cert" {
+  type        = string
+  default     = ""
+  description = "COS certificate"
+}
+
+variable "cos_tls_key" {
+  type        = string
+  default     = ""
+  description = "COS certificate key"
+}
+
+variable "cos_user" {
+  description = "The name of the Juju user of the COS deployment."
+  type        = string
+  default     = "admin"
+}
+
+variable "grafana_size" {
+  description = "Storage size for the grafana database"
+  type        = string
+  default     = "10G"
+}
+
+variable "loki_active_index_directory_size" {
+  description = "Storage size for the active index directory for Loki"
+  type        = string
+  default     = "10G"
+}
+
+variable "loki_chunks_size" {
+  description = "Storage size for the Loki chucks storage"
+  type        = string
+  default     = "500G"
+}
+
+variable "model" {
+  description = "The name of the Juju model to deploy to"
+  type        = string
+}
+
+variable "prometheus_size" {
+  description = "Storage size for the Prometheus database"
+  type        = string
+  default     = "500G"
+}
+
+variable "traefik_size" {
+  description = "Storage size for the Traefik storage"
+  type        = string
+  default     = "10G"
+}

From 0b4b7818d30016701c9ac221b9e42a8da7c31d6a Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 31 Mar 2026 11:20:54 +0200
Subject: [PATCH 02/62] Adapt object storage charm modules, remove manual tls

---
 .../charms/azure-storage-integrator/README.md | 27 +++---
 .../azure-storage-integrator/applications.tf  | 21 -----
 .../azure-storage-integrator/integrations.tf  | 30 ------
 .../charms/azure-storage-integrator/locals.tf |  2 -
 .../charms/azure-storage-integrator/main.tf   | 44 ++++++++-
 .../azure-storage-integrator/outputs.tf       | 20 ++--
 .../azure-storage-integrator/providers.tf     |  9 +-
 .../azure-storage-integrator/resources.tf     | 30 ------
 .../azure-storage-integrator/variables.tf     | 72 +++++++++++----
 .../terraform/charms/manual-tls/README.md     | 41 ---------
 .../terraform/charms/manual-tls/locals.tf     |  2 -
 releases/terraform/charms/manual-tls/main.tf  | 91 -------------------
 .../terraform/charms/manual-tls/outputs.tf    | 16 ----
 .../terraform/charms/manual-tls/providers.tf  |  6 --
 .../terraform/charms/manual-tls/variables.tf  | 50 ----------
 .../charms/s3-integrator-v1/README.md         | 44 +++++++++
 .../terraform/charms/s3-integrator-v1/main.tf | 23 +++++
 .../charms/s3-integrator-v1/outputs.tf        | 22 +++++
 .../charms/s3-integrator-v1/providers.tf      |  5 +
 .../terraform.tf                              |  0
 .../charms/s3-integrator-v1/variables.tf      | 67 ++++++++++++++
 .../terraform/charms/s3-integrator/README.md  | 42 ---------
 .../charms/s3-integrator/applications.tf      | 20 ----
 .../charms/s3-integrator/integrations.tf      | 30 ------
 .../terraform/charms/s3-integrator/locals.tf  |  2 -
 .../terraform/charms/s3-integrator/main.tf    |  2 -
 .../terraform/charms/s3-integrator/outputs.tf | 21 -----
 .../charms/s3-integrator/providers.tf         |  6 --
 .../charms/s3-integrator/resources.tf         | 12 ---
 .../charms/s3-integrator/terraform.tf         | 10 --
 .../charms/s3-integrator/variables.tf         | 39 --------
 31 files changed, 286 insertions(+), 520 deletions(-)
 delete mode 100644 releases/terraform/charms/azure-storage-integrator/applications.tf
 delete mode 100644 releases/terraform/charms/azure-storage-integrator/integrations.tf
 delete mode 100644 releases/terraform/charms/azure-storage-integrator/locals.tf
 delete mode 100644 releases/terraform/charms/azure-storage-integrator/resources.tf
 delete mode 100644 releases/terraform/charms/manual-tls/README.md
 delete mode 100644 releases/terraform/charms/manual-tls/locals.tf
 delete mode 100644 releases/terraform/charms/manual-tls/main.tf
 delete mode 100644 releases/terraform/charms/manual-tls/outputs.tf
 delete mode 100644 releases/terraform/charms/manual-tls/providers.tf
 delete mode 100644 releases/terraform/charms/manual-tls/variables.tf
 create mode 100644 releases/terraform/charms/s3-integrator-v1/README.md
 create mode 100644 releases/terraform/charms/s3-integrator-v1/main.tf
 create mode 100644 releases/terraform/charms/s3-integrator-v1/outputs.tf
 create mode 100644 releases/terraform/charms/s3-integrator-v1/providers.tf
 rename releases/terraform/charms/{manual-tls => s3-integrator-v1}/terraform.tf (100%)
 create mode 100644 releases/terraform/charms/s3-integrator-v1/variables.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/README.md
 delete mode 100644 releases/terraform/charms/s3-integrator/applications.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/integrations.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/locals.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/main.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/outputs.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/providers.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/resources.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/terraform.tf
 delete mode 100644 releases/terraform/charms/s3-integrator/variables.tf

diff --git a/releases/terraform/charms/azure-storage-integrator/README.md b/releases/terraform/charms/azure-storage-integrator/README.md
index d8926087..48cfeeee 100644
--- a/releases/terraform/charms/azure-storage-integrator/README.md
+++ b/releases/terraform/charms/azure-storage-integrator/README.md
@@ -19,26 +19,29 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [juju_access_secret.azure_blob_storage_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
+| [juju_access_secret.azure_storage_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
 | [juju_application.azure_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_integration.azure_storage_integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.history_server_azure_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_secret.azure_blob_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
-| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
+| [juju_offer.azure_storage_credentials](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_secret.azure_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_azure_storage"></a> [azure\_storage](#input\_azure\_storage) | Azure Object storage information | <pre>object({<br/>    container       = optional(string, "azurecontainer")<br/>    storage_account = optional(string, "azurestorageaccount")<br/>    protocol        = optional(string, "abfss")<br/>    secret_key      = optional(string, "secret-key")<br/>    path            = optional(string, "spark-events")<br/>  })</pre> | `{}` | no |
-| <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | n/a | yes |
-| <a name="input_model"></a> [model](#input\_model) | Name of the Spark Juju Model to deploy to | `string` | n/a | yes |
-| <a name="input_spark_charms"></a> [spark\_charms](#input\_spark\_charms) | Names of the Spark applications in the Spark Juju model. | `map(string)` | n/a | yes |
+| <a name="input_app_name"></a> [app\_name](#input\_app\_name) | Name to give the deployed application. | `string` | `"azure-storage-integrator"` | no |
+| <a name="input_azure_storage_secret_key"></a> [azure\_storage\_secret\_key](#input\_azure\_storage\_secret\_key) | Secret key to the Azure Storage account. | `string` | n/a | yes |
+| <a name="input_base"></a> [base](#input\_base) | The operating system on which to deploy | `string` | `"ubuntu@22.04"` | no |
+| <a name="input_channel"></a> [channel](#input\_channel) | Channel of the charm. | `string` | `"latest/edge"` | no |
+| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    connection-protocol = optional(string, "abfss")<br/>    container           = optional(string, "azurecontainer")<br/>    endpoint            = optional(string, "")<br/>    path                = optional(string, "spark-events")<br/>    resource-group      = optional(string, "azurerg")<br/>    storage_account     = optional(string, "azurestorageaccount")<br/>  })</pre> | `{}` | no |
+| <a name="input_constraints"></a> [constraints](#input\_constraints) | String listing constraints for this application. | `string` | `null` | no |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
+| <a name="input_revision"></a> [revision](#input\_revision) | Revision number of the charm. | `number` | `null` | no |
+| <a name="input_units"></a> [units](#input\_units) | Unit count. | `number` | `1` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| <a name="output_application"></a> [application](#output\_application) | The deployed application name. |
-| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
-| <a name="output_provides"></a> [provides](#output\_provides) | Relation endpoint provided by this module. |
+| <a name="output_application"></a> [application](#output\_application) | Object representing the deployed application. |
+| <a name="output_offers"></a> [offers](#output\_offers) | Map of all offers exposed by the single charm. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Provides endpoints. |
diff --git a/releases/terraform/charms/azure-storage-integrator/applications.tf b/releases/terraform/charms/azure-storage-integrator/applications.tf
deleted file mode 100644
index 64d1485a..00000000
--- a/releases/terraform/charms/azure-storage-integrator/applications.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "azure_storage" {
-  name  = "azure-storage"
-  model_uuid = data.juju_model.spark.id
-  charm {
-    name     = "azure-storage-integrator"
-    channel  = "latest/edge"
-    revision = var.azure_storage_revision
-  }
-  config = {
-    container           = var.azure_storage.container
-    storage-account     = var.azure_storage.storage_account
-    path                = var.azure_storage.path
-    connection-protocol = var.azure_storage.protocol
-    credentials         = "secret:${juju_secret.azure_blob_storage_secret.secret_id}"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/terraform/charms/azure-storage-integrator/integrations.tf b/releases/terraform/charms/azure-storage-integrator/integrations.tf
deleted file mode 100644
index b008ad80..00000000
--- a/releases/terraform/charms/azure-storage-integrator/integrations.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "history_server_azure_storage" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.azure_storage.name
-    endpoint = "azure-storage-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "azure-storage-credentials"
-  }
-}
-
-resource "juju_integration" "azure_storage_integration_hub" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.azure_storage.name
-    endpoint = "azure-storage-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "azure-storage-credentials"
-  }
-}
diff --git a/releases/terraform/charms/azure-storage-integrator/locals.tf b/releases/terraform/charms/azure-storage-integrator/locals.tf
deleted file mode 100644
index 44f3a0f6..00000000
--- a/releases/terraform/charms/azure-storage-integrator/locals.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Local values for $(basename "$moduleDir").
-# Define computed values used in this module.
diff --git a/releases/terraform/charms/azure-storage-integrator/main.tf b/releases/terraform/charms/azure-storage-integrator/main.tf
index 46aca146..ebf47196 100644
--- a/releases/terraform/charms/azure-storage-integrator/main.tf
+++ b/releases/terraform/charms/azure-storage-integrator/main.tf
@@ -1,2 +1,42 @@
-# Main Terraform code for $(basename "$moduleDir").
-# Module content is structured across *.tf files in this directory.
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "azure_storage" {
+  name       = var.app_name
+  model_uuid = var.model_uuid
+  charm {
+    name     = "azure-storage-integrator"
+    channel  = var.channel
+    revision = var.revision
+  }
+  config = merge(var.config, {
+    credentials = "secret:${juju_secret.azure_storage_secret.secret_id}"
+  })
+  units       = var.units
+  constraints = var.constraints
+}
+
+# TODO: Secret creation must be handled at the product level
+resource "juju_secret" "azure_storage_secret" {
+  model_uuid = var.model_uuid
+  name       = "azure_storage_secret"
+  value = {
+    secret-key = var.azure_storage_secret_key
+  }
+  info = "This is the secret key for the Azure storage account"
+}
+
+resource "juju_access_secret" "azure_storage_secret_access" {
+  model_uuid = var.model_uuid
+  applications = [
+    juju_application.azure_storage.name
+  ]
+  secret_id = juju_secret.azure_storage_secret.secret_id
+}
+
+resource "juju_offer" "azure_storage_credentials" {
+  name             = "azure-storage-credentials"
+  model_uuid       = var.model_uuid
+  application_name = var.app_name
+  endpoints        = ["azure-storage-credentials"]
+}
diff --git a/releases/terraform/charms/azure-storage-integrator/outputs.tf b/releases/terraform/charms/azure-storage-integrator/outputs.tf
index 0a4eec05..88c604b1 100644
--- a/releases/terraform/charms/azure-storage-integrator/outputs.tf
+++ b/releases/terraform/charms/azure-storage-integrator/outputs.tf
@@ -1,21 +1,23 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
+# 
+output "application" {
+  description = "Object representing the deployed application."
+  value       = juju_application.azure_storage
+}
 
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
+output "offers" {
+  description = "Map of all offers exposed by the single charm."
   value = {
-    azure_storage = juju_application.azure_storage.name
+    azure_storage_credentials = juju_offer.azure_storage_credentials.url
   }
 }
 
-output "application" {
-  description = "The deployed application name."
-  value       = juju_application.azure_storage.name
-}
 
 output "provides" {
-  description = "Relation endpoint provided by this module."
+  description = "Provides endpoints."
   value = {
-    "azure-storage-credentials" = juju_application.azure_storage.name
+    azure_storage_credentials = "azure-storage-credentials"
   }
 }
+
diff --git a/releases/terraform/charms/azure-storage-integrator/providers.tf b/releases/terraform/charms/azure-storage-integrator/providers.tf
index 0693f27b..58f9c66b 100644
--- a/releases/terraform/charms/azure-storage-integrator/providers.tf
+++ b/releases/terraform/charms/azure-storage-integrator/providers.tf
@@ -1,6 +1,5 @@
-# Provider configuration for module $(basename "$moduleDir").
-# Terraform block moved to terraform.tf and should not be in providers.tf.
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
 
-# provider "juju" {
-#   # Configure provider here if required by the module.
-# }
+# Provider configuration for azure-storage-integrator.
+# Provider blocks, if any, should be defined here.
diff --git a/releases/terraform/charms/azure-storage-integrator/resources.tf b/releases/terraform/charms/azure-storage-integrator/resources.tf
deleted file mode 100644
index e342298a..00000000
--- a/releases/terraform/charms/azure-storage-integrator/resources.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-locals {
-  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
-}
-
-data "juju_model" "spark" {
-  name = var.model
-}
-
-resource "juju_secret" "azure_blob_storage_secret" {
-  model_uuid = local.target_model_uuid
-  name  = "azure_blob_storage_secret"
-  value = {
-    secret-key = var.azure_storage.secret_key
-  }
-  info = "This is the secret key for the Azure storage account"
-}
-
-resource "juju_access_secret" "azure_blob_storage_secret_access" {
-  model_uuid = local.target_model_uuid
-  applications = [
-    juju_application.azure_storage.name
-  ]
-  # Use the secret_id from your secret resource or data source.
-  secret_id = juju_secret.azure_blob_storage_secret.secret_id
-}
diff --git a/releases/terraform/charms/azure-storage-integrator/variables.tf b/releases/terraform/charms/azure-storage-integrator/variables.tf
index 04072b59..5f6d933e 100644
--- a/releases/terraform/charms/azure-storage-integrator/variables.tf
+++ b/releases/terraform/charms/azure-storage-integrator/variables.tf
@@ -1,37 +1,71 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-variable "azure_storage" {
-  description = "Azure Object storage information"
-  type = object({
-    container       = optional(string, "azurecontainer")
-    storage_account = optional(string, "azurestorageaccount")
-    protocol        = optional(string, "abfss")
-    secret_key      = optional(string, "secret-key")
-    path            = optional(string, "spark-events")
-  })
-  default = {}
+variable "app_name" {
+  description = "Name to give the deployed application."
+  type        = string
+  default     = "azure-storage-integrator"
+  nullable    = false
 }
 
-variable "azure_storage_revision" {
-  description = "Charm revision for azure-storage-integrator"
-  type        = number
+variable "base" {
+  description = "The operating system on which to deploy"
+  type        = string
+  default     = "ubuntu@22.04"
   nullable    = false
 }
 
-variable "model" {
-  description = "Name of the Spark Juju Model to deploy to (fallback if model_uuid is not provided)."
+variable "channel" {
+  description = "Channel of the charm."
   type        = string
+  default     = "latest/edge" # TODO: Update to stable once we have the new release
   nullable    = false
 }
 
+variable "config" {
+  description = "Map for configuration options."
+  type = object({
+    connection-protocol = optional(string, "abfss")
+    container           = optional(string, "azurecontainer")
+    endpoint            = optional(string, "")
+    path                = optional(string, "spark-events")
+    resource-group      = optional(string, "azurerg")
+    storage_account     = optional(string, "azurestorageaccount")
+  })
+  default = {
+  }
+}
+
+
+variable "constraints" {
+  description = "String listing constraints for this application."
+  type        = string
+  default     = null
+}
+
 variable "model_uuid" {
-  description = "Optional existing Juju model UUID to deploy to."
+  description = "Reference to an existing model uuid."
   type        = string
+  nullable    = false
+}
+
+
+variable "revision" {
+  description = "Revision number of the charm."
+  type        = number
   default     = null
 }
 
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
+variable "units" {
+  description = "Unit count."
+  type        = number
+  default     = 1
+}
+
+# We should eventually handle this differently 
+variable "azure_storage_secret_key" {
+  description = "Secret key to the Azure Storage account."
+  type        = string
+  nullable    = false
+  sensitive   = true
 }
diff --git a/releases/terraform/charms/manual-tls/README.md b/releases/terraform/charms/manual-tls/README.md
deleted file mode 100644
index bf0eb31e..00000000
--- a/releases/terraform/charms/manual-tls/README.md
+++ /dev/null
@@ -1,41 +0,0 @@
-## Requirements
-
-| Name | Version |
-|------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
-
-## Providers
-
-| Name | Version |
-|------|---------|
-| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 >=1.0.0 |
-
-## Modules
-
-No modules.
-
-## Resources
-
-| Name | Type |
-|------|------|
-| [juju_application.certificates](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| <a name="input_app_name"></a> [app\_name](#input\_app\_name) | n/a | `string` | `"certificates"` | no |
-| <a name="input_channel"></a> [channel](#input\_channel) | n/a | `string` | `"latest/stable"` | no |
-| <a name="input_example"></a> [example](#input\_example) | Example variable | `string` | `""` | no |
-| <a name="input_model"></a> [model](#input\_model) | n/a | `string` | n/a | yes |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| <a name="output_app_name"></a> [app\_name](#output\_app\_name) | Name of the deployed application. |
-| <a name="output_example"></a> [example](#output\_example) | Example output |
-| <a name="output_provides"></a> [provides](#output\_provides) | n/a |
diff --git a/releases/terraform/charms/manual-tls/locals.tf b/releases/terraform/charms/manual-tls/locals.tf
deleted file mode 100644
index 44f3a0f6..00000000
--- a/releases/terraform/charms/manual-tls/locals.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Local values for $(basename "$moduleDir").
-# Define computed values used in this module.
diff --git a/releases/terraform/charms/manual-tls/main.tf b/releases/terraform/charms/manual-tls/main.tf
deleted file mode 100644
index b6dff313..00000000
--- a/releases/terraform/charms/manual-tls/main.tf
+++ /dev/null
@@ -1,91 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# This module is but a placeholder until we get an official TF module for the manual TLS operator
-# For testing purpose.
-
-variable "model" {
-  description = "Target Juju model name (fallback if model_uuid is not provided)."
-  type        = string
-  default     = "spark"
-}
-
-variable "model_uuid" {
-  description = "Optional existing Juju model UUID to deploy to."
-  type        = string
-  default     = null
-}
-
-variable "channel" {
-  description = "Charm channel."
-  type        = string
-  default     = "latest/stable"
-}
-
-variable "app_name" {
-  description = "Name of the deployed application."
-  type        = string
-  default     = "certificates"
-}
-
-variable "config" {
-  description = "Charm configuration map."
-  type        = map(string)
-  default     = {}
-}
-
-variable "constraints" {
-  description = "Juju constraints."
-  type        = string
-  default     = "arch=amd64"
-}
-
-variable "revision" {
-  description = "Charm revision number."
-  type        = number
-  default     = null
-}
-
-variable "units" {
-  description = "Number of units."
-  type        = number
-  default     = 1
-}
-
-locals {
-  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.target[0].id
-}
-
-data "juju_model" "target" {
-  count = var.model_uuid == null ? 1 : 0
-  name  = var.model
-}
-
-resource "juju_application" "certificates" {
-  name       = var.app_name
-  model_uuid = local.target_model_uuid
-
-  charm {
-    name     = "manual-tls-certificates"
-    channel  = var.channel
-    revision = var.revision
-  }
-
-  config      = var.config
-  constraints = var.constraints
-  units       = var.units
-  trust       = true
-}
-
-output "application" {
-  description = "Object representing the deployed application."
-  value = {
-    name = juju_application.certificates.name
-  }
-}
-
-output "provides" {
-  value = {
-    certificates = juju_application.certificates.name
-  }
-}
diff --git a/releases/terraform/charms/manual-tls/outputs.tf b/releases/terraform/charms/manual-tls/outputs.tf
deleted file mode 100644
index 8af3a313..00000000
--- a/releases/terraform/charms/manual-tls/outputs.tf
+++ /dev/null
@@ -1,16 +0,0 @@
-# Outputs for $(basename "$moduleDir").
-
-output "application" {
-  description = "Object representing the deployed application."
-  value = {
-    name = juju_application.certificates.name
-  }
-}
-
-output "provides" {
-  description = "Relation endpoints provided by the manual-tls charm."
-  value = {
-    certificates = juju_application.certificates.name
-  }
-}
-
diff --git a/releases/terraform/charms/manual-tls/providers.tf b/releases/terraform/charms/manual-tls/providers.tf
deleted file mode 100644
index 2ae9c8a3..00000000
--- a/releases/terraform/charms/manual-tls/providers.tf
+++ /dev/null
@@ -1,6 +0,0 @@
-# Provider configuration for module $(basename "$moduleDir").
-# Root module should configure provider credentials and connection.
-
-# provider "juju" {
-#   # Configure provider here if required by the module.
-# }
diff --git a/releases/terraform/charms/manual-tls/variables.tf b/releases/terraform/charms/manual-tls/variables.tf
deleted file mode 100644
index a9d09057..00000000
--- a/releases/terraform/charms/manual-tls/variables.tf
+++ /dev/null
@@ -1,50 +0,0 @@
-# Variables for $(basename "$moduleDir").
-
-variable "app_name" {
-  description = "Name to give the deployed application."
-  type        = string
-  default     = "certificates"
-}
-
-variable "channel" {
-  description = "Channel for the charm."
-  type        = string
-  default     = "latest/stable"
-}
-
-variable "config" {
-  description = "Application config for the charm."
-  type        = map(string)
-  default     = {}
-}
-
-variable "constraints" {
-  description = "Juju constraints for the charm deployment."
-  type        = string
-  default     = "arch=amd64"
-}
-
-variable "model" {
-  description = "Target Juju model name (fallback if model_uuid is not provided)."
-  type        = string
-  default     = "spark"
-}
-
-variable "model_uuid" {
-  description = "Optional existing Juju model UUID to deploy the charm into."
-  type        = string
-  default     = null
-}
-
-variable "revision" {
-  description = "Charm revision number (null means latest)."
-  type        = number
-  default     = null
-}
-
-variable "units" {
-  description = "Number of units for the application."
-  type        = number
-  default     = 1
-}
-
diff --git a/releases/terraform/charms/s3-integrator-v1/README.md b/releases/terraform/charms/s3-integrator-v1/README.md
new file mode 100644
index 00000000..725b258b
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator-v1/README.md
@@ -0,0 +1,44 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_application.s3_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_offer.s3_credentials](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_app_name"></a> [app\_name](#input\_app\_name) | Name to give the deployed application. | `string` | `"s3-integrator"` | no |
+| <a name="input_base"></a> [base](#input\_base) | The operating system on which to deploy | `string` | `"ubuntu@22.04"` | no |
+| <a name="input_channel"></a> [channel](#input\_channel) | Channel of the charm. | `string` | `"1/stable"` | no |
+| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    attributes                          = optional(string, "")<br/>    bucket                              = optional(string, "spark-bucket")<br/>    endpoint                            = optional(string, "https://s3.amazonaws.com")<br/>    experimental-delete-older-than-days = optional(number, 14)<br/>    path                                = optional(string, "path/")<br/>    region                              = optional(string, "us-east-1")<br/>    s3-api-version                      = optional(string, "2")<br/>    s3-uri-style                        = optional(string, "2")<br/>    storage-class                       = optional(string, "")<br/>    tls-ca-chain                        = optional(string, "")<br/>  })</pre> | `{}` | no |
+| <a name="input_constraints"></a> [constraints](#input\_constraints) | String listing constraints for this application. | `string` | `null` | no |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
+| <a name="input_revision"></a> [revision](#input\_revision) | Revision number of the charm. | `number` | `null` | no |
+| <a name="input_units"></a> [units](#input\_units) | Unit count. | `number` | `1` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_application"></a> [application](#output\_application) | Object representing the deployed application. |
+| <a name="output_offers"></a> [offers](#output\_offers) | Map of all offers exposed by the single charm. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Provides endpoints. |
diff --git a/releases/terraform/charms/s3-integrator-v1/main.tf b/releases/terraform/charms/s3-integrator-v1/main.tf
new file mode 100644
index 00000000..5b07ace9
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator-v1/main.tf
@@ -0,0 +1,23 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+
+resource "juju_application" "s3_integrator" {
+  name       = var.app_name
+  model_uuid = var.model_uuid
+  charm {
+    name     = "s3-integrator"
+    channel  = var.channel
+    revision = var.revision
+  }
+  config      = var.config
+  units       = var.units
+  constraints = var.constraints
+}
+
+resource "juju_offer" "s3_credentials" {
+  name             = "s3-credentials"
+  model_uuid       = var.model_uuid
+  application_name = var.app_name
+  endpoints        = ["s3-credentials"]
+}
diff --git a/releases/terraform/charms/s3-integrator-v1/outputs.tf b/releases/terraform/charms/s3-integrator-v1/outputs.tf
new file mode 100644
index 00000000..37d889bf
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator-v1/outputs.tf
@@ -0,0 +1,22 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "application" {
+  description = "Object representing the deployed application."
+  value       = juju_application.s3_integrator
+}
+
+output "offers" {
+  description = "Map of all offers exposed by the single charm."
+  value = {
+    s3_credentials = juju_offer.s3_credentials.url
+  }
+}
+
+
+output "provides" {
+  description = "Provides endpoints."
+  value = {
+    s3_credentials = "s3-credentials"
+  }
+}
diff --git a/releases/terraform/charms/s3-integrator-v1/providers.tf b/releases/terraform/charms/s3-integrator-v1/providers.tf
new file mode 100644
index 00000000..efca3ca1
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator-v1/providers.tf
@@ -0,0 +1,5 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Provider configuration for s3-integrator-v1.
+# Provider blocks, if any, should be defined here.
diff --git a/releases/terraform/charms/manual-tls/terraform.tf b/releases/terraform/charms/s3-integrator-v1/terraform.tf
similarity index 100%
rename from releases/terraform/charms/manual-tls/terraform.tf
rename to releases/terraform/charms/s3-integrator-v1/terraform.tf
diff --git a/releases/terraform/charms/s3-integrator-v1/variables.tf b/releases/terraform/charms/s3-integrator-v1/variables.tf
new file mode 100644
index 00000000..d9b7e0f6
--- /dev/null
+++ b/releases/terraform/charms/s3-integrator-v1/variables.tf
@@ -0,0 +1,67 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "app_name" {
+  description = "Name to give the deployed application."
+  type        = string
+  default     = "s3-integrator"
+  nullable    = false
+}
+
+variable "base" {
+  description = "The operating system on which to deploy"
+  type        = string
+  default     = "ubuntu@22.04"
+  nullable    = false
+}
+
+variable "channel" {
+  description = "Channel of the charm."
+  type        = string
+  default     = "1/stable"
+  nullable    = false
+}
+
+variable "config" {
+  description = "Map for configuration options."
+  type = object({
+    attributes                          = optional(string, "")
+    bucket                              = optional(string, "spark-bucket")
+    endpoint                            = optional(string, "https://s3.amazonaws.com")
+    experimental-delete-older-than-days = optional(number, 14)
+    path                                = optional(string, "path/")
+    region                              = optional(string, "us-east-1")
+    s3-api-version                      = optional(string, "2")
+    s3-uri-style                        = optional(string, "2")
+    storage-class                       = optional(string, "")
+    tls-ca-chain                        = optional(string, "")
+  })
+  default = {
+  }
+}
+
+
+variable "constraints" {
+  description = "String listing constraints for this application."
+  type        = string
+  default     = null
+}
+
+variable "model_uuid" {
+  description = "Reference to an existing model uuid."
+  type        = string
+  nullable    = false
+}
+
+
+variable "revision" {
+  description = "Revision number of the charm."
+  type        = number
+  default     = null
+}
+
+variable "units" {
+  description = "Unit count."
+  type        = number
+  default     = 1
+}
diff --git a/releases/terraform/charms/s3-integrator/README.md b/releases/terraform/charms/s3-integrator/README.md
deleted file mode 100644
index dbd80b99..00000000
--- a/releases/terraform/charms/s3-integrator/README.md
+++ /dev/null
@@ -1,42 +0,0 @@
-## Requirements
-
-| Name | Version |
-|------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
-
-## Providers
-
-| Name | Version |
-|------|---------|
-| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
-
-## Modules
-
-No modules.
-
-## Resources
-
-| Name | Type |
-|------|------|
-| [juju_application.s3](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_integration.s3_history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.s3_integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
-
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| <a name="input_model"></a> [model](#input\_model) | Name of the Spark Juju Model to deploy to | `string` | n/a | yes |
-| <a name="input_s3"></a> [s3](#input\_s3) | S3 Bucket information | <pre>object({<br/>    bucket   = optional(string, "spark-bucket")<br/>    endpoint = optional(string, "https://s3.amazonaws.com")<br/>    region   = optional(string, "us-east-1")<br/>  })</pre> | <pre>{<br/>  "bucket": "spark-bucket",<br/>  "endpoint": "https://s3.amazonaws.com",<br/>  "region": "us-east-1"<br/>}</pre> | no |
-| <a name="input_s3_revision"></a> [s3\_revision](#input\_s3\_revision) | Charm revision for s3-integrator | `number` | n/a | yes |
-| <a name="input_spark_charms"></a> [spark\_charms](#input\_spark\_charms) | Names of the Spark applications in the Spark Juju model. | `map(string)` | n/a | yes |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| <a name="output_application"></a> [application](#output\_application) | The deployed application name. |
-| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
-| <a name="output_provides"></a> [provides](#output\_provides) | Relation endpoint provided by this module. |
diff --git a/releases/terraform/charms/s3-integrator/applications.tf b/releases/terraform/charms/s3-integrator/applications.tf
deleted file mode 100644
index e56ec9eb..00000000
--- a/releases/terraform/charms/s3-integrator/applications.tf
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "s3" {
-  name  = "s3"
-  model_uuid = data.juju_model.spark.id
-  charm {
-    name     = "s3-integrator"
-    channel  = "1/stable"
-    revision = var.s3_revision
-  }
-  config = {
-    path     = "spark-events"
-    bucket   = var.s3.bucket
-    endpoint = var.s3.endpoint
-    region   = var.s3.region
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/terraform/charms/s3-integrator/integrations.tf b/releases/terraform/charms/s3-integrator/integrations.tf
deleted file mode 100644
index c4bda8b4..00000000
--- a/releases/terraform/charms/s3-integrator/integrations.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "s3_history_server" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.s3.name
-    endpoint = "s3-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "s3-credentials"
-  }
-}
-
-resource "juju_integration" "s3_integration_hub" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.s3.name
-    endpoint = "s3-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "s3-credentials"
-  }
-}
diff --git a/releases/terraform/charms/s3-integrator/locals.tf b/releases/terraform/charms/s3-integrator/locals.tf
deleted file mode 100644
index 44f3a0f6..00000000
--- a/releases/terraform/charms/s3-integrator/locals.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Local values for $(basename "$moduleDir").
-# Define computed values used in this module.
diff --git a/releases/terraform/charms/s3-integrator/main.tf b/releases/terraform/charms/s3-integrator/main.tf
deleted file mode 100644
index 46aca146..00000000
--- a/releases/terraform/charms/s3-integrator/main.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Main Terraform code for $(basename "$moduleDir").
-# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/charms/s3-integrator/outputs.tf b/releases/terraform/charms/s3-integrator/outputs.tf
deleted file mode 100644
index 05253e4f..00000000
--- a/releases/terraform/charms/s3-integrator/outputs.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    s3 = juju_application.s3.name
-  }
-}
-
-output "application" {
-  description = "The deployed application name."
-  value       = juju_application.s3.name
-}
-
-output "provides" {
-  description = "Relation endpoint provided by this module."
-  value = {
-    "s3-credentials" = juju_application.s3.name
-  }
-}
diff --git a/releases/terraform/charms/s3-integrator/providers.tf b/releases/terraform/charms/s3-integrator/providers.tf
deleted file mode 100644
index 0693f27b..00000000
--- a/releases/terraform/charms/s3-integrator/providers.tf
+++ /dev/null
@@ -1,6 +0,0 @@
-# Provider configuration for module $(basename "$moduleDir").
-# Terraform block moved to terraform.tf and should not be in providers.tf.
-
-# provider "juju" {
-#   # Configure provider here if required by the module.
-# }
diff --git a/releases/terraform/charms/s3-integrator/resources.tf b/releases/terraform/charms/s3-integrator/resources.tf
deleted file mode 100644
index 0ddad034..00000000
--- a/releases/terraform/charms/s3-integrator/resources.tf
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-locals {
-  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
-}
-
-data "juju_model" "spark" {
-  name = var.model
-}
diff --git a/releases/terraform/charms/s3-integrator/terraform.tf b/releases/terraform/charms/s3-integrator/terraform.tf
deleted file mode 100644
index 6412125a..00000000
--- a/releases/terraform/charms/s3-integrator/terraform.tf
+++ /dev/null
@@ -1,10 +0,0 @@
-terraform {
-  required_version = ">=1.0.0"
-
-  required_providers {
-    juju = {
-      source  = "juju/juju"
-      version = ">=1.0.0"
-    }
-  }
-}
diff --git a/releases/terraform/charms/s3-integrator/variables.tf b/releases/terraform/charms/s3-integrator/variables.tf
deleted file mode 100644
index 31f13080..00000000
--- a/releases/terraform/charms/s3-integrator/variables.tf
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-variable "model" {
-  description = "Name of the Spark Juju Model to deploy to (fallback if model_uuid is not provided)."
-  type        = string
-  nullable    = false
-}
-
-variable "model_uuid" {
-  description = "Optional existing Juju model UUID to deploy to."
-  type        = string
-  default     = null
-}
-
-variable "s3" {
-  description = "S3 Bucket information"
-  type = object({
-    bucket   = optional(string, "spark-bucket")
-    endpoint = optional(string, "https://s3.amazonaws.com")
-    region   = optional(string, "us-east-1")
-  })
-  default = {
-    bucket   = "spark-bucket"
-    endpoint = "https://s3.amazonaws.com"
-    region   = "us-east-1"
-  }
-}
-
-variable "s3_revision" {
-  description = "Charm revision for s3-integrator"
-  type        = number
-  nullable    = false
-}
-
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
-}

From 8352cb8aa9ac417a60698d89f88f569b54f21632 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 31 Mar 2026 17:34:38 +0200
Subject: [PATCH 03/62] Adapt zk and data-integrator charm modules

---
 .../charms/data-integrator/README.md          | 43 ++++++++++++
 .../terraform/charms/data-integrator/main.tf  | 17 +++++
 .../charms/data-integrator/outputs.tf         | 19 ++++++
 .../charms/data-integrator/providers.tf       |  5 ++
 .../charms/data-integrator/terraform.tf       | 13 ++++
 .../charms/data-integrator/variables.tf       | 55 ++++++++++++++++
 releases/terraform/charms/zookeeper/README.md | 45 +++++++++++++
 releases/terraform/charms/zookeeper/main.tf   | 23 +++++++
 .../terraform/charms/zookeeper/outputs.tf     | 19 ++++++
 .../terraform/charms/zookeeper/providers.tf   |  5 ++
 .../terraform/charms/zookeeper/terraform.tf   | 13 ++++
 .../terraform/charms/zookeeper/variables.tf   | 65 +++++++++++++++++++
 12 files changed, 322 insertions(+)
 create mode 100644 releases/terraform/charms/data-integrator/README.md
 create mode 100644 releases/terraform/charms/data-integrator/main.tf
 create mode 100644 releases/terraform/charms/data-integrator/outputs.tf
 create mode 100644 releases/terraform/charms/data-integrator/providers.tf
 create mode 100644 releases/terraform/charms/data-integrator/terraform.tf
 create mode 100644 releases/terraform/charms/data-integrator/variables.tf
 create mode 100644 releases/terraform/charms/zookeeper/README.md
 create mode 100644 releases/terraform/charms/zookeeper/main.tf
 create mode 100644 releases/terraform/charms/zookeeper/outputs.tf
 create mode 100644 releases/terraform/charms/zookeeper/providers.tf
 create mode 100644 releases/terraform/charms/zookeeper/terraform.tf
 create mode 100644 releases/terraform/charms/zookeeper/variables.tf

diff --git a/releases/terraform/charms/data-integrator/README.md b/releases/terraform/charms/data-integrator/README.md
new file mode 100644
index 00000000..40a2b260
--- /dev/null
+++ b/releases/terraform/charms/data-integrator/README.md
@@ -0,0 +1,43 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_application.data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_app_name"></a> [app\_name](#input\_app\_name) | Name to give the deployed application. | `string` | `"data-integrator"` | no |
+| <a name="input_base"></a> [base](#input\_base) | The operating system on which to deploy | `string` | `"ubuntu@24.04"` | no |
+| <a name="input_channel"></a> [channel](#input\_channel) | Channel of the charm. | `string` | `"latest/stable"` | no |
+| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    database-name = optional(string, "integrator")<br/>  })</pre> | `{}` | no |
+| <a name="input_constraints"></a> [constraints](#input\_constraints) | String listing constraints for this application. | `string` | `"arch=amd64"` | no |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
+| <a name="input_revision"></a> [revision](#input\_revision) | Revision number of the charm. | `number` | `null` | no |
+| <a name="input_units"></a> [units](#input\_units) | Unit count. | `number` | `1` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_application"></a> [application](#output\_application) | Object representing the deployed application. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provided endpoints. |
+| <a name="output_requires"></a> [requires](#output\_requires) | Map of all the required endpoints. |
diff --git a/releases/terraform/charms/data-integrator/main.tf b/releases/terraform/charms/data-integrator/main.tf
new file mode 100644
index 00000000..a4e30d35
--- /dev/null
+++ b/releases/terraform/charms/data-integrator/main.tf
@@ -0,0 +1,17 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "data_integrator" {
+  name       = var.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "data-integrator"
+    channel  = var.channel
+    revision = var.revision
+  }
+
+  config      = var.config
+  units       = var.units
+  constraints = var.constraints
+}
diff --git a/releases/terraform/charms/data-integrator/outputs.tf b/releases/terraform/charms/data-integrator/outputs.tf
new file mode 100644
index 00000000..425f291c
--- /dev/null
+++ b/releases/terraform/charms/data-integrator/outputs.tf
@@ -0,0 +1,19 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "application" {
+  description = "Object representing the deployed application."
+  value       = juju_application.data_integrator
+}
+
+output "provides" {
+  description = "Map of all the provided endpoints."
+  value       = {}
+}
+
+output "requires" {
+  description = "Map of all the required endpoints."
+  value = {
+    kyuubi = "kyuubi"
+  }
+}
diff --git a/releases/terraform/charms/data-integrator/providers.tf b/releases/terraform/charms/data-integrator/providers.tf
new file mode 100644
index 00000000..bb5bbccd
--- /dev/null
+++ b/releases/terraform/charms/data-integrator/providers.tf
@@ -0,0 +1,5 @@
+# Copyright 2026  Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Provider configuration for data-integrator.
+# Provider blocks, if any, should be defined here.
diff --git a/releases/terraform/charms/data-integrator/terraform.tf b/releases/terraform/charms/data-integrator/terraform.tf
new file mode 100644
index 00000000..63a76877
--- /dev/null
+++ b/releases/terraform/charms/data-integrator/terraform.tf
@@ -0,0 +1,13 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/charms/data-integrator/variables.tf b/releases/terraform/charms/data-integrator/variables.tf
new file mode 100644
index 00000000..ddcab111
--- /dev/null
+++ b/releases/terraform/charms/data-integrator/variables.tf
@@ -0,0 +1,55 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "app_name" {
+  description = "Name to give the deployed application."
+  type        = string
+  default     = "data-integrator"
+  nullable    = false
+}
+
+variable "base" {
+  description = "The operating system on which to deploy"
+  type        = string
+  default     = "ubuntu@24.04"
+  nullable    = false
+}
+
+variable "channel" {
+  description = "Channel of the charm."
+  type        = string
+  default     = "latest/stable"
+  nullable    = false
+}
+
+variable "config" {
+  description = "Map for configuration options."
+  type = object({
+    database-name = optional(string, "integrator")
+  })
+  default = {}
+}
+
+variable "constraints" {
+  description = "String listing constraints for this application."
+  type        = string
+  default     = "arch=amd64"
+}
+
+variable "model_uuid" {
+  description = "Reference to an existing model uuid."
+  type        = string
+  nullable    = false
+}
+
+variable "revision" {
+  description = "Revision number of the charm."
+  type        = number
+  default     = null
+}
+
+variable "units" {
+  description = "Unit count."
+  type        = number
+  default     = 1
+}
diff --git a/releases/terraform/charms/zookeeper/README.md b/releases/terraform/charms/zookeeper/README.md
new file mode 100644
index 00000000..20e53f4b
--- /dev/null
+++ b/releases/terraform/charms/zookeeper/README.md
@@ -0,0 +1,45 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_application.zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_app_name"></a> [app\_name](#input\_app\_name) | Name to give the deployed application. | `string` | `"zookeeper"` | no |
+| <a name="input_base"></a> [base](#input\_base) | The operating system on which to deploy | `string` | `"ubuntu@22.04"` | no |
+| <a name="input_channel"></a> [channel](#input\_channel) | Channel of the charm. | `string` | `"3/stable"` | no |
+| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | `map(string)` | `{}` | no |
+| <a name="input_constraints"></a> [constraints](#input\_constraints) | String listing constraints for this application. | `string` | `"arch=amd64"` | no |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
+| <a name="input_resources"></a> [resources](#input\_resources) | Resources to attach to the application (e.g. OCI images). | `map(string)` | `null` | no |
+| <a name="input_revision"></a> [revision](#input\_revision) | Revision number of the charm. | `number` | `null` | no |
+| <a name="input_storage_size"></a> [storage\_size](#input\_storage\_size) | Storage size for the zookeeper data. | `string` | `"10G"` | no |
+| <a name="input_units"></a> [units](#input\_units) | Unit count. 3 units are recommended for high availability. | `number` | `3` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_application"></a> [application](#output\_application) | Object representing the deployed application. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provided endpoints. |
+| <a name="output_requires"></a> [requires](#output\_requires) | Map of all the required endpoints. |
diff --git a/releases/terraform/charms/zookeeper/main.tf b/releases/terraform/charms/zookeeper/main.tf
new file mode 100644
index 00000000..5a81d24b
--- /dev/null
+++ b/releases/terraform/charms/zookeeper/main.tf
@@ -0,0 +1,23 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "zookeeper" {
+  name       = var.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "zookeeper-k8s"
+    channel  = var.channel
+    revision = var.revision
+  }
+
+  config = var.config
+
+  storage_directives = {
+    zookeeper = var.storage_size
+  }
+
+  resources   = var.resources
+  units       = var.units
+  constraints = var.constraints
+}
diff --git a/releases/terraform/charms/zookeeper/outputs.tf b/releases/terraform/charms/zookeeper/outputs.tf
new file mode 100644
index 00000000..bb25a17f
--- /dev/null
+++ b/releases/terraform/charms/zookeeper/outputs.tf
@@ -0,0 +1,19 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "application" {
+  description = "Object representing the deployed application."
+  value       = juju_application.zookeeper
+}
+
+output "provides" {
+  description = "Map of all the provided endpoints."
+  value = {
+    zookeeper = "zookeeper"
+  }
+}
+
+output "requires" {
+  description = "Map of all the required endpoints."
+  value       = {}
+}
diff --git a/releases/terraform/charms/zookeeper/providers.tf b/releases/terraform/charms/zookeeper/providers.tf
new file mode 100644
index 00000000..528856dd
--- /dev/null
+++ b/releases/terraform/charms/zookeeper/providers.tf
@@ -0,0 +1,5 @@
+# Copyright 2026  Canonical Ltd.
+# See LICENSE file for licensing details.
+
+# Provider configuration for zookeeper.
+# Provider blocks, if any, should be defined here.
diff --git a/releases/terraform/charms/zookeeper/terraform.tf b/releases/terraform/charms/zookeeper/terraform.tf
new file mode 100644
index 00000000..63a76877
--- /dev/null
+++ b/releases/terraform/charms/zookeeper/terraform.tf
@@ -0,0 +1,13 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/releases/terraform/charms/zookeeper/variables.tf b/releases/terraform/charms/zookeeper/variables.tf
new file mode 100644
index 00000000..09fc9f02
--- /dev/null
+++ b/releases/terraform/charms/zookeeper/variables.tf
@@ -0,0 +1,65 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "app_name" {
+  description = "Name to give the deployed application."
+  type        = string
+  default     = "zookeeper"
+  nullable    = false
+}
+
+variable "base" {
+  description = "The operating system on which to deploy"
+  type        = string
+  default     = "ubuntu@22.04"
+  nullable    = false
+}
+
+variable "channel" {
+  description = "Channel of the charm."
+  type        = string
+  default     = "3/stable"
+  nullable    = false
+}
+
+variable "config" {
+  description = "Map for configuration options."
+  type        = map(string)
+  default     = {}
+}
+
+variable "constraints" {
+  description = "String listing constraints for this application."
+  type        = string
+  default     = "arch=amd64"
+}
+
+variable "model_uuid" {
+  description = "Reference to an existing model uuid."
+  type        = string
+  nullable    = false
+}
+
+variable "resources" {
+  description = "Resources to attach to the application (e.g. OCI images)."
+  type        = map(string)
+  default     = null
+}
+
+variable "revision" {
+  description = "Revision number of the charm."
+  type        = number
+  default     = null
+}
+
+variable "storage_size" {
+  description = "Storage size for the zookeeper data."
+  type        = string
+  default     = "10G"
+}
+
+variable "units" {
+  description = "Unit count. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+}

From ab974deb4192682a076fcb9329f9946929a81eec Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 31 Mar 2026 18:12:13 +0200
Subject: [PATCH 04/62] Initial working 3.4 bundle

---
 .../components/spark-3.4/applications.tf      | 237 ++++-----
 .../components/spark-3.4/integrations.tf      |  66 ++-
 .../terraform/components/spark-3.4/locals.tf  |  37 +-
 .../terraform/components/spark-3.4/main.tf    |   2 +-
 .../terraform/components/spark-3.4/offers.tf  |  11 +-
 .../terraform/components/spark-3.4/outputs.tf |  51 +-
 .../components/spark-3.4/providers.tf         |   9 +-
 .../components/spark-3.4/resources.tf         |  14 +-
 .../components/spark-3.4/terraform.tf         |   4 +-
 .../components/spark-3.4/variables.tf         | 454 +++++++++---------
 .../products/charmed-spark-3.4/README.md      |   2 +-
 .../products/charmed-spark-3.4/locals.tf      |  38 +-
 .../products/charmed-spark-3.4/main.tf        | 334 ++++++++-----
 .../products/charmed-spark-3.4/outputs.tf     |  53 +-
 .../products/charmed-spark-3.4/terraform.tf   |   4 +-
 .../products/charmed-spark-3.4/variables.tf   |  31 +-
 16 files changed, 738 insertions(+), 609 deletions(-)

diff --git a/releases/terraform/components/spark-3.4/applications.tf b/releases/terraform/components/spark-3.4/applications.tf
index 3943ad1c..d9419543 100644
--- a/releases/terraform/components/spark-3.4/applications.tf
+++ b/releases/terraform/components/spark-3.4/applications.tf
@@ -2,189 +2,118 @@
 # See LICENSE file for licensing details.
 
 resource "juju_application" "history_server" {
-  name  = "history-server"
-  model_uuid = data.juju_model.spark.id
+  name       = var.history_server.app_name
+  model_uuid = var.model_uuid
 
   charm {
     name     = "spark-history-server-k8s"
-    channel  = "3/stable"
-    revision = var.history_server_revision
+    channel  = var.history_server.channel
+    revision = var.history_server.revision != null ? var.history_server.revision : local.revisions.history_server
   }
 
-  resources = var.history_server_image
-  units     = 1
-
-  constraints = "arch=amd64"
+  config      = var.history_server.config
+  constraints = var.history_server.constraints
+  resources = merge({
+    spark-history-server-image = local.images.history_server
+    },
+    var.history_server.resources == null ? {} : var.history_server.resources
+  )
+  units = var.history_server.units
 }
 
 resource "juju_application" "kyuubi" {
-  name  = "kyuubi"
-  model_uuid = data.juju_model.spark.id
+  name       = var.kyuubi.app_name
+  model_uuid = var.model_uuid
 
   charm {
     name     = "kyuubi-k8s"
-    channel  = "3.5/stable"
-    revision = var.kyuubi_revision
+    channel  = var.kyuubi.channel
+    revision = var.kyuubi.revision != null ? var.kyuubi.revision : local.revisions.kyuubi
   }
 
-  resources = var.kyuubi_image
-
-  config = merge(
-    {
-      enable-dynamic-allocation  = var.enable_dynamic_allocation
-      expose-external            = "loadbalancer"
-      gpu-enable                 = var.kyuubi_gpu_enable
-      gpu-engine-executors-limit = var.kyuubi_gpu_engine_executors_limit
-      gpu-pinned-memory          = var.kyuubi_gpu_pinned_memory
-      namespace                  = data.juju_model.spark.name
-      profile                    = var.kyuubi_profile
-      service-account            = var.kyuubi_user
-    },
-    var.kyuubi_k8s_node_selectors == null ? {} : {
-      k8s-node-selectors = var.kyuubi_k8s_node_selectors
-    },
-    var.kyuubi_loadbalancer_extra_annotations == null ? {} : {
-      loadbalancer-extra-annotations = var.kyuubi_loadbalancer_extra_annotations
-    },
-    var.tls_private_key == null ? {} : {
-      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+  config      = var.kyuubi.config # TODO: parametrize
+  constraints = var.kyuubi.constraints
+  resources = merge({
+    kyuubi-image = local.images.kyuubi
     },
-    var.admin_password == null ? {} : {
-      system-users = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    },
-    var.kyuubi_executor_cores == null ? {} : {
-      executor-cores = var.kyuubi_executor_cores
-    },
-    var.kyuubi_executor_memory == null ? {} : {
-      executor-memory = var.kyuubi_executor_memory
-    },
-    var.kyuubi_driver_pod_template == null ? {} : {
-      driver-pod-template = var.kyuubi_driver_pod_template
-    },
-    var.kyuubi_executor_pod_template == null ? {} : {
-      executor-pod-template = var.kyuubi_executor_pod_template
-    }
+    var.kyuubi.resources == null ? {} : var.kyuubi.resources
   )
-
-  units = var.kyuubi_units
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "kyuubi_users" {
-  name  = "kyuubi-users"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "postgresql-k8s"
-    channel  = "14/stable"
-    revision = var.kyuubi_users_revision
-  }
-
-  storage_directives = {
-    pgdata = var.kyuubi_users_size
-  }
-
-  resources = var.kyuubi_users_image
-
-  units = 1
   trust = true
+  units = var.kyuubi.units
+
+  # config = merge(
+  #   {
+  #     enable-dynamic-allocation  = var.enable_dynamic_allocation
+  #     expose-external            = "loadbalancer"
+  #     gpu-enable                 = var.kyuubi_gpu_enable
+  #     gpu-engine-executors-limit = var.kyuubi_gpu_engine_executors_limit
+  #     gpu-pinned-memory          = var.kyuubi_gpu_pinned_memory
+  #     namespace                  = var.model_uuid # TODO: parametrize
+  #     profile                    = var.kyuubi_profile
+  #     service-account            = var.kyuubi_user
+  #   },
+  #   var.kyuubi_k8s_node_selectors == null ? {} : {
+  #     k8s-node-selectors = var.kyuubi_k8s_node_selectors
+  #   },
+  #   var.kyuubi_loadbalancer_extra_annotations == null ? {} : {
+  #     loadbalancer-extra-annotations = var.kyuubi_loadbalancer_extra_annotations
+  #   },
+  #   var.tls_private_key == null ? {} : {
+  #     tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+  #   },
+  #   var.admin_password == null ? {} : {
+  #     system-users = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+  #   },
+  #   var.kyuubi_executor_cores == null ? {} : {
+  #     executor-cores = var.kyuubi_executor_cores
+  #   },
+  #   var.kyuubi_executor_memory == null ? {} : {
+  #     executor-memory = var.kyuubi_executor_memory
+  #   },
+  #   var.kyuubi_driver_pod_template == null ? {} : {
+  #     driver-pod-template = var.kyuubi_driver_pod_template
+  #   },
+  #   var.kyuubi_executor_pod_template == null ? {} : {
+  #     executor-pod-template = var.kyuubi_executor_pod_template
+  #   }
+  # )
 
-  constraints = "arch=amd64"
 }
 
-resource "juju_application" "metastore" {
-  name  = "metastore"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "postgresql-k8s"
-    channel  = "14/stable"
-    revision = var.metastore_revision
-  }
-
-  storage_directives = {
-    pgdata = var.kyuubi_users_size
-  }
-
-
-  resources = var.metastore_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
 
 resource "juju_application" "integration_hub" {
-  name  = "integration-hub"
-  model_uuid = data.juju_model.spark.id
+  name       = var.integration_hub.app_name
+  model_uuid = var.model_uuid
 
   charm {
     name     = "spark-integration-hub-k8s"
-    channel  = "3/stable"
-    revision = var.integration_hub_revision
+    channel  = var.integration_hub.channel
+    revision = var.integration_hub.revision != null ? var.integration_hub.revision : local.revisions.integration_hub
   }
 
-  config = merge(
-    {
-      enable-dynamic-allocation = var.enable_dynamic_allocation
-    },
-    var.driver_pod_template == null ? {} : {
-      driver-pod-template = var.driver_pod_template
+  config      = var.integration_hub.config
+  constraints = var.integration_hub.constraints
+  resources = merge({
+    integration-hub-image = local.images.integration_hub
     },
-    var.executor_pod_template == null ? {} : {
-      executor-pod-template = var.executor_pod_template
-    },
-    var.integration_hub_monitored_service_accounts == null ? {} : {
-      monitored-service-accounts = var.integration_hub_monitored_service_accounts
-    }
+    var.integration_hub.resources == null ? {} : var.integration_hub.resources
   )
-
-  resources = var.integration_hub_image
-
-  units = 1
   trust = true
-
-  constraints = "arch=amd64"
+  units = var.integration_hub.units
+  # config = merge(
+  #   {
+  #     enable-dynamic-allocation = var.enable_dynamic_allocation
+  #   },
+  #   var.driver_pod_template == null ? {} : {
+  #     driver-pod-template = var.driver_pod_template
+  #   },
+  #   var.executor_pod_template == null ? {} : {
+  #     executor-pod-template = var.executor_pod_template
+  #   },
+  #   var.integration_hub_monitored_service_accounts == null ? {} : {
+  #     monitored-service-accounts = var.integration_hub_monitored_service_accounts
+  #   }
+  # )
 }
 
-resource "juju_application" "zookeeper" {
-  name  = "zookeeper"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "zookeeper-k8s"
-    channel  = "3/stable"
-    revision = var.zookeeper_revision
-  }
 
-  storage_directives = {
-    zookeeper = var.zookeeper_size
-  }
-
-
-  resources = var.zookeeper_image
-
-  units       = var.zookeeper_units
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "data_integrator" {
-  name  = "data-integrator"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "data-integrator"
-    channel  = "latest/stable"
-    revision = var.data_integrator_revision
-  }
-
-  config = {
-    database-name = "integrator"
-  }
-
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/terraform/components/spark-3.4/integrations.tf b/releases/terraform/components/spark-3.4/integrations.tf
index 8031e0d5..4bc8fc38 100644
--- a/releases/terraform/components/spark-3.4/integrations.tf
+++ b/releases/terraform/components/spark-3.4/integrations.tf
@@ -2,11 +2,11 @@
 # See LICENSE file for licensing details.
 
 resource "juju_integration" "kyuubi_metastore" {
-  model_uuid = data.juju_model.spark.id
+  model_uuid = var.model_uuid
 
   application {
-    name     = juju_application.metastore.name
-    endpoint = "database"
+    name     = var.metastore_endpoint.name
+    endpoint = var.metastore_endpoint.endpoint
   }
 
   application {
@@ -16,21 +16,21 @@ resource "juju_integration" "kyuubi_metastore" {
 }
 
 resource "juju_integration" "kyuubi_users" {
-  model_uuid = data.juju_model.spark.id
+  model_uuid = var.model_uuid
 
   application {
-    name     = juju_application.kyuubi_users.name
-    endpoint = "database"
+    name     = juju_application.kyuubi.name
+    endpoint = "auth-db"
   }
 
   application {
-    name     = juju_application.kyuubi.name
-    endpoint = "auth-db"
+    name     = var.users_db_endpoint.name
+    endpoint = var.users_db_endpoint.endpoint
   }
 }
 
 resource "juju_integration" "kyuubi_service_account" {
-  model_uuid = data.juju_model.spark.id
+  model_uuid = var.model_uuid
 
   application {
     name     = juju_application.kyuubi.name
@@ -44,21 +44,21 @@ resource "juju_integration" "kyuubi_service_account" {
 }
 
 resource "juju_integration" "kyuubi_zookeeper" {
-  model_uuid = data.juju_model.spark.id
+  model_uuid = var.model_uuid
 
   application {
-    name     = juju_application.zookeeper.name
+    name     = juju_application.kyuubi.name
     endpoint = "zookeeper"
   }
 
   application {
-    name     = juju_application.kyuubi.name
-    endpoint = "zookeeper"
+    name     = var.zookeeper_endpoint.name
+    endpoint = var.zookeeper_endpoint.endpoint
   }
 }
 
 resource "juju_integration" "kyuubi_tls" {
-  model_uuid = data.juju_model.spark.id
+  model_uuid = var.model_uuid
 
   application {
     name     = juju_application.kyuubi.name
@@ -66,13 +66,13 @@ resource "juju_integration" "kyuubi_tls" {
   }
 
   application {
-    name     = var.tls_app_name
-    endpoint = var.tls_certificates_endpoint
+    name     = var.tls_endpoint.name
+    endpoint = var.tls_endpoint.endpoint
   }
 }
 
 resource "juju_integration" "kyuubi_data_integrator" {
-  model_uuid = data.juju_model.spark.id
+  model_uuid = var.model_uuid
 
   application {
     name     = juju_application.kyuubi.name
@@ -80,7 +80,35 @@ resource "juju_integration" "kyuubi_data_integrator" {
   }
 
   application {
-    name     = juju_application.data_integrator.name
-    endpoint = "kyuubi"
+    name     = var.data_integrator_endpoint.name
+    endpoint = var.data_integrator_endpoint.endpoint
+  }
+}
+
+resource "juju_integration" "integration_hub_object_storage" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.integration_hub.name
+    endpoint = var.object_storage_endpoint.endpoint
+  }
+
+  application {
+    name     = var.object_storage_endpoint.name
+    endpoint = var.object_storage_endpoint.endpoint
+  }
+}
+
+resource "juju_integration" "history_server_object_storage" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.history_server.name
+    endpoint = var.object_storage_endpoint.endpoint
+  }
+
+  application {
+    name     = var.object_storage_endpoint.name
+    endpoint = var.object_storage_endpoint.endpoint
   }
 }
diff --git a/releases/terraform/components/spark-3.4/locals.tf b/releases/terraform/components/spark-3.4/locals.tf
index 22eaae95..56ec0837 100644
--- a/releases/terraform/components/spark-3.4/locals.tf
+++ b/releases/terraform/components/spark-3.4/locals.tf
@@ -6,38 +6,13 @@ locals {
     history_server  = 94
     integration_hub = 113
     kyuubi          = 140
-    kyuubi_users    = 495
-    metastore       = 495
-    zookeeper       = 78
-    data_integrator = 179
-    s3              = 145
-    azure_storage   = 15
-    grafana_agent   = 121
-    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
-    cos_configuration = 65
-    pushgateway       = 27
-    scrape_config     = 67
   }
   images = {
-    history_server = {
-      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
-    } # rev23, spark-version: 3.5.7, release date 2026-02-27
-    integration_hub = {
-      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
-    } # rev11, release date 2026-03-02
-    kyuubi = {
-      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e"
-    } # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
-    kyuubi_users = {
-      postgresql-image = 165
-    }
-    metastore = {
-      postgresql-image = 165
-    }
-    zookeeper = {
-      zookeeper-image = 34
-    }
+    history_server = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
+    # rev23, spark-version: 3.5.7, release date 2026-02-27
+    integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
+    # rev11, release date 2026-03-02
+    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e"
+    # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
   }
 }
diff --git a/releases/terraform/components/spark-3.4/main.tf b/releases/terraform/components/spark-3.4/main.tf
index 46aca146..5b43ee7f 100644
--- a/releases/terraform/components/spark-3.4/main.tf
+++ b/releases/terraform/components/spark-3.4/main.tf
@@ -1,2 +1,2 @@
-# Main Terraform code for $(basename "$moduleDir").
+# Main Terraform code for spark-3.4.
 # Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/components/spark-3.4/offers.tf b/releases/terraform/components/spark-3.4/offers.tf
index 46f9b8dc..b9d57b96 100644
--- a/releases/terraform/components/spark-3.4/offers.tf
+++ b/releases/terraform/components/spark-3.4/offers.tf
@@ -1,11 +1,8 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
 resource "juju_offer" "integration_hub" {
-  model_uuid = data.juju_model.spark.id
+  model_uuid       = var.model_uuid
   application_name = juju_application.integration_hub.name
   endpoints        = ["spark-service-account"]
 }
-
-resource "juju_offer" "metastore" {
-  model_uuid = data.juju_model.spark.id
-  application_name = juju_application.metastore.name
-  endpoints        = ["database"]
-}
\ No newline at end of file
diff --git a/releases/terraform/components/spark-3.4/outputs.tf b/releases/terraform/components/spark-3.4/outputs.tf
index 01714e58..38c32ba8 100644
--- a/releases/terraform/components/spark-3.4/outputs.tf
+++ b/releases/terraform/components/spark-3.4/outputs.tf
@@ -6,29 +6,58 @@ output "charms" {
   value = {
     history_server  = juju_application.history_server.name
     kyuubi          = juju_application.kyuubi.name
-    kyuubi_users    = juju_application.kyuubi_users.name
-    metastore       = juju_application.metastore.name
     integration_hub = juju_application.integration_hub.name
-    zookeeper       = juju_application.zookeeper.name
-    certificates    = var.tls_app_name
   }
 }
 
 output "components" {
   description = "List of the deployed applications for this component module."
   value = [
-    { name = juju_application.history_server.name, type = "history_server" },
-    { name = juju_application.kyuubi.name, type = "kyuubi" },
-    { name = juju_application.kyuubi_users.name, type = "kyuubi_users" },
-    { name = juju_application.metastore.name, type = "metastore" },
-    { name = juju_application.integration_hub.name, type = "integration_hub" },
-    { name = juju_application.zookeeper.name, type = "zookeeper" },
+    juju_application.history_server,
+    juju_application.integration_hub,
+    juju_application.kyuubi,
   ]
 }
 
+output "provides" {
+  description = "Map of all the provides endpoints."
+  value = {
+    integration_hub_spark_service_account = {
+      name     = juju_application.integration_hub.name
+      endpoint = "spark-service-account"
+    }
+    # TODO: Kyuubi
+  }
+}
+
+output "requires" {
+  description = "Map of the requires endpoints."
+  value = {
+    kyuubi_certificates = {
+      name     = juju_application.kyuubi.name
+      endpoint = "certificates"
+    }
+    kyuubi_metastore_db = {
+      name     = juju_application.kyuubi.name
+      endpoint = "metastore-db"
+    }
+    kyuubi_auth_db = {
+      name     = juju_application.kyuubi.name
+      endpoint = "auth-db"
+    }
+    kyuubi_zookeeper = {
+      name     = juju_application.kyuubi.name
+      endpoint = "zookeeper"
+    }
+    kyuubi_jdbc = {
+      name     = juju_application.kyuubi.name
+      endpoint = "jdbc"
+    }
+  }
+}
+
 output "offers" {
   value = {
     hub_service_account = juju_offer.integration_hub
-    metastore_database  = juju_offer.metastore
   }
 }
diff --git a/releases/terraform/components/spark-3.4/providers.tf b/releases/terraform/components/spark-3.4/providers.tf
index 0693f27b..1b0ee6a1 100644
--- a/releases/terraform/components/spark-3.4/providers.tf
+++ b/releases/terraform/components/spark-3.4/providers.tf
@@ -1,6 +1,5 @@
-# Provider configuration for module $(basename "$moduleDir").
-# Terraform block moved to terraform.tf and should not be in providers.tf.
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
 
-# provider "juju" {
-#   # Configure provider here if required by the module.
-# }
+# Provider configuration for spark-3.4.
+# Provider blocks, if any, should be defined here.
diff --git a/releases/terraform/components/spark-3.4/resources.tf b/releases/terraform/components/spark-3.4/resources.tf
index e25d743f..90a96fc7 100644
--- a/releases/terraform/components/spark-3.4/resources.tf
+++ b/releases/terraform/components/spark-3.4/resources.tf
@@ -4,17 +4,17 @@
 # Define juju resources (model, secrets, storage)
 
 locals {
-  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
+  target_model_uuid = var.model_uuid
 }
 
-data "juju_model" "spark" {
-  name = var.model
-}
+# data "juju_model" "spark" {
+#   name = var.model
+# }
 
 resource "juju_secret" "system_users_and_private_key_secret" {
-  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
   model_uuid = local.target_model_uuid
-  name  = "system_users_and_private_key_secret"
+  name       = "system_users_and_private_key_secret"
   value = merge(
     var.admin_password == null ? {} : {
       admin = var.admin_password
@@ -27,7 +27,7 @@ resource "juju_secret" "system_users_and_private_key_secret" {
 }
 
 resource "juju_access_secret" "system_users_and_private_key_secret_access" {
-  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
   model_uuid = local.target_model_uuid
   applications = [
     juju_application.kyuubi.name
diff --git a/releases/terraform/components/spark-3.4/terraform.tf b/releases/terraform/components/spark-3.4/terraform.tf
index 9a82f31b..6412125a 100644
--- a/releases/terraform/components/spark-3.4/terraform.tf
+++ b/releases/terraform/components/spark-3.4/terraform.tf
@@ -1,8 +1,10 @@
 terraform {
+  required_version = ">=1.0.0"
+
   required_providers {
     juju = {
       source  = "juju/juju"
-      version = ">=0.20.0"
+      version = ">=1.0.0"
     }
   }
 }
diff --git a/releases/terraform/components/spark-3.4/variables.tf b/releases/terraform/components/spark-3.4/variables.tf
index 673a1de8..d10839d9 100644
--- a/releases/terraform/components/spark-3.4/variables.tf
+++ b/releases/terraform/components/spark-3.4/variables.tf
@@ -1,214 +1,240 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-# Define spark-specific variables
-
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "data_integrator_revision" {
-  description = "Charm revision for data-integrator"
-  type        = number
-  nullable    = false
-}
-
-variable "driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "enable_dynamic_allocation" {
-  description = "Enable dynamic allocation of pods for Spark jobs."
-  type        = bool
-  default     = false
-}
-
-variable "executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "history_server_image" {
-  description = "Image for spark-history-server-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "history_server_revision" {
-  description = "Charm revision for spark-history-server-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "integration_hub_image" {
-  description = "Image for spark-integration-hub-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "integration_hub_monitored_service_accounts" {
-  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-  type        = string
-  default     = null
-}
-
-variable "integration_hub_revision" {
-  description = "Charm revision for spark-integration-hub-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_cores" {
-  description = "Set kyuubi executor pods cpu cores."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_memory" {
-  description = "Set kyuubi executor pods memory (in GB)."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_gpu_enable" {
-  description = "Enable GPU acceleration for SparkSQLEngine."
-  type        = bool
-  nullable    = false
-}
-
-variable "kyuubi_gpu_engine_executors_limit" {
-  description = "Limit the number of GPUs an engine can schedule executor pods on."
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_gpu_pinned_memory" {
-  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_image" {
-  description = "Image for kyuubi-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_k8s_node_selectors" {
-  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_loadbalancer_extra_annotations" {
-  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_profile" {
-  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-  type        = string
-  nullable    = false
-  default     = "production"
-}
-
-variable "kyuubi_revision" {
-  description = "Charm revision for kyuubi-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_units" {
-  description = "Number of Kyuubi units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "kyuubi_user" {
-  description = "User name to be used for running Kyuubi engines."
-  type        = string
-  nullable    = false
-  default     = "kyuubi-spark-engine"
-}
-
-variable "kyuubi_users_image" {
-  description = "Image for postgresql-k8s (auth-db)"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_users_revision" {
-  description = "Charm revision for postgresql-k8s (auth-db)"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_users_size" {
-  description = "Storage size for the Kyuubi users database"
+variable "model_uuid" {
+  description = "Reference to an existing model uuid."
   type        = string
-  default     = "1G"
-}
-
-variable "metastore_image" {
-  description = "Image for postgresql-k8s (metastore)"
-  type        = map(string)
   nullable    = false
 }
 
-variable "metastore_revision" {
-  description = "Charm revision for postgresql-k8s (metastore)"
-  type        = number
-  nullable    = false
-}
-
-variable "metastore_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
+variable "history_server" {
+  type = object({
+    app_name    = optional(string, "history-server")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "3/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {
+    app_name    = "history-server"
+    base        = "ubuntu@22.04"
+    constraints = "arch=amd64"
+    units       = 1
+  }
+}
+
+variable "integration_hub" {
+  type = object({
+    app_name    = optional(string, "integration-hub")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "3/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {
+    app_name    = "integration-hub"
+    base        = "ubuntu@22.04"
+    constraints = "arch=amd64"
+    units       = 1
+  }
+}
+
+
+variable "kyuubi" {
+  type = object({
+    app_name    = optional(string, "kyuubi")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "3.4/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {
+    app_name    = "kyuubi"
+    base        = "ubuntu@22.04"
+    constraints = "arch=amd64"
+    units       = 1
+  }
 }
 
-variable "model" {
-  description = "Name of the Juju Model to deploy to (fallback if model_uuid is not provided)."
-  type        = string
-  nullable    = false
-}
 
-variable "model_uuid" {
-  description = "Optional existing Juju model UUID to deploy to."
+variable "admin_password" {
+  description = "The password for the admin user."
   type        = string
+  sensitive   = true
   default     = null
 }
 
-variable "tls_app_name" {
-  description = "Name of the application providing the `tls-certificates` interface."
-  type        = string
-  nullable    = false
-}
 
-variable "tls_certificates_endpoint" {
-  description = "Name of the endpoint providing the `tls-certificates` interface."
-  type        = string
-  nullable    = false
+# variable "driver_pod_template" {
+#   description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
+#   type        = string
+#   default     = null
+# }
+
+# variable "enable_dynamic_allocation" {
+#   description = "Enable dynamic allocation of pods for Spark jobs."
+#   type        = bool
+#   default     = false
+# }
+
+# variable "executor_pod_template" {
+#   description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
+#   type        = string
+#   default     = null
+# }
+
+# variable "history_server_image" {
+#   description = "Image for spark-history-server-k8s"
+#   type        = map(string)
+#   nullable    = false
+# }
+
+# variable "history_server_revision" {
+#   description = "Charm revision for spark-history-server-k8s"
+#   type        = number
+#   nullable    = false
+# }
+
+# variable "integration_hub_image" {
+#   description = "Image for spark-integration-hub-k8s"
+#   type        = map(string)
+#   nullable    = false
+# }
+
+# variable "integration_hub_monitored_service_accounts" {
+#   description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
+#   type        = string
+#   default     = null
+# }
+
+# variable "integration_hub_revision" {
+#   description = "Charm revision for spark-integration-hub-k8s"
+#   type        = number
+#   nullable    = false
+# }
+
+# variable "kyuubi_driver_pod_template" {
+#   description = "Define K8s driver pod from a file accessible in the object storage."
+#   type        = string
+#   default     = null
+# }
+
+# variable "kyuubi_executor_cores" {
+#   description = "Set kyuubi executor pods cpu cores."
+#   type        = number
+#   default     = null
+# }
+
+# variable "kyuubi_executor_memory" {
+#   description = "Set kyuubi executor pods memory (in GB)."
+#   type        = number
+#   default     = null
+# }
+
+# variable "kyuubi_executor_pod_template" {
+#   description = "Define K8s executor pod from a file accessible in the object storage."
+#   type        = string
+#   default     = null
+# }
+
+# variable "kyuubi_gpu_enable" {
+#   description = "Enable GPU acceleration for SparkSQLEngine."
+#   type        = bool
+#   nullable    = false
+# }
+
+# variable "kyuubi_gpu_engine_executors_limit" {
+#   description = "Limit the number of GPUs an engine can schedule executor pods on."
+#   type        = number
+#   nullable    = false
+# }
+
+# variable "kyuubi_gpu_pinned_memory" {
+#   description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
+#   type        = number
+#   nullable    = false
+# }
+
+# variable "kyuubi_image" {
+#   description = "Image for kyuubi-k8s"
+#   type        = map(string)
+#   nullable    = false
+# }
+
+# variable "kyuubi_k8s_node_selectors" {
+#   description = "Comma separated label:value selectors for K8s pods in Kyuubi."
+#   type        = string
+#   default     = null
+# }
+
+# variable "kyuubi_loadbalancer_extra_annotations" {
+#   description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
+#   type        = string
+#   default     = null
+# }
+
+# variable "kyuubi_profile" {
+#   description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
+#   type        = string
+#   nullable    = false
+#   default     = "production"
+# }
+
+# variable "kyuubi_revision" {
+#   description = "Charm revision for kyuubi-k8s"
+#   type        = number
+#   nullable    = false
+# }
+
+# variable "kyuubi_units" {
+#   description = "Number of Kyuubi units. 3 units are recommended for high availability."
+#   type        = number
+#   default     = 3
+#   nullable    = false
+# }
+
+# variable "kyuubi_user" {
+#   description = "User name to be used for running Kyuubi engines."
+#   type        = string
+#   nullable    = false
+#   default     = "kyuubi-spark-engine"
+# }
+
+
+
+
+
+variable "data_integrator_endpoint" {
+  description = "In-model endpoint for the data-integrator application."
+  type = object({
+    name     = string
+    endpoint = optional(string, "kyuubi")
+  })
+}
+
+variable "metastore_endpoint" {
+  description = "In-model endpoint for the metastore (postgresql-k8s) application."
+  type = object({
+    name     = string
+    endpoint = optional(string, "database")
+  })
+}
+
+variable "tls_endpoint" {
+  description = "In-model endpoint for the TLS certificates provider."
+  type = object({
+    name     = string
+    endpoint = optional(string, "certificates")
+  })
 }
 
 variable "tls_private_key" {
@@ -218,27 +244,27 @@ variable "tls_private_key" {
   default     = null
 }
 
-variable "zookeeper_image" {
-  description = "Image for zookeeper-k8s"
-  type        = map(string)
-  nullable    = false
+variable "users_db_endpoint" {
+  description = "In-model endpoint for the Kyuubi users database (postgresql-k8s) application."
+  type = object({
+    name     = string
+    endpoint = optional(string, "database")
+  })
 }
 
-variable "zookeeper_revision" {
-  description = "Charm revision for zookeeper-k8s"
-  type        = number
-  nullable    = false
+variable "zookeeper_endpoint" {
+  description = "In-model endpoint for the ZooKeeper application."
+  type = object({
+    name     = string
+    endpoint = optional(string, "zookeeper")
+  })
 }
 
-variable "zookeeper_size" {
-  description = "Storage size for the zookeeper unit"
-  type        = string
-  default     = "10G"
+variable "object_storage_endpoint" {
+  description = "In-model endpoint for the object storage integrator application."
+  type = object({
+    name     = string
+    endpoint = optional(string, "s3_credentials")
+  })
 }
 
-variable "zookeeper_units" {
-  description = "Number of zookeeper units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
diff --git a/releases/terraform/products/charmed-spark-3.4/README.md b/releases/terraform/products/charmed-spark-3.4/README.md
index 40f903f3..6d28ad0f 100644
--- a/releases/terraform/products/charmed-spark-3.4/README.md
+++ b/releases/terraform/products/charmed-spark-3.4/README.md
@@ -17,7 +17,7 @@
 | <a name="module_azure_storage"></a> [azure\_storage](#module\_azure\_storage) | ./charms/azure-storage-integrator | n/a |
 | <a name="module_bundled_cos"></a> [bundled\_cos](#module\_bundled\_cos) | ./products/cos | n/a |
 | <a name="module_observability"></a> [observability](#module\_observability) | ./components/observability | n/a |
-| <a name="module_s3"></a> [s3](#module\_s3) | ./charms/s3-integrator | n/a |
+| <a name="module_s3"></a> [s3](#module\_s3) | ./charms/s3-integrator-v1 | n/a |
 | <a name="module_spark"></a> [spark](#module\_spark) | ./components/spark-3.4 | n/a |
 | <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev326 |
 
diff --git a/releases/terraform/products/charmed-spark-3.4/locals.tf b/releases/terraform/products/charmed-spark-3.4/locals.tf
index 034c5a66..2cfcfc42 100644
--- a/releases/terraform/products/charmed-spark-3.4/locals.tf
+++ b/releases/terraform/products/charmed-spark-3.4/locals.tf
@@ -6,38 +6,30 @@ locals {
     history_server  = 94
     integration_hub = 113
     kyuubi          = 140
-    kyuubi_users    = 495
-    metastore       = 495
+    kyuubi_users    = 774
+    metastore       = 774
     zookeeper       = 78
-    data_integrator = 179
-    s3              = 145
+    data_integrator = 362
+    s3              = 330
+    ssc             = 586
     azure_storage   = 15
     grafana_agent   = 121
     # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
     # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
     cos_configuration = 65
     pushgateway       = 27
     scrape_config     = 67
   }
   images = {
-    history_server = {
-      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
-    } # rev23, spark-version: 3.5.7, release date 2026-02-27
-    integration_hub = {
-      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
-    } # rev11, release date 2026-03-02
-    kyuubi = {
-      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e"
-    } # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
-    kyuubi_users = {
-      postgresql-image = 165
-    }
-    metastore = {
-      postgresql-image = 165
-    }
-    zookeeper = {
-      zookeeper-image = 34
-    }
+    history_server = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
+    # rev23, spark-version: 3.5.7, release date 2026-02-27
+    integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
+    # rev11, release date 2026-03-02
+    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e"
+    # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
+    kyuubi_users = "184"
+    metastore    = "184"
+    zookeeper    = "34"
   }
 }
diff --git a/releases/terraform/products/charmed-spark-3.4/main.tf b/releases/terraform/products/charmed-spark-3.4/main.tf
index d5c4f24b..9d28b0a6 100644
--- a/releases/terraform/products/charmed-spark-3.4/main.tf
+++ b/releases/terraform/products/charmed-spark-3.4/main.tf
@@ -2,145 +2,251 @@
 # See LICENSE file for licensing details.
 
 resource "juju_model" "spark" {
-  count      = (var.model_uuid == null && var.create_model == true) ? 1 : 0
-  name       = var.model
+  count = (var.model_uuid == null && var.create_model == true) ? 1 : 0
+
+  name       = var.spark_model_name
   credential = var.K8S_CREDENTIAL
   cloud {
     name = var.K8S_CLOUD
   }
 }
 
-resource "juju_model" "cos" {
-  count      = var.cos.deployed == "bundled" && var.model_uuid == null && var.create_model == true ? 1 : 0
-  name       = var.cos.model
-  credential = var.K8S_CREDENTIAL
-  cloud {
-    name = var.K8S_CLOUD
+# resource "juju_model" "cos" {
+#   count = var.cos.deployed == "bundled" && var.model_uuid == null && var.create_model == true ? 1 : 0
+
+#   name       = var.cos.model
+#   credential = var.K8S_CREDENTIAL
+#   cloud {
+#     name = var.K8S_CLOUD
+#   }
+# }
+
+module "ssc" {
+  depends_on = [juju_model.spark]
+  source     = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev586"
+  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+
+  base    = "ubuntu@24.04"
+  channel = "1/stable"
+  config = {
+    ca-common-name = var.certificate_common_name
   }
+  constraints = "arch=amd64"
+  revision    = local.revisions.ssc
+  units       = 1
+
 }
 
-locals {
-  active_model = coalesce(var.model_uuid, var.model)
+module "kyuubi_users" {
+  depends_on = [juju_model.spark]
+  source     = "git::https://github.com/canonical/postgresql-k8s-operator//terraform?ref=rev774"
+  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+
+  app_name    = "kyuubi-users"
+  base        = "ubuntu@22.04"
+  channel     = "14/stable"
+  constraints = "arch=amd64"
+  revision    = local.revisions.kyuubi_users
+  resources = {
+    postgresql-image = local.images.kyuubi_users
+  }
+  storage_directives = {
+    pgdata = var.kyuubi_users_size
+  }
+  units = 1
 }
 
-data "juju_model" "spark" {
-  name = var.model
+module "metastore" {
+  depends_on = [juju_model.spark]
+  source     = "git::https://github.com/canonical/postgresql-k8s-operator//terraform?ref=rev774"
+  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+
+  app_name    = "metastore"
+  base        = "ubuntu@22.04"
+  channel     = "14/stable"
+  constraints = "arch=amd64"
+  revision    = local.revisions.metastore
+  resources = {
+    postgresql-image = local.images.metastore
+  }
+  storage_directives = {
+    pgdata = var.metastore_size
+  }
+  units = 1
 }
 
-module "ssc" {
-  depends_on  = [juju_model.spark]
-  source      = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev586"
-  model_uuid  = coalesce(var.model_uuid, data.juju_model.spark.id)
-  app_name    = "certificates"
-  channel     = "1/stable"
-  revision    = 317
+
+module "zk" {
+  depends_on = [juju_model.spark]
+  source     = "../../charms/zookeeper"
+  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+
+  base        = "ubuntu@22.04"
+  channel     = "3/stable"
   constraints = "arch=amd64"
-  base        = "ubuntu@24.04"
-  units       = 1
-  config = {
-    ca-common-name = var.certificate_common_name
+  revision    = local.revisions.zookeeper
+  resources = {
+    zookeeper-image = local.images.zookeeper
   }
+  units = var.zookeeper_units
 }
 
-module "spark" {
-  depends_on                                 = [juju_model.spark, module.ssc]
-  source                                     = "../../components/spark-3.4"
-  model                                      = local.active_model
-  kyuubi_user                                = var.kyuubi_user
-  kyuubi_profile                             = var.kyuubi_profile
-  admin_password                             = var.admin_password
-  tls_private_key                            = var.tls_private_key
-  enable_dynamic_allocation                  = var.enable_dynamic_allocation
-  kyuubi_k8s_node_selectors                  = var.kyuubi_k8s_node_selectors
-  kyuubi_loadbalancer_extra_annotations      = var.kyuubi_loadbalancer_extra_annotations
-  kyuubi_gpu_enable                          = var.kyuubi_gpu_enable
-  kyuubi_gpu_engine_executors_limit          = var.kyuubi_gpu_engine_executors_limit
-  kyuubi_gpu_pinned_memory                   = var.kyuubi_gpu_pinned_memory
-  kyuubi_driver_pod_template                 = var.kyuubi_driver_pod_template
-  kyuubi_executor_pod_template               = var.kyuubi_executor_pod_template
-  kyuubi_executor_cores                      = var.kyuubi_executor_cores
-  kyuubi_executor_memory                     = var.kyuubi_executor_memory
-  driver_pod_template                        = var.driver_pod_template
-  executor_pod_template                      = var.executor_pod_template
-  integration_hub_monitored_service_accounts = var.integration_hub_monitored_service_accounts
-
-  tls_app_name              = module.ssc.app_name
-  tls_certificates_endpoint = module.ssc.provides.certificates
-  zookeeper_units           = var.zookeeper_units
-  kyuubi_units              = var.kyuubi_units
-
-  history_server_revision  = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server
-  history_server_image     = var.history_server_image != null ? var.history_server_image : local.images.history_server
-  integration_hub_revision = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub
-  integration_hub_image    = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub
-  kyuubi_revision          = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi
-  kyuubi_image             = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi
-  kyuubi_users_revision    = var.kyuubi_users_revision != null ? var.kyuubi_users_revision : local.revisions.kyuubi_users
-  kyuubi_users_image       = var.kyuubi_users_image != null ? var.kyuubi_users_image : local.images.kyuubi_users
-  metastore_revision       = var.metastore_revision != null ? var.metastore_revision : local.revisions.metastore
-  metastore_image          = var.metastore_image != null ? var.metastore_image : local.images.metastore
-  zookeeper_revision       = var.zookeeper_revision != null ? var.zookeeper_revision : local.revisions.zookeeper
-  zookeeper_image          = var.zookeeper_image != null ? var.zookeeper_image : local.images.zookeeper
-  data_integrator_revision = var.data_integrator_revision != null ? var.data_integrator_revision : local.revisions.data_integrator
-
-  kyuubi_users_size = var.kyuubi_users_size
-  metastore_size    = var.metastore_size
-  zookeeper_size    = var.zookeeper_size
+module "data_integrator" {
+  depends_on = [juju_model.spark]
+  source     = "../../charms/data-integrator"
+  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+
+  base        = "ubuntu@24.04"
+  channel     = "latest/stable"
+  constraints = "arch=amd64"
+  revision    = local.revisions.data_integrator
 }
 
 module "azure_storage" {
-  depends_on    = [module.spark]
-  count         = var.storage_backend == "azure_storage" ? 1 : 0
-  source = "../../charms/azure-storage-integrator"
-  model_uuid    = local.active_model
-  spark_charms  = module.spark.charms
-  azure_storage = var.azure_storage
-
-  azure_storage_revision = var.azure_storage_revision != null ? var.azure_storage_revision : local.revisions.azure_storage
+  depends_on = [juju_model.spark]
+  count      = var.storage_backend == "azure_storage" ? 1 : 0
+  source     = "../../charms/azure-storage-integrator"
+  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+
+  azure_storage_secret_key = var.azure_storage.secret_key
+  base                     = "ubuntu@22.04"
+  channel                  = "latest/edge"
+  config = {
+    connection-protocol = var.azure_storage.protocol
+    container           = var.azure_storage.container
+    storage_account     = var.azure_storage.storage_account
+  }
+  constraints = "arch=amd64"
+  revision    = local.revisions.azure_storage
 }
 
 module "s3" {
-  depends_on   = [module.spark]
-  count        = var.storage_backend == "s3" ? 1 : 0
-  source = "../../charms/s3-integrator"
-  model_uuid   = local.active_model
-  spark_charms = module.spark.charms
-  s3           = var.s3
-
-  s3_revision = var.s3_revision != null ? var.s3_revision : local.revisions.s3
-}
+  depends_on = [juju_model.spark]
+  count      = var.storage_backend == "s3" ? 1 : 0
+  source     = "../../charms/s3-integrator-v1"
+  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
 
-module "bundled_cos" {
-  depends_on   = [juju_model.cos]
-  count        = var.cos.deployed == "bundled" ? 1 : 0
-  source = "../cos"
-  model        = var.cos.model
-  cos_tls_ca   = var.cos.tls.ca
-  cos_tls_cert = var.cos.tls.cert
-  cos_tls_key  = var.cos.tls.key
-
-  alertmanager_size                = var.alertmanager_size
-  grafana_size                     = var.grafana_size
-  loki_active_index_directory_size = var.loki_active_index_directory_size
-  loki_chunks_size                 = var.loki_chunks_size
-  prometheus_size                  = var.prometheus_size
-  traefik_size                     = var.traefik_size
+  base    = "ubuntu@22.04"
+  channel = "1/stable"
+  config = {
+    bucket   = var.s3.bucket
+    endpoint = var.s3.endpoint
+    region   = var.s3.region
+  }
+  constraints = "arch=amd64"
+  revision    = local.revisions.s3
 }
 
+# module "bundled_cos" {
+#   depends_on   = [juju_model.cos]
+#   count        = var.cos.deployed == "bundled" ? 1 : 0
+#   source       = "../cos"
+#   model_uuid   = juju_model.cos.uuid
+#   cos_tls_ca   = var.cos.tls.ca
+#   cos_tls_cert = var.cos.tls.cert
+#   cos_tls_key  = var.cos.tls.key
+
+#   alertmanager_size                = var.alertmanager_size
+#   grafana_size                     = var.grafana_size
+#   loki_active_index_directory_size = var.loki_active_index_directory_size
+#   loki_chunks_size                 = var.loki_chunks_size
+#   prometheus_size                  = var.prometheus_size
+#   traefik_size                     = var.traefik_size
+# }
+
+module "spark" {
+  depends_on = [
+    juju_model.spark,
+    module.azure_storage,
+    module.data_integrator,
+    module.kyuubi_users,
+    module.metastore,
+    module.s3,
+    module.ssc,
+    module.zk
+  ]
+  source     = "../../components/spark-3.4"
+  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+
+  history_server  = {}
+  integration_hub = {}
+  kyuubi          = {}
+
+  # kyuubi_user                                = var.kyuubi_user
+  # kyuubi_profile                             = var.kyuubi_profile
+  # admin_password                             = var.admin_password
+  # tls_private_key                            = var.tls_private_key
+  # enable_dynamic_allocation                  = var.enable_dynamic_allocation
+  # kyuubi_k8s_node_selectors                  = var.kyuubi_k8s_node_selectors
+  # kyuubi_loadbalancer_extra_annotations      = var.kyuubi_loadbalancer_extra_annotations
+  # kyuubi_gpu_enable                          = var.kyuubi_gpu_enable
+  # kyuubi_gpu_engine_executors_limit          = var.kyuubi_gpu_engine_executors_limit
+  # kyuubi_gpu_pinned_memory                   = var.kyuubi_gpu_pinned_memory
+  # kyuubi_driver_pod_template                 = var.kyuubi_driver_pod_template
+  # kyuubi_executor_pod_template               = var.kyuubi_executor_pod_template
+  # kyuubi_executor_cores                      = var.kyuubi_executor_cores
+  # kyuubi_executor_memory                     = var.kyuubi_executor_memory
+  # driver_pod_template                        = var.driver_pod_template
+  # executor_pod_template                      = var.executor_pod_template
+  # integration_hub_monitored_service_accounts = var.integration_hub_monitored_service_accounts
+
+  tls_endpoint = {
+    name     = module.ssc.app_name
+    endpoint = module.ssc.provides.certificates
+  }
+
+  metastore_endpoint = {
+    name     = module.metastore.app_name
+    endpoint = module.metastore.provides.database
+  }
+
+  users_db_endpoint = {
+    name     = module.kyuubi_users.app_name
+    endpoint = module.kyuubi_users.provides.database
+  }
+
+  zookeeper_endpoint = {
+    name     = module.zk.application.name
+    endpoint = module.zk.provides.zookeeper
+  }
+
+  data_integrator_endpoint = {
+    name     = module.data_integrator.application.name
+    endpoint = module.data_integrator.requires.kyuubi
+  }
+
+  object_storage_endpoint = {
+    name     = module.s3 != null ? module.s3[0].application.name : module.azure_storage[0].application.name
+    endpoint = module.s3 != null ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
+  }
+
+  # kyuubi_units = var.kyuubi_units
+
+  # history_server_revision  = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server
+  # history_server_image     = var.history_server_image != null ? var.history_server_image : local.images.history_server
+  # integration_hub_revision = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub
+  # integration_hub_image    = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub
+  # kyuubi_revision          = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi
+  # kyuubi_image             = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi
 
-module "observability" {
-  depends_on       = [module.spark, module.bundled_cos]
-  count            = var.cos.deployed == "no" ? 0 : 1
-  source = "../../components/observability"
-  dashboards_offer = var.cos.deployed == "external" ? var.cos.offers.dashboard : one(module.bundled_cos[*].dashboards_offer)
-  logging_offer    = var.cos.deployed == "external" ? var.cos.offers.logging : one(module.bundled_cos[*].logging_offer)
-  metrics_offer    = var.cos.deployed == "external" ? var.cos.offers.metrics : one(module.bundled_cos[*].metrics_offer)
-  spark_model      = var.model
-  model_uuid       = local.active_model
-  spark_charms     = module.spark.charms
-
-  grafana_agent_revision     = var.grafana_agent_revision != null ? var.grafana_agent_revision : local.revisions.grafana_agent
-  cos_configuration_revision = var.cos_configuration_revision != null ? var.cos_configuration_revision : local.revisions.cos_configuration
-  pushgateway_revision       = var.pushgateway_revision != null ? var.pushgateway_revision : local.revisions.pushgateway
-  scrape_config_revision     = var.scrape_config_revision != null ? var.scrape_config_revision : local.revisions.scrape_config
 }
+
+
+
+
+# module "observability" {
+#   depends_on       = [module.spark, module.bundled_cos]
+#   count            = var.cos.deployed == "no" ? 0 : 1
+#   source           = "../../components/observability"
+#   dashboards_offer = var.cos.deployed == "external" ? var.cos.offers.dashboard : one(module.bundled_cos[*].dashboards_offer)
+#   logging_offer    = var.cos.deployed == "external" ? var.cos.offers.logging : one(module.bundled_cos[*].logging_offer)
+#   metrics_offer    = var.cos.deployed == "external" ? var.cos.offers.metrics : one(module.bundled_cos[*].metrics_offer)
+#   spark_model      = var.model
+#   model_uuid       = local.active_model
+#   spark_charms     = module.spark.charms
+
+#   grafana_agent_revision     = var.grafana_agent_revision != null ? var.grafana_agent_revision : local.revisions.grafana_agent
+#   cos_configuration_revision = var.cos_configuration_revision != null ? var.cos_configuration_revision : local.revisions.cos_configuration
+#   pushgateway_revision       = var.pushgateway_revision != null ? var.pushgateway_revision : local.revisions.pushgateway
+#   scrape_config_revision     = var.scrape_config_revision != null ? var.scrape_config_revision : local.revisions.scrape_config
+# }
diff --git a/releases/terraform/products/charmed-spark-3.4/outputs.tf b/releases/terraform/products/charmed-spark-3.4/outputs.tf
index f1a2b99f..9e3a8ad3 100644
--- a/releases/terraform/products/charmed-spark-3.4/outputs.tf
+++ b/releases/terraform/products/charmed-spark-3.4/outputs.tf
@@ -1,14 +1,6 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = merge(
-    concat(
-      module.spark[*].charms, module.azure_storage[*].charms, module.s3[*].charms, module.bundled_cos[*].charms, module.observability[*].charms
-    )...
-  )
-}
 
 output "components" {
   description = "List of component modules (spark and optional addons) and their deployed charms/offers."
@@ -28,16 +20,16 @@ output "components" {
       charms = try(module.s3.charms, [])
       offers = try(module.s3.offers, {})
     },
-    {
-      name   = "bundled_cos"
-      charms = try(module.bundled_cos.charms, [])
-      offers = try(module.bundled_cos.offers, {})
-    },
-    {
-      name   = "observability"
-      charms = try(module.observability.charms, [])
-      offers = try(module.observability.offers, {})
-    }
+    # {
+    #   name   = "bundled_cos"
+    #   charms = try(module.bundled_cos.charms, [])
+    #   offers = try(module.bundled_cos.offers, {})
+    # },
+    # {
+    #   name   = "observability"
+    #   charms = try(module.observability.charms, [])
+    #   offers = try(module.observability.offers, {})
+    # }
   ]
 }
 
@@ -49,3 +41,28 @@ output "offers" {
     )...
   )
 }
+
+output "metadata" {
+  description = "Metadata of the product deployment."
+  value = {
+    version     = "3.4"
+    deployed_at = timestamp()
+    updated_at  = timestamp()
+  }
+}
+
+output "models" {
+  description = "Map of the models and the components deployed in each model."
+  value = {
+    spark = {
+      model_uuid = var.model_uuid
+      components = {
+        spark         = module.spark
+        azure_storage = try(one(module.azure_storage), null)
+        s3            = try(one(module.s3), null)
+        # observability = try(one(module.observability), null)
+        # bundled_cos   = try(one(module.bundled_cos), null)
+      }
+    }
+  }
+}
diff --git a/releases/terraform/products/charmed-spark-3.4/terraform.tf b/releases/terraform/products/charmed-spark-3.4/terraform.tf
index 6ae144c8..975cf66e 100644
--- a/releases/terraform/products/charmed-spark-3.4/terraform.tf
+++ b/releases/terraform/products/charmed-spark-3.4/terraform.tf
@@ -2,10 +2,12 @@
 # See LICENSE file for licensing details.
 
 terraform {
+  required_version = ">=1.0.0"
+
   required_providers {
     juju = {
       source  = "juju/juju"
-      version = ">=0.20.0"
+      version = ">=1.0.0"
     }
   }
 }
diff --git a/releases/terraform/products/charmed-spark-3.4/variables.tf b/releases/terraform/products/charmed-spark-3.4/variables.tf
index 57c95ffe..8c97ee74 100644
--- a/releases/terraform/products/charmed-spark-3.4/variables.tf
+++ b/releases/terraform/products/charmed-spark-3.4/variables.tf
@@ -164,6 +164,17 @@ variable "K8S_CREDENTIAL" {
   default     = "microk8s"
 }
 
+variable "juju_controller" {
+  description = "Controller information: endpoint, username, password and CA certificate."
+  type = object({
+    endpoint = string
+    username = string
+    password = string
+    ca       = string
+  })
+  default = null
+}
+
 variable "kyuubi_driver_pod_template" {
   description = "Define K8s driver pod from a file accessible in the object storage."
   type        = string
@@ -280,6 +291,12 @@ variable "loki_chunks_size" {
   default     = "500G"
 }
 
+variable "logging_config" {
+  description = "Logging configuration to be used."
+  type        = string
+  default     = ""
+}
+
 variable "metastore_image" {
   description = "Image for postgresql-k8s (metastore)"
   type        = map(string)
@@ -298,10 +315,10 @@ variable "metastore_size" {
   default     = "10G"
 }
 
-variable "model" {
+variable "spark_model_name" {
   description = "The name of the juju model to deploy Spark to"
   type        = string
-  default     = "spark"
+  default     = "spark-test"
 }
 
 variable "model_uuid" {
@@ -322,6 +339,16 @@ variable "pushgateway_revision" {
   default     = null
 }
 
+variable "proxy" {
+  description = "Proxy information for the deployment."
+  type = object({
+    http     = optional(string, "")
+    https    = optional(string, "")
+    no_proxy = optional(string, "")
+  })
+  default = {}
+}
+
 variable "s3" {
   description = "S3 Bucket information"
   type = object({

From c498609e7fe9df819d6b488ea93e5071c92d3355 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 1 Apr 2026 09:42:31 +0200
Subject: [PATCH 05/62] Move terraform bundle to root level

---
 resources/grafana/spark_dashboard.json        | 1733 +++++++++++++++++
 .../charms/azure-storage-integrator/README.md |    0
 .../charms/azure-storage-integrator/main.tf   |    0
 .../azure-storage-integrator/outputs.tf       |    0
 .../azure-storage-integrator/providers.tf     |    0
 .../azure-storage-integrator/terraform.tf     |    0
 .../azure-storage-integrator/variables.tf     |    0
 .../charms/data-integrator/README.md          |    0
 .../charms/data-integrator/main.tf            |    0
 .../charms/data-integrator/outputs.tf         |    0
 .../charms/data-integrator/providers.tf       |    0
 .../charms/data-integrator/terraform.tf       |    0
 .../charms/data-integrator/variables.tf       |    0
 .../charms/s3-integrator-v1/README.md         |    0
 .../charms/s3-integrator-v1/main.tf           |    0
 .../charms/s3-integrator-v1/outputs.tf        |    0
 .../charms/s3-integrator-v1/providers.tf      |    0
 .../charms/s3-integrator-v1/terraform.tf      |    0
 .../charms/s3-integrator-v1/variables.tf      |    0
 .../charms/zookeeper/README.md                |    0
 .../charms/zookeeper/main.tf                  |    0
 .../charms/zookeeper/outputs.tf               |    0
 .../charms/zookeeper/providers.tf             |    0
 .../charms/zookeeper/terraform.tf             |    0
 .../charms/zookeeper/variables.tf             |    0
 .../components/spark-3.4/README.md            |    0
 .../components/spark-3.4/applications.tf      |    0
 .../components/spark-3.4/integrations.tf      |    0
 .../components/spark-3.4/locals.tf            |    0
 .../components/spark-3.4/main.tf              |    0
 .../components/spark-3.4/offers.tf            |    0
 .../components/spark-3.4/outputs.tf           |    0
 .../components/spark-3.4/providers.tf         |    0
 .../components/spark-3.4/resources.tf         |    0
 .../components/spark-3.4/terraform.tf         |    0
 .../components/spark-3.4/variables.tf         |    0
 .../products/charmed-spark-3.4/README.md      |    0
 .../products/charmed-spark-3.4/locals.tf      |    0
 .../products/charmed-spark-3.4/main.tf        |    0
 .../products/charmed-spark-3.4/outputs.tf     |    0
 .../products/charmed-spark-3.4/providers.tf   |    0
 .../products/charmed-spark-3.4/terraform.tf   |    0
 .../products/charmed-spark-3.4/variables.tf   |    0
 43 files changed, 1733 insertions(+)
 create mode 100644 resources/grafana/spark_dashboard.json
 rename {releases/terraform => terraform}/charms/azure-storage-integrator/README.md (100%)
 rename {releases/terraform => terraform}/charms/azure-storage-integrator/main.tf (100%)
 rename {releases/terraform => terraform}/charms/azure-storage-integrator/outputs.tf (100%)
 rename {releases/terraform => terraform}/charms/azure-storage-integrator/providers.tf (100%)
 rename {releases/terraform => terraform}/charms/azure-storage-integrator/terraform.tf (100%)
 rename {releases/terraform => terraform}/charms/azure-storage-integrator/variables.tf (100%)
 rename {releases/terraform => terraform}/charms/data-integrator/README.md (100%)
 rename {releases/terraform => terraform}/charms/data-integrator/main.tf (100%)
 rename {releases/terraform => terraform}/charms/data-integrator/outputs.tf (100%)
 rename {releases/terraform => terraform}/charms/data-integrator/providers.tf (100%)
 rename {releases/terraform => terraform}/charms/data-integrator/terraform.tf (100%)
 rename {releases/terraform => terraform}/charms/data-integrator/variables.tf (100%)
 rename {releases/terraform => terraform}/charms/s3-integrator-v1/README.md (100%)
 rename {releases/terraform => terraform}/charms/s3-integrator-v1/main.tf (100%)
 rename {releases/terraform => terraform}/charms/s3-integrator-v1/outputs.tf (100%)
 rename {releases/terraform => terraform}/charms/s3-integrator-v1/providers.tf (100%)
 rename {releases/terraform => terraform}/charms/s3-integrator-v1/terraform.tf (100%)
 rename {releases/terraform => terraform}/charms/s3-integrator-v1/variables.tf (100%)
 rename {releases/terraform => terraform}/charms/zookeeper/README.md (100%)
 rename {releases/terraform => terraform}/charms/zookeeper/main.tf (100%)
 rename {releases/terraform => terraform}/charms/zookeeper/outputs.tf (100%)
 rename {releases/terraform => terraform}/charms/zookeeper/providers.tf (100%)
 rename {releases/terraform => terraform}/charms/zookeeper/terraform.tf (100%)
 rename {releases/terraform => terraform}/charms/zookeeper/variables.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/README.md (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/applications.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/integrations.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/locals.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/main.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/offers.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/outputs.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/providers.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/resources.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/terraform.tf (100%)
 rename {releases/terraform => terraform}/components/spark-3.4/variables.tf (100%)
 rename {releases/terraform => terraform}/products/charmed-spark-3.4/README.md (100%)
 rename {releases/terraform => terraform}/products/charmed-spark-3.4/locals.tf (100%)
 rename {releases/terraform => terraform}/products/charmed-spark-3.4/main.tf (100%)
 rename {releases/terraform => terraform}/products/charmed-spark-3.4/outputs.tf (100%)
 rename {releases/terraform => terraform}/products/charmed-spark-3.4/providers.tf (100%)
 rename {releases/terraform => terraform}/products/charmed-spark-3.4/terraform.tf (100%)
 rename {releases/terraform => terraform}/products/charmed-spark-3.4/variables.tf (100%)

diff --git a/resources/grafana/spark_dashboard.json b/resources/grafana/spark_dashboard.json
new file mode 100644
index 00000000..f34cd1d8
--- /dev/null
+++ b/resources/grafana/spark_dashboard.json
@@ -0,0 +1,1733 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "panels": [],
+      "title": "Block Manager (Driver)",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decmbytes"
+        },
+        "overrides": [
+          {
+            "__systemRef": "hideSeriesFrom",
+            "matcher": {
+              "id": "byNames",
+              "options": {
+                "mode": "exclude",
+                "names": [
+                  "Max Memory (MB)"
+                ],
+                "prefix": "All except:",
+                "readOnly": true
+              }
+            },
+            "properties": [
+              {
+                "id": "custom.hideFrom",
+                "value": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": true
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "BlockManager_memory_maxMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "instant": false,
+          "legendFormat": "Max Memory (MB)",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "BlockManager_memory_maxOffHeapMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "hide": false,
+          "legendFormat": "Max Off Heap Memory (MB)",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "BlockManager_memory_maxOnHeapMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "hide": false,
+          "legendFormat": "Max On Heap Memory (MB)",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "BlockManager_memory_remainingMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "hide": false,
+          "legendFormat": "Remaining Memory (MB)",
+          "range": true,
+          "refId": "D"
+        }
+      ],
+      "title": "Block Manager Memory",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "decmbytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "BlockManager_disk_diskSpaceUsed_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "legendFormat": "Disk Memory (MB)",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Block Manager Disk Memory",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 10
+      },
+      "id": 10,
+      "panels": [],
+      "title": "Hive External Catalog (Driver)",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 11
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "HiveExternalCatalog_fileCacheHits{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "instant": false,
+          "legendFormat": "File Cache Hits",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "HiveExternalCatalog_filesDiscovered{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "File Discovered",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "HiveExternalCatalog_hiveClientCalls{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Client Calls",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "HiveExternalCatalog_parallelListingJobCount{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Parallel Listing Jobs",
+          "range": true,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "HiveExternalCatalog_partitionsFetched{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "Partitions Fetched",
+          "range": true,
+          "refId": "E"
+        }
+      ],
+      "title": "Hive External Catalog",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 19
+      },
+      "id": 14,
+      "panels": [],
+      "title": "Dag Scheduler (Driver)",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "id": 16,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "DAGScheduler_job_activeJobs{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "instant": false,
+          "legendFormat": "Active Jobs",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "exemplar": false,
+          "expr": "DAGScheduler_job_allJobs{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "All Jobs",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "DAG Schedule Jobs",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "id": 18,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "DAGScheduler_stage_failedStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "legendFormat": "Failed Stages",
+          "range": true,
+          "refId": "FailedStages"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "DAGScheduler_stage_runningStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "Running Stages",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "DAGScheduler_stage_waitingStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "Waiting Stages",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Stages Status",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "id": 20,
+      "panels": [],
+      "title": "Memory Metrics (Driver)",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 29
+      },
+      "id": 22,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "ExecutorMetrics_DirectPoolMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "legendFormat": "Direct Pool Memory",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "ExecutorMetrics_JVMHeapMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "JVM Heap Memory",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "ExecutorMetrics_JVMOffHeapMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "JVM Off Heap Memory",
+          "range": true,
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "ExecutorMetrics_OffHeapExecutionMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "JVM Off Heap Execution Memory",
+          "range": true,
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "ExecutorMetrics_OnHeapStorageMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "JVM On Heap Storage Memory",
+          "range": true,
+          "refId": "E"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "ExecutorMetrics_OnHeapExecutionMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "JVM On Heap Execution Memory",
+          "range": true,
+          "refId": "F"
+        }
+      ],
+      "title": "JVM Executor Memory",
+      "type": "timeseries"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 37
+      },
+      "id": 24,
+      "panels": [],
+      "title": "Executors Metrics",
+      "type": "row"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 38
+      },
+      "id": 26,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "executor_bytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
+          "legendFormat": "Bytes Reads (#Executor {{number}})",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "executor_bytesWritten{role=\"executor\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "Bytes Written (#Executor {{number}})",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Executor Read and Writes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 38
+      },
+      "id": 28,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "ExecutorMetrics_JVMHeapMemory{role=\"executor\", exported_job=~\"$spark_job_id\"}",
+          "legendFormat": "JVM Heap Memory #{{number}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "ExecutorMetrics_JVMOffHeapMemory{role=\"executor\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "JVM Off Heap Memory #{{number}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "Memory",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 46
+      },
+      "id": 30,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "rate(executor_runTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
+          "legendFormat": "Runtime (Executor #{{number}})",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Executors Runtime",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 46
+      },
+      "id": 32,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "executor_shuffleBytesWritten{role=\"executor\", exported_job=~\"$spark_job_id\"}",
+          "legendFormat": "Shuffle Bytes Writes (Executor #{{number}})",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "executor_shuffleLocalBytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "Shuffle Local Bytes Reads (Executor #{{number}})",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "executor_shuffleRemoteBytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
+          "hide": false,
+          "legendFormat": "Shuffle Remote Bytes Reads (Executor #{{number}})",
+          "range": true,
+          "refId": "C"
+        }
+      ],
+      "title": "Shuffle Reads/Writes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "__systemRef": "hideSeriesFrom",
+            "matcher": {
+              "id": "byNames",
+              "options": {
+                "mode": "exclude",
+                "names": [
+                  "Minor GC Count (Executor #1"
+                ],
+                "prefix": "All except:",
+                "readOnly": true
+              }
+            },
+            "properties": [
+              {
+                "id": "custom.hideFrom",
+                "value": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": true
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 54
+      },
+      "id": 34,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "increase(ExecutorMetrics_MinorGCCount{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
+          "legendFormat": "Minor GC Count (Executor #{{number}}",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "increase(ExecutorMetrics_MajorGCCount{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
+          "hide": false,
+          "legendFormat": "Major GC Count (Executor #{{number}}",
+          "range": true,
+          "refId": "B"
+        }
+      ],
+      "title": "GC counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${prometheusds}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 54
+      },
+      "id": 36,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "rate(ExecutorMetrics_MajorGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
+          "legendFormat": "Major GC time (Executor #{{number}})",
+          "range": true,
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "rate(ExecutorMetrics_MinorGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
+          "hide": false,
+          "legendFormat": "Minor GC time (Executor #{{number}})",
+          "range": true,
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${prometheusds}"
+          },
+          "editorMode": "builder",
+          "expr": "rate(ExecutorMetrics_TotalGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
+          "hide": false,
+          "legendFormat": "Total GC time (Executor #{{number}})",
+          "range": true,
+          "refId": "C"
+        }
+      ],
+      "title": "GC Time Spend",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 37,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "hide": 0,
+        "includeAll": true,
+        "label": "Loki datasource",
+        "multi": true,
+        "name": "lokids",
+        "options": [],
+        "query": "loki",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "uid": "${prometheusds}"
+        },
+        "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Juju unit",
+        "multi": true,
+        "name": "juju_unit",
+        "options": [],
+        "query": {
+          "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "juju_cos_348d0940-98ce-4ac5-8dcc-4dce43d9c620_prometheus_0",
+          "value": "juju_cos_348d0940-98ce-4ac5-8dcc-4dce43d9c620_prometheus_0"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Prometheus datasource",
+        "multi": false,
+        "name": "prometheusds",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${prometheusds}"
+        },
+        "definition": "label_values(up,juju_model)",
+        "description": "The",
+        "hide": 0,
+        "includeAll": true,
+        "label": "juju model",
+        "multi": true,
+        "name": "juju_model",
+        "options": [],
+        "query": {
+          "query": "label_values(up,juju_model)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${prometheusds}"
+        },
+        "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Juju model uuid",
+        "multi": true,
+        "name": "juju_model_uuid",
+        "options": [],
+        "query": {
+          "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${prometheusds}"
+        },
+        "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "juju application",
+        "multi": true,
+        "name": "juju_application",
+        "options": [],
+        "query": {
+          "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": ".*",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "${prometheusds}"
+        },
+        "definition": "label_values(push_time_seconds{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},exported_job)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Spark Job Id",
+        "multi": true,
+        "name": "spark_job_id",
+        "options": [],
+        "query": {
+          "query": "label_values(push_time_seconds{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},exported_job)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-12h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Spark dashboard",
+  "uid": "ZyD93JGSk",
+  "version": 1,
+  "weekStart": ""
+}
\ No newline at end of file
diff --git a/releases/terraform/charms/azure-storage-integrator/README.md b/terraform/charms/azure-storage-integrator/README.md
similarity index 100%
rename from releases/terraform/charms/azure-storage-integrator/README.md
rename to terraform/charms/azure-storage-integrator/README.md
diff --git a/releases/terraform/charms/azure-storage-integrator/main.tf b/terraform/charms/azure-storage-integrator/main.tf
similarity index 100%
rename from releases/terraform/charms/azure-storage-integrator/main.tf
rename to terraform/charms/azure-storage-integrator/main.tf
diff --git a/releases/terraform/charms/azure-storage-integrator/outputs.tf b/terraform/charms/azure-storage-integrator/outputs.tf
similarity index 100%
rename from releases/terraform/charms/azure-storage-integrator/outputs.tf
rename to terraform/charms/azure-storage-integrator/outputs.tf
diff --git a/releases/terraform/charms/azure-storage-integrator/providers.tf b/terraform/charms/azure-storage-integrator/providers.tf
similarity index 100%
rename from releases/terraform/charms/azure-storage-integrator/providers.tf
rename to terraform/charms/azure-storage-integrator/providers.tf
diff --git a/releases/terraform/charms/azure-storage-integrator/terraform.tf b/terraform/charms/azure-storage-integrator/terraform.tf
similarity index 100%
rename from releases/terraform/charms/azure-storage-integrator/terraform.tf
rename to terraform/charms/azure-storage-integrator/terraform.tf
diff --git a/releases/terraform/charms/azure-storage-integrator/variables.tf b/terraform/charms/azure-storage-integrator/variables.tf
similarity index 100%
rename from releases/terraform/charms/azure-storage-integrator/variables.tf
rename to terraform/charms/azure-storage-integrator/variables.tf
diff --git a/releases/terraform/charms/data-integrator/README.md b/terraform/charms/data-integrator/README.md
similarity index 100%
rename from releases/terraform/charms/data-integrator/README.md
rename to terraform/charms/data-integrator/README.md
diff --git a/releases/terraform/charms/data-integrator/main.tf b/terraform/charms/data-integrator/main.tf
similarity index 100%
rename from releases/terraform/charms/data-integrator/main.tf
rename to terraform/charms/data-integrator/main.tf
diff --git a/releases/terraform/charms/data-integrator/outputs.tf b/terraform/charms/data-integrator/outputs.tf
similarity index 100%
rename from releases/terraform/charms/data-integrator/outputs.tf
rename to terraform/charms/data-integrator/outputs.tf
diff --git a/releases/terraform/charms/data-integrator/providers.tf b/terraform/charms/data-integrator/providers.tf
similarity index 100%
rename from releases/terraform/charms/data-integrator/providers.tf
rename to terraform/charms/data-integrator/providers.tf
diff --git a/releases/terraform/charms/data-integrator/terraform.tf b/terraform/charms/data-integrator/terraform.tf
similarity index 100%
rename from releases/terraform/charms/data-integrator/terraform.tf
rename to terraform/charms/data-integrator/terraform.tf
diff --git a/releases/terraform/charms/data-integrator/variables.tf b/terraform/charms/data-integrator/variables.tf
similarity index 100%
rename from releases/terraform/charms/data-integrator/variables.tf
rename to terraform/charms/data-integrator/variables.tf
diff --git a/releases/terraform/charms/s3-integrator-v1/README.md b/terraform/charms/s3-integrator-v1/README.md
similarity index 100%
rename from releases/terraform/charms/s3-integrator-v1/README.md
rename to terraform/charms/s3-integrator-v1/README.md
diff --git a/releases/terraform/charms/s3-integrator-v1/main.tf b/terraform/charms/s3-integrator-v1/main.tf
similarity index 100%
rename from releases/terraform/charms/s3-integrator-v1/main.tf
rename to terraform/charms/s3-integrator-v1/main.tf
diff --git a/releases/terraform/charms/s3-integrator-v1/outputs.tf b/terraform/charms/s3-integrator-v1/outputs.tf
similarity index 100%
rename from releases/terraform/charms/s3-integrator-v1/outputs.tf
rename to terraform/charms/s3-integrator-v1/outputs.tf
diff --git a/releases/terraform/charms/s3-integrator-v1/providers.tf b/terraform/charms/s3-integrator-v1/providers.tf
similarity index 100%
rename from releases/terraform/charms/s3-integrator-v1/providers.tf
rename to terraform/charms/s3-integrator-v1/providers.tf
diff --git a/releases/terraform/charms/s3-integrator-v1/terraform.tf b/terraform/charms/s3-integrator-v1/terraform.tf
similarity index 100%
rename from releases/terraform/charms/s3-integrator-v1/terraform.tf
rename to terraform/charms/s3-integrator-v1/terraform.tf
diff --git a/releases/terraform/charms/s3-integrator-v1/variables.tf b/terraform/charms/s3-integrator-v1/variables.tf
similarity index 100%
rename from releases/terraform/charms/s3-integrator-v1/variables.tf
rename to terraform/charms/s3-integrator-v1/variables.tf
diff --git a/releases/terraform/charms/zookeeper/README.md b/terraform/charms/zookeeper/README.md
similarity index 100%
rename from releases/terraform/charms/zookeeper/README.md
rename to terraform/charms/zookeeper/README.md
diff --git a/releases/terraform/charms/zookeeper/main.tf b/terraform/charms/zookeeper/main.tf
similarity index 100%
rename from releases/terraform/charms/zookeeper/main.tf
rename to terraform/charms/zookeeper/main.tf
diff --git a/releases/terraform/charms/zookeeper/outputs.tf b/terraform/charms/zookeeper/outputs.tf
similarity index 100%
rename from releases/terraform/charms/zookeeper/outputs.tf
rename to terraform/charms/zookeeper/outputs.tf
diff --git a/releases/terraform/charms/zookeeper/providers.tf b/terraform/charms/zookeeper/providers.tf
similarity index 100%
rename from releases/terraform/charms/zookeeper/providers.tf
rename to terraform/charms/zookeeper/providers.tf
diff --git a/releases/terraform/charms/zookeeper/terraform.tf b/terraform/charms/zookeeper/terraform.tf
similarity index 100%
rename from releases/terraform/charms/zookeeper/terraform.tf
rename to terraform/charms/zookeeper/terraform.tf
diff --git a/releases/terraform/charms/zookeeper/variables.tf b/terraform/charms/zookeeper/variables.tf
similarity index 100%
rename from releases/terraform/charms/zookeeper/variables.tf
rename to terraform/charms/zookeeper/variables.tf
diff --git a/releases/terraform/components/spark-3.4/README.md b/terraform/components/spark-3.4/README.md
similarity index 100%
rename from releases/terraform/components/spark-3.4/README.md
rename to terraform/components/spark-3.4/README.md
diff --git a/releases/terraform/components/spark-3.4/applications.tf b/terraform/components/spark-3.4/applications.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/applications.tf
rename to terraform/components/spark-3.4/applications.tf
diff --git a/releases/terraform/components/spark-3.4/integrations.tf b/terraform/components/spark-3.4/integrations.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/integrations.tf
rename to terraform/components/spark-3.4/integrations.tf
diff --git a/releases/terraform/components/spark-3.4/locals.tf b/terraform/components/spark-3.4/locals.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/locals.tf
rename to terraform/components/spark-3.4/locals.tf
diff --git a/releases/terraform/components/spark-3.4/main.tf b/terraform/components/spark-3.4/main.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/main.tf
rename to terraform/components/spark-3.4/main.tf
diff --git a/releases/terraform/components/spark-3.4/offers.tf b/terraform/components/spark-3.4/offers.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/offers.tf
rename to terraform/components/spark-3.4/offers.tf
diff --git a/releases/terraform/components/spark-3.4/outputs.tf b/terraform/components/spark-3.4/outputs.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/outputs.tf
rename to terraform/components/spark-3.4/outputs.tf
diff --git a/releases/terraform/components/spark-3.4/providers.tf b/terraform/components/spark-3.4/providers.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/providers.tf
rename to terraform/components/spark-3.4/providers.tf
diff --git a/releases/terraform/components/spark-3.4/resources.tf b/terraform/components/spark-3.4/resources.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/resources.tf
rename to terraform/components/spark-3.4/resources.tf
diff --git a/releases/terraform/components/spark-3.4/terraform.tf b/terraform/components/spark-3.4/terraform.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/terraform.tf
rename to terraform/components/spark-3.4/terraform.tf
diff --git a/releases/terraform/components/spark-3.4/variables.tf b/terraform/components/spark-3.4/variables.tf
similarity index 100%
rename from releases/terraform/components/spark-3.4/variables.tf
rename to terraform/components/spark-3.4/variables.tf
diff --git a/releases/terraform/products/charmed-spark-3.4/README.md b/terraform/products/charmed-spark-3.4/README.md
similarity index 100%
rename from releases/terraform/products/charmed-spark-3.4/README.md
rename to terraform/products/charmed-spark-3.4/README.md
diff --git a/releases/terraform/products/charmed-spark-3.4/locals.tf b/terraform/products/charmed-spark-3.4/locals.tf
similarity index 100%
rename from releases/terraform/products/charmed-spark-3.4/locals.tf
rename to terraform/products/charmed-spark-3.4/locals.tf
diff --git a/releases/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
similarity index 100%
rename from releases/terraform/products/charmed-spark-3.4/main.tf
rename to terraform/products/charmed-spark-3.4/main.tf
diff --git a/releases/terraform/products/charmed-spark-3.4/outputs.tf b/terraform/products/charmed-spark-3.4/outputs.tf
similarity index 100%
rename from releases/terraform/products/charmed-spark-3.4/outputs.tf
rename to terraform/products/charmed-spark-3.4/outputs.tf
diff --git a/releases/terraform/products/charmed-spark-3.4/providers.tf b/terraform/products/charmed-spark-3.4/providers.tf
similarity index 100%
rename from releases/terraform/products/charmed-spark-3.4/providers.tf
rename to terraform/products/charmed-spark-3.4/providers.tf
diff --git a/releases/terraform/products/charmed-spark-3.4/terraform.tf b/terraform/products/charmed-spark-3.4/terraform.tf
similarity index 100%
rename from releases/terraform/products/charmed-spark-3.4/terraform.tf
rename to terraform/products/charmed-spark-3.4/terraform.tf
diff --git a/releases/terraform/products/charmed-spark-3.4/variables.tf b/terraform/products/charmed-spark-3.4/variables.tf
similarity index 100%
rename from releases/terraform/products/charmed-spark-3.4/variables.tf
rename to terraform/products/charmed-spark-3.4/variables.tf

From d66c2d5bca298d8f52146ce4c344b6b4b9022147 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 1 Apr 2026 10:31:09 +0200
Subject: [PATCH 06/62] Fix product models output

---
 terraform/components/spark-3.4/README.md      | 69 ++++++-------------
 terraform/components/spark-3.4/main.tf        |  2 -
 terraform/components/spark-3.4/offers.tf      |  8 ++-
 terraform/components/spark-3.4/outputs.tf     | 42 +++++------
 .../products/charmed-spark-3.4/README.md      | 29 ++++----
 terraform/products/charmed-spark-3.4/main.tf  |  8 +--
 .../products/charmed-spark-3.4/outputs.tf     | 69 ++++++++-----------
 7 files changed, 93 insertions(+), 134 deletions(-)
 delete mode 100644 terraform/components/spark-3.4/main.tf

diff --git a/terraform/components/spark-3.4/README.md b/terraform/components/spark-3.4/README.md
index 6de53b81..3f42f49b 100644
--- a/terraform/components/spark-3.4/README.md
+++ b/terraform/components/spark-3.4/README.md
@@ -2,13 +2,14 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=0.20.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_juju"></a> [juju](#provider\_juju) | >=0.20.0 |
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
 ## Modules
 
@@ -19,71 +20,43 @@ No modules.
 | Name | Type |
 |------|------|
 | [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
-| [juju_application.data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_application.history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_application.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_application.kyuubi](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.history_server_object_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.integration_hub_object_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
 | [juju_integration.kyuubi_data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
 | [juju_integration.kyuubi_metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
 | [juju_integration.kyuubi_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
 | [juju_integration.kyuubi_tls](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
 | [juju_integration.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
 | [juju_integration.kyuubi_zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_offer.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_offer.metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_offer.integration_hub_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_offer.kyuubi_jdbc](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
 | [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
-| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
-| <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | n/a | yes |
-| <a name="input_driver_pod_template"></a> [driver\_pod\_template](#input\_driver\_pod\_template) | Define K8s driver pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
-| <a name="input_enable_dynamic_allocation"></a> [enable\_dynamic\_allocation](#input\_enable\_dynamic\_allocation) | Enable dynamic allocation of pods for Spark jobs. | `bool` | `false` | no |
-| <a name="input_executor_pod_template"></a> [executor\_pod\_template](#input\_executor\_pod\_template) | Define K8s executor pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
-| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `map(string)` | n/a | yes |
-| <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | n/a | yes |
-| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | n/a | yes |
-| <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
-| <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | n/a | yes |
-| <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
-| <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
-| <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
-| <a name="input_kyuubi_executor_pod_template"></a> [kyuubi\_executor\_pod\_template](#input\_kyuubi\_executor\_pod\_template) | Define K8s executor pod from a file accessible in the object storage. | `string` | `null` | no |
-| <a name="input_kyuubi_gpu_enable"></a> [kyuubi\_gpu\_enable](#input\_kyuubi\_gpu\_enable) | Enable GPU acceleration for SparkSQLEngine. | `bool` | n/a | yes |
-| <a name="input_kyuubi_gpu_engine_executors_limit"></a> [kyuubi\_gpu\_engine\_executors\_limit](#input\_kyuubi\_gpu\_engine\_executors\_limit) | Limit the number of GPUs an engine can schedule executor pods on. | `number` | n/a | yes |
-| <a name="input_kyuubi_gpu_pinned_memory"></a> [kyuubi\_gpu\_pinned\_memory](#input\_kyuubi\_gpu\_pinned\_memory) | Set the host memory (in GB) per executor reserved for fast data transfer on GPUs. | `number` | n/a | yes |
-| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `map(string)` | n/a | yes |
-| <a name="input_kyuubi_k8s_node_selectors"></a> [kyuubi\_k8s\_node\_selectors](#input\_kyuubi\_k8s\_node\_selectors) | Comma separated label:value selectors for K8s pods in Kyuubi. | `string` | `null` | no |
-| <a name="input_kyuubi_loadbalancer_extra_annotations"></a> [kyuubi\_loadbalancer\_extra\_annotations](#input\_kyuubi\_loadbalancer\_extra\_annotations) | Optional extra annotations to be supplied to the load balancer service in Kyuubi. | `string` | `null` | no |
-| <a name="input_kyuubi_profile"></a> [kyuubi\_profile](#input\_kyuubi\_profile) | The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'. | `string` | `"production"` | no |
-| <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | n/a | yes |
-| <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
-| <a name="input_kyuubi_user"></a> [kyuubi\_user](#input\_kyuubi\_user) | User name to be used for running Kyuubi engines. | `string` | `"kyuubi-spark-engine"` | no |
-| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | n/a | yes |
-| <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | n/a | yes |
-| <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
-| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | n/a | yes |
-| <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | n/a | yes |
-| <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
-| <a name="input_model"></a> [model](#input\_model) | Name of the Juju Model to deploy to. | `string` | n/a | yes |
-| <a name="input_tls_app_name"></a> [tls\_app\_name](#input\_tls\_app\_name) | Name of the application providing the `tls-certificates` interface. | `string` | n/a | yes |
-| <a name="input_tls_certificates_endpoint"></a> [tls\_certificates\_endpoint](#input\_tls\_certificates\_endpoint) | Name of the endpoint providing the `tls-certificates` interface. | `string` | n/a | yes |
+| <a name="input_data_integrator_endpoint"></a> [data\_integrator\_endpoint](#input\_data\_integrator\_endpoint) | In-model endpoint for the data-integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "kyuubi")<br/>  })</pre> | n/a | yes |
+| <a name="input_history_server"></a> [history\_server](#input\_history\_server) | n/a | <pre>object({<br/>    app_name    = optional(string, "history-server")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "history-server",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_integration_hub"></a> [integration\_hub](#input\_integration\_hub) | n/a | <pre>object({<br/>    app_name    = optional(string, "integration-hub")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "integration-hub",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_kyuubi"></a> [kyuubi](#input\_kyuubi) | n/a | <pre>object({<br/>    app_name    = optional(string, "kyuubi")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3.4/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "kyuubi",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_metastore_endpoint"></a> [metastore\_endpoint](#input\_metastore\_endpoint) | In-model endpoint for the metastore (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "database")<br/>  })</pre> | n/a | yes |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
+| <a name="input_object_storage_endpoint"></a> [object\_storage\_endpoint](#input\_object\_storage\_endpoint) | In-model endpoint for the object storage integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "s3_credentials")<br/>  })</pre> | n/a | yes |
+| <a name="input_tls_endpoint"></a> [tls\_endpoint](#input\_tls\_endpoint) | In-model endpoint for the TLS certificates provider. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "certificates")<br/>  })</pre> | n/a | yes |
 | <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
-| <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | n/a | yes |
-| <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | n/a | yes |
-| <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the zookeeper unit | `string` | `"10G"` | no |
-| <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
+| <a name="input_users_db_endpoint"></a> [users\_db\_endpoint](#input\_users\_db\_endpoint) | In-model endpoint for the Kyuubi users database (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "database")<br/>  })</pre> | n/a | yes |
+| <a name="input_zookeeper_endpoint"></a> [zookeeper\_endpoint](#input\_zookeeper\_endpoint) | In-model endpoint for the ZooKeeper application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "zookeeper")<br/>  })</pre> | n/a | yes |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
-| <a name="output_components"></a> [components](#output\_components) | List of the deployed applications for this component module. |
-| <a name="output_offers"></a> [offers](#output\_offers) | n/a |
+| <a name="output_components"></a> [components](#output\_components) | Map of the deployed applications for this component module. |
+| <a name="output_offers"></a> [offers](#output\_offers) | Map of all offers exposed by the component's charms. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provides endpoints. |
+| <a name="output_requires"></a> [requires](#output\_requires) | Map of the requires endpoints. |
diff --git a/terraform/components/spark-3.4/main.tf b/terraform/components/spark-3.4/main.tf
deleted file mode 100644
index 5b43ee7f..00000000
--- a/terraform/components/spark-3.4/main.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Main Terraform code for spark-3.4.
-# Module content is structured across *.tf files in this directory.
diff --git a/terraform/components/spark-3.4/offers.tf b/terraform/components/spark-3.4/offers.tf
index b9d57b96..21bd6a62 100644
--- a/terraform/components/spark-3.4/offers.tf
+++ b/terraform/components/spark-3.4/offers.tf
@@ -1,8 +1,14 @@
 # Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-resource "juju_offer" "integration_hub" {
+resource "juju_offer" "integration_hub_service_account" {
   model_uuid       = var.model_uuid
   application_name = juju_application.integration_hub.name
   endpoints        = ["spark-service-account"]
 }
+
+resource "juju_offer" "kyuubi_jdbc" {
+  model_uuid       = var.model_uuid
+  application_name = juju_application.kyuubi.name
+  endpoints        = ["jdbc"]
+}
diff --git a/terraform/components/spark-3.4/outputs.tf b/terraform/components/spark-3.4/outputs.tf
index 38c32ba8..cffe6085 100644
--- a/terraform/components/spark-3.4/outputs.tf
+++ b/terraform/components/spark-3.4/outputs.tf
@@ -1,32 +1,12 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    history_server  = juju_application.history_server.name
-    kyuubi          = juju_application.kyuubi.name
-    integration_hub = juju_application.integration_hub.name
-  }
-}
-
 output "components" {
-  description = "List of the deployed applications for this component module."
-  value = [
-    juju_application.history_server,
-    juju_application.integration_hub,
-    juju_application.kyuubi,
-  ]
-}
-
-output "provides" {
-  description = "Map of all the provides endpoints."
+  description = "Map of the deployed applications for this component module."
   value = {
-    integration_hub_spark_service_account = {
-      name     = juju_application.integration_hub.name
-      endpoint = "spark-service-account"
-    }
-    # TODO: Kyuubi
+    history_server  = juju_application.history_server,
+    integration_hub = juju_application.integration_hub,
+    kyuubi          = juju_application.kyuubi,
   }
 }
 
@@ -49,6 +29,16 @@ output "requires" {
       name     = juju_application.kyuubi.name
       endpoint = "zookeeper"
     }
+  }
+}
+
+output "provides" {
+  description = "Map of all the provides endpoints."
+  value = {
+    integration_hub_service_account = {
+      name     = juju_application.integration_hub.name
+      endpoint = "spark-service-account"
+    }
     kyuubi_jdbc = {
       name     = juju_application.kyuubi.name
       endpoint = "jdbc"
@@ -57,7 +47,9 @@ output "requires" {
 }
 
 output "offers" {
+  description = "Map of all offers exposed by the component's charms."
   value = {
-    hub_service_account = juju_offer.integration_hub
+    integration_hub_service_account = juju_offer.integration_hub_service_account
+    kyuubi_jdbc                     = juju_offer.kyuubi_jdbc
   }
 }
diff --git a/terraform/products/charmed-spark-3.4/README.md b/terraform/products/charmed-spark-3.4/README.md
index 6d28ad0f..4eda79c2 100644
--- a/terraform/products/charmed-spark-3.4/README.md
+++ b/terraform/products/charmed-spark-3.4/README.md
@@ -2,30 +2,32 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=0.20.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_juju"></a> [juju](#provider\_juju) | >=0.20.0 |
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_azure_storage"></a> [azure\_storage](#module\_azure\_storage) | ./charms/azure-storage-integrator | n/a |
-| <a name="module_bundled_cos"></a> [bundled\_cos](#module\_bundled\_cos) | ./products/cos | n/a |
-| <a name="module_observability"></a> [observability](#module\_observability) | ./components/observability | n/a |
-| <a name="module_s3"></a> [s3](#module\_s3) | ./charms/s3-integrator-v1 | n/a |
-| <a name="module_spark"></a> [spark](#module\_spark) | ./components/spark-3.4 | n/a |
-| <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev326 |
+| <a name="module_azure_storage"></a> [azure\_storage](#module\_azure\_storage) | ../../charms/azure-storage-integrator | n/a |
+| <a name="module_data_integrator"></a> [data\_integrator](#module\_data\_integrator) | ../../charms/data-integrator | n/a |
+| <a name="module_kyuubi_users"></a> [kyuubi\_users](#module\_kyuubi\_users) | git::https://github.com/canonical/postgresql-k8s-operator//terraform | rev774 |
+| <a name="module_metastore"></a> [metastore](#module\_metastore) | git::https://github.com/canonical/postgresql-k8s-operator//terraform | rev774 |
+| <a name="module_s3"></a> [s3](#module\_s3) | ../../charms/s3-integrator-v1 | n/a |
+| <a name="module_spark"></a> [spark](#module\_spark) | ../../components/spark-3.4 | n/a |
+| <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev586 |
+| <a name="module_zookeeper"></a> [zookeeper](#module\_zookeeper) | ../../charms/zookeeper | n/a |
 
 ## Resources
 
 | Name | Type |
 |------|------|
-| [juju_model.cos](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
 | [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
 
 ## Inputs
@@ -53,6 +55,7 @@
 | <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | `null` | no |
 | <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
 | <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | `null` | no |
+| <a name="input_juju_controller"></a> [juju\_controller](#input\_juju\_controller) | Controller information: endpoint, username, password and CA certificate. | <pre>object({<br/>    endpoint = string<br/>    username = string<br/>    password = string<br/>    ca       = string<br/>  })</pre> | `null` | no |
 | <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
 | <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
 | <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
@@ -70,18 +73,20 @@
 | <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | `null` | no |
 | <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | `null` | no |
 | <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
+| <a name="input_logging_config"></a> [logging\_config](#input\_logging\_config) | Logging configuration to be used. | `string` | `""` | no |
 | <a name="input_loki_active_index_directory_size"></a> [loki\_active\_index\_directory\_size](#input\_loki\_active\_index\_directory\_size) | Storage size for the active index directory for Loki | `string` | `"10G"` | no |
 | <a name="input_loki_chunks_size"></a> [loki\_chunks\_size](#input\_loki\_chunks\_size) | Storage size for the Loki chucks storage | `string` | `"500G"` | no |
 | <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | `null` | no |
 | <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | `null` | no |
 | <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
-| <a name="input_model"></a> [model](#input\_model) | The name of the juju model to deploy Spark to | `string` | `"spark"` | no |
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules. | `string` | `null` | no |
 | <a name="input_prometheus_size"></a> [prometheus\_size](#input\_prometheus\_size) | Storage size for the Prometheus database | `string` | `"500G"` | no |
+| <a name="input_proxy"></a> [proxy](#input\_proxy) | Proxy information for the deployment. | <pre>object({<br/>    http     = optional(string, "")<br/>    https    = optional(string, "")<br/>    no_proxy = optional(string, "")<br/>  })</pre> | `{}` | no |
 | <a name="input_pushgateway_revision"></a> [pushgateway\_revision](#input\_pushgateway\_revision) | Charm revision for prometheus-pushgateway-k8s | `number` | `null` | no |
 | <a name="input_s3"></a> [s3](#input\_s3) | S3 Bucket information | <pre>object({<br/>    bucket   = optional(string, "spark-test")<br/>    endpoint = optional(string, "https://s3.amazonaws.com")<br/>    region   = optional(string, "us-east-1")<br/>  })</pre> | `{}` | no |
 | <a name="input_s3_revision"></a> [s3\_revision](#input\_s3\_revision) | Charm revision for s3-integrator | `number` | `null` | no |
 | <a name="input_scrape_config_revision"></a> [scrape\_config\_revision](#input\_scrape\_config\_revision) | Charm revision for prometheus-scrape-config-k8s | `number` | `null` | no |
+| <a name="input_spark_model_name"></a> [spark\_model\_name](#input\_spark\_model\_name) | The name of the juju model to deploy Spark to | `string` | `"spark-test"` | no |
 | <a name="input_storage_backend"></a> [storage\_backend](#input\_storage\_backend) | Storage backend to be used | `string` | `"s3"` | no |
 | <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The file path of the private key to use for TLS certificates. | `string` | `null` | no |
 | <a name="input_traefik_size"></a> [traefik\_size](#input\_traefik\_size) | Storage size for the Traefik storage | `string` | `"10G"` | no |
@@ -94,6 +99,6 @@
 
 | Name | Description |
 |------|-------------|
-| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
-| <a name="output_components"></a> [components](#output\_components) | List of component modules (spark and optional addons) and their deployed charms/offers. |
+| <a name="output_metadata"></a> [metadata](#output\_metadata) | Metadata of the product deployment. |
+| <a name="output_models"></a> [models](#output\_models) | Maps of the models and the components deployed in each model. |
 | <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index 9d28b0a6..fa00f6d1 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -76,7 +76,7 @@ module "metastore" {
 }
 
 
-module "zk" {
+module "zookeeper" {
   depends_on = [juju_model.spark]
   source     = "../../charms/zookeeper"
   model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
@@ -163,7 +163,7 @@ module "spark" {
     module.metastore,
     module.s3,
     module.ssc,
-    module.zk
+    module.zookeeper
   ]
   source     = "../../components/spark-3.4"
   model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
@@ -206,8 +206,8 @@ module "spark" {
   }
 
   zookeeper_endpoint = {
-    name     = module.zk.application.name
-    endpoint = module.zk.provides.zookeeper
+    name     = module.zookeeper.application.name
+    endpoint = module.zookeeper.provides.zookeeper
   }
 
   data_integrator_endpoint = {
diff --git a/terraform/products/charmed-spark-3.4/outputs.tf b/terraform/products/charmed-spark-3.4/outputs.tf
index 9e3a8ad3..089e4cfb 100644
--- a/terraform/products/charmed-spark-3.4/outputs.tf
+++ b/terraform/products/charmed-spark-3.4/outputs.tf
@@ -2,37 +2,6 @@
 # See LICENSE file for licensing details.
 
 
-output "components" {
-  description = "List of component modules (spark and optional addons) and their deployed charms/offers."
-  value = [
-    {
-      name   = "spark"
-      charms = module.spark.charms
-      offers = module.spark.offers
-    },
-    {
-      name   = "azure_storage"
-      charms = try(module.azure_storage.charms, [])
-      offers = try(module.azure_storage.offers, {})
-    },
-    {
-      name   = "s3"
-      charms = try(module.s3.charms, [])
-      offers = try(module.s3.offers, {})
-    },
-    # {
-    #   name   = "bundled_cos"
-    #   charms = try(module.bundled_cos.charms, [])
-    #   offers = try(module.bundled_cos.offers, {})
-    # },
-    # {
-    #   name   = "observability"
-    #   charms = try(module.observability.charms, [])
-    #   offers = try(module.observability.offers, {})
-    # }
-  ]
-}
-
 output "offers" {
   description = "The name and url of the various offers being exposed"
   value = merge(
@@ -52,17 +21,33 @@ output "metadata" {
 }
 
 output "models" {
-  description = "Map of the models and the components deployed in each model."
-  value = {
-    spark = {
-      model_uuid = var.model_uuid
-      components = {
-        spark         = module.spark
-        azure_storage = try(one(module.azure_storage), null)
-        s3            = try(one(module.s3), null)
-        # observability = try(one(module.observability), null)
-        # bundled_cos   = try(one(module.bundled_cos), null)
+  description = "Maps of the models and the components deployed in each model."
+  value = [{
+    model_uuid = var.model_uuid
+    components = merge(concat(
+      module.spark.components,
+      {
+        zookeeper = module.zookeeper.application
+      },
+      {
+        data_integrator = module.data_integrator.application
+      },
+      {
+        self_signed_certificates = module.ssc.app_name # TODO: expose application
+      },
+      {
+        metastore = module.metastore.app_name # TODO: expose application
+      },
+      {
+        kyuubi_users = module.kyuubi_users.app_name # TODO: expose application
+      },
+      module.azure_storage == null ? {} : {
+        azure_storage = module.azure_storage[0].application
+      },
+      module.s3 == null ? {} : {
+        s3 = module.s3[0].application
       }
+    ))
     }
-  }
+  ]
 }

From e733eaa2bb0433dc071cfe952a0f0afc63260431 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 1 Apr 2026 14:56:33 +0200
Subject: [PATCH 07/62] Migrate aks setup to root level resources

---
 resources/aks-setup/cluster.tf        |  32 +++++++
 resources/aks-setup/network.tf        | 117 ++++++++++++++++++++++++++
 resources/aks-setup/outputs.tf        |  12 +++
 resources/aks-setup/providers.tf      |  18 ++++
 resources/aks-setup/resource-group.tf |   7 ++
 resources/aks-setup/variables.tf      |  23 +++++
 6 files changed, 209 insertions(+)
 create mode 100644 resources/aks-setup/cluster.tf
 create mode 100644 resources/aks-setup/network.tf
 create mode 100644 resources/aks-setup/outputs.tf
 create mode 100644 resources/aks-setup/providers.tf
 create mode 100644 resources/aks-setup/resource-group.tf
 create mode 100644 resources/aks-setup/variables.tf

diff --git a/resources/aks-setup/cluster.tf b/resources/aks-setup/cluster.tf
new file mode 100644
index 00000000..0cb82b77
--- /dev/null
+++ b/resources/aks-setup/cluster.tf
@@ -0,0 +1,32 @@
+
+resource "azurerm_kubernetes_cluster" "prod-k8s" {
+  location            = azurerm_resource_group.rg.location
+  name                = var.cluster_name
+  resource_group_name = azurerm_resource_group.rg.name
+  dns_prefix          = "testaksclusterprefix"
+  kubernetes_version  = "1.31.7"
+
+  identity {
+    type = "SystemAssigned"
+  }
+
+  default_node_pool {
+    name           = "default"
+    vm_size        = "Standard_DS2_v2"
+    node_count     = var.num_nodes
+    vnet_subnet_id = azurerm_subnet.infrastructure.id
+  }
+
+  auto_scaler_profile {
+    expander = "least-waste" # if not least waste
+    #ignore-daemonsets-utilization = true
+    scale_down_utilization_threshold = 0.2
+    scale_down_unneeded              = "30m"
+  }
+
+  network_profile {
+    network_plugin = "kubenet"
+    service_cidr   = "10.1.0.0/16" # Ensure this does not overlap with any subnets
+    dns_service_ip = "10.1.0.10"   # Hardcoded DNS service IP
+  }
+}
diff --git a/resources/aks-setup/network.tf b/resources/aks-setup/network.tf
new file mode 100644
index 00000000..96597e21
--- /dev/null
+++ b/resources/aks-setup/network.tf
@@ -0,0 +1,117 @@
+resource "azurerm_virtual_network" "vn" {
+  name                = "TestSparkAKSVN"
+  address_space       = ["10.0.0.0/16"] # Define the address space for the VNet
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
+}
+
+
+resource "azurerm_subnet" "infrastructure" {
+  name                 = "testsparkaks-subnet"
+  virtual_network_name = azurerm_virtual_network.vn.name
+  resource_group_name  = azurerm_resource_group.rg.name
+  address_prefixes     = ["10.0.10.0/24"]
+}
+
+
+resource "azurerm_nat_gateway" "natgw" {
+  location            = azurerm_resource_group.rg.location
+  name                = "testsparkaks-nat-gateway"
+  resource_group_name = azurerm_resource_group.rg.name
+  sku_name            = "Standard"
+
+}
+
+
+resource "azurerm_public_ip" "natip" {
+  name                = "testsparkaks-nat-gateway-ip"
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
+  allocation_method   = "Static"
+  sku                 = "Standard"
+}
+
+
+resource "azurerm_nat_gateway_public_ip_association" "example" {
+  nat_gateway_id       = azurerm_nat_gateway.natgw.id
+  public_ip_address_id = azurerm_public_ip.natip.id
+}
+
+
+resource "azurerm_subnet_nat_gateway_association" "natinfra" {
+  subnet_id      = azurerm_subnet.infrastructure.id
+  nat_gateway_id = azurerm_nat_gateway.natgw.id
+}
+
+
+resource "azurerm_network_security_group" "infra_sec_group" {
+  name                = "TestAKSInfrastructureSecurityGroup"
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
+
+  security_rule {
+    name                       = "AllowAnyCustomAnyOutbound"
+    priority                   = 110
+    direction                  = "Outbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "*"
+    source_address_prefix      = "*"
+    destination_address_prefix = "*"
+  }
+
+  security_rule {
+    name                       = "AllowNATInbound"
+    priority                   = 120
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "*"
+    source_address_prefix      = azurerm_public_ip.natip.ip_address
+    destination_address_prefix = "*"
+  }
+
+  security_rule {
+    name                       = "Allow80AnyInbound"
+    priority                   = 130
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "80"
+    source_address_prefix      = "*"
+    destination_address_prefix = "*"
+  }
+
+  security_rule {
+    name                       = "Allow443AnyInbound"
+    priority                   = 140
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "443"
+    source_address_prefix      = "*"
+    destination_address_prefix = "*"
+  }
+
+  security_rule {
+    name                       = "Allow8080AnyInbound"
+    priority                   = 150
+    direction                  = "Inbound"
+    access                     = "Allow"
+    protocol                   = "Tcp"
+    source_port_range          = "*"
+    destination_port_range     = "8080"
+    source_address_prefix      = "*"
+    destination_address_prefix = "*"
+  }
+}
+
+
+resource "azurerm_subnet_network_security_group_association" "infra-association" {
+  subnet_id                 = azurerm_subnet.infrastructure.id
+  network_security_group_id = azurerm_network_security_group.infra_sec_group.id
+}
diff --git a/resources/aks-setup/outputs.tf b/resources/aks-setup/outputs.tf
new file mode 100644
index 00000000..8837c5e9
--- /dev/null
+++ b/resources/aks-setup/outputs.tf
@@ -0,0 +1,12 @@
+
+output "resource_group_name" {
+  value       = azurerm_resource_group.rg.name
+  description = "The name of the resource group"
+}
+
+
+output "aks_cluster_name" {
+  value       = azurerm_kubernetes_cluster.prod-k8s.name
+  description = "The name of the AKS cluster"
+}
+
diff --git a/resources/aks-setup/providers.tf b/resources/aks-setup/providers.tf
new file mode 100644
index 00000000..a9beee3c
--- /dev/null
+++ b/resources/aks-setup/providers.tf
@@ -0,0 +1,18 @@
+terraform {
+  required_version = ">=1.0"
+
+  required_providers {
+    azapi = {
+      source  = "azure/azapi"
+      version = "~>1.5"
+    }
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = "~>3.0"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
diff --git a/resources/aks-setup/resource-group.tf b/resources/aks-setup/resource-group.tf
new file mode 100644
index 00000000..0a50405b
--- /dev/null
+++ b/resources/aks-setup/resource-group.tf
@@ -0,0 +1,7 @@
+resource "azurerm_resource_group" "rg" {
+  name     = var.resource_group_name
+  location = var.region
+}
+
+
+
diff --git a/resources/aks-setup/variables.tf b/resources/aks-setup/variables.tf
new file mode 100644
index 00000000..9cbf6fcc
--- /dev/null
+++ b/resources/aks-setup/variables.tf
@@ -0,0 +1,23 @@
+variable "num_nodes" {
+  description = "Number of nodes in the AKS cluster"
+  type        = number
+  default     = 3
+}
+
+variable "cluster_name" {
+  description = "The name of the AKS cluster to be created"
+  type        = string
+  default     = "TestAKSCluster"
+}
+
+variable "region" {
+  description = "The region in which cloud resources are to be created"
+  type        = string
+  default     = "East US"
+}
+
+variable "resource_group_name" {
+  description = "The name of the resource group under which the resources will be created"
+  type        = string
+  default     = "TestSparkAKSRG"
+}
\ No newline at end of file

From 30e352a7eb3ed79d8ef70f8a97336aaf50488402 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 1 Apr 2026 14:57:01 +0200
Subject: [PATCH 08/62] Remove previous bundles

---
 .../resources/grafana/spark_dashboard.json    | 1733 -----------------
 releases/3.4/terraform/README.md              |  194 --
 .../terraform/external/cos/applications.tf    |  103 -
 .../terraform/external/cos/integrations.tf    |  300 ---
 .../3.4/terraform/external/cos/outputs.tf     |   30 -
 .../3.4/terraform/external/cos/providers.tf   |   13 -
 .../3.4/terraform/external/cos/resources.tf   |   27 -
 .../3.4/terraform/external/cos/variables.tf   |   69 -
 releases/3.4/terraform/external/tls/main.tf   |   58 -
 releases/3.4/terraform/main.tf                |  136 --
 .../modules/azure-storage/applications.tf     |   21 -
 .../modules/azure-storage/integrations.tf     |   30 -
 .../modules/azure-storage/outputs.tf          |    9 -
 .../modules/azure-storage/providers.tf        |   13 -
 .../modules/azure-storage/resources.tf        |   26 -
 .../modules/azure-storage/variables.tf        |   31 -
 .../modules/observability/applications.tf     |   64 -
 .../modules/observability/integrations.tf     |  236 ---
 .../modules/observability/outputs.tf          |   12 -
 .../modules/observability/providers.tf        |   13 -
 .../modules/observability/resources.tf        |    8 -
 .../modules/observability/variables.tf        |   75 -
 .../3.4/terraform/modules/s3/applications.tf  |   20 -
 .../3.4/terraform/modules/s3/integrations.tf  |   30 -
 releases/3.4/terraform/modules/s3/outputs.tf  |    9 -
 .../3.4/terraform/modules/s3/providers.tf     |   13 -
 .../3.4/terraform/modules/s3/resources.tf     |    8 -
 .../3.4/terraform/modules/s3/variables.tf     |   34 -
 .../terraform/modules/spark/applications.tf   |  190 --
 .../terraform/modules/spark/integrations.tf   |   86 -
 .../3.4/terraform/modules/spark/offers.tf     |   11 -
 .../3.4/terraform/modules/spark/outputs.tf    |   22 -
 .../3.4/terraform/modules/spark/providers.tf  |   13 -
 .../3.4/terraform/modules/spark/resources.tf  |   32 -
 .../3.4/terraform/modules/spark/variables.tf  |  241 ---
 releases/3.4/terraform/outputs.tf             |   21 -
 releases/3.4/terraform/variables.tf           |  396 ----
 releases/3.4/terraform/versions.tf            |   53 -
 releases/3.4/yaml/README.md                   |    5 -
 .../3.4/yaml/bundle-azure-storage.yaml.j2     |  112 --
 releases/3.4/yaml/bundle.yaml.j2              |  113 --
 releases/3.4/yaml/charmcraft.yaml             |    4 -
 .../yaml/docs-resources/aks-setup/cluster.tf  |   32 -
 .../yaml/docs-resources/aks-setup/network.tf  |  117 --
 .../yaml/docs-resources/aks-setup/outputs.tf  |   12 -
 .../docs-resources/aks-setup/providers.tf     |   18 -
 .../aks-setup/resource-group.tf               |    7 -
 .../docs-resources/aks-setup/variables.tf     |   23 -
 releases/3.4/yaml/metadata.yaml               |   21 -
 .../3.4/yaml/overlays/cos-integration.yaml.j2 |   94 -
 .../resources/grafana/spark_dashboard.json    | 1733 -----------------
 releases/3.5/terraform/README.md              |  194 --
 .../terraform/external/cos/applications.tf    |  103 -
 .../terraform/external/cos/integrations.tf    |  300 ---
 .../3.5/terraform/external/cos/outputs.tf     |   30 -
 .../3.5/terraform/external/cos/providers.tf   |   13 -
 .../3.5/terraform/external/cos/resources.tf   |   27 -
 .../3.5/terraform/external/cos/variables.tf   |   69 -
 releases/3.5/terraform/external/tls/main.tf   |   58 -
 releases/3.5/terraform/main.tf                |  137 --
 .../modules/azure-storage/applications.tf     |   21 -
 .../modules/azure-storage/integrations.tf     |   30 -
 .../modules/azure-storage/outputs.tf          |    9 -
 .../modules/azure-storage/providers.tf        |   13 -
 .../modules/azure-storage/resources.tf        |   26 -
 .../modules/azure-storage/variables.tf        |   31 -
 .../modules/observability/applications.tf     |   64 -
 .../modules/observability/integrations.tf     |  236 ---
 .../modules/observability/outputs.tf          |   12 -
 .../modules/observability/providers.tf        |   13 -
 .../modules/observability/resources.tf        |    8 -
 .../modules/observability/variables.tf        |   75 -
 .../3.5/terraform/modules/s3/applications.tf  |   20 -
 .../3.5/terraform/modules/s3/integrations.tf  |   30 -
 releases/3.5/terraform/modules/s3/outputs.tf  |    9 -
 .../3.5/terraform/modules/s3/providers.tf     |   13 -
 .../3.5/terraform/modules/s3/resources.tf     |    8 -
 .../3.5/terraform/modules/s3/variables.tf     |   34 -
 .../terraform/modules/spark/applications.tf   |  190 --
 .../terraform/modules/spark/integrations.tf   |   86 -
 .../3.5/terraform/modules/spark/offers.tf     |   11 -
 .../3.5/terraform/modules/spark/outputs.tf    |   22 -
 .../3.5/terraform/modules/spark/providers.tf  |   13 -
 .../3.5/terraform/modules/spark/resources.tf  |   32 -
 .../3.5/terraform/modules/spark/variables.tf  |  241 ---
 releases/3.5/terraform/outputs.tf             |   21 -
 releases/3.5/terraform/variables.tf           |  396 ----
 releases/3.5/terraform/versions.tf            |   53 -
 releases/3.5/yaml/README.md                   |    5 -
 .../3.5/yaml/bundle-azure-storage.yaml.j2     |  112 --
 releases/3.5/yaml/bundle.yaml.j2              |  113 --
 releases/3.5/yaml/charmcraft.yaml             |    4 -
 releases/3.5/yaml/metadata.yaml               |   21 -
 .../3.5/yaml/overlays/cos-integration.yaml.j2 |   94 -
 .../components/observability/README.md        |   67 -
 .../components/observability/applications.tf  |   64 -
 .../components/observability/integrations.tf  |  236 ---
 .../components/observability/locals.tf        |    2 -
 .../components/observability/main.tf          |    2 -
 .../components/observability/outputs.tf       |   21 -
 .../components/observability/providers.tf     |    6 -
 .../components/observability/resources.tf     |   12 -
 .../components/observability/terraform.tf     |   10 -
 .../components/observability/variables.tf     |   81 -
 .../terraform/components/spark-3.5/README.md  |   90 -
 .../components/spark-3.5/applications.tf      |  190 --
 .../components/spark-3.5/integrations.tf      |   86 -
 .../terraform/components/spark-3.5/locals.tf  |   43 -
 .../terraform/components/spark-3.5/main.tf    |    2 -
 .../terraform/components/spark-3.5/offers.tf  |   11 -
 .../terraform/components/spark-3.5/outputs.tf |   34 -
 .../components/spark-3.5/providers.tf         |    6 -
 .../components/spark-3.5/resources.tf         |   36 -
 .../components/spark-3.5/terraform.tf         |   10 -
 .../components/spark-3.5/variables.tf         |  244 ---
 .../products/charmed-spark-3.5/README.md      |  100 -
 .../products/charmed-spark-3.5/locals.tf      |   43 -
 .../products/charmed-spark-3.5/main.tf        |  146 --
 .../products/charmed-spark-3.5/outputs.tf     |   51 -
 .../products/charmed-spark-3.5/providers.tf   |    6 -
 .../products/charmed-spark-3.5/terraform.tf   |   13 -
 .../products/charmed-spark-3.5/variables.tf   |  395 ----
 releases/terraform/products/cos/README.md     |   77 -
 .../terraform/products/cos/applications.tf    |  103 -
 .../terraform/products/cos/integrations.tf    |  300 ---
 releases/terraform/products/cos/locals.tf     |    2 -
 releases/terraform/products/cos/main.tf       |    2 -
 releases/terraform/products/cos/outputs.tf    |   30 -
 releases/terraform/products/cos/providers.tf  |    6 -
 releases/terraform/products/cos/resources.tf  |   27 -
 releases/terraform/products/cos/terraform.tf  |   10 -
 releases/terraform/products/cos/variables.tf  |   69 -
 132 files changed, 12301 deletions(-)
 delete mode 100644 releases/3.4/resources/grafana/spark_dashboard.json
 delete mode 100644 releases/3.4/terraform/README.md
 delete mode 100644 releases/3.4/terraform/external/cos/applications.tf
 delete mode 100644 releases/3.4/terraform/external/cos/integrations.tf
 delete mode 100644 releases/3.4/terraform/external/cos/outputs.tf
 delete mode 100644 releases/3.4/terraform/external/cos/providers.tf
 delete mode 100644 releases/3.4/terraform/external/cos/resources.tf
 delete mode 100644 releases/3.4/terraform/external/cos/variables.tf
 delete mode 100644 releases/3.4/terraform/external/tls/main.tf
 delete mode 100644 releases/3.4/terraform/main.tf
 delete mode 100644 releases/3.4/terraform/modules/azure-storage/applications.tf
 delete mode 100644 releases/3.4/terraform/modules/azure-storage/integrations.tf
 delete mode 100644 releases/3.4/terraform/modules/azure-storage/outputs.tf
 delete mode 100644 releases/3.4/terraform/modules/azure-storage/providers.tf
 delete mode 100644 releases/3.4/terraform/modules/azure-storage/resources.tf
 delete mode 100644 releases/3.4/terraform/modules/azure-storage/variables.tf
 delete mode 100644 releases/3.4/terraform/modules/observability/applications.tf
 delete mode 100644 releases/3.4/terraform/modules/observability/integrations.tf
 delete mode 100644 releases/3.4/terraform/modules/observability/outputs.tf
 delete mode 100644 releases/3.4/terraform/modules/observability/providers.tf
 delete mode 100644 releases/3.4/terraform/modules/observability/resources.tf
 delete mode 100644 releases/3.4/terraform/modules/observability/variables.tf
 delete mode 100644 releases/3.4/terraform/modules/s3/applications.tf
 delete mode 100644 releases/3.4/terraform/modules/s3/integrations.tf
 delete mode 100644 releases/3.4/terraform/modules/s3/outputs.tf
 delete mode 100644 releases/3.4/terraform/modules/s3/providers.tf
 delete mode 100644 releases/3.4/terraform/modules/s3/resources.tf
 delete mode 100644 releases/3.4/terraform/modules/s3/variables.tf
 delete mode 100644 releases/3.4/terraform/modules/spark/applications.tf
 delete mode 100644 releases/3.4/terraform/modules/spark/integrations.tf
 delete mode 100644 releases/3.4/terraform/modules/spark/offers.tf
 delete mode 100644 releases/3.4/terraform/modules/spark/outputs.tf
 delete mode 100644 releases/3.4/terraform/modules/spark/providers.tf
 delete mode 100644 releases/3.4/terraform/modules/spark/resources.tf
 delete mode 100644 releases/3.4/terraform/modules/spark/variables.tf
 delete mode 100644 releases/3.4/terraform/outputs.tf
 delete mode 100644 releases/3.4/terraform/variables.tf
 delete mode 100644 releases/3.4/terraform/versions.tf
 delete mode 100644 releases/3.4/yaml/README.md
 delete mode 100644 releases/3.4/yaml/bundle-azure-storage.yaml.j2
 delete mode 100644 releases/3.4/yaml/bundle.yaml.j2
 delete mode 100644 releases/3.4/yaml/charmcraft.yaml
 delete mode 100644 releases/3.4/yaml/docs-resources/aks-setup/cluster.tf
 delete mode 100644 releases/3.4/yaml/docs-resources/aks-setup/network.tf
 delete mode 100644 releases/3.4/yaml/docs-resources/aks-setup/outputs.tf
 delete mode 100644 releases/3.4/yaml/docs-resources/aks-setup/providers.tf
 delete mode 100644 releases/3.4/yaml/docs-resources/aks-setup/resource-group.tf
 delete mode 100644 releases/3.4/yaml/docs-resources/aks-setup/variables.tf
 delete mode 100644 releases/3.4/yaml/metadata.yaml
 delete mode 100644 releases/3.4/yaml/overlays/cos-integration.yaml.j2
 delete mode 100644 releases/3.5/resources/grafana/spark_dashboard.json
 delete mode 100644 releases/3.5/terraform/README.md
 delete mode 100644 releases/3.5/terraform/external/cos/applications.tf
 delete mode 100644 releases/3.5/terraform/external/cos/integrations.tf
 delete mode 100644 releases/3.5/terraform/external/cos/outputs.tf
 delete mode 100644 releases/3.5/terraform/external/cos/providers.tf
 delete mode 100644 releases/3.5/terraform/external/cos/resources.tf
 delete mode 100644 releases/3.5/terraform/external/cos/variables.tf
 delete mode 100644 releases/3.5/terraform/external/tls/main.tf
 delete mode 100644 releases/3.5/terraform/main.tf
 delete mode 100644 releases/3.5/terraform/modules/azure-storage/applications.tf
 delete mode 100644 releases/3.5/terraform/modules/azure-storage/integrations.tf
 delete mode 100644 releases/3.5/terraform/modules/azure-storage/outputs.tf
 delete mode 100644 releases/3.5/terraform/modules/azure-storage/providers.tf
 delete mode 100644 releases/3.5/terraform/modules/azure-storage/resources.tf
 delete mode 100644 releases/3.5/terraform/modules/azure-storage/variables.tf
 delete mode 100644 releases/3.5/terraform/modules/observability/applications.tf
 delete mode 100644 releases/3.5/terraform/modules/observability/integrations.tf
 delete mode 100644 releases/3.5/terraform/modules/observability/outputs.tf
 delete mode 100644 releases/3.5/terraform/modules/observability/providers.tf
 delete mode 100644 releases/3.5/terraform/modules/observability/resources.tf
 delete mode 100644 releases/3.5/terraform/modules/observability/variables.tf
 delete mode 100644 releases/3.5/terraform/modules/s3/applications.tf
 delete mode 100644 releases/3.5/terraform/modules/s3/integrations.tf
 delete mode 100644 releases/3.5/terraform/modules/s3/outputs.tf
 delete mode 100644 releases/3.5/terraform/modules/s3/providers.tf
 delete mode 100644 releases/3.5/terraform/modules/s3/resources.tf
 delete mode 100644 releases/3.5/terraform/modules/s3/variables.tf
 delete mode 100644 releases/3.5/terraform/modules/spark/applications.tf
 delete mode 100644 releases/3.5/terraform/modules/spark/integrations.tf
 delete mode 100644 releases/3.5/terraform/modules/spark/offers.tf
 delete mode 100644 releases/3.5/terraform/modules/spark/outputs.tf
 delete mode 100644 releases/3.5/terraform/modules/spark/providers.tf
 delete mode 100644 releases/3.5/terraform/modules/spark/resources.tf
 delete mode 100644 releases/3.5/terraform/modules/spark/variables.tf
 delete mode 100644 releases/3.5/terraform/outputs.tf
 delete mode 100644 releases/3.5/terraform/variables.tf
 delete mode 100644 releases/3.5/terraform/versions.tf
 delete mode 100644 releases/3.5/yaml/README.md
 delete mode 100644 releases/3.5/yaml/bundle-azure-storage.yaml.j2
 delete mode 100644 releases/3.5/yaml/bundle.yaml.j2
 delete mode 100644 releases/3.5/yaml/charmcraft.yaml
 delete mode 100644 releases/3.5/yaml/metadata.yaml
 delete mode 100644 releases/3.5/yaml/overlays/cos-integration.yaml.j2
 delete mode 100644 releases/terraform/components/observability/README.md
 delete mode 100644 releases/terraform/components/observability/applications.tf
 delete mode 100644 releases/terraform/components/observability/integrations.tf
 delete mode 100644 releases/terraform/components/observability/locals.tf
 delete mode 100644 releases/terraform/components/observability/main.tf
 delete mode 100644 releases/terraform/components/observability/outputs.tf
 delete mode 100644 releases/terraform/components/observability/providers.tf
 delete mode 100644 releases/terraform/components/observability/resources.tf
 delete mode 100644 releases/terraform/components/observability/terraform.tf
 delete mode 100644 releases/terraform/components/observability/variables.tf
 delete mode 100644 releases/terraform/components/spark-3.5/README.md
 delete mode 100644 releases/terraform/components/spark-3.5/applications.tf
 delete mode 100644 releases/terraform/components/spark-3.5/integrations.tf
 delete mode 100644 releases/terraform/components/spark-3.5/locals.tf
 delete mode 100644 releases/terraform/components/spark-3.5/main.tf
 delete mode 100644 releases/terraform/components/spark-3.5/offers.tf
 delete mode 100644 releases/terraform/components/spark-3.5/outputs.tf
 delete mode 100644 releases/terraform/components/spark-3.5/providers.tf
 delete mode 100644 releases/terraform/components/spark-3.5/resources.tf
 delete mode 100644 releases/terraform/components/spark-3.5/terraform.tf
 delete mode 100644 releases/terraform/components/spark-3.5/variables.tf
 delete mode 100644 releases/terraform/products/charmed-spark-3.5/README.md
 delete mode 100644 releases/terraform/products/charmed-spark-3.5/locals.tf
 delete mode 100644 releases/terraform/products/charmed-spark-3.5/main.tf
 delete mode 100644 releases/terraform/products/charmed-spark-3.5/outputs.tf
 delete mode 100644 releases/terraform/products/charmed-spark-3.5/providers.tf
 delete mode 100644 releases/terraform/products/charmed-spark-3.5/terraform.tf
 delete mode 100644 releases/terraform/products/charmed-spark-3.5/variables.tf
 delete mode 100644 releases/terraform/products/cos/README.md
 delete mode 100644 releases/terraform/products/cos/applications.tf
 delete mode 100644 releases/terraform/products/cos/integrations.tf
 delete mode 100644 releases/terraform/products/cos/locals.tf
 delete mode 100644 releases/terraform/products/cos/main.tf
 delete mode 100644 releases/terraform/products/cos/outputs.tf
 delete mode 100644 releases/terraform/products/cos/providers.tf
 delete mode 100644 releases/terraform/products/cos/resources.tf
 delete mode 100644 releases/terraform/products/cos/terraform.tf
 delete mode 100644 releases/terraform/products/cos/variables.tf

diff --git a/releases/3.4/resources/grafana/spark_dashboard.json b/releases/3.4/resources/grafana/spark_dashboard.json
deleted file mode 100644
index f34cd1d8..00000000
--- a/releases/3.4/resources/grafana/spark_dashboard.json
+++ /dev/null
@@ -1,1733 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": [],
-          "type": "dashboard"
-        },
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 4,
-      "panels": [],
-      "title": "Block Manager (Driver)",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "decmbytes"
-        },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "Max Memory (MB)"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 1
-      },
-      "id": 2,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "BlockManager_memory_maxMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "instant": false,
-          "legendFormat": "Max Memory (MB)",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "BlockManager_memory_maxOffHeapMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "legendFormat": "Max Off Heap Memory (MB)",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "BlockManager_memory_maxOnHeapMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "legendFormat": "Max On Heap Memory (MB)",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "BlockManager_memory_remainingMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "legendFormat": "Remaining Memory (MB)",
-          "range": true,
-          "refId": "D"
-        }
-      ],
-      "title": "Block Manager Memory",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "decmbytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 1
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "BlockManager_disk_diskSpaceUsed_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "legendFormat": "Disk Memory (MB)",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Block Manager Disk Memory",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 10
-      },
-      "id": 10,
-      "panels": [],
-      "title": "Hive External Catalog (Driver)",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 12,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_fileCacheHits{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "instant": false,
-          "legendFormat": "File Cache Hits",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_filesDiscovered{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "File Discovered",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_hiveClientCalls{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "Client Calls",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_parallelListingJobCount{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "Parallel Listing Jobs",
-          "range": true,
-          "refId": "D"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_partitionsFetched{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "Partitions Fetched",
-          "range": true,
-          "refId": "E"
-        }
-      ],
-      "title": "Hive External Catalog",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 19
-      },
-      "id": 14,
-      "panels": [],
-      "title": "Dag Scheduler (Driver)",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 20
-      },
-      "id": 16,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "DAGScheduler_job_activeJobs{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "instant": false,
-          "legendFormat": "Active Jobs",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "DAGScheduler_job_allJobs{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "All Jobs",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DAG Schedule Jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 20
-      },
-      "id": 18,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "DAGScheduler_stage_failedStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "Failed Stages",
-          "range": true,
-          "refId": "FailedStages"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "DAGScheduler_stage_runningStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Running Stages",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "DAGScheduler_stage_waitingStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Waiting Stages",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Stages Status",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 28
-      },
-      "id": 20,
-      "panels": [],
-      "title": "Memory Metrics (Driver)",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 29
-      },
-      "id": 22,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_DirectPoolMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "Direct Pool Memory",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_JVMHeapMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM Heap Memory",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_JVMOffHeapMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM Off Heap Memory",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_OffHeapExecutionMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM Off Heap Execution Memory",
-          "range": true,
-          "refId": "D"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_OnHeapStorageMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM On Heap Storage Memory",
-          "range": true,
-          "refId": "E"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_OnHeapExecutionMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM On Heap Execution Memory",
-          "range": true,
-          "refId": "F"
-        }
-      ],
-      "title": "JVM Executor Memory",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 37
-      },
-      "id": 24,
-      "panels": [],
-      "title": "Executors Metrics",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 38
-      },
-      "id": 26,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_bytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "Bytes Reads (#Executor {{number}})",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_bytesWritten{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Bytes Written (#Executor {{number}})",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Executor Read and Writes",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 38
-      },
-      "id": 28,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_JVMHeapMemory{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "JVM Heap Memory #{{number}}",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_JVMOffHeapMemory{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM Off Heap Memory #{{number}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Memory",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 46
-      },
-      "id": 30,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "rate(executor_runTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "legendFormat": "Runtime (Executor #{{number}})",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Executors Runtime",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 46
-      },
-      "id": 32,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_shuffleBytesWritten{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "Shuffle Bytes Writes (Executor #{{number}})",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_shuffleLocalBytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Shuffle Local Bytes Reads (Executor #{{number}})",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_shuffleRemoteBytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Shuffle Remote Bytes Reads (Executor #{{number}})",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Shuffle Reads/Writes",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "Minor GC Count (Executor #1"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 54
-      },
-      "id": 34,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "increase(ExecutorMetrics_MinorGCCount{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "legendFormat": "Minor GC Count (Executor #{{number}}",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "increase(ExecutorMetrics_MajorGCCount{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "Major GC Count (Executor #{{number}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "GC counts",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 54
-      },
-      "id": 36,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "rate(ExecutorMetrics_MajorGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "legendFormat": "Major GC time (Executor #{{number}})",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "rate(ExecutorMetrics_MinorGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "Minor GC time (Executor #{{number}})",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "rate(ExecutorMetrics_TotalGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "Total GC time (Executor #{{number}})",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "GC Time Spend",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": false,
-  "schemaVersion": 37,
-  "style": "dark",
-  "tags": [],
-  "templating": {
-    "list": [
-      {
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "hide": 0,
-        "includeAll": true,
-        "label": "Loki datasource",
-        "multi": true,
-        "name": "lokids",
-        "options": [],
-        "query": "loki",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Juju unit",
-        "multi": true,
-        "name": "juju_unit",
-        "options": [],
-        "query": {
-          "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "tagValuesQuery": "",
-        "tags": [],
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      },
-      {
-        "current": {
-          "selected": false,
-          "text": "juju_cos_348d0940-98ce-4ac5-8dcc-4dce43d9c620_prometheus_0",
-          "value": "juju_cos_348d0940-98ce-4ac5-8dcc-4dce43d9c620_prometheus_0"
-        },
-        "hide": 0,
-        "includeAll": false,
-        "label": "Prometheus datasource",
-        "multi": false,
-        "name": "prometheusds",
-        "options": [],
-        "query": "prometheus",
-        "queryValue": "",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(up,juju_model)",
-        "description": "The",
-        "hide": 0,
-        "includeAll": true,
-        "label": "juju model",
-        "multi": true,
-        "name": "juju_model",
-        "options": [],
-        "query": {
-          "query": "label_values(up,juju_model)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Juju model uuid",
-        "multi": true,
-        "name": "juju_model_uuid",
-        "options": [],
-        "query": {
-          "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "juju application",
-        "multi": true,
-        "name": "juju_application",
-        "options": [],
-        "query": {
-          "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(push_time_seconds{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},exported_job)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Spark Job Id",
-        "multi": true,
-        "name": "spark_job_id",
-        "options": [],
-        "query": {
-          "query": "label_values(push_time_seconds{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},exported_job)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      }
-    ]
-  },
-  "time": {
-    "from": "now-12h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "Spark dashboard",
-  "uid": "ZyD93JGSk",
-  "version": 1,
-  "weekStart": ""
-}
\ No newline at end of file
diff --git a/releases/3.4/terraform/README.md b/releases/3.4/terraform/README.md
deleted file mode 100644
index d9a5c726..00000000
--- a/releases/3.4/terraform/README.md
+++ /dev/null
@@ -1,194 +0,0 @@
-# Spark bundle Terraform modules
-
-These Terraform modules aim to facilitate the deployment of the Charmed Spark solution, using the using the [Terraform juju provider](https://github.com/juju/terraform-provider-juju/). For more information, refer to the provider [documentation](https://registry.terraform.io/providers/juju/juju/latest/docs).
-
-The solution consists of the following modules:
-
-- `spark`: main spark module with integration-hub, history-server, kyuubi
-- `s3` and `azure-storage`: storage backend. Having one of them is necessary, and they are mutually exclusive
-- `observabiliby`: parts of the observability stack to be deployed alongside spark (config, pushgateway, etc.)
-
-> [!NOTE]
-> The `s3` module itself doesn't act as an S3 object storage system. For the solution to be functional, the `s3-integrator` charm needs to point to an S3-like storage.
-> The same applies to `azure-storage`.
-
-## Requirements
-
-This module requires a `juju` model to be available. Refer to the [usage section](#usage) below for more details.
-
-## Top-level inputs
-
-| Variable                                   | Description                                                                     | Type   | Nullable | Default             |
-| ------------------------------------------ | ------------------------------------------------------------------------------- | ------ | -------- | ------------------- |
-| K8S_CLOUD                                  | The kubernetes juju cloud name.                                                 | string | True     | microk8s            |
-| K8S_CREDENTIAL                             | The name of the kubernetes juju credential.                                     | string | True     | microk8s            |
-| model                                      | The name of the juju model to deploy Spark to                                   | string | True     | spark               |
-| create_model                               | Should terraform create the Juju models? If set to false, assume the [...]      | bool   | False    | True                |
-| cos                                        | Observability settings                                                          | map    | True     | See variables.tf    |
-| storage_backend                            | Storage backend to be used                                                      | string | True     | s3                  |
-| s3                                         | S3 Bucket information                                                           | map    | True     | See variables.tf    |
-| azure_storage                              | Azure Object storage information                                                | map    | True     | See variables.tf    |
-| admin_password                             | The password for the admin user.                                                | string | True     | null                |
-| tls_private_key                            | The file path of the private key to use for TLS certificates.                   | string | True     | null                |
-| kyuubi_profile                             | The profile to be used for Kyuubi; should be one of 'testing', 'staging' [...]  | string | False    | production          |
-| kyuubi_user                                | Define the user to be used for running Kyuubi enginers                          | string | True     | kyuubi-spark-engine |
-| enable_dynamic_allocation                  | Enable dynamic allocation of pods for Spark jobs.                               | bool   | True     | False               |
-| kyuubi_k8s_node_selectors                  | Comma separated label:value selectors for K8s pods in Kyuubi.                   | string | True     | null                |
-| kyuubi_loadbalancer_extra_annotations      | Optional extra annotations to be supplied to the load balancer service in [...] | string | True     | null                |
-| kyuubi_gpu_enable                          | Enable GPU acceleration for SparkSQLEngine.                                     | bool   | True     | False               |
-| kyuubi_gpu_engine_executors_limit          | Limit the number of GPUs an engine can schedule executor pods on.               | number | True     | 1                   |
-| kyuubi_gpu_pinned_memory                   | Set the host memory (in GB) per executor reserved for fast data transfer [...]  | number | True     | 1                   |
-| kyuubi_driver_pod_template                 | Define K8s driver pod from a file accessible in the object storage.             | string | True     | null                |
-| kyuubi_executor_pod_template               | Define K8s executor pod from a file accessible in the object storage.           | string | True     | null                |
-| kyuubi_executor_cores                      | Set kyuubi executor pods cpu cores.                                             | number | True     | null                |
-| kyuubi_executor_memory                     | Set kyuubi executor pods memory (in GB).                                        | number | True     | null                |
-| driver_pod_template                        | Define K8s driver pod from a file accessible to the `spark-submit` process.     | string | True     | null                |
-| executor_pod_template                      | Define K8s executor pod from a file accessible to the `spark-submit` process.   | string | True     | null                |
-| integration_hub_monitored_service_accounts | Comma-separated patterns for namespaces and service accounts to monitor [...]   | string | True     | null                |
-| zookeeper_units                            | Define the number of zookeeper units. 3 units are recommended for high [...]    | number | False    | 3                   |
-| history_server_revision                    | Charm revision for spark-history-server-k8s                                     | number | True     | null                |
-| history_server_image                       | Image for spark-history-server-k8s                                              | map    | True     | null                |
-| integration_hub_revision                   | Charm revision for spark-integration-hub-k8s                                    | number | True     | null                |
-| integration_hub_image                      | Image for spark-integration-hub-k8s                                             | map    | True     | null                |
-| kyuubi_revision                            | Charm revision for kyuubi-k8s                                                   | number | True     | null                |
-| kyuubi_image                               | Image for kyuubi-k8s                                                            | map    | True     | null                |
-| kyuubi_units                               | Number of Kyuubi units. 3 units are recommended for high availability.          | number | False    | 3                   |
-| kyuubi_users_revision                      | Charm revision for postgresql-k8s (auth-db)                                     | number | True     | null                |
-| kyuubi_users_image                         | Image for postgresql-k8s (auth-db)                                              | map    | True     | null                |
-| metastore_revision                         | Charm revision for postgresql-k8s (metastore)                                   | number | True     | null                |
-| metastore_image                            | Image for postgresql-k8s (metastore)                                            | map    | True     | null                |
-| zookeeper_revision                         | Charm revision for zookeeper-k8s                                                | number | True     | null                |
-| zookeeper_image                            | Image for zookeeper-k8s                                                         | map    | True     | null                |
-| data_integrator_revision                   | Charm revision for data-integrator                                              | number | True     | null                |
-| s3_revision                                | Charm revision for s3-integrator                                                | number | True     | null                |
-| azure_storage_revision                     | Charm revision for azure-storage-integrator                                     | number | True     | null                |
-| grafana_agent_revision                     | Charm revision for grafana-agent-k8s                                            | number | True     | null                |
-| cos_configuration_revision                 | Charm revision for cos-configuration-k8s                                        | number | True     | null                |
-| pushgateway_revision                       | Charm revision for prometheus-pushgateway-k8s                                   | number | True     | null                |
-| scrape_config_revision                     | Charm revision for prometheus-scrape-config-k8s                                 | number | True     | null                |
-| certificate_common_name                    | Common name for the certificate to be used in self-signed                       | string | True     | charmed-spark       |
-
-## Usage
-
-### Quick start
-
-Here is what a minimal deployment could look like:
-
-```hcl
-resource "juju_model" "spark" {
-  name = "spark"
-}
-
-module "ssc" {
-  depends_on  = [juju_model.spark]
-  source      = "git::https://github.com/canonical/self-signed-certificates-operator//terraform"
-  model       = juju_model.spark.name
-  channel     = "latest/stable"
-  revision    = 163
-  constraints = "arch=amd64"
-  base        = "ubuntu@22.04"
-  units       = 1
-}
-
-module "spark" {
-  depends_on                = [juju_model.spark]
-  source                    = "git::https://github.com/canonical/spark-k8s-bundle//releases/3.4/terraform/modules/spark"
-  model                     = juju_model.spark.name
-  tls_app_name              = module.ssc.app_name
-  tls_certificates_endpoint = module.ssc.provides.certificates
-}
-
-module "azure" {
-  depends_on   = [juju_model.spark]
-  source       = "git::https://github.com/canonical/spark-k8s-bundle//releases/3.4/terraform/modules/azure-storage"
-  model        = juju_model.spark.name
-  spark_charms = module.spark.charms
-  azure = {
-    storage_account = "storage_account"
-    storage_secret  = "storage_secret"
-  }
-}
-```
-
-## Development
-
-The features are split into multiples submodules, where each file has a dedicated role:
-
-- `applications.tf` -> Speak for itself
-- `integrations.tf` -> Same
-- `resources.tf` -> Define juju models, secrets, cross model offers
-  each module can also include `providers.tf`, `variables.tf` and `outputs.tf`. The latter is especially useful to pass values back to a different module.
-
-We defined a dependency chain such as `azure-storage`, `observability` and `s3` depend on `spark`.
-By depending on `spark`, they can use whatever output `spark` defines.\
-This means that using this structure, a new optional module just takes care of defining its applications and their integrations with the existing resources, without any change needed from the existing modules.
-
-## Versions
-
-| Module        | Charm                        | Channel       | revision |
-| ------------- | ---------------------------- | ------------- | -------- |
-| azure         | azure-storage-integrator     | latest/edge   | 15       |
-| observability | grafana-agent-k8s            | 1/stable      | 121      |
-| observability | cos-configuration-k8s        | 1/stable      | 65       |
-| observability | prometheus-pushgateway-k8s   | 1/stable      | 27       |
-| observability | prometheus-scrape-config-k8s | 1/stable      | 67       |
-| spark         | spark-history-server-k8s     | 3/stable      | 94       |
-| spark         | spark-integration-hub-k8s    | 3/stable      | 113      |
-| spark         | kyuubi-k8s                   | 3.4/stable    | 140      |
-| spark         | postgresql-k8s               | 14/stable     | 495      |
-| spark         | postgresql-k8s               | 14/stable     | 495      |
-| spark         | zookeeper-k8s                | 3/stable      | 78       |
-| spark         | data-integrator              | latest/stable | 179      |
-| s3            | s3-integrator                | 1/stable      | 145      |
-
-### Updating the inputs table
-
-1. Run `tox -e tfinputs` from the `python` folder.
-2. Paste the relevant output table in the README.
-
-### Updating the version table
-
-1. Make sure that you work from a clean slate (no `terraform.tfstate`)\
-   `terraform plan -out "tfplan"`
-2. Convert to json `terraform show -json "tfplan" > tfplan.json`
-3. Run the following python script
-
-```py
-import json
-import re
-from typing import TypedDict
-
-
-class Charm(TypedDict):
-    name: str
-    channel: str
-    revision: int
-
-
-with open("tfplan.json", "r") as f:
-    plan = json.load(f)
-
-root = plan["planned_values"]["root_module"]
-
-charms_summary = []
-for module in root["child_modules"]:
-    if match := re.match(r"module\.(?P<name>\w+)\[?", module["address"]):
-        name = match.group("name")
-    else:
-        name = module["address"]
-    for resource in module["resources"]:
-        if not resource["type"] == "juju_application":
-            continue
-        charm = Charm(resource["values"]["charm"][0])
-        charms_summary.append([name, charm["name"], charm["channel"], charm["revision"]])
-
-
-data = [["Module", "Charm", "Channel", "revision"]] + charms_summary
-md_table = "\n".join(
-    [
-        "| " + " | ".join(map(str, row)) + " |"
-        for row in [data[0], ["---"] * len(data[0])] + data[1:]
-    ]
-)
-print(md_table)
-```
diff --git a/releases/3.4/terraform/external/cos/applications.tf b/releases/3.4/terraform/external/cos/applications.tf
deleted file mode 100644
index c736f2e2..00000000
--- a/releases/3.4/terraform/external/cos/applications.tf
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "alertmanager" {
-  name  = "alertmanager"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "alertmanager-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    data = var.alertmanager_size
-  }
-}
-
-resource "juju_application" "catalogue" {
-  name  = "catalogue"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "catalogue-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  config = {
-    "description" : "Canonical Observability Stack Lite"
-  }
-}
-
-resource "juju_application" "grafana" {
-  name  = "grafana"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "grafana-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    database = var.grafana_size
-  }
-}
-
-resource "juju_application" "loki" {
-  name  = "loki"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "loki-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    active-index-directory = var.loki_active_index_directory_size
-    loki-chunks            = var.loki_chunks_size
-  }
-}
-
-resource "juju_application" "prometheus" {
-  name  = "prometheus"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "prometheus-k8s"
-    channel = "1/stable"
-  }
-  config = {
-    "metrics_retention_time" : "90d"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    database = var.prometheus_size
-  }
-}
-
-resource "juju_application" "traefik" {
-  name  = "traefik"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name = "traefik-k8s"
-    # FIXME(stable): Use different track when available
-    channel = "latest/stable"
-  }
-  config = {
-    "tls-cert" : var.cos_tls_cert,
-    "tls-key" : var.cos_tls_key,
-    "tls-ca" : var.cos_tls_ca
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    configurations = var.traefik_size
-  }
-}
-
diff --git a/releases/3.4/terraform/external/cos/integrations.tf b/releases/3.4/terraform/external/cos/integrations.tf
deleted file mode 100644
index e656c361..00000000
--- a/releases/3.4/terraform/external/cos/integrations.tf
+++ /dev/null
@@ -1,300 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-
-
-
-resource "juju_integration" "traefik-grafana" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "traefik-route"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-prometheus" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress-per-unit"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-loki" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress-per-unit"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-alertmanager" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "prometheus-alertmanager-alerting" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "alertmanager"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "alerting"
-  }
-}
-
-
-resource "juju_integration" "grafana-prometheus-source" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "grafana-loki-source" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "grafana-alertmanager-source" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "loki-alertmanager" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "alertmanager"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "alerting"
-  }
-}
-
-
-resource "juju_integration" "prometheus-traefik" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-alertmanager-metrics" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "self-metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-loki" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-grafana" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "grafana-loki-dashboard" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "grafana-prometheus-dashboard" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "grafana-alertmanager-dashboard" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "catalogue-traefik" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "ingress"
-  }
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress"
-  }
-}
-
-
-resource "juju_integration" "catalogue-grafana" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "catalogue"
-  }
-}
-
-
-resource "juju_integration" "catalogue-prometheus" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "catalogue"
-  }
-}
-
-
-resource "juju_integration" "catalogue-alertmanager" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "catalogue"
-  }
-}
diff --git a/releases/3.4/terraform/external/cos/outputs.tf b/releases/3.4/terraform/external/cos/outputs.tf
deleted file mode 100644
index d1cc844b..00000000
--- a/releases/3.4/terraform/external/cos/outputs.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    alertmanager = juju_application.alertmanager.name
-    catalogue    = juju_application.catalogue.name
-    grafana      = juju_application.grafana.name
-    loki         = juju_application.loki.name
-    prometheus   = juju_application.prometheus.name
-  }
-}
-
-output "user" {
-  description = "The name of the Juju user of the COS deployment."
-  value       = var.cos_user
-}
-
-output "dashboards_offer" {
-  value = juju_offer.grafana_dashboards.url
-}
-
-output "metrics_offer" {
-  value = juju_offer.prometheus_receive_remote_write.url
-}
-
-output "logging_offer" {
-  value = juju_offer.loki_logging.url
-}
diff --git a/releases/3.4/terraform/external/cos/providers.tf b/releases/3.4/terraform/external/cos/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.4/terraform/external/cos/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.4/terraform/external/cos/resources.tf b/releases/3.4/terraform/external/cos/resources.tf
deleted file mode 100644
index c1244915..00000000
--- a/releases/3.4/terraform/external/cos/resources.tf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "cos" {
-  name = var.model
-}
-
-resource "juju_offer" "prometheus_receive_remote_write" {
-  model            = data.juju_model.cos.name
-  application_name = juju_application.prometheus.name
-  endpoints        = ["receive-remote-write"]
-}
-
-resource "juju_offer" "grafana_dashboards" {
-  model            = data.juju_model.cos.name
-  application_name = juju_application.grafana.name
-  endpoints        = ["grafana-dashboard"]
-}
-
-resource "juju_offer" "loki_logging" {
-  model            = data.juju_model.cos.name
-  application_name = juju_application.loki.name
-  endpoints        = ["logging"]
-}
-
diff --git a/releases/3.4/terraform/external/cos/variables.tf b/releases/3.4/terraform/external/cos/variables.tf
deleted file mode 100644
index c702af41..00000000
--- a/releases/3.4/terraform/external/cos/variables.tf
+++ /dev/null
@@ -1,69 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define cos-specific variables
-
-variable "model" {
-  description = "The name of the Juju model to deploy to"
-  type        = string
-}
-
-variable "cos_user" {
-  description = "The name of the Juju user of the COS deployment."
-  type        = string
-  default     = "admin"
-}
-
-variable "cos_tls_cert" {
-  type        = string
-  default     = ""
-  description = "COS certificate"
-}
-
-variable "cos_tls_key" {
-  type        = string
-  default     = ""
-  description = "COS certificate key"
-}
-
-variable "cos_tls_ca" {
-  type        = string
-  default     = ""
-  description = "COS CA certificate"
-}
-
-variable "alertmanager_size" {
-  description = "Storage size for the alertmanager database"
-  type        = string
-  default     = "10G"
-}
-
-variable "grafana_size" {
-  description = "Storage size for the grafana database"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_active_index_directory_size" {
-  description = "Storage size for the active index directory for Loki"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_chunks_size" {
-  description = "Storage size for the Loki chucks storage"
-  type        = string
-  default     = "500G"
-}
-
-variable "prometheus_size" {
-  description = "Storage size for the Prometheus database"
-  type        = string
-  default     = "500G"
-}
-
-variable "traefik_size" {
-  description = "Storage size for the Traefik storage"
-  type        = string
-  default     = "10G"
-}
diff --git a/releases/3.4/terraform/external/tls/main.tf b/releases/3.4/terraform/external/tls/main.tf
deleted file mode 100644
index 843a22d8..00000000
--- a/releases/3.4/terraform/external/tls/main.tf
+++ /dev/null
@@ -1,58 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# This module is but a placeholder until we get an official TF module for the manual TLS operator
-# For testing purpose.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = ">=0.16.0"
-      source  = "juju/juju"
-    }
-  }
-}
-
-variable "model" {
-  type     = string
-  nullable = false
-}
-
-variable "channel" {
-  type    = string
-  default = "latest/stable"
-}
-
-variable "app_name" {
-  type     = string
-  default  = "certificates"
-  nullable = false
-}
-
-resource "juju_application" "certificates" {
-  name  = var.app_name
-  model = var.model
-
-  charm {
-    name    = "manual-tls-certificates"
-    channel = var.channel
-  }
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-output "app_name" {
-  description = "Name of the deployed application."
-  value       = juju_application.certificates.name
-}
-
-output "provides" {
-  value = {
-    certificates = "certificates"
-  }
-}
diff --git a/releases/3.4/terraform/main.tf b/releases/3.4/terraform/main.tf
deleted file mode 100644
index 602f00bd..00000000
--- a/releases/3.4/terraform/main.tf
+++ /dev/null
@@ -1,136 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_model" "spark" {
-  count      = var.create_model == true ? 1 : 0
-  name       = var.model
-  credential = var.K8S_CREDENTIAL
-  cloud {
-    name = var.K8S_CLOUD
-  }
-}
-
-resource "juju_model" "cos" {
-  count      = var.cos.deployed == "bundled" && var.create_model == true ? 1 : 0
-  name       = var.cos.model
-  credential = var.K8S_CREDENTIAL
-  cloud {
-    name = var.K8S_CLOUD
-  }
-}
-
-module "ssc" {
-  depends_on  = [juju_model.spark]
-  source      = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev326"
-  model       = var.model
-  app_name    = "certificates"
-  channel     = "1/stable"
-  revision    = 317
-  constraints = "arch=amd64"
-  base        = "ubuntu@24.04"
-  units       = 1
-  config = {
-    ca-common-name = var.certificate_common_name
-  }
-}
-
-module "spark" {
-  depends_on                                 = [juju_model.spark, module.ssc]
-  source                                     = "./modules/spark"
-  model                                      = var.model
-  kyuubi_user                                = var.kyuubi_user
-  kyuubi_profile                             = var.kyuubi_profile
-  admin_password                             = var.admin_password
-  tls_private_key                            = var.tls_private_key
-  enable_dynamic_allocation                  = var.enable_dynamic_allocation
-  kyuubi_k8s_node_selectors                  = var.kyuubi_k8s_node_selectors
-  kyuubi_loadbalancer_extra_annotations      = var.kyuubi_loadbalancer_extra_annotations
-  kyuubi_gpu_enable                          = var.kyuubi_gpu_enable
-  kyuubi_gpu_engine_executors_limit          = var.kyuubi_gpu_engine_executors_limit
-  kyuubi_gpu_pinned_memory                   = var.kyuubi_gpu_pinned_memory
-  kyuubi_driver_pod_template                 = var.kyuubi_driver_pod_template
-  kyuubi_executor_pod_template               = var.kyuubi_executor_pod_template
-  kyuubi_executor_cores                      = var.kyuubi_executor_cores
-  kyuubi_executor_memory                     = var.kyuubi_executor_memory
-  driver_pod_template                        = var.driver_pod_template
-  executor_pod_template                      = var.executor_pod_template
-  integration_hub_monitored_service_accounts = var.integration_hub_monitored_service_accounts
-  tls_app_name                               = module.ssc.app_name
-  tls_certificates_endpoint                  = module.ssc.provides.certificates
-  zookeeper_units                            = var.zookeeper_units
-  kyuubi_units                               = var.kyuubi_units
-
-  history_server_revision  = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server
-  history_server_image     = var.history_server_image != null ? var.history_server_image : local.images.history_server
-  integration_hub_revision = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub
-  integration_hub_image    = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub
-  kyuubi_revision          = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi
-  kyuubi_image             = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi
-  kyuubi_users_revision    = var.kyuubi_users_revision != null ? var.kyuubi_users_revision : local.revisions.kyuubi_users
-  kyuubi_users_image       = var.kyuubi_users_image != null ? var.kyuubi_users_image : local.images.kyuubi_users
-  metastore_revision       = var.metastore_revision != null ? var.metastore_revision : local.revisions.metastore
-  metastore_image          = var.metastore_image != null ? var.metastore_image : local.images.metastore
-  zookeeper_revision       = var.zookeeper_revision != null ? var.zookeeper_revision : local.revisions.zookeeper
-  zookeeper_image          = var.zookeeper_image != null ? var.zookeeper_image : local.images.zookeeper
-  data_integrator_revision = var.data_integrator_revision != null ? var.data_integrator_revision : local.revisions.data_integrator
-
-  kyuubi_users_size = var.kyuubi_users_size
-  metastore_size    = var.metastore_size
-  zookeeper_size    = var.zookeeper_size
-}
-
-module "azure_storage" {
-  depends_on    = [module.spark]
-  count         = var.storage_backend == "azure_storage" ? 1 : 0
-  source        = "./modules/azure-storage"
-  model         = var.model
-  spark_charms  = module.spark.charms
-  azure_storage = var.azure_storage
-
-  azure_storage_revision = var.azure_storage_revision != null ? var.azure_storage_revision : local.revisions.azure_storage
-}
-
-module "s3" {
-  depends_on   = [module.spark]
-  count        = var.storage_backend == "s3" ? 1 : 0
-  source       = "./modules/s3"
-  model        = var.model
-  spark_charms = module.spark.charms
-  s3           = var.s3
-
-  s3_revision = var.s3_revision != null ? var.s3_revision : local.revisions.s3
-}
-
-module "bundled_cos" {
-  depends_on   = [juju_model.cos]
-  count        = var.cos.deployed == "bundled" ? 1 : 0
-  source       = "./external/cos"
-  model        = var.cos.model
-  cos_tls_ca   = var.cos.tls.ca
-  cos_tls_cert = var.cos.tls.cert
-  cos_tls_key  = var.cos.tls.key
-
-  alertmanager_size                = var.alertmanager_size
-  grafana_size                     = var.grafana_size
-  loki_active_index_directory_size = var.loki_active_index_directory_size
-  loki_chunks_size                 = var.loki_chunks_size
-  prometheus_size                  = var.prometheus_size
-  traefik_size                     = var.traefik_size
-}
-
-
-module "observability" {
-  depends_on       = [module.spark, module.bundled_cos]
-  count            = var.cos.deployed == "no" ? 0 : 1
-  source           = "./modules/observability"
-  dashboards_offer = var.cos.deployed == "external" ? var.cos.offers.dashboard : one(module.bundled_cos[*].dashboards_offer)
-  logging_offer    = var.cos.deployed == "external" ? var.cos.offers.logging : one(module.bundled_cos[*].logging_offer)
-  metrics_offer    = var.cos.deployed == "external" ? var.cos.offers.metrics : one(module.bundled_cos[*].metrics_offer)
-  spark_model      = var.model
-  spark_charms     = module.spark.charms
-
-  grafana_agent_revision     = var.grafana_agent_revision != null ? var.grafana_agent_revision : local.revisions.grafana_agent
-  cos_configuration_revision = var.cos_configuration_revision != null ? var.cos_configuration_revision : local.revisions.cos_configuration
-  pushgateway_revision       = var.pushgateway_revision != null ? var.pushgateway_revision : local.revisions.pushgateway
-  scrape_config_revision     = var.scrape_config_revision != null ? var.scrape_config_revision : local.revisions.scrape_config
-}
diff --git a/releases/3.4/terraform/modules/azure-storage/applications.tf b/releases/3.4/terraform/modules/azure-storage/applications.tf
deleted file mode 100644
index 642df05d..00000000
--- a/releases/3.4/terraform/modules/azure-storage/applications.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "azure_storage" {
-  name  = "azure-storage"
-  model = var.model
-  charm {
-    name     = "azure-storage-integrator"
-    channel  = "latest/edge"
-    revision = var.azure_storage_revision
-  }
-  config = {
-    container           = var.azure_storage.container
-    storage-account     = var.azure_storage.storage_account
-    path                = var.azure_storage.path
-    connection-protocol = var.azure_storage.protocol
-    credentials         = "secret:${juju_secret.azure_blob_storage_secret.secret_id}"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/3.4/terraform/modules/azure-storage/integrations.tf b/releases/3.4/terraform/modules/azure-storage/integrations.tf
deleted file mode 100644
index 0611ee3c..00000000
--- a/releases/3.4/terraform/modules/azure-storage/integrations.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "history_server_azure_storage" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.azure_storage.name
-    endpoint = "azure-storage-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "azure-storage-credentials"
-  }
-}
-
-resource "juju_integration" "azure_storage_integration_hub" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.azure_storage.name
-    endpoint = "azure-storage-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "azure-storage-credentials"
-  }
-}
diff --git a/releases/3.4/terraform/modules/azure-storage/outputs.tf b/releases/3.4/terraform/modules/azure-storage/outputs.tf
deleted file mode 100644
index beff36af..00000000
--- a/releases/3.4/terraform/modules/azure-storage/outputs.tf
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    azure_storage = juju_application.azure_storage.name
-  }
-}
diff --git a/releases/3.4/terraform/modules/azure-storage/providers.tf b/releases/3.4/terraform/modules/azure-storage/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.4/terraform/modules/azure-storage/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.4/terraform/modules/azure-storage/resources.tf b/releases/3.4/terraform/modules/azure-storage/resources.tf
deleted file mode 100644
index ee5b10b5..00000000
--- a/releases/3.4/terraform/modules/azure-storage/resources.tf
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "spark" {
-  name = var.model
-}
-
-resource "juju_secret" "azure_blob_storage_secret" {
-  model = var.model
-  name  = "azure_blob_storage_secret"
-  value = {
-    secret-key = var.azure_storage.secret_key
-  }
-  info = "This is the secret key for the Azure storage account"
-}
-
-resource "juju_access_secret" "azure_blob_storage_secret_access" {
-  model = var.model
-  applications = [
-    juju_application.azure_storage.name
-  ]
-  # Use the secret_id from your secret resource or data source.
-  secret_id = juju_secret.azure_blob_storage_secret.secret_id
-}
diff --git a/releases/3.4/terraform/modules/azure-storage/variables.tf b/releases/3.4/terraform/modules/azure-storage/variables.tf
deleted file mode 100644
index 8383e474..00000000
--- a/releases/3.4/terraform/modules/azure-storage/variables.tf
+++ /dev/null
@@ -1,31 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-variable "model" {
-  description = "Name of the Spark Juju Model to deploy to"
-  type        = string
-  nullable    = false
-}
-
-variable "azure_storage" {
-  description = "Azure Object storage information"
-  type = object({
-    container       = optional(string, "azurecontainer")
-    storage_account = optional(string, "azurestorageaccount")
-    protocol        = optional(string, "abfss")
-    secret_key      = optional(string, "secret-key")
-    path            = optional(string, "spark-events")
-  })
-  default = {}
-}
-
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
-}
-
-variable "azure_storage_revision" {
-  description = "Charm revision for azure-storage-integrator"
-  type        = number
-  nullable    = false
-}
diff --git a/releases/3.4/terraform/modules/observability/applications.tf b/releases/3.4/terraform/modules/observability/applications.tf
deleted file mode 100644
index e7778be6..00000000
--- a/releases/3.4/terraform/modules/observability/applications.tf
+++ /dev/null
@@ -1,64 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "grafana_agent" {
-  name  = "grafana-agent"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "grafana-agent-k8s"
-    channel  = "1/stable"
-    revision = var.grafana_agent_revision
-  }
-  units       = 1
-  trust       = true
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "cos_configuration" {
-  name  = "cos-configuration"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "cos-configuration-k8s"
-    channel  = "1/stable"
-    revision = var.cos_configuration_revision
-  }
-  config = {
-    git_branch              = "main"
-    git_depth               = "1"
-    git_repo                = var.config_repo
-    grafana_dashboards_path = var.config_grafana_path
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "pushgateway" {
-  name  = "pushgateway"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "prometheus-pushgateway-k8s"
-    channel  = "1/stable"
-    revision = var.pushgateway_revision
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    pushgateway-store = var.pushgateway_size
-  }
-}
-
-resource "juju_application" "scrape_config" {
-  name  = "scrape-config"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "prometheus-scrape-config-k8s"
-    channel  = "1/stable"
-    revision = var.scrape_config_revision
-  }
-  config = {
-    scrape_interval = "10s"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
-
diff --git a/releases/3.4/terraform/modules/observability/integrations.tf b/releases/3.4/terraform/modules/observability/integrations.tf
deleted file mode 100644
index 0101cf88..00000000
--- a/releases/3.4/terraform/modules/observability/integrations.tf
+++ /dev/null
@@ -1,236 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-data "juju_offer" "grafana_dashboards" {
-  url = var.dashboards_offer
-}
-
-data "juju_offer" "prometheus_receive_remote_write" {
-  url = var.metrics_offer
-}
-
-data "juju_offer" "loki_logging" {
-  url = var.logging_offer
-}
-
-resource "juju_integration" "cos_configuration_grafana_agent" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.cos_configuration.name
-    endpoint = "grafana-dashboards"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "pushgateway_scrape_config" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.pushgateway.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.scrape_config.name
-    endpoint = "configurable-scrape-jobs"
-  }
-}
-
-resource "juju_integration" "scrape_config_grafana_agent" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.scrape_config.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "grafana_agent_grafana_dashboards" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-provider"
-  }
-
-  application {
-    offer_url = data.juju_offer.grafana_dashboards.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-
-resource "juju_integration" "grafana_agent_prometheus" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "send-remote-write"
-  }
-
-  application {
-    offer_url = data.juju_offer.prometheus_receive_remote_write.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-
-resource "juju_integration" "grafana_agent_loki" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-consumer"
-  }
-
-  application {
-    offer_url = data.juju_offer.loki_logging.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-resource "juju_integration" "pushgateway_integration_hub" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.pushgateway.name
-    endpoint = "push-endpoint"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "cos"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_dashboard" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_logging" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_metrics" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_metrics" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_dashboards" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_logging" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
-
-resource "juju_integration" "integration_hub_grafana_agent_logging" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
diff --git a/releases/3.4/terraform/modules/observability/outputs.tf b/releases/3.4/terraform/modules/observability/outputs.tf
deleted file mode 100644
index 603d65b2..00000000
--- a/releases/3.4/terraform/modules/observability/outputs.tf
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    grafana_agent     = juju_application.grafana_agent.name
-    cos_configuration = juju_application.cos_configuration.name
-    pushgateway       = juju_application.pushgateway.name
-    scrape_config     = juju_application.scrape_config.name
-  }
-}
diff --git a/releases/3.4/terraform/modules/observability/providers.tf b/releases/3.4/terraform/modules/observability/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.4/terraform/modules/observability/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.4/terraform/modules/observability/resources.tf b/releases/3.4/terraform/modules/observability/resources.tf
deleted file mode 100644
index ec198a06..00000000
--- a/releases/3.4/terraform/modules/observability/resources.tf
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "spark" {
-  name = var.spark_model
-}
diff --git a/releases/3.4/terraform/modules/observability/variables.tf b/releases/3.4/terraform/modules/observability/variables.tf
deleted file mode 100644
index 19a2d92d..00000000
--- a/releases/3.4/terraform/modules/observability/variables.tf
+++ /dev/null
@@ -1,75 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-variable "config_repo" {
-  description = "COS configuration repo URL."
-  type        = string
-  default     = "https://github.com/canonical/spark-k8s-bundle"
-  nullable    = false
-}
-
-variable "config_grafana_path" {
-  description = "Grafana dashboard path from configuration repo root."
-  type        = string
-  default     = "releases/3.4/resources/grafana/"
-  nullable    = false
-}
-
-variable "spark_model" {
-  description = "Name of the Spark Juju model."
-  type        = string
-  nullable    = false
-}
-
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
-}
-
-variable "dashboards_offer" {
-  description = "URL of the `grafana_dashboard` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "metrics_offer" {
-  description = "URL of the `prometheus_remote_write` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "logging_offer" {
-  description = "URL of the `loki_push_api` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "grafana_agent_revision" {
-  description = "Charm revision for grafana-agent-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "cos_configuration_revision" {
-  description = "Charm revision for cos-configuration-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "pushgateway_revision" {
-  description = "Charm revision for prometheus-pushgateway-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "scrape_config_revision" {
-  description = "Charm revision for prometheus-scrape-config-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "pushgateway_size" {
-  description = "Storage size for the pushgateway database"
-  type        = string
-  default     = "10G"
-}
diff --git a/releases/3.4/terraform/modules/s3/applications.tf b/releases/3.4/terraform/modules/s3/applications.tf
deleted file mode 100644
index 44f1aa25..00000000
--- a/releases/3.4/terraform/modules/s3/applications.tf
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "s3" {
-  name  = "s3"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "s3-integrator"
-    channel  = "1/stable"
-    revision = var.s3_revision
-  }
-  config = {
-    path     = "spark-events"
-    bucket   = var.s3.bucket
-    endpoint = var.s3.endpoint
-    region   = var.s3.region
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/3.4/terraform/modules/s3/integrations.tf b/releases/3.4/terraform/modules/s3/integrations.tf
deleted file mode 100644
index cd520fc7..00000000
--- a/releases/3.4/terraform/modules/s3/integrations.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "s3_history_server" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.s3.name
-    endpoint = "s3-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "s3-credentials"
-  }
-}
-
-resource "juju_integration" "s3_integration_hub" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.s3.name
-    endpoint = "s3-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "s3-credentials"
-  }
-}
diff --git a/releases/3.4/terraform/modules/s3/outputs.tf b/releases/3.4/terraform/modules/s3/outputs.tf
deleted file mode 100644
index 77aa54aa..00000000
--- a/releases/3.4/terraform/modules/s3/outputs.tf
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    s3 = juju_application.s3.name
-  }
-}
diff --git a/releases/3.4/terraform/modules/s3/providers.tf b/releases/3.4/terraform/modules/s3/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.4/terraform/modules/s3/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.4/terraform/modules/s3/resources.tf b/releases/3.4/terraform/modules/s3/resources.tf
deleted file mode 100644
index 2e2dfbde..00000000
--- a/releases/3.4/terraform/modules/s3/resources.tf
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "spark" {
-  name = var.model
-}
diff --git a/releases/3.4/terraform/modules/s3/variables.tf b/releases/3.4/terraform/modules/s3/variables.tf
deleted file mode 100644
index 89b34681..00000000
--- a/releases/3.4/terraform/modules/s3/variables.tf
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-variable "model" {
-  description = "Name of the Spark Juju Model to deploy to"
-  type        = string
-  nullable    = false
-}
-
-variable "s3" {
-  description = "S3 Bucket information"
-  type = object({
-    bucket   = optional(string, "spark-bucket")
-    endpoint = optional(string, "https://s3.amazonaws.com")
-    region   = optional(string, "us-east-1")
-  })
-  default = {
-    bucket   = "spark-bucket"
-    endpoint = "https://s3.amazonaws.com"
-    region   = "us-east-1"
-  }
-}
-
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
-}
-
-variable "s3_revision" {
-  description = "Charm revision for s3-integrator"
-  type        = number
-  nullable    = false
-}
-
diff --git a/releases/3.4/terraform/modules/spark/applications.tf b/releases/3.4/terraform/modules/spark/applications.tf
deleted file mode 100644
index 4cd06e60..00000000
--- a/releases/3.4/terraform/modules/spark/applications.tf
+++ /dev/null
@@ -1,190 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "history_server" {
-  name  = "history-server"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "spark-history-server-k8s"
-    channel  = "3/stable"
-    revision = var.history_server_revision
-  }
-
-  resources = var.history_server_image
-  units     = 1
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "kyuubi" {
-  name  = "kyuubi"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "kyuubi-k8s"
-    channel  = "3.4/stable"
-    revision = var.kyuubi_revision
-  }
-
-  resources = var.kyuubi_image
-
-  config = merge(
-    {
-      enable-dynamic-allocation  = var.enable_dynamic_allocation
-      expose-external            = "loadbalancer"
-      gpu-enable                 = var.kyuubi_gpu_enable
-      gpu-engine-executors-limit = var.kyuubi_gpu_engine_executors_limit
-      gpu-pinned-memory          = var.kyuubi_gpu_pinned_memory
-      namespace                  = data.juju_model.spark.name
-      profile                    = var.kyuubi_profile
-      service-account            = var.kyuubi_user
-    },
-    var.kyuubi_k8s_node_selectors == null ? {} : {
-      k8s-node-selectors = var.kyuubi_k8s_node_selectors
-    },
-    var.kyuubi_loadbalancer_extra_annotations == null ? {} : {
-      loadbalancer-extra-annotations = var.kyuubi_loadbalancer_extra_annotations
-    },
-    var.tls_private_key == null ? {} : {
-      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    },
-    var.admin_password == null ? {} : {
-      system-users = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    },
-    var.kyuubi_executor_cores == null ? {} : {
-      executor-cores = var.kyuubi_executor_cores
-    },
-    var.kyuubi_executor_memory == null ? {} : {
-      executor-memory = var.kyuubi_executor_memory
-    },
-    var.kyuubi_driver_pod_template == null ? {} : {
-      driver-pod-template = var.kyuubi_driver_pod_template
-    },
-    var.kyuubi_executor_pod_template == null ? {} : {
-      executor-pod-template = var.kyuubi_executor_pod_template
-    }
-  )
-
-  units = var.kyuubi_units
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "kyuubi_users" {
-  name  = "kyuubi-users"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "postgresql-k8s"
-    channel  = "14/stable"
-    revision = var.kyuubi_users_revision
-  }
-
-  storage_directives = {
-    pgdata = var.kyuubi_users_size
-  }
-
-  resources = var.kyuubi_users_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "metastore" {
-  name  = "metastore"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "postgresql-k8s"
-    channel  = "14/stable"
-    revision = var.metastore_revision
-  }
-
-  storage_directives = {
-    pgdata = var.kyuubi_users_size
-  }
-
-
-  resources = var.metastore_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "integration_hub" {
-  name  = "integration-hub"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "spark-integration-hub-k8s"
-    channel  = "3/stable"
-    revision = var.integration_hub_revision
-  }
-
-  config = merge(
-    {
-      enable-dynamic-allocation = var.enable_dynamic_allocation
-    },
-    var.driver_pod_template == null ? {} : {
-      driver-pod-template = var.driver_pod_template
-    },
-    var.executor_pod_template == null ? {} : {
-      executor-pod-template = var.executor_pod_template
-    },
-    var.integration_hub_monitored_service_accounts == null ? {} : {
-      monitored-service-accounts = var.integration_hub_monitored_service_accounts
-    }
-  )
-
-  resources = var.integration_hub_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "zookeeper" {
-  name  = "zookeeper"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "zookeeper-k8s"
-    channel  = "3/stable"
-    revision = var.zookeeper_revision
-  }
-
-  storage_directives = {
-    zookeeper = var.zookeeper_size
-  }
-
-
-  resources = var.zookeeper_image
-
-  units       = var.zookeeper_units
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "data_integrator" {
-  name  = "data-integrator"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "data-integrator"
-    channel  = "latest/stable"
-    revision = var.data_integrator_revision
-  }
-
-  config = {
-    database-name = "integrator"
-  }
-
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/3.4/terraform/modules/spark/integrations.tf b/releases/3.4/terraform/modules/spark/integrations.tf
deleted file mode 100644
index 37f1e566..00000000
--- a/releases/3.4/terraform/modules/spark/integrations.tf
+++ /dev/null
@@ -1,86 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "kyuubi_metastore" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.metastore.name
-    endpoint = "database"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "metastore-db"
-  }
-}
-
-resource "juju_integration" "kyuubi_users" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.kyuubi_users.name
-    endpoint = "database"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "auth-db"
-  }
-}
-
-resource "juju_integration" "kyuubi_service_account" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "spark-service-account"
-  }
-
-  application {
-    name     = juju_application.integration_hub.name
-    endpoint = "spark-service-account"
-  }
-}
-
-resource "juju_integration" "kyuubi_zookeeper" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.zookeeper.name
-    endpoint = "zookeeper"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "zookeeper"
-  }
-}
-
-resource "juju_integration" "kyuubi_tls" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "certificates"
-  }
-
-  application {
-    name     = var.tls_app_name
-    endpoint = var.tls_certificates_endpoint
-  }
-}
-
-resource "juju_integration" "kyuubi_data_integrator" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "jdbc"
-  }
-
-  application {
-    name     = juju_application.data_integrator.name
-    endpoint = "kyuubi"
-  }
-}
diff --git a/releases/3.4/terraform/modules/spark/offers.tf b/releases/3.4/terraform/modules/spark/offers.tf
deleted file mode 100644
index 40f06528..00000000
--- a/releases/3.4/terraform/modules/spark/offers.tf
+++ /dev/null
@@ -1,11 +0,0 @@
-resource "juju_offer" "integration_hub" {
-  model            = data.juju_model.spark.name
-  application_name = juju_application.integration_hub.name
-  endpoints        = ["spark-service-account"]
-}
-
-resource "juju_offer" "metastore" {
-  model            = data.juju_model.spark.name
-  application_name = juju_application.metastore.name
-  endpoints        = ["database"]
-}
\ No newline at end of file
diff --git a/releases/3.4/terraform/modules/spark/outputs.tf b/releases/3.4/terraform/modules/spark/outputs.tf
deleted file mode 100644
index 010f6461..00000000
--- a/releases/3.4/terraform/modules/spark/outputs.tf
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    history_server  = juju_application.history_server.name
-    kyuubi          = juju_application.kyuubi.name
-    kyuubi_users    = juju_application.kyuubi_users.name
-    metastore       = juju_application.metastore.name
-    integration_hub = juju_application.integration_hub.name
-    zookeeper       = juju_application.zookeeper.name
-    certificates    = var.tls_app_name
-  }
-}
-
-output "offers" {
-  value = {
-    hub_service_account = juju_offer.integration_hub
-    metastore_database  = juju_offer.metastore
-  }
-}
diff --git a/releases/3.4/terraform/modules/spark/providers.tf b/releases/3.4/terraform/modules/spark/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.4/terraform/modules/spark/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.4/terraform/modules/spark/resources.tf b/releases/3.4/terraform/modules/spark/resources.tf
deleted file mode 100644
index 8b73939b..00000000
--- a/releases/3.4/terraform/modules/spark/resources.tf
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "spark" {
-  name = var.model
-}
-
-resource "juju_secret" "system_users_and_private_key_secret" {
-  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model = var.model
-  name  = "system_users_and_private_key_secret"
-  value = merge(
-    var.admin_password == null ? {} : {
-      admin = var.admin_password
-    },
-    var.tls_private_key == null ? {} : {
-      private-key = var.tls_private_key
-    }
-  )
-  info = "This secret contains password for admin user and the TLS private key."
-}
-
-resource "juju_access_secret" "system_users_and_private_key_secret_access" {
-  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model = var.model
-  applications = [
-    juju_application.kyuubi.name
-  ]
-  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
-}
diff --git a/releases/3.4/terraform/modules/spark/variables.tf b/releases/3.4/terraform/modules/spark/variables.tf
deleted file mode 100644
index 6e442a55..00000000
--- a/releases/3.4/terraform/modules/spark/variables.tf
+++ /dev/null
@@ -1,241 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define spark-specific variables
-
-variable "model" {
-  description = "Name of the Juju Model to deploy to."
-  type        = string
-  nullable    = false
-}
-
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "tls_private_key" {
-  description = "The private key to be used for TLS certificates."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "kyuubi_profile" {
-  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-  type        = string
-  nullable    = false
-  default     = "production"
-}
-
-variable "kyuubi_user" {
-  description = "User name to be used for running Kyuubi engines."
-  type        = string
-  nullable    = false
-  default     = "kyuubi-spark-engine"
-}
-
-variable "enable_dynamic_allocation" {
-  description = "Enable dynamic allocation of pods for Spark jobs."
-  type        = bool
-  default     = false
-}
-
-variable "kyuubi_k8s_node_selectors" {
-  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_loadbalancer_extra_annotations" {
-  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_gpu_enable" {
-  description = "Enable GPU acceleration for SparkSQLEngine."
-  type        = bool
-  nullable    = false
-}
-
-variable "kyuubi_gpu_engine_executors_limit" {
-  description = "Limit the number of GPUs an engine can schedule executor pods on."
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_gpu_pinned_memory" {
-  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_cores" {
-  description = "Set kyuubi executor pods cpu cores."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_memory" {
-  description = "Set kyuubi executor pods memory (in GB)."
-  type        = number
-  default     = null
-}
-
-variable "driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "integration_hub_monitored_service_accounts" {
-  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-  type        = string
-  default     = null
-}
-
-variable "zookeeper_units" {
-  description = "Number of zookeeper units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "tls_app_name" {
-  description = "Name of the application providing the `tls-certificates` interface."
-  type        = string
-  nullable    = false
-}
-
-variable "tls_certificates_endpoint" {
-  description = "Name of the endpoint providing the `tls-certificates` interface."
-  type        = string
-  nullable    = false
-}
-
-variable "history_server_revision" {
-  description = "Charm revision for spark-history-server-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "history_server_image" {
-  description = "Image for spark-history-server-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "integration_hub_revision" {
-  description = "Charm revision for spark-integration-hub-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_units" {
-  description = "Number of Kyuubi units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "integration_hub_image" {
-  description = "Image for spark-integration-hub-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_revision" {
-  description = "Charm revision for kyuubi-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_image" {
-  description = "Image for kyuubi-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_users_revision" {
-  description = "Charm revision for postgresql-k8s (auth-db)"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_users_image" {
-  description = "Image for postgresql-k8s (auth-db)"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_users_size" {
-  description = "Storage size for the Kyuubi users database"
-  type        = string
-  default     = "1G"
-}
-
-
-variable "metastore_revision" {
-  description = "Charm revision for postgresql-k8s (metastore)"
-  type        = number
-  nullable    = false
-}
-
-variable "metastore_image" {
-  description = "Image for postgresql-k8s (metastore)"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "metastore_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-
-variable "zookeeper_revision" {
-  description = "Charm revision for zookeeper-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "zookeeper_image" {
-  description = "Image for zookeeper-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "zookeeper_size" {
-  description = "Storage size for the zookeeper unit"
-  type        = string
-  default     = "10G"
-}
-
-
-variable "data_integrator_revision" {
-  description = "Charm revision for data-integrator"
-  type        = number
-  nullable    = false
-}
diff --git a/releases/3.4/terraform/outputs.tf b/releases/3.4/terraform/outputs.tf
deleted file mode 100644
index 2febb798..00000000
--- a/releases/3.4/terraform/outputs.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = merge(
-    concat(
-      module.spark[*].charms, module.azure_storage[*].charms, module.s3[*].charms, module.bundled_cos[*].charms, module.observability[*].charms
-    )...
-  )
-}
-
-output "offers" {
-  description = "The name and url of the various offers being exposed"
-  value = merge(
-    concat(
-      module.spark[*].offers
-    )...
-  )
-}
-
diff --git a/releases/3.4/terraform/variables.tf b/releases/3.4/terraform/variables.tf
deleted file mode 100644
index ccd64031..00000000
--- a/releases/3.4/terraform/variables.tf
+++ /dev/null
@@ -1,396 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Juju
-
-variable "K8S_CLOUD" {
-  type        = string
-  description = "The kubernetes juju cloud name."
-  default     = "microk8s"
-}
-
-variable "K8S_CREDENTIAL" {
-  type        = string
-  description = "The name of the kubernetes juju credential."
-  default     = "microk8s"
-}
-
-# Models
-
-variable "model" {
-  description = "The name of the juju model to deploy Spark to"
-  type        = string
-  default     = "spark"
-}
-
-variable "create_model" {
-  description = "Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism."
-  type        = bool
-  default     = true
-  nullable    = false
-}
-
-# cos specifics
-
-variable "cos" {
-  description = "Observability settings"
-  type = object({
-    model    = optional(string, "cos")
-    deployed = optional(string, "bundled")
-    offers = optional(object({
-      dashboard = optional(string, null),
-      metrics   = optional(string, null),
-      logging   = optional(string, null)
-    }), {}),
-    tls = optional(object({
-      cert = optional(string, "")
-      key  = optional(string, "")
-      ca   = optional(string, "")
-    }), {})
-  })
-  default = { model = "cos", deployed = "bundled", offers = {}, tls = {} }
-
-  validation {
-    condition     = contains(["external", "bundled", "no"], var.cos.deployed)
-    error_message = "Valid values for var: cos.deployed are (external, bundled, no)"
-  }
-
-  validation {
-    condition = var.cos.deployed != "external" || alltrue([
-      var.cos.offers.dashboard != null,
-      var.cos.offers.metrics != null,
-      var.cos.offers.logging != null,
-    ])
-    error_message = "When using external cos, please define all offers variables"
-  }
-
-}
-
-# Storage
-
-variable "storage_backend" {
-  type        = string
-  description = "Storage backend to be used"
-
-  validation {
-    condition     = contains(["azure_storage", "s3"], var.storage_backend)
-    error_message = "Valid values for var: test_variable are (s3, azure_storage)."
-  }
-
-  default = "s3"
-}
-
-variable "s3" {
-  description = "S3 Bucket information"
-  type = object({
-    bucket   = optional(string, "spark-test")
-    endpoint = optional(string, "https://s3.amazonaws.com")
-    region   = optional(string, "us-east-1")
-  })
-  default = {}
-}
-
-variable "azure_storage" {
-  description = "Azure Object storage information"
-  type = object({
-    container       = optional(string, "azurecontainer")
-    storage_account = optional(string, "azurestorageaccount")
-    secret_key      = optional(string, "azurestoragesecret")
-    protocol        = optional(string, "abfss")
-  })
-  default = {}
-}
-
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "tls_private_key" {
-  description = "The file path of the private key to use for TLS certificates."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "kyuubi_profile" {
-  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-  type        = string
-  nullable    = false
-  default     = "production"
-}
-
-variable "kyuubi_user" {
-  description = "Define the user to be used for running Kyuubi enginers"
-  type        = string
-  default     = "kyuubi-spark-engine"
-}
-
-variable "enable_dynamic_allocation" {
-  description = "Enable dynamic allocation of pods for Spark jobs."
-  type        = bool
-  default     = false
-}
-
-variable "kyuubi_k8s_node_selectors" {
-  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_loadbalancer_extra_annotations" {
-  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_gpu_enable" {
-  description = "Enable GPU acceleration for SparkSQLEngine."
-  type        = bool
-  default     = false
-}
-
-variable "kyuubi_gpu_engine_executors_limit" {
-  description = "Limit the number of GPUs an engine can schedule executor pods on."
-  type        = number
-  default     = 1
-}
-
-variable "kyuubi_gpu_pinned_memory" {
-  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-  type        = number
-  default     = 1
-}
-
-variable "kyuubi_driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_cores" {
-  description = "Set kyuubi executor pods cpu cores."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_memory" {
-  description = "Set kyuubi executor pods memory (in GB)."
-  type        = number
-  default     = null
-}
-
-variable "driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "integration_hub_monitored_service_accounts" {
-  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-  type        = string
-  default     = null
-}
-
-variable "zookeeper_units" {
-  description = "Define the number of zookeeper units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "history_server_revision" {
-  description = "Charm revision for spark-history-server-k8s"
-  type        = number
-  default     = null
-}
-
-variable "history_server_image" {
-  description = "Image for spark-history-server-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "integration_hub_revision" {
-  description = "Charm revision for spark-integration-hub-k8s"
-  type        = number
-  default     = null
-}
-
-variable "integration_hub_image" {
-  description = "Image for spark-integration-hub-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "kyuubi_revision" {
-  description = "Charm revision for kyuubi-k8s"
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_image" {
-  description = "Image for kyuubi-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "kyuubi_units" {
-  description = "Number of Kyuubi units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "kyuubi_users_revision" {
-  description = "Charm revision for postgresql-k8s (auth-db)"
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_users_image" {
-  description = "Image for postgresql-k8s (auth-db)"
-  type        = map(string)
-  default     = null
-}
-
-variable "metastore_revision" {
-  description = "Charm revision for postgresql-k8s (metastore)"
-  type        = number
-  default     = null
-}
-
-variable "metastore_image" {
-  description = "Image for postgresql-k8s (metastore)"
-  type        = map(string)
-  default     = null
-}
-
-variable "zookeeper_revision" {
-  description = "Charm revision for zookeeper-k8s"
-  type        = number
-  default     = null
-}
-
-variable "zookeeper_image" {
-  description = "Image for zookeeper-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "data_integrator_revision" {
-  description = "Charm revision for data-integrator"
-  type        = number
-  default     = null
-}
-
-variable "s3_revision" {
-  description = "Charm revision for s3-integrator"
-  type        = number
-  default     = null
-}
-
-variable "azure_storage_revision" {
-  description = "Charm revision for azure-storage-integrator"
-  type        = number
-  default     = null
-}
-
-variable "grafana_agent_revision" {
-  description = "Charm revision for grafana-agent-k8s"
-  type        = number
-  default     = null
-}
-
-variable "cos_configuration_revision" {
-  description = "Charm revision for cos-configuration-k8s"
-  type        = number
-  default     = null
-}
-
-variable "pushgateway_revision" {
-  description = "Charm revision for prometheus-pushgateway-k8s"
-  type        = number
-  default     = null
-}
-
-variable "scrape_config_revision" {
-  description = "Charm revision for prometheus-scrape-config-k8s"
-  type        = number
-  default     = null
-}
-
-variable "certificate_common_name" {
-  description = "Common name for the certificate to be used in self-signed"
-  type        = string
-  default     = "charmed-spark"
-}
-
-variable "kyuubi_users_size" {
-  description = "Storage size for the Kyuubi users database"
-  type        = string
-  default     = "1G"
-}
-
-variable "metastore_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-variable "zookeeper_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-
-variable "alertmanager_size" {
-  description = "Storage size for the alertmanager database"
-  type        = string
-  default     = "10G"
-}
-
-variable "grafana_size" {
-  description = "Storage size for the grafana database"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_active_index_directory_size" {
-  description = "Storage size for the active index directory for Loki"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_chunks_size" {
-  description = "Storage size for the Loki chucks storage"
-  type        = string
-  default     = "500G"
-}
-
-variable "prometheus_size" {
-  description = "Storage size for the Prometheus database"
-  type        = string
-  default     = "500G"
-}
-
-variable "traefik_size" {
-  description = "Storage size for the Traefik storage"
-  type        = string
-  default     = "10G"
-}
diff --git a/releases/3.4/terraform/versions.tf b/releases/3.4/terraform/versions.tf
deleted file mode 100644
index 0dfc21c7..00000000
--- a/releases/3.4/terraform/versions.tf
+++ /dev/null
@@ -1,53 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_providers {
-    juju = {
-      source  = "juju/juju"
-      version = ">=0.20.0"
-    }
-  }
-}
-
-locals {
-  revisions = {
-    history_server  = 96
-    integration_hub = 119
-    kyuubi          = 149
-    kyuubi_users    = 495
-    metastore       = 495
-    zookeeper       = 78
-    data_integrator = 362
-    s3              = 330
-    azure_storage   = 270
-    grafana_agent   = 121
-    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
-    cos_configuration = 65
-    pushgateway       = 27
-    scrape_config     = 67
-  }
-  images = {
-    history_server = {
-      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70"
-    } # rev24,  spark-version: 3.5.7, release date 16/03/2026 
-    integration_hub = {
-      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134"
-    } # rev13, release date 19/03/2026
-    kyuubi = {
-      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:60e81dd2e9be50d4a857ce987935341c8518c784c7de36e814fc41dd3451d58f"
-    } # rev19, spark-version: 3.4.4, kyuubi-version: 1.10.3, release date 17/03/2026
-    kyuubi_users = {
-      postgresql-image = 165
-    }
-    metastore = {
-      postgresql-image = 165
-    }
-    zookeeper = {
-      zookeeper-image = 34
-    }
-  }
-}
-
diff --git a/releases/3.4/yaml/README.md b/releases/3.4/yaml/README.md
deleted file mode 100644
index 5601fa6d..00000000
--- a/releases/3.4/yaml/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Charmed Apache Spark Kubernetes Bundle
-
-## A fast and fault-tolerant, real-time event streaming platform!
-
-...
\ No newline at end of file
diff --git a/releases/3.4/yaml/bundle-azure-storage.yaml.j2 b/releases/3.4/yaml/bundle-azure-storage.yaml.j2
deleted file mode 100644
index dd85f266..00000000
--- a/releases/3.4/yaml/bundle-azure-storage.yaml.j2
+++ /dev/null
@@ -1,112 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-bundle: kubernetes
-applications:
-  kyuubi:
-    charm: kyuubi-k8s
-    channel: 3.4/stable
-    revision: 140
-    resources:
-      # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
-      kyuubi-image: ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e
-    scale: 3
-    options:
-      namespace: {{ namespace }}
-      service-account: {{ service_account }}
-      expose-external: loadbalancer
-      profile: {{ kyuubi_profile }}
-    constraints: arch=amd64
-    trust: true
-  kyuubi-users:
-    charm: postgresql-k8s
-    channel: 14/stable
-    revision: 495
-    resources:
-      postgresql-image: 165
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pgdata: kubernetes,1,1024M
-    trust: true
-  zookeeper:
-    charm: zookeeper-k8s
-    channel: 3/stable
-    revision: 78
-    scale: {{ zookeeper_units or 3}}
-    constraints: arch=amd64
-    resources:
-      zookeeper-image: 34
-  metastore:
-    charm: postgresql-k8s
-    channel: 14/stable
-    revision: 495
-    resources:
-      postgresql-image: 165
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pgdata: kubernetes,1,1024M
-    trust: true
-  azure-storage:
-    charm: azure-storage-integrator
-    channel: latest/edge
-    revision: 15
-    scale: 1
-    constraints: arch=amd64
-    options:
-      container: {{ container }}
-      storage-account: {{ storage_account }}
-      path: spark-events
-      connection-protocol: abfss
-  history-server:
-    charm: spark-history-server-k8s
-    channel: 3.4/edge
-    revision: 94
-    resources:
-      # rev23,  spark-version: 3.5.7, release date 2026-02-27 
-      spark-history-server-image: ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706
-    scale: 1
-    constraints: arch=amd64
-  integration-hub:
-    charm: spark-integration-hub-k8s
-    channel: 3/stable
-    revision: 113
-    resources:
-      # rev 11, release date 2026-03-02
-      integration-hub-image: ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa
-    scale: 1
-    constraints: arch=amd64
-    trust: true
-  certificates:
-    charm: self-signed-certificates
-    channel: 1/stable
-    revision: 317
-    scale: 1
-    constraints: arch=amd64
-    trust: true
-  data-integrator:
-    charm: data-integrator
-    channel: latest/stable
-    revision: 181
-    scale: 1
-    constraints: arch=amd64
-    options:
-      database-name: integrator
-relations:
-- - integration-hub:azure-storage-credentials
-  - azure-storage:azure-storage-credentials
-- - history-server:azure-storage-credentials
-  - azure-storage:azure-storage-credentials
-- - kyuubi:metastore-db
-  - metastore:database
-- - kyuubi:auth-db
-  - kyuubi-users:database
-- - kyuubi:spark-service-account
-  - integration-hub:spark-service-account
-- - kyuubi:zookeeper
-  - zookeeper:zookeeper
-- - kyuubi:certificates
-  - certificates:certificates
-- - kyuubi:jdbc
-  - data-integrator:kyuubi
diff --git a/releases/3.4/yaml/bundle.yaml.j2 b/releases/3.4/yaml/bundle.yaml.j2
deleted file mode 100644
index 15dfab3e..00000000
--- a/releases/3.4/yaml/bundle.yaml.j2
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-bundle: kubernetes
-applications:
-  kyuubi:
-    charm: kyuubi-k8s
-    channel: 3.4/stable
-    revision: 149
-    resources:
-      # rev19, spark-version: 3.4.4, kyuubi-version: 1.10.3, release date 17/03/2026
-      kyuubi-image: ghcr.io/canonical/charmed-spark-kyuubi@sha256:60e81dd2e9be50d4a857ce987935341c8518c784c7de36e814fc41dd3451d58f
-    scale: 3
-    options:
-      namespace: {{ namespace }}
-      service-account: {{ service_account }}
-      expose-external: loadbalancer
-      profile: {{ kyuubi_profile }}
-    constraints: arch=amd64
-    trust: true
-  kyuubi-users:
-    charm: postgresql-k8s
-    channel: 14/stable
-    revision: 495
-    resources:
-      postgresql-image: 165
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pgdata: kubernetes,1,{{ kyuubi_user_size }}
-    trust: true
-  zookeeper:
-    charm: zookeeper-k8s
-    channel: 3/stable
-    revision: 78
-    scale: {{ zookeeper_units or 3}}
-    constraints: arch=amd64
-    resources:
-      zookeeper-image: 34
-    storage:
-      zookeeper: kubernetes,1,{{ zookeeper_size }}
-  metastore:
-    charm: postgresql-k8s
-    channel: 14/stable
-    revision: 495
-    resources:
-      postgresql-image: 165
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pgdata: kubernetes,1,{{ metastore_size }}
-    trust: true
-  s3:
-    charm: s3-integrator
-    channel: 1/stable
-    revision: 145
-    scale: 1
-    options:
-      bucket: {{ bucket }}
-      endpoint: {{ s3_endpoint }}
-      path: spark-events
-    constraints: arch=amd64
-  history-server:
-    charm: spark-history-server-k8s
-    channel: 3/stable
-    revision: 94
-    resources:
-      # rev24,  spark-version: 3.5.7, release date 16/03/2026 
-      spark-history-server-image: ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70
-    scale: 1
-    constraints: arch=amd64
-  integration-hub:
-    charm: spark-integration-hub-k8s
-    channel: 3/stable
-    revision: 119
-    resources:
-      # rev 13, release date 19/03/2026
-      integration-hub-image: ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134
-    scale: 1
-    constraints: arch=amd64
-    trust: true
-  certificates:
-    charm: self-signed-certificates
-    channel: 1/stable
-    revision: 317
-    scale: 1
-    constraints: arch=amd64
-    trust: true
-  data-integrator:
-    charm: data-integrator
-    channel: latest/stable
-    revision: 179
-    scale: 1
-    constraints: arch=amd64
-    options:
-      database-name: integrator
-relations:
-- - integration-hub:s3-credentials
-  - s3:s3-credentials
-- - history-server:s3-credentials
-  - s3:s3-credentials
-- - kyuubi:metastore-db
-  - metastore:database
-- - kyuubi:auth-db
-  - kyuubi-users:database
-- - kyuubi:spark-service-account
-  - integration-hub:spark-service-account
-- - kyuubi:zookeeper
-  - zookeeper:zookeeper
-- - kyuubi:certificates
-  - certificates:certificates
-- - kyuubi:jdbc
-  - data-integrator:kyuubi
diff --git a/releases/3.4/yaml/charmcraft.yaml b/releases/3.4/yaml/charmcraft.yaml
deleted file mode 100644
index 12cb8fd1..00000000
--- a/releases/3.4/yaml/charmcraft.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-type: bundle
\ No newline at end of file
diff --git a/releases/3.4/yaml/docs-resources/aks-setup/cluster.tf b/releases/3.4/yaml/docs-resources/aks-setup/cluster.tf
deleted file mode 100644
index 0cb82b77..00000000
--- a/releases/3.4/yaml/docs-resources/aks-setup/cluster.tf
+++ /dev/null
@@ -1,32 +0,0 @@
-
-resource "azurerm_kubernetes_cluster" "prod-k8s" {
-  location            = azurerm_resource_group.rg.location
-  name                = var.cluster_name
-  resource_group_name = azurerm_resource_group.rg.name
-  dns_prefix          = "testaksclusterprefix"
-  kubernetes_version  = "1.31.7"
-
-  identity {
-    type = "SystemAssigned"
-  }
-
-  default_node_pool {
-    name           = "default"
-    vm_size        = "Standard_DS2_v2"
-    node_count     = var.num_nodes
-    vnet_subnet_id = azurerm_subnet.infrastructure.id
-  }
-
-  auto_scaler_profile {
-    expander = "least-waste" # if not least waste
-    #ignore-daemonsets-utilization = true
-    scale_down_utilization_threshold = 0.2
-    scale_down_unneeded              = "30m"
-  }
-
-  network_profile {
-    network_plugin = "kubenet"
-    service_cidr   = "10.1.0.0/16" # Ensure this does not overlap with any subnets
-    dns_service_ip = "10.1.0.10"   # Hardcoded DNS service IP
-  }
-}
diff --git a/releases/3.4/yaml/docs-resources/aks-setup/network.tf b/releases/3.4/yaml/docs-resources/aks-setup/network.tf
deleted file mode 100644
index 96597e21..00000000
--- a/releases/3.4/yaml/docs-resources/aks-setup/network.tf
+++ /dev/null
@@ -1,117 +0,0 @@
-resource "azurerm_virtual_network" "vn" {
-  name                = "TestSparkAKSVN"
-  address_space       = ["10.0.0.0/16"] # Define the address space for the VNet
-  location            = azurerm_resource_group.rg.location
-  resource_group_name = azurerm_resource_group.rg.name
-}
-
-
-resource "azurerm_subnet" "infrastructure" {
-  name                 = "testsparkaks-subnet"
-  virtual_network_name = azurerm_virtual_network.vn.name
-  resource_group_name  = azurerm_resource_group.rg.name
-  address_prefixes     = ["10.0.10.0/24"]
-}
-
-
-resource "azurerm_nat_gateway" "natgw" {
-  location            = azurerm_resource_group.rg.location
-  name                = "testsparkaks-nat-gateway"
-  resource_group_name = azurerm_resource_group.rg.name
-  sku_name            = "Standard"
-
-}
-
-
-resource "azurerm_public_ip" "natip" {
-  name                = "testsparkaks-nat-gateway-ip"
-  location            = azurerm_resource_group.rg.location
-  resource_group_name = azurerm_resource_group.rg.name
-  allocation_method   = "Static"
-  sku                 = "Standard"
-}
-
-
-resource "azurerm_nat_gateway_public_ip_association" "example" {
-  nat_gateway_id       = azurerm_nat_gateway.natgw.id
-  public_ip_address_id = azurerm_public_ip.natip.id
-}
-
-
-resource "azurerm_subnet_nat_gateway_association" "natinfra" {
-  subnet_id      = azurerm_subnet.infrastructure.id
-  nat_gateway_id = azurerm_nat_gateway.natgw.id
-}
-
-
-resource "azurerm_network_security_group" "infra_sec_group" {
-  name                = "TestAKSInfrastructureSecurityGroup"
-  location            = azurerm_resource_group.rg.location
-  resource_group_name = azurerm_resource_group.rg.name
-
-  security_rule {
-    name                       = "AllowAnyCustomAnyOutbound"
-    priority                   = 110
-    direction                  = "Outbound"
-    access                     = "Allow"
-    protocol                   = "Tcp"
-    source_port_range          = "*"
-    destination_port_range     = "*"
-    source_address_prefix      = "*"
-    destination_address_prefix = "*"
-  }
-
-  security_rule {
-    name                       = "AllowNATInbound"
-    priority                   = 120
-    direction                  = "Inbound"
-    access                     = "Allow"
-    protocol                   = "Tcp"
-    source_port_range          = "*"
-    destination_port_range     = "*"
-    source_address_prefix      = azurerm_public_ip.natip.ip_address
-    destination_address_prefix = "*"
-  }
-
-  security_rule {
-    name                       = "Allow80AnyInbound"
-    priority                   = 130
-    direction                  = "Inbound"
-    access                     = "Allow"
-    protocol                   = "Tcp"
-    source_port_range          = "*"
-    destination_port_range     = "80"
-    source_address_prefix      = "*"
-    destination_address_prefix = "*"
-  }
-
-  security_rule {
-    name                       = "Allow443AnyInbound"
-    priority                   = 140
-    direction                  = "Inbound"
-    access                     = "Allow"
-    protocol                   = "Tcp"
-    source_port_range          = "*"
-    destination_port_range     = "443"
-    source_address_prefix      = "*"
-    destination_address_prefix = "*"
-  }
-
-  security_rule {
-    name                       = "Allow8080AnyInbound"
-    priority                   = 150
-    direction                  = "Inbound"
-    access                     = "Allow"
-    protocol                   = "Tcp"
-    source_port_range          = "*"
-    destination_port_range     = "8080"
-    source_address_prefix      = "*"
-    destination_address_prefix = "*"
-  }
-}
-
-
-resource "azurerm_subnet_network_security_group_association" "infra-association" {
-  subnet_id                 = azurerm_subnet.infrastructure.id
-  network_security_group_id = azurerm_network_security_group.infra_sec_group.id
-}
diff --git a/releases/3.4/yaml/docs-resources/aks-setup/outputs.tf b/releases/3.4/yaml/docs-resources/aks-setup/outputs.tf
deleted file mode 100644
index 8837c5e9..00000000
--- a/releases/3.4/yaml/docs-resources/aks-setup/outputs.tf
+++ /dev/null
@@ -1,12 +0,0 @@
-
-output "resource_group_name" {
-  value       = azurerm_resource_group.rg.name
-  description = "The name of the resource group"
-}
-
-
-output "aks_cluster_name" {
-  value       = azurerm_kubernetes_cluster.prod-k8s.name
-  description = "The name of the AKS cluster"
-}
-
diff --git a/releases/3.4/yaml/docs-resources/aks-setup/providers.tf b/releases/3.4/yaml/docs-resources/aks-setup/providers.tf
deleted file mode 100644
index a9beee3c..00000000
--- a/releases/3.4/yaml/docs-resources/aks-setup/providers.tf
+++ /dev/null
@@ -1,18 +0,0 @@
-terraform {
-  required_version = ">=1.0"
-
-  required_providers {
-    azapi = {
-      source  = "azure/azapi"
-      version = "~>1.5"
-    }
-    azurerm = {
-      source  = "hashicorp/azurerm"
-      version = "~>3.0"
-    }
-  }
-}
-
-provider "azurerm" {
-  features {}
-}
diff --git a/releases/3.4/yaml/docs-resources/aks-setup/resource-group.tf b/releases/3.4/yaml/docs-resources/aks-setup/resource-group.tf
deleted file mode 100644
index 0a50405b..00000000
--- a/releases/3.4/yaml/docs-resources/aks-setup/resource-group.tf
+++ /dev/null
@@ -1,7 +0,0 @@
-resource "azurerm_resource_group" "rg" {
-  name     = var.resource_group_name
-  location = var.region
-}
-
-
-
diff --git a/releases/3.4/yaml/docs-resources/aks-setup/variables.tf b/releases/3.4/yaml/docs-resources/aks-setup/variables.tf
deleted file mode 100644
index 9cbf6fcc..00000000
--- a/releases/3.4/yaml/docs-resources/aks-setup/variables.tf
+++ /dev/null
@@ -1,23 +0,0 @@
-variable "num_nodes" {
-  description = "Number of nodes in the AKS cluster"
-  type        = number
-  default     = 3
-}
-
-variable "cluster_name" {
-  description = "The name of the AKS cluster to be created"
-  type        = string
-  default     = "TestAKSCluster"
-}
-
-variable "region" {
-  description = "The region in which cloud resources are to be created"
-  type        = string
-  default     = "East US"
-}
-
-variable "resource_group_name" {
-  description = "The name of the resource group under which the resources will be created"
-  type        = string
-  default     = "TestSparkAKSRG"
-}
\ No newline at end of file
diff --git a/releases/3.4/yaml/metadata.yaml b/releases/3.4/yaml/metadata.yaml
deleted file mode 100644
index 6a1d78b1..00000000
--- a/releases/3.4/yaml/metadata.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-name: spark-k8s-bundle
-display-name: Charmed Kafka
-description: |
-  Apache Spark is an distributed processing engine, designed to scale out your data 
-  analysis and machine learning tasks. This bundles charms to enable a Apache Spark 
-  deployment on K8s, featuring integration with object storage, monitoring,
-  seamless and integrated configuration.
-summary: Charmed Apache Spark bundle
-docs: https://discourse.charmhub.io/t/charmed-spark-documentation/8963
-source: https://github.com/canonical/spark-k8s-bundle
-issues: https://github.com/canonical/spark-k8s-bundle/issues
-website:
-  - https://ubuntu.com/data/spark
-  - https://canonical.com/data/docs/spark/k8s
-  - https://github.com/canonical/spark-k8s-bundle
-  - https://matrix.to/#/%23charmhub-data-platform%3Aubuntu.com
-maintainers:
-  - Canonical Data Platform <data-platform@lists.launchpad.net>
\ No newline at end of file
diff --git a/releases/3.4/yaml/overlays/cos-integration.yaml.j2 b/releases/3.4/yaml/overlays/cos-integration.yaml.j2
deleted file mode 100644
index a557816c..00000000
--- a/releases/3.4/yaml/overlays/cos-integration.yaml.j2
+++ /dev/null
@@ -1,94 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-bundle: kubernetes
-saas:
-  grafana-dashboards:
-    url: {{ cos_controller|default('micro', true) }}:admin/{{ cos_model|default('cos', true) }}.grafana-dashboards
-  prometheus-receive-remote-write:
-    url: {{ cos_controller|default('micro', true) }}:admin/{{ cos_model|default('cos', true) }}.prometheus-receive-remote-write
-  traefik:
-    url: {{ cos_controller|default('micro', true) }}:admin/{{ cos_model|default('cos', true) }}.traefik
-  loki-logging:
-    url: {{ cos_controller|default('micro', true) }}:admin/{{ cos_model|default('cos', true) }}.loki-logging
-applications:
-  agent:
-    charm: grafana-agent-k8s
-    channel: 1/stable
-    revision: 121
-    resources:
-      agent-image: 46
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      data: kubernetes,1,1024M
-    trust: true
-  cos-configuration:
-    charm: cos-configuration-k8s
-    channel: 1/stable
-    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
-    revision: 65
-    base: ubuntu@20.04/stable
-    resources:
-      git-sync-image: 35
-    scale: 1
-    options:
-      git_branch: main
-      git_depth: 1
-      git_repo: https://github.com/canonical/spark-k8s-bundle
-      grafana_dashboards_path: releases/3.4/resources/grafana/
-    constraints: arch=amd64
-    storage:
-      content-from-git: kubernetes,1,1024M
-  pushgateway:
-    charm: prometheus-pushgateway-k8s
-    channel: 1/stable
-    revision: 27
-    resources:
-      pushgateway-image: 11
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pushgateway-store: kubernetes,1,1024M
-  scrape-config:
-    charm: prometheus-scrape-config-k8s
-    channel: 1/stable
-    revision: 67
-    base: ubuntu@20.04/stable
-    scale: 1
-    options:
-      scrape_interval: 10s
-    constraints: arch=amd64
-relations:
-- - cos-configuration:grafana-dashboards
-  - agent:grafana-dashboards-consumer
-- - scrape-config:configurable-scrape-jobs
-  - pushgateway:metrics-endpoint
-- - scrape-config:metrics-endpoint
-  - agent:metrics-endpoint
-- - pushgateway:push-endpoint
-  - integration-hub:cos
-- - agent:grafana-dashboards-provider
-  - grafana-dashboards:grafana-dashboard
-- - agent:send-remote-write
-  - prometheus-receive-remote-write:receive-remote-write
-- - agent:logging-consumer
-  - loki-logging:logging
-- - history-server:ingress
-  - traefik:ingress
-- - history-server:metrics-endpoint
-  - agent:metrics-endpoint
-- - history-server:grafana-dashboard
-  - agent:grafana-dashboards-consumer
-- - history-server:logging
-  - agent:logging-provider
-- - kyuubi:metrics-endpoint
-  - agent:metrics-endpoint
-- - kyuubi:grafana-dashboard
-  - agent:grafana-dashboards-consumer
-- - kyuubi:logging
-  - agent:logging-provider
-- - integration-hub:logging
-  - agent:logging-provider
diff --git a/releases/3.5/resources/grafana/spark_dashboard.json b/releases/3.5/resources/grafana/spark_dashboard.json
deleted file mode 100644
index f34cd1d8..00000000
--- a/releases/3.5/resources/grafana/spark_dashboard.json
+++ /dev/null
@@ -1,1733 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "builtIn": 1,
-        "datasource": {
-          "type": "grafana",
-          "uid": "-- Grafana --"
-        },
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "target": {
-          "limit": 100,
-          "matchAny": false,
-          "tags": [],
-          "type": "dashboard"
-        },
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "links": [],
-  "liveNow": false,
-  "panels": [
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 4,
-      "panels": [],
-      "title": "Block Manager (Driver)",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "decmbytes"
-        },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "Max Memory (MB)"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 1
-      },
-      "id": 2,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "BlockManager_memory_maxMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "instant": false,
-          "legendFormat": "Max Memory (MB)",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "BlockManager_memory_maxOffHeapMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "legendFormat": "Max Off Heap Memory (MB)",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "BlockManager_memory_maxOnHeapMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "legendFormat": "Max On Heap Memory (MB)",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "BlockManager_memory_remainingMem_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "legendFormat": "Remaining Memory (MB)",
-          "range": true,
-          "refId": "D"
-        }
-      ],
-      "title": "Block Manager Memory",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "decmbytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 1
-      },
-      "id": 6,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "BlockManager_disk_diskSpaceUsed_MB{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "legendFormat": "Disk Memory (MB)",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Block Manager Disk Memory",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 10
-      },
-      "id": 10,
-      "panels": [],
-      "title": "Hive External Catalog (Driver)",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 11
-      },
-      "id": 12,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_fileCacheHits{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "instant": false,
-          "legendFormat": "File Cache Hits",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_filesDiscovered{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "File Discovered",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_hiveClientCalls{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "Client Calls",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_parallelListingJobCount{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "Parallel Listing Jobs",
-          "range": true,
-          "refId": "D"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "HiveExternalCatalog_partitionsFetched{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "Partitions Fetched",
-          "range": true,
-          "refId": "E"
-        }
-      ],
-      "title": "Hive External Catalog",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 19
-      },
-      "id": 14,
-      "panels": [],
-      "title": "Dag Scheduler (Driver)",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 20
-      },
-      "id": 16,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "DAGScheduler_job_activeJobs{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "instant": false,
-          "legendFormat": "Active Jobs",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "DAGScheduler_job_allJobs{exported_job=~\"$spark_job_id\",juju_application=~\"$juju_application\",juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_unit=~\"$juju_unit\",role=\"driver\"}",
-          "hide": false,
-          "instant": false,
-          "legendFormat": "All Jobs",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "DAG Schedule Jobs",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 20
-      },
-      "id": 18,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "DAGScheduler_stage_failedStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "Failed Stages",
-          "range": true,
-          "refId": "FailedStages"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "DAGScheduler_stage_runningStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Running Stages",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "DAGScheduler_stage_waitingStages{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Waiting Stages",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Stages Status",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 28
-      },
-      "id": 20,
-      "panels": [],
-      "title": "Memory Metrics (Driver)",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 29
-      },
-      "id": 22,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_DirectPoolMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "Direct Pool Memory",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_JVMHeapMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM Heap Memory",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_JVMOffHeapMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM Off Heap Memory",
-          "range": true,
-          "refId": "C"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_OffHeapExecutionMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM Off Heap Execution Memory",
-          "range": true,
-          "refId": "D"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_OnHeapStorageMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM On Heap Storage Memory",
-          "range": true,
-          "refId": "E"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_OnHeapExecutionMemory{role=\"driver\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM On Heap Execution Memory",
-          "range": true,
-          "refId": "F"
-        }
-      ],
-      "title": "JVM Executor Memory",
-      "type": "timeseries"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 37
-      },
-      "id": 24,
-      "panels": [],
-      "title": "Executors Metrics",
-      "type": "row"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 38
-      },
-      "id": 26,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_bytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "Bytes Reads (#Executor {{number}})",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_bytesWritten{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Bytes Written (#Executor {{number}})",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Executor Read and Writes",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          },
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 38
-      },
-      "id": 28,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_JVMHeapMemory{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "JVM Heap Memory #{{number}}",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "ExecutorMetrics_JVMOffHeapMemory{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "JVM Off Heap Memory #{{number}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "Memory",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 46
-      },
-      "id": 30,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "rate(executor_runTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "legendFormat": "Runtime (Executor #{{number}})",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Executors Runtime",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 46
-      },
-      "id": 32,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_shuffleBytesWritten{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "legendFormat": "Shuffle Bytes Writes (Executor #{{number}})",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_shuffleLocalBytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Shuffle Local Bytes Reads (Executor #{{number}})",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "executor_shuffleRemoteBytesRead{role=\"executor\", exported_job=~\"$spark_job_id\"}",
-          "hide": false,
-          "legendFormat": "Shuffle Remote Bytes Reads (Executor #{{number}})",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "Shuffle Reads/Writes",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": [
-          {
-            "__systemRef": "hideSeriesFrom",
-            "matcher": {
-              "id": "byNames",
-              "options": {
-                "mode": "exclude",
-                "names": [
-                  "Minor GC Count (Executor #1"
-                ],
-                "prefix": "All except:",
-                "readOnly": true
-              }
-            },
-            "properties": [
-              {
-                "id": "custom.hideFrom",
-                "value": {
-                  "legend": false,
-                  "tooltip": false,
-                  "viz": true
-                }
-              }
-            ]
-          }
-        ]
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 54
-      },
-      "id": 34,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "increase(ExecutorMetrics_MinorGCCount{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "legendFormat": "Minor GC Count (Executor #{{number}}",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "increase(ExecutorMetrics_MajorGCCount{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "Major GC Count (Executor #{{number}}",
-          "range": true,
-          "refId": "B"
-        }
-      ],
-      "title": "GC counts",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "${prometheusds}"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 12,
-        "y": 54
-      },
-      "id": 36,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "rate(ExecutorMetrics_MajorGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "legendFormat": "Major GC time (Executor #{{number}})",
-          "range": true,
-          "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "rate(ExecutorMetrics_MinorGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "Minor GC time (Executor #{{number}})",
-          "range": true,
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "${prometheusds}"
-          },
-          "editorMode": "builder",
-          "expr": "rate(ExecutorMetrics_TotalGCTime{role=\"executor\", exported_job=~\"$spark_job_id\"}[$__rate_interval])",
-          "hide": false,
-          "legendFormat": "Total GC time (Executor #{{number}})",
-          "range": true,
-          "refId": "C"
-        }
-      ],
-      "title": "GC Time Spend",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": false,
-  "schemaVersion": 37,
-  "style": "dark",
-  "tags": [],
-  "templating": {
-    "list": [
-      {
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "hide": 0,
-        "includeAll": true,
-        "label": "Loki datasource",
-        "multi": true,
-        "name": "lokids",
-        "options": [],
-        "query": "loki",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Juju unit",
-        "multi": true,
-        "name": "juju_unit",
-        "options": [],
-        "query": {
-          "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},juju_unit)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "tagValuesQuery": "",
-        "tags": [],
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      },
-      {
-        "current": {
-          "selected": false,
-          "text": "juju_cos_348d0940-98ce-4ac5-8dcc-4dce43d9c620_prometheus_0",
-          "value": "juju_cos_348d0940-98ce-4ac5-8dcc-4dce43d9c620_prometheus_0"
-        },
-        "hide": 0,
-        "includeAll": false,
-        "label": "Prometheus datasource",
-        "multi": false,
-        "name": "prometheusds",
-        "options": [],
-        "query": "prometheus",
-        "queryValue": "",
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "type": "datasource"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(up,juju_model)",
-        "description": "The",
-        "hide": 0,
-        "includeAll": true,
-        "label": "juju model",
-        "multi": true,
-        "name": "juju_model",
-        "options": [],
-        "query": {
-          "query": "label_values(up,juju_model)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Juju model uuid",
-        "multi": true,
-        "name": "juju_model_uuid",
-        "options": [],
-        "query": {
-          "query": "label_values(up{juju_model=~\"$juju_model\"},juju_model_uuid)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "juju application",
-        "multi": true,
-        "name": "juju_application",
-        "options": [],
-        "query": {
-          "query": "label_values(up{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\"},juju_application)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      },
-      {
-        "allValue": ".*",
-        "current": {
-          "selected": false,
-          "text": "All",
-          "value": "$__all"
-        },
-        "datasource": {
-          "type": "prometheus",
-          "uid": "${prometheusds}"
-        },
-        "definition": "label_values(push_time_seconds{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},exported_job)",
-        "hide": 0,
-        "includeAll": true,
-        "label": "Spark Job Id",
-        "multi": true,
-        "name": "spark_job_id",
-        "options": [],
-        "query": {
-          "query": "label_values(push_time_seconds{juju_model=~\"$juju_model\",juju_model_uuid=~\"$juju_model_uuid\",juju_application=~\"$juju_application\"},exported_job)",
-          "refId": "StandardVariableQuery"
-        },
-        "refresh": 1,
-        "regex": "",
-        "skipUrlSync": false,
-        "sort": 0,
-        "type": "query"
-      }
-    ]
-  },
-  "time": {
-    "from": "now-12h",
-    "to": "now"
-  },
-  "timepicker": {},
-  "timezone": "",
-  "title": "Spark dashboard",
-  "uid": "ZyD93JGSk",
-  "version": 1,
-  "weekStart": ""
-}
\ No newline at end of file
diff --git a/releases/3.5/terraform/README.md b/releases/3.5/terraform/README.md
deleted file mode 100644
index d0749e04..00000000
--- a/releases/3.5/terraform/README.md
+++ /dev/null
@@ -1,194 +0,0 @@
-# Spark bundle Terraform modules
-
-These Terraform modules aim to facilitate the deployment of the Charmed Spark solution, using the using the [Terraform juju provider](https://github.com/juju/terraform-provider-juju/). For more information, refer to the provider [documentation](https://registry.terraform.io/providers/juju/juju/latest/docs).
-
-The solution consists of the following modules:
-
-- `spark`: main spark module with integration-hub, history-server, kyuubi
-- `s3` and `azure-storage`: storage backend. Having one of them is necessary, and they are mutually exclusive
-- `observabiliby`: parts of the observability stack to be deployed alongside spark (config, pushgateway, etc.)
-
-> [!NOTE]
-> The `s3` module itself doesn't act as an S3 object storage system. For the solution to be functional, the `s3-integrator` charm needs to point to an S3-like storage.
-> The same applies to `azure-storage`.
-
-## Requirements
-
-This module requires a `juju` model to be available. Refer to the [usage section](#usage) below for more details.
-
-## Top-level inputs
-
-| Variable                                   | Description                                                                     | Type   | Nullable | Default             |
-| ------------------------------------------ | ------------------------------------------------------------------------------- | ------ | -------- | ------------------- |
-| K8S_CLOUD                                  | The kubernetes juju cloud name.                                                 | string | True     | microk8s            |
-| K8S_CREDENTIAL                             | The name of the kubernetes juju credential.                                     | string | True     | microk8s            |
-| model                                      | The name of the juju model to deploy Spark to                                   | string | True     | spark               |
-| create_model                               | Should terraform create the Juju models? If set to false, assume the [...]      | bool   | False    | True                |
-| cos                                        | Observability settings                                                          | map    | True     | See variables.tf    |
-| storage_backend                            | Storage backend to be used                                                      | string | True     | s3                  |
-| s3                                         | S3 Bucket information                                                           | map    | True     | See variables.tf    |
-| azure_storage                              | Azure Object storage information                                                | map    | True     | See variables.tf    |
-| admin_password                             | The password for the admin user.                                                | string | True     | null                |
-| tls_private_key                            | The file path of the private key to use for TLS certificates.                   | string | True     | null                |
-| kyuubi_profile                             | The profile to be used for Kyuubi; should be one of 'testing', 'staging' [...]  | string | False    | production          |
-| kyuubi_user                                | Define the user to be used for running Kyuubi enginers                          | string | True     | kyuubi-spark-engine |
-| enable_dynamic_allocation                  | Enable dynamic allocation of pods for Spark jobs.                               | bool   | True     | False               |
-| kyuubi_k8s_node_selectors                  | Comma separated label:value selectors for K8s pods in Kyuubi.                   | string | True     | null                |
-| kyuubi_loadbalancer_extra_annotations      | Optional extra annotations to be supplied to the load balancer service in [...] | string | True     | null                |
-| kyuubi_gpu_enable                          | Enable GPU acceleration for SparkSQLEngine.                                     | bool   | True     | False               |
-| kyuubi_gpu_engine_executors_limit          | Limit the number of GPUs an engine can schedule executor pods on.               | number | True     | 1                   |
-| kyuubi_gpu_pinned_memory                   | Set the host memory (in GB) per executor reserved for fast data transfer [...]  | number | True     | 1                   |
-| kyuubi_driver_pod_template                 | Define K8s driver pod from a file accessible in the object storage.             | string | True     | null                |
-| kyuubi_executor_pod_template               | Define K8s executor pod from a file accessible in the object storage.           | string | True     | null                |
-| kyuubi_executor_cores                      | Set kyuubi executor pods cpu cores.                                             | number | True     | null                |
-| kyuubi_executor_memory                     | Set kyuubi executor pods memory (in GB).                                        | number | True     | null                |
-| driver_pod_template                        | Define K8s driver pod from a file accessible to the `spark-submit` process.     | string | True     | null                |
-| executor_pod_template                      | Define K8s executor pod from a file accessible to the `spark-submit` process.   | string | True     | null                |
-| integration_hub_monitored_service_accounts | Comma-separated patterns for namespaces and service accounts to monitor [...]   | string | True     | null                |
-| zookeeper_units                            | Define the number of zookeeper units. 3 units are recommended for high [...]    | number | False    | 3                   |
-| history_server_revision                    | Charm revision for spark-history-server-k8s                                     | number | True     | null                |
-| history_server_image                       | Image for spark-history-server-k8s                                              | map    | True     | null                |
-| integration_hub_revision                   | Charm revision for spark-integration-hub-k8s                                    | number | True     | null                |
-| integration_hub_image                      | Image for spark-integration-hub-k8s                                             | map    | True     | null                |
-| kyuubi_revision                            | Charm revision for kyuubi-k8s                                                   | number | True     | null                |
-| kyuubi_image                               | Image for kyuubi-k8s                                                            | map    | True     | null                |
-| kyuubi_units                               | Number of Kyuubi units. 3 units are recommended for high availability.          | number | False    | 3                   |
-| kyuubi_users_revision                      | Charm revision for postgresql-k8s (auth-db)                                     | number | True     | null                |
-| kyuubi_users_image                         | Image for postgresql-k8s (auth-db)                                              | map    | True     | null                |
-| metastore_revision                         | Charm revision for postgresql-k8s (metastore)                                   | number | True     | null                |
-| metastore_image                            | Image for postgresql-k8s (metastore)                                            | map    | True     | null                |
-| zookeeper_revision                         | Charm revision for zookeeper-k8s                                                | number | True     | null                |
-| zookeeper_image                            | Image for zookeeper-k8s                                                         | map    | True     | null                |
-| data_integrator_revision                   | Charm revision for data-integrator                                              | number | True     | null                |
-| s3_revision                                | Charm revision for s3-integrator                                                | number | True     | null                |
-| azure_storage_revision                     | Charm revision for azure-storage-integrator                                     | number | True     | null                |
-| grafana_agent_revision                     | Charm revision for grafana-agent-k8s                                            | number | True     | null                |
-| cos_configuration_revision                 | Charm revision for cos-configuration-k8s                                        | number | True     | null                |
-| pushgateway_revision                       | Charm revision for prometheus-pushgateway-k8s                                   | number | True     | null                |
-| scrape_config_revision                     | Charm revision for prometheus-scrape-config-k8s                                 | number | True     | null                |
-| certificate_common_name                    | Common name for the certificate to be used in self-signed                       | string | True     | charmed-spark       |
-
-## Usage
-
-### Quick start
-
-Here is what a minimal deployment could look like:
-
-```hcl
-resource "juju_model" "spark" {
-  name = "spark"
-}
-
-module "ssc" {
-  depends_on  = [juju_model.spark]
-  source      = "git::https://github.com/canonical/self-signed-certificates-operator//terraform"
-  model       = juju_model.spark.name
-  channel     = "latest/stable"
-  revision    = 163
-  constraints = "arch=amd64"
-  base        = "ubuntu@22.04"
-  units       = 1
-}
-
-module "spark" {
-  depends_on                = [juju_model.spark]
-  source                    = "git::https://github.com/canonical/spark-k8s-bundle//releases/3.5/terraform/modules/spark"
-  model                     = juju_model.spark.name
-  tls_app_name              = module.ssc.app_name
-  tls_certificates_endpoint = module.ssc.provides.certificates
-}
-
-module "azure" {
-  depends_on   = [juju_model.spark]
-  source       = "git::https://github.com/canonical/spark-k8s-bundle//releases/3.5/terraform/modules/azure-storage"
-  model        = juju_model.spark.name
-  spark_charms = module.spark.charms
-  azure = {
-    storage_account = "storage_account"
-    storage_secret  = "storage_secret"
-  }
-}
-```
-
-## Development
-
-The features are split into multiples submodules, where each file has a dedicated role:
-
-- `applications.tf` -> Speak for itself
-- `integrations.tf` -> Same
-- `resources.tf` -> Define juju models, secrets, cross model offers
-  each module can also include `providers.tf`, `variables.tf` and `outputs.tf`. The latter is especially useful to pass values back to a different module.
-
-We defined a dependency chain such as `azure-storage`, `observability` and `s3` depend on `spark`.
-By depending on `spark`, they can use whatever output `spark` defines.\
-This means that using this structure, a new optional module just takes care of defining its applications and their integrations with the existing resources, without any change needed from the existing modules.
-
-## Versions
-
-| Module        | Charm                        | Channel       | revision |
-| ------------- | ---------------------------- | ------------- | -------- |
-| azure         | azure-storage-integrator     | latest/edge   | 15       |
-| observability | grafana-agent-k8s            | 1/stable      | 121      |
-| observability | cos-configuration-k8s        | 1/stable      | 65       |
-| observability | prometheus-pushgateway-k8s   | 1/stable      | 27       |
-| observability | prometheus-scrape-config-k8s | 1/stable      | 67       |
-| spark         | spark-history-server-k8s     | 3/stable      | 94       |
-| spark         | spark-integration-hub-k8s    | 3/stable      | 113      |
-| spark         | kyuubi-k8s                   | 3.5/stable    | 142      |
-| spark         | postgresql-k8s               | 14/stable     | 495      |
-| spark         | postgresql-k8s               | 14/stable     | 495      |
-| spark         | zookeeper-k8s                | 3/stable      | 78       |
-| spark         | data-integrator              | latest/stable | 179      |
-| s3            | s3-integrator                | 1/stable      | 145      |
-
-### Updating the inputs table
-
-1. Run `tox -e tfinputs` from the `python` folder.
-2. Paste the relevant output table in the README.
-
-### Updating the version table
-
-1. Make sure that you work from a clean slate (no `terraform.tfstate`)\
-   `terraform plan -out "tfplan"`
-2. Convert to json `terraform show -json "tfplan" > tfplan.json`
-3. Run the following python script
-
-```py
-import json
-import re
-from typing import TypedDict
-
-
-class Charm(TypedDict):
-    name: str
-    channel: str
-    revision: int
-
-
-with open("tfplan.json", "r") as f:
-    plan = json.load(f)
-
-root = plan["planned_values"]["root_module"]
-
-charms_summary = []
-for module in root["child_modules"]:
-    if match := re.match(r"module\.(?P<name>\w+)\[?", module["address"]):
-        name = match.group("name")
-    else:
-        name = module["address"]
-    for resource in module["resources"]:
-        if not resource["type"] == "juju_application":
-            continue
-        charm = Charm(resource["values"]["charm"][0])
-        charms_summary.append([name, charm["name"], charm["channel"], charm["revision"]])
-
-
-data = [["Module", "Charm", "Channel", "revision"]] + charms_summary
-md_table = "\n".join(
-    [
-        "| " + " | ".join(map(str, row)) + " |"
-        for row in [data[0], ["---"] * len(data[0])] + data[1:]
-    ]
-)
-print(md_table)
-```
diff --git a/releases/3.5/terraform/external/cos/applications.tf b/releases/3.5/terraform/external/cos/applications.tf
deleted file mode 100644
index c736f2e2..00000000
--- a/releases/3.5/terraform/external/cos/applications.tf
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "alertmanager" {
-  name  = "alertmanager"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "alertmanager-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    data = var.alertmanager_size
-  }
-}
-
-resource "juju_application" "catalogue" {
-  name  = "catalogue"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "catalogue-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  config = {
-    "description" : "Canonical Observability Stack Lite"
-  }
-}
-
-resource "juju_application" "grafana" {
-  name  = "grafana"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "grafana-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    database = var.grafana_size
-  }
-}
-
-resource "juju_application" "loki" {
-  name  = "loki"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "loki-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    active-index-directory = var.loki_active_index_directory_size
-    loki-chunks            = var.loki_chunks_size
-  }
-}
-
-resource "juju_application" "prometheus" {
-  name  = "prometheus"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name    = "prometheus-k8s"
-    channel = "1/stable"
-  }
-  config = {
-    "metrics_retention_time" : "90d"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    database = var.prometheus_size
-  }
-}
-
-resource "juju_application" "traefik" {
-  name  = "traefik"
-  trust = true
-  model = data.juju_model.cos.name
-  charm {
-    name = "traefik-k8s"
-    # FIXME(stable): Use different track when available
-    channel = "latest/stable"
-  }
-  config = {
-    "tls-cert" : var.cos_tls_cert,
-    "tls-key" : var.cos_tls_key,
-    "tls-ca" : var.cos_tls_ca
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    configurations = var.traefik_size
-  }
-}
-
diff --git a/releases/3.5/terraform/external/cos/integrations.tf b/releases/3.5/terraform/external/cos/integrations.tf
deleted file mode 100644
index e656c361..00000000
--- a/releases/3.5/terraform/external/cos/integrations.tf
+++ /dev/null
@@ -1,300 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-
-
-
-resource "juju_integration" "traefik-grafana" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "traefik-route"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-prometheus" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress-per-unit"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-loki" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress-per-unit"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-alertmanager" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "prometheus-alertmanager-alerting" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "alertmanager"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "alerting"
-  }
-}
-
-
-resource "juju_integration" "grafana-prometheus-source" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "grafana-loki-source" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "grafana-alertmanager-source" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "loki-alertmanager" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "alertmanager"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "alerting"
-  }
-}
-
-
-resource "juju_integration" "prometheus-traefik" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-alertmanager-metrics" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "self-metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-loki" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-grafana" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "grafana-loki-dashboard" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "grafana-prometheus-dashboard" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "grafana-alertmanager-dashboard" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "catalogue-traefik" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "ingress"
-  }
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress"
-  }
-}
-
-
-resource "juju_integration" "catalogue-grafana" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "catalogue"
-  }
-}
-
-
-resource "juju_integration" "catalogue-prometheus" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "catalogue"
-  }
-}
-
-
-resource "juju_integration" "catalogue-alertmanager" {
-  model = data.juju_model.cos.name
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "catalogue"
-  }
-}
diff --git a/releases/3.5/terraform/external/cos/outputs.tf b/releases/3.5/terraform/external/cos/outputs.tf
deleted file mode 100644
index d1cc844b..00000000
--- a/releases/3.5/terraform/external/cos/outputs.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    alertmanager = juju_application.alertmanager.name
-    catalogue    = juju_application.catalogue.name
-    grafana      = juju_application.grafana.name
-    loki         = juju_application.loki.name
-    prometheus   = juju_application.prometheus.name
-  }
-}
-
-output "user" {
-  description = "The name of the Juju user of the COS deployment."
-  value       = var.cos_user
-}
-
-output "dashboards_offer" {
-  value = juju_offer.grafana_dashboards.url
-}
-
-output "metrics_offer" {
-  value = juju_offer.prometheus_receive_remote_write.url
-}
-
-output "logging_offer" {
-  value = juju_offer.loki_logging.url
-}
diff --git a/releases/3.5/terraform/external/cos/providers.tf b/releases/3.5/terraform/external/cos/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.5/terraform/external/cos/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.5/terraform/external/cos/resources.tf b/releases/3.5/terraform/external/cos/resources.tf
deleted file mode 100644
index c1244915..00000000
--- a/releases/3.5/terraform/external/cos/resources.tf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "cos" {
-  name = var.model
-}
-
-resource "juju_offer" "prometheus_receive_remote_write" {
-  model            = data.juju_model.cos.name
-  application_name = juju_application.prometheus.name
-  endpoints        = ["receive-remote-write"]
-}
-
-resource "juju_offer" "grafana_dashboards" {
-  model            = data.juju_model.cos.name
-  application_name = juju_application.grafana.name
-  endpoints        = ["grafana-dashboard"]
-}
-
-resource "juju_offer" "loki_logging" {
-  model            = data.juju_model.cos.name
-  application_name = juju_application.loki.name
-  endpoints        = ["logging"]
-}
-
diff --git a/releases/3.5/terraform/external/cos/variables.tf b/releases/3.5/terraform/external/cos/variables.tf
deleted file mode 100644
index c702af41..00000000
--- a/releases/3.5/terraform/external/cos/variables.tf
+++ /dev/null
@@ -1,69 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define cos-specific variables
-
-variable "model" {
-  description = "The name of the Juju model to deploy to"
-  type        = string
-}
-
-variable "cos_user" {
-  description = "The name of the Juju user of the COS deployment."
-  type        = string
-  default     = "admin"
-}
-
-variable "cos_tls_cert" {
-  type        = string
-  default     = ""
-  description = "COS certificate"
-}
-
-variable "cos_tls_key" {
-  type        = string
-  default     = ""
-  description = "COS certificate key"
-}
-
-variable "cos_tls_ca" {
-  type        = string
-  default     = ""
-  description = "COS CA certificate"
-}
-
-variable "alertmanager_size" {
-  description = "Storage size for the alertmanager database"
-  type        = string
-  default     = "10G"
-}
-
-variable "grafana_size" {
-  description = "Storage size for the grafana database"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_active_index_directory_size" {
-  description = "Storage size for the active index directory for Loki"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_chunks_size" {
-  description = "Storage size for the Loki chucks storage"
-  type        = string
-  default     = "500G"
-}
-
-variable "prometheus_size" {
-  description = "Storage size for the Prometheus database"
-  type        = string
-  default     = "500G"
-}
-
-variable "traefik_size" {
-  description = "Storage size for the Traefik storage"
-  type        = string
-  default     = "10G"
-}
diff --git a/releases/3.5/terraform/external/tls/main.tf b/releases/3.5/terraform/external/tls/main.tf
deleted file mode 100644
index 843a22d8..00000000
--- a/releases/3.5/terraform/external/tls/main.tf
+++ /dev/null
@@ -1,58 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# This module is but a placeholder until we get an official TF module for the manual TLS operator
-# For testing purpose.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = ">=0.16.0"
-      source  = "juju/juju"
-    }
-  }
-}
-
-variable "model" {
-  type     = string
-  nullable = false
-}
-
-variable "channel" {
-  type    = string
-  default = "latest/stable"
-}
-
-variable "app_name" {
-  type     = string
-  default  = "certificates"
-  nullable = false
-}
-
-resource "juju_application" "certificates" {
-  name  = var.app_name
-  model = var.model
-
-  charm {
-    name    = "manual-tls-certificates"
-    channel = var.channel
-  }
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-output "app_name" {
-  description = "Name of the deployed application."
-  value       = juju_application.certificates.name
-}
-
-output "provides" {
-  value = {
-    certificates = "certificates"
-  }
-}
diff --git a/releases/3.5/terraform/main.tf b/releases/3.5/terraform/main.tf
deleted file mode 100644
index 5379903d..00000000
--- a/releases/3.5/terraform/main.tf
+++ /dev/null
@@ -1,137 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_model" "spark" {
-  count      = var.create_model == true ? 1 : 0
-  name       = var.model
-  credential = var.K8S_CREDENTIAL
-  cloud {
-    name = var.K8S_CLOUD
-  }
-}
-
-resource "juju_model" "cos" {
-  count      = var.cos.deployed == "bundled" && var.create_model == true ? 1 : 0
-  name       = var.cos.model
-  credential = var.K8S_CREDENTIAL
-  cloud {
-    name = var.K8S_CLOUD
-  }
-}
-
-module "ssc" {
-  depends_on  = [juju_model.spark]
-  source      = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev326"
-  model       = var.model
-  app_name    = "certificates"
-  channel     = "1/stable"
-  revision    = 317
-  constraints = "arch=amd64"
-  base        = "ubuntu@24.04"
-  units       = 1
-  config = {
-    ca-common-name = var.certificate_common_name
-  }
-}
-
-module "spark" {
-  depends_on                                 = [juju_model.spark, module.ssc]
-  source                                     = "./modules/spark"
-  model                                      = var.model
-  kyuubi_user                                = var.kyuubi_user
-  kyuubi_profile                             = var.kyuubi_profile
-  admin_password                             = var.admin_password
-  tls_private_key                            = var.tls_private_key
-  enable_dynamic_allocation                  = var.enable_dynamic_allocation
-  kyuubi_k8s_node_selectors                  = var.kyuubi_k8s_node_selectors
-  kyuubi_loadbalancer_extra_annotations      = var.kyuubi_loadbalancer_extra_annotations
-  kyuubi_gpu_enable                          = var.kyuubi_gpu_enable
-  kyuubi_gpu_engine_executors_limit          = var.kyuubi_gpu_engine_executors_limit
-  kyuubi_gpu_pinned_memory                   = var.kyuubi_gpu_pinned_memory
-  kyuubi_driver_pod_template                 = var.kyuubi_driver_pod_template
-  kyuubi_executor_pod_template               = var.kyuubi_executor_pod_template
-  kyuubi_executor_cores                      = var.kyuubi_executor_cores
-  kyuubi_executor_memory                     = var.kyuubi_executor_memory
-  driver_pod_template                        = var.driver_pod_template
-  executor_pod_template                      = var.executor_pod_template
-  integration_hub_monitored_service_accounts = var.integration_hub_monitored_service_accounts
-
-  tls_app_name              = module.ssc.app_name
-  tls_certificates_endpoint = module.ssc.provides.certificates
-  zookeeper_units           = var.zookeeper_units
-  kyuubi_units              = var.kyuubi_units
-
-  history_server_revision  = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server
-  history_server_image     = var.history_server_image != null ? var.history_server_image : local.images.history_server
-  integration_hub_revision = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub
-  integration_hub_image    = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub
-  kyuubi_revision          = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi
-  kyuubi_image             = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi
-  kyuubi_users_revision    = var.kyuubi_users_revision != null ? var.kyuubi_users_revision : local.revisions.kyuubi_users
-  kyuubi_users_image       = var.kyuubi_users_image != null ? var.kyuubi_users_image : local.images.kyuubi_users
-  metastore_revision       = var.metastore_revision != null ? var.metastore_revision : local.revisions.metastore
-  metastore_image          = var.metastore_image != null ? var.metastore_image : local.images.metastore
-  zookeeper_revision       = var.zookeeper_revision != null ? var.zookeeper_revision : local.revisions.zookeeper
-  zookeeper_image          = var.zookeeper_image != null ? var.zookeeper_image : local.images.zookeeper
-  data_integrator_revision = var.data_integrator_revision != null ? var.data_integrator_revision : local.revisions.data_integrator
-
-  kyuubi_users_size = var.kyuubi_users_size
-  metastore_size    = var.metastore_size
-  zookeeper_size    = var.zookeeper_size
-}
-
-module "azure_storage" {
-  depends_on    = [module.spark]
-  count         = var.storage_backend == "azure_storage" ? 1 : 0
-  source        = "./modules/azure-storage"
-  model         = var.model
-  spark_charms  = module.spark.charms
-  azure_storage = var.azure_storage
-
-  azure_storage_revision = var.azure_storage_revision != null ? var.azure_storage_revision : local.revisions.azure_storage
-}
-
-module "s3" {
-  depends_on   = [module.spark]
-  count        = var.storage_backend == "s3" ? 1 : 0
-  source       = "./modules/s3"
-  model        = var.model
-  spark_charms = module.spark.charms
-  s3           = var.s3
-
-  s3_revision = var.s3_revision != null ? var.s3_revision : local.revisions.s3
-}
-
-module "bundled_cos" {
-  depends_on   = [juju_model.cos]
-  count        = var.cos.deployed == "bundled" ? 1 : 0
-  source       = "./external/cos"
-  model        = var.cos.model
-  cos_tls_ca   = var.cos.tls.ca
-  cos_tls_cert = var.cos.tls.cert
-  cos_tls_key  = var.cos.tls.key
-
-  alertmanager_size                = var.alertmanager_size
-  grafana_size                     = var.grafana_size
-  loki_active_index_directory_size = var.loki_active_index_directory_size
-  loki_chunks_size                 = var.loki_chunks_size
-  prometheus_size                  = var.prometheus_size
-  traefik_size                     = var.traefik_size
-}
-
-
-module "observability" {
-  depends_on       = [module.spark, module.bundled_cos]
-  count            = var.cos.deployed == "no" ? 0 : 1
-  source           = "./modules/observability"
-  dashboards_offer = var.cos.deployed == "external" ? var.cos.offers.dashboard : one(module.bundled_cos[*].dashboards_offer)
-  logging_offer    = var.cos.deployed == "external" ? var.cos.offers.logging : one(module.bundled_cos[*].logging_offer)
-  metrics_offer    = var.cos.deployed == "external" ? var.cos.offers.metrics : one(module.bundled_cos[*].metrics_offer)
-  spark_model      = var.model
-  spark_charms     = module.spark.charms
-
-  grafana_agent_revision     = var.grafana_agent_revision != null ? var.grafana_agent_revision : local.revisions.grafana_agent
-  cos_configuration_revision = var.cos_configuration_revision != null ? var.cos_configuration_revision : local.revisions.cos_configuration
-  pushgateway_revision       = var.pushgateway_revision != null ? var.pushgateway_revision : local.revisions.pushgateway
-  scrape_config_revision     = var.scrape_config_revision != null ? var.scrape_config_revision : local.revisions.scrape_config
-}
diff --git a/releases/3.5/terraform/modules/azure-storage/applications.tf b/releases/3.5/terraform/modules/azure-storage/applications.tf
deleted file mode 100644
index 642df05d..00000000
--- a/releases/3.5/terraform/modules/azure-storage/applications.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "azure_storage" {
-  name  = "azure-storage"
-  model = var.model
-  charm {
-    name     = "azure-storage-integrator"
-    channel  = "latest/edge"
-    revision = var.azure_storage_revision
-  }
-  config = {
-    container           = var.azure_storage.container
-    storage-account     = var.azure_storage.storage_account
-    path                = var.azure_storage.path
-    connection-protocol = var.azure_storage.protocol
-    credentials         = "secret:${juju_secret.azure_blob_storage_secret.secret_id}"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/3.5/terraform/modules/azure-storage/integrations.tf b/releases/3.5/terraform/modules/azure-storage/integrations.tf
deleted file mode 100644
index 0611ee3c..00000000
--- a/releases/3.5/terraform/modules/azure-storage/integrations.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "history_server_azure_storage" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.azure_storage.name
-    endpoint = "azure-storage-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "azure-storage-credentials"
-  }
-}
-
-resource "juju_integration" "azure_storage_integration_hub" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.azure_storage.name
-    endpoint = "azure-storage-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "azure-storage-credentials"
-  }
-}
diff --git a/releases/3.5/terraform/modules/azure-storage/outputs.tf b/releases/3.5/terraform/modules/azure-storage/outputs.tf
deleted file mode 100644
index beff36af..00000000
--- a/releases/3.5/terraform/modules/azure-storage/outputs.tf
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    azure_storage = juju_application.azure_storage.name
-  }
-}
diff --git a/releases/3.5/terraform/modules/azure-storage/providers.tf b/releases/3.5/terraform/modules/azure-storage/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.5/terraform/modules/azure-storage/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.5/terraform/modules/azure-storage/resources.tf b/releases/3.5/terraform/modules/azure-storage/resources.tf
deleted file mode 100644
index ee5b10b5..00000000
--- a/releases/3.5/terraform/modules/azure-storage/resources.tf
+++ /dev/null
@@ -1,26 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "spark" {
-  name = var.model
-}
-
-resource "juju_secret" "azure_blob_storage_secret" {
-  model = var.model
-  name  = "azure_blob_storage_secret"
-  value = {
-    secret-key = var.azure_storage.secret_key
-  }
-  info = "This is the secret key for the Azure storage account"
-}
-
-resource "juju_access_secret" "azure_blob_storage_secret_access" {
-  model = var.model
-  applications = [
-    juju_application.azure_storage.name
-  ]
-  # Use the secret_id from your secret resource or data source.
-  secret_id = juju_secret.azure_blob_storage_secret.secret_id
-}
diff --git a/releases/3.5/terraform/modules/azure-storage/variables.tf b/releases/3.5/terraform/modules/azure-storage/variables.tf
deleted file mode 100644
index 8383e474..00000000
--- a/releases/3.5/terraform/modules/azure-storage/variables.tf
+++ /dev/null
@@ -1,31 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-variable "model" {
-  description = "Name of the Spark Juju Model to deploy to"
-  type        = string
-  nullable    = false
-}
-
-variable "azure_storage" {
-  description = "Azure Object storage information"
-  type = object({
-    container       = optional(string, "azurecontainer")
-    storage_account = optional(string, "azurestorageaccount")
-    protocol        = optional(string, "abfss")
-    secret_key      = optional(string, "secret-key")
-    path            = optional(string, "spark-events")
-  })
-  default = {}
-}
-
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
-}
-
-variable "azure_storage_revision" {
-  description = "Charm revision for azure-storage-integrator"
-  type        = number
-  nullable    = false
-}
diff --git a/releases/3.5/terraform/modules/observability/applications.tf b/releases/3.5/terraform/modules/observability/applications.tf
deleted file mode 100644
index e7778be6..00000000
--- a/releases/3.5/terraform/modules/observability/applications.tf
+++ /dev/null
@@ -1,64 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "grafana_agent" {
-  name  = "grafana-agent"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "grafana-agent-k8s"
-    channel  = "1/stable"
-    revision = var.grafana_agent_revision
-  }
-  units       = 1
-  trust       = true
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "cos_configuration" {
-  name  = "cos-configuration"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "cos-configuration-k8s"
-    channel  = "1/stable"
-    revision = var.cos_configuration_revision
-  }
-  config = {
-    git_branch              = "main"
-    git_depth               = "1"
-    git_repo                = var.config_repo
-    grafana_dashboards_path = var.config_grafana_path
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "pushgateway" {
-  name  = "pushgateway"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "prometheus-pushgateway-k8s"
-    channel  = "1/stable"
-    revision = var.pushgateway_revision
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    pushgateway-store = var.pushgateway_size
-  }
-}
-
-resource "juju_application" "scrape_config" {
-  name  = "scrape-config"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "prometheus-scrape-config-k8s"
-    channel  = "1/stable"
-    revision = var.scrape_config_revision
-  }
-  config = {
-    scrape_interval = "10s"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
-
diff --git a/releases/3.5/terraform/modules/observability/integrations.tf b/releases/3.5/terraform/modules/observability/integrations.tf
deleted file mode 100644
index 0101cf88..00000000
--- a/releases/3.5/terraform/modules/observability/integrations.tf
+++ /dev/null
@@ -1,236 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-data "juju_offer" "grafana_dashboards" {
-  url = var.dashboards_offer
-}
-
-data "juju_offer" "prometheus_receive_remote_write" {
-  url = var.metrics_offer
-}
-
-data "juju_offer" "loki_logging" {
-  url = var.logging_offer
-}
-
-resource "juju_integration" "cos_configuration_grafana_agent" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.cos_configuration.name
-    endpoint = "grafana-dashboards"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "pushgateway_scrape_config" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.pushgateway.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.scrape_config.name
-    endpoint = "configurable-scrape-jobs"
-  }
-}
-
-resource "juju_integration" "scrape_config_grafana_agent" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.scrape_config.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "grafana_agent_grafana_dashboards" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-provider"
-  }
-
-  application {
-    offer_url = data.juju_offer.grafana_dashboards.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-
-resource "juju_integration" "grafana_agent_prometheus" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "send-remote-write"
-  }
-
-  application {
-    offer_url = data.juju_offer.prometheus_receive_remote_write.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-
-resource "juju_integration" "grafana_agent_loki" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-consumer"
-  }
-
-  application {
-    offer_url = data.juju_offer.loki_logging.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-resource "juju_integration" "pushgateway_integration_hub" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.pushgateway.name
-    endpoint = "push-endpoint"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "cos"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_dashboard" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_logging" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_metrics" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_metrics" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_dashboards" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_logging" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
-
-resource "juju_integration" "integration_hub_grafana_agent_logging" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
diff --git a/releases/3.5/terraform/modules/observability/outputs.tf b/releases/3.5/terraform/modules/observability/outputs.tf
deleted file mode 100644
index 603d65b2..00000000
--- a/releases/3.5/terraform/modules/observability/outputs.tf
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    grafana_agent     = juju_application.grafana_agent.name
-    cos_configuration = juju_application.cos_configuration.name
-    pushgateway       = juju_application.pushgateway.name
-    scrape_config     = juju_application.scrape_config.name
-  }
-}
diff --git a/releases/3.5/terraform/modules/observability/providers.tf b/releases/3.5/terraform/modules/observability/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.5/terraform/modules/observability/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.5/terraform/modules/observability/resources.tf b/releases/3.5/terraform/modules/observability/resources.tf
deleted file mode 100644
index ec198a06..00000000
--- a/releases/3.5/terraform/modules/observability/resources.tf
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "spark" {
-  name = var.spark_model
-}
diff --git a/releases/3.5/terraform/modules/observability/variables.tf b/releases/3.5/terraform/modules/observability/variables.tf
deleted file mode 100644
index 19a2d92d..00000000
--- a/releases/3.5/terraform/modules/observability/variables.tf
+++ /dev/null
@@ -1,75 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-variable "config_repo" {
-  description = "COS configuration repo URL."
-  type        = string
-  default     = "https://github.com/canonical/spark-k8s-bundle"
-  nullable    = false
-}
-
-variable "config_grafana_path" {
-  description = "Grafana dashboard path from configuration repo root."
-  type        = string
-  default     = "releases/3.4/resources/grafana/"
-  nullable    = false
-}
-
-variable "spark_model" {
-  description = "Name of the Spark Juju model."
-  type        = string
-  nullable    = false
-}
-
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
-}
-
-variable "dashboards_offer" {
-  description = "URL of the `grafana_dashboard` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "metrics_offer" {
-  description = "URL of the `prometheus_remote_write` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "logging_offer" {
-  description = "URL of the `loki_push_api` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "grafana_agent_revision" {
-  description = "Charm revision for grafana-agent-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "cos_configuration_revision" {
-  description = "Charm revision for cos-configuration-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "pushgateway_revision" {
-  description = "Charm revision for prometheus-pushgateway-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "scrape_config_revision" {
-  description = "Charm revision for prometheus-scrape-config-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "pushgateway_size" {
-  description = "Storage size for the pushgateway database"
-  type        = string
-  default     = "10G"
-}
diff --git a/releases/3.5/terraform/modules/s3/applications.tf b/releases/3.5/terraform/modules/s3/applications.tf
deleted file mode 100644
index 44f1aa25..00000000
--- a/releases/3.5/terraform/modules/s3/applications.tf
+++ /dev/null
@@ -1,20 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "s3" {
-  name  = "s3"
-  model = data.juju_model.spark.name
-  charm {
-    name     = "s3-integrator"
-    channel  = "1/stable"
-    revision = var.s3_revision
-  }
-  config = {
-    path     = "spark-events"
-    bucket   = var.s3.bucket
-    endpoint = var.s3.endpoint
-    region   = var.s3.region
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/3.5/terraform/modules/s3/integrations.tf b/releases/3.5/terraform/modules/s3/integrations.tf
deleted file mode 100644
index cd520fc7..00000000
--- a/releases/3.5/terraform/modules/s3/integrations.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "s3_history_server" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.s3.name
-    endpoint = "s3-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "s3-credentials"
-  }
-}
-
-resource "juju_integration" "s3_integration_hub" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.s3.name
-    endpoint = "s3-credentials"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "s3-credentials"
-  }
-}
diff --git a/releases/3.5/terraform/modules/s3/outputs.tf b/releases/3.5/terraform/modules/s3/outputs.tf
deleted file mode 100644
index 77aa54aa..00000000
--- a/releases/3.5/terraform/modules/s3/outputs.tf
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    s3 = juju_application.s3.name
-  }
-}
diff --git a/releases/3.5/terraform/modules/s3/providers.tf b/releases/3.5/terraform/modules/s3/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.5/terraform/modules/s3/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.5/terraform/modules/s3/resources.tf b/releases/3.5/terraform/modules/s3/resources.tf
deleted file mode 100644
index 2e2dfbde..00000000
--- a/releases/3.5/terraform/modules/s3/resources.tf
+++ /dev/null
@@ -1,8 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "spark" {
-  name = var.model
-}
diff --git a/releases/3.5/terraform/modules/s3/variables.tf b/releases/3.5/terraform/modules/s3/variables.tf
deleted file mode 100644
index 89b34681..00000000
--- a/releases/3.5/terraform/modules/s3/variables.tf
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-variable "model" {
-  description = "Name of the Spark Juju Model to deploy to"
-  type        = string
-  nullable    = false
-}
-
-variable "s3" {
-  description = "S3 Bucket information"
-  type = object({
-    bucket   = optional(string, "spark-bucket")
-    endpoint = optional(string, "https://s3.amazonaws.com")
-    region   = optional(string, "us-east-1")
-  })
-  default = {
-    bucket   = "spark-bucket"
-    endpoint = "https://s3.amazonaws.com"
-    region   = "us-east-1"
-  }
-}
-
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
-}
-
-variable "s3_revision" {
-  description = "Charm revision for s3-integrator"
-  type        = number
-  nullable    = false
-}
-
diff --git a/releases/3.5/terraform/modules/spark/applications.tf b/releases/3.5/terraform/modules/spark/applications.tf
deleted file mode 100644
index 445cb2f6..00000000
--- a/releases/3.5/terraform/modules/spark/applications.tf
+++ /dev/null
@@ -1,190 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "history_server" {
-  name  = "history-server"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "spark-history-server-k8s"
-    channel  = "3/stable"
-    revision = var.history_server_revision
-  }
-
-  resources = var.history_server_image
-  units     = 1
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "kyuubi" {
-  name  = "kyuubi"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "kyuubi-k8s"
-    channel  = "3.5/stable"
-    revision = var.kyuubi_revision
-  }
-
-  resources = var.kyuubi_image
-
-  config = merge(
-    {
-      enable-dynamic-allocation  = var.enable_dynamic_allocation
-      expose-external            = "loadbalancer"
-      gpu-enable                 = var.kyuubi_gpu_enable
-      gpu-engine-executors-limit = var.kyuubi_gpu_engine_executors_limit
-      gpu-pinned-memory          = var.kyuubi_gpu_pinned_memory
-      namespace                  = data.juju_model.spark.name
-      profile                    = var.kyuubi_profile
-      service-account            = var.kyuubi_user
-    },
-    var.kyuubi_k8s_node_selectors == null ? {} : {
-      k8s-node-selectors = var.kyuubi_k8s_node_selectors
-    },
-    var.kyuubi_loadbalancer_extra_annotations == null ? {} : {
-      loadbalancer-extra-annotations = var.kyuubi_loadbalancer_extra_annotations
-    },
-    var.tls_private_key == null ? {} : {
-      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    },
-    var.admin_password == null ? {} : {
-      system-users = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    },
-    var.kyuubi_executor_cores == null ? {} : {
-      executor-cores = var.kyuubi_executor_cores
-    },
-    var.kyuubi_executor_memory == null ? {} : {
-      executor-memory = var.kyuubi_executor_memory
-    },
-    var.kyuubi_driver_pod_template == null ? {} : {
-      driver-pod-template = var.kyuubi_driver_pod_template
-    },
-    var.kyuubi_executor_pod_template == null ? {} : {
-      executor-pod-template = var.kyuubi_executor_pod_template
-    }
-  )
-
-  units = var.kyuubi_units
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "kyuubi_users" {
-  name  = "kyuubi-users"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "postgresql-k8s"
-    channel  = "14/stable"
-    revision = var.kyuubi_users_revision
-  }
-
-  storage_directives = {
-    pgdata = var.kyuubi_users_size
-  }
-
-  resources = var.kyuubi_users_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "metastore" {
-  name  = "metastore"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "postgresql-k8s"
-    channel  = "14/stable"
-    revision = var.metastore_revision
-  }
-
-  storage_directives = {
-    pgdata = var.kyuubi_users_size
-  }
-
-
-  resources = var.metastore_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "integration_hub" {
-  name  = "integration-hub"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "spark-integration-hub-k8s"
-    channel  = "3/stable"
-    revision = var.integration_hub_revision
-  }
-
-  config = merge(
-    {
-      enable-dynamic-allocation = var.enable_dynamic_allocation
-    },
-    var.driver_pod_template == null ? {} : {
-      driver-pod-template = var.driver_pod_template
-    },
-    var.executor_pod_template == null ? {} : {
-      executor-pod-template = var.executor_pod_template
-    },
-    var.integration_hub_monitored_service_accounts == null ? {} : {
-      monitored-service-accounts = var.integration_hub_monitored_service_accounts
-    }
-  )
-
-  resources = var.integration_hub_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "zookeeper" {
-  name  = "zookeeper"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "zookeeper-k8s"
-    channel  = "3/stable"
-    revision = var.zookeeper_revision
-  }
-
-  storage_directives = {
-    zookeeper = var.zookeeper_size
-  }
-
-
-  resources = var.zookeeper_image
-
-  units       = var.zookeeper_units
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "data_integrator" {
-  name  = "data-integrator"
-  model = data.juju_model.spark.name
-
-  charm {
-    name     = "data-integrator"
-    channel  = "latest/stable"
-    revision = var.data_integrator_revision
-  }
-
-  config = {
-    database-name = "integrator"
-  }
-
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/3.5/terraform/modules/spark/integrations.tf b/releases/3.5/terraform/modules/spark/integrations.tf
deleted file mode 100644
index 37f1e566..00000000
--- a/releases/3.5/terraform/modules/spark/integrations.tf
+++ /dev/null
@@ -1,86 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "kyuubi_metastore" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.metastore.name
-    endpoint = "database"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "metastore-db"
-  }
-}
-
-resource "juju_integration" "kyuubi_users" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.kyuubi_users.name
-    endpoint = "database"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "auth-db"
-  }
-}
-
-resource "juju_integration" "kyuubi_service_account" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "spark-service-account"
-  }
-
-  application {
-    name     = juju_application.integration_hub.name
-    endpoint = "spark-service-account"
-  }
-}
-
-resource "juju_integration" "kyuubi_zookeeper" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.zookeeper.name
-    endpoint = "zookeeper"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "zookeeper"
-  }
-}
-
-resource "juju_integration" "kyuubi_tls" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "certificates"
-  }
-
-  application {
-    name     = var.tls_app_name
-    endpoint = var.tls_certificates_endpoint
-  }
-}
-
-resource "juju_integration" "kyuubi_data_integrator" {
-  model = data.juju_model.spark.name
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "jdbc"
-  }
-
-  application {
-    name     = juju_application.data_integrator.name
-    endpoint = "kyuubi"
-  }
-}
diff --git a/releases/3.5/terraform/modules/spark/offers.tf b/releases/3.5/terraform/modules/spark/offers.tf
deleted file mode 100644
index 40f06528..00000000
--- a/releases/3.5/terraform/modules/spark/offers.tf
+++ /dev/null
@@ -1,11 +0,0 @@
-resource "juju_offer" "integration_hub" {
-  model            = data.juju_model.spark.name
-  application_name = juju_application.integration_hub.name
-  endpoints        = ["spark-service-account"]
-}
-
-resource "juju_offer" "metastore" {
-  model            = data.juju_model.spark.name
-  application_name = juju_application.metastore.name
-  endpoints        = ["database"]
-}
\ No newline at end of file
diff --git a/releases/3.5/terraform/modules/spark/outputs.tf b/releases/3.5/terraform/modules/spark/outputs.tf
deleted file mode 100644
index 010f6461..00000000
--- a/releases/3.5/terraform/modules/spark/outputs.tf
+++ /dev/null
@@ -1,22 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    history_server  = juju_application.history_server.name
-    kyuubi          = juju_application.kyuubi.name
-    kyuubi_users    = juju_application.kyuubi_users.name
-    metastore       = juju_application.metastore.name
-    integration_hub = juju_application.integration_hub.name
-    zookeeper       = juju_application.zookeeper.name
-    certificates    = var.tls_app_name
-  }
-}
-
-output "offers" {
-  value = {
-    hub_service_account = juju_offer.integration_hub
-    metastore_database  = juju_offer.metastore
-  }
-}
diff --git a/releases/3.5/terraform/modules/spark/providers.tf b/releases/3.5/terraform/modules/spark/providers.tf
deleted file mode 100644
index d94b3f97..00000000
--- a/releases/3.5/terraform/modules/spark/providers.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.7.3"
-
-  required_providers {
-    juju = {
-      version = "<1.0.0"
-      source  = "juju/juju"
-    }
-  }
-}
diff --git a/releases/3.5/terraform/modules/spark/resources.tf b/releases/3.5/terraform/modules/spark/resources.tf
deleted file mode 100644
index 8b73939b..00000000
--- a/releases/3.5/terraform/modules/spark/resources.tf
+++ /dev/null
@@ -1,32 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "spark" {
-  name = var.model
-}
-
-resource "juju_secret" "system_users_and_private_key_secret" {
-  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model = var.model
-  name  = "system_users_and_private_key_secret"
-  value = merge(
-    var.admin_password == null ? {} : {
-      admin = var.admin_password
-    },
-    var.tls_private_key == null ? {} : {
-      private-key = var.tls_private_key
-    }
-  )
-  info = "This secret contains password for admin user and the TLS private key."
-}
-
-resource "juju_access_secret" "system_users_and_private_key_secret_access" {
-  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model = var.model
-  applications = [
-    juju_application.kyuubi.name
-  ]
-  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
-}
diff --git a/releases/3.5/terraform/modules/spark/variables.tf b/releases/3.5/terraform/modules/spark/variables.tf
deleted file mode 100644
index 6e442a55..00000000
--- a/releases/3.5/terraform/modules/spark/variables.tf
+++ /dev/null
@@ -1,241 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define spark-specific variables
-
-variable "model" {
-  description = "Name of the Juju Model to deploy to."
-  type        = string
-  nullable    = false
-}
-
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "tls_private_key" {
-  description = "The private key to be used for TLS certificates."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "kyuubi_profile" {
-  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-  type        = string
-  nullable    = false
-  default     = "production"
-}
-
-variable "kyuubi_user" {
-  description = "User name to be used for running Kyuubi engines."
-  type        = string
-  nullable    = false
-  default     = "kyuubi-spark-engine"
-}
-
-variable "enable_dynamic_allocation" {
-  description = "Enable dynamic allocation of pods for Spark jobs."
-  type        = bool
-  default     = false
-}
-
-variable "kyuubi_k8s_node_selectors" {
-  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_loadbalancer_extra_annotations" {
-  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_gpu_enable" {
-  description = "Enable GPU acceleration for SparkSQLEngine."
-  type        = bool
-  nullable    = false
-}
-
-variable "kyuubi_gpu_engine_executors_limit" {
-  description = "Limit the number of GPUs an engine can schedule executor pods on."
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_gpu_pinned_memory" {
-  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_cores" {
-  description = "Set kyuubi executor pods cpu cores."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_memory" {
-  description = "Set kyuubi executor pods memory (in GB)."
-  type        = number
-  default     = null
-}
-
-variable "driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "integration_hub_monitored_service_accounts" {
-  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-  type        = string
-  default     = null
-}
-
-variable "zookeeper_units" {
-  description = "Number of zookeeper units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "tls_app_name" {
-  description = "Name of the application providing the `tls-certificates` interface."
-  type        = string
-  nullable    = false
-}
-
-variable "tls_certificates_endpoint" {
-  description = "Name of the endpoint providing the `tls-certificates` interface."
-  type        = string
-  nullable    = false
-}
-
-variable "history_server_revision" {
-  description = "Charm revision for spark-history-server-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "history_server_image" {
-  description = "Image for spark-history-server-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "integration_hub_revision" {
-  description = "Charm revision for spark-integration-hub-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_units" {
-  description = "Number of Kyuubi units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "integration_hub_image" {
-  description = "Image for spark-integration-hub-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_revision" {
-  description = "Charm revision for kyuubi-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_image" {
-  description = "Image for kyuubi-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_users_revision" {
-  description = "Charm revision for postgresql-k8s (auth-db)"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_users_image" {
-  description = "Image for postgresql-k8s (auth-db)"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_users_size" {
-  description = "Storage size for the Kyuubi users database"
-  type        = string
-  default     = "1G"
-}
-
-
-variable "metastore_revision" {
-  description = "Charm revision for postgresql-k8s (metastore)"
-  type        = number
-  nullable    = false
-}
-
-variable "metastore_image" {
-  description = "Image for postgresql-k8s (metastore)"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "metastore_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-
-variable "zookeeper_revision" {
-  description = "Charm revision for zookeeper-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "zookeeper_image" {
-  description = "Image for zookeeper-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "zookeeper_size" {
-  description = "Storage size for the zookeeper unit"
-  type        = string
-  default     = "10G"
-}
-
-
-variable "data_integrator_revision" {
-  description = "Charm revision for data-integrator"
-  type        = number
-  nullable    = false
-}
diff --git a/releases/3.5/terraform/outputs.tf b/releases/3.5/terraform/outputs.tf
deleted file mode 100644
index 2febb798..00000000
--- a/releases/3.5/terraform/outputs.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = merge(
-    concat(
-      module.spark[*].charms, module.azure_storage[*].charms, module.s3[*].charms, module.bundled_cos[*].charms, module.observability[*].charms
-    )...
-  )
-}
-
-output "offers" {
-  description = "The name and url of the various offers being exposed"
-  value = merge(
-    concat(
-      module.spark[*].offers
-    )...
-  )
-}
-
diff --git a/releases/3.5/terraform/variables.tf b/releases/3.5/terraform/variables.tf
deleted file mode 100644
index ccd64031..00000000
--- a/releases/3.5/terraform/variables.tf
+++ /dev/null
@@ -1,396 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Juju
-
-variable "K8S_CLOUD" {
-  type        = string
-  description = "The kubernetes juju cloud name."
-  default     = "microk8s"
-}
-
-variable "K8S_CREDENTIAL" {
-  type        = string
-  description = "The name of the kubernetes juju credential."
-  default     = "microk8s"
-}
-
-# Models
-
-variable "model" {
-  description = "The name of the juju model to deploy Spark to"
-  type        = string
-  default     = "spark"
-}
-
-variable "create_model" {
-  description = "Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism."
-  type        = bool
-  default     = true
-  nullable    = false
-}
-
-# cos specifics
-
-variable "cos" {
-  description = "Observability settings"
-  type = object({
-    model    = optional(string, "cos")
-    deployed = optional(string, "bundled")
-    offers = optional(object({
-      dashboard = optional(string, null),
-      metrics   = optional(string, null),
-      logging   = optional(string, null)
-    }), {}),
-    tls = optional(object({
-      cert = optional(string, "")
-      key  = optional(string, "")
-      ca   = optional(string, "")
-    }), {})
-  })
-  default = { model = "cos", deployed = "bundled", offers = {}, tls = {} }
-
-  validation {
-    condition     = contains(["external", "bundled", "no"], var.cos.deployed)
-    error_message = "Valid values for var: cos.deployed are (external, bundled, no)"
-  }
-
-  validation {
-    condition = var.cos.deployed != "external" || alltrue([
-      var.cos.offers.dashboard != null,
-      var.cos.offers.metrics != null,
-      var.cos.offers.logging != null,
-    ])
-    error_message = "When using external cos, please define all offers variables"
-  }
-
-}
-
-# Storage
-
-variable "storage_backend" {
-  type        = string
-  description = "Storage backend to be used"
-
-  validation {
-    condition     = contains(["azure_storage", "s3"], var.storage_backend)
-    error_message = "Valid values for var: test_variable are (s3, azure_storage)."
-  }
-
-  default = "s3"
-}
-
-variable "s3" {
-  description = "S3 Bucket information"
-  type = object({
-    bucket   = optional(string, "spark-test")
-    endpoint = optional(string, "https://s3.amazonaws.com")
-    region   = optional(string, "us-east-1")
-  })
-  default = {}
-}
-
-variable "azure_storage" {
-  description = "Azure Object storage information"
-  type = object({
-    container       = optional(string, "azurecontainer")
-    storage_account = optional(string, "azurestorageaccount")
-    secret_key      = optional(string, "azurestoragesecret")
-    protocol        = optional(string, "abfss")
-  })
-  default = {}
-}
-
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "tls_private_key" {
-  description = "The file path of the private key to use for TLS certificates."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "kyuubi_profile" {
-  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-  type        = string
-  nullable    = false
-  default     = "production"
-}
-
-variable "kyuubi_user" {
-  description = "Define the user to be used for running Kyuubi enginers"
-  type        = string
-  default     = "kyuubi-spark-engine"
-}
-
-variable "enable_dynamic_allocation" {
-  description = "Enable dynamic allocation of pods for Spark jobs."
-  type        = bool
-  default     = false
-}
-
-variable "kyuubi_k8s_node_selectors" {
-  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_loadbalancer_extra_annotations" {
-  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_gpu_enable" {
-  description = "Enable GPU acceleration for SparkSQLEngine."
-  type        = bool
-  default     = false
-}
-
-variable "kyuubi_gpu_engine_executors_limit" {
-  description = "Limit the number of GPUs an engine can schedule executor pods on."
-  type        = number
-  default     = 1
-}
-
-variable "kyuubi_gpu_pinned_memory" {
-  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-  type        = number
-  default     = 1
-}
-
-variable "kyuubi_driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_cores" {
-  description = "Set kyuubi executor pods cpu cores."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_memory" {
-  description = "Set kyuubi executor pods memory (in GB)."
-  type        = number
-  default     = null
-}
-
-variable "driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "integration_hub_monitored_service_accounts" {
-  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-  type        = string
-  default     = null
-}
-
-variable "zookeeper_units" {
-  description = "Define the number of zookeeper units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "history_server_revision" {
-  description = "Charm revision for spark-history-server-k8s"
-  type        = number
-  default     = null
-}
-
-variable "history_server_image" {
-  description = "Image for spark-history-server-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "integration_hub_revision" {
-  description = "Charm revision for spark-integration-hub-k8s"
-  type        = number
-  default     = null
-}
-
-variable "integration_hub_image" {
-  description = "Image for spark-integration-hub-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "kyuubi_revision" {
-  description = "Charm revision for kyuubi-k8s"
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_image" {
-  description = "Image for kyuubi-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "kyuubi_units" {
-  description = "Number of Kyuubi units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "kyuubi_users_revision" {
-  description = "Charm revision for postgresql-k8s (auth-db)"
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_users_image" {
-  description = "Image for postgresql-k8s (auth-db)"
-  type        = map(string)
-  default     = null
-}
-
-variable "metastore_revision" {
-  description = "Charm revision for postgresql-k8s (metastore)"
-  type        = number
-  default     = null
-}
-
-variable "metastore_image" {
-  description = "Image for postgresql-k8s (metastore)"
-  type        = map(string)
-  default     = null
-}
-
-variable "zookeeper_revision" {
-  description = "Charm revision for zookeeper-k8s"
-  type        = number
-  default     = null
-}
-
-variable "zookeeper_image" {
-  description = "Image for zookeeper-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "data_integrator_revision" {
-  description = "Charm revision for data-integrator"
-  type        = number
-  default     = null
-}
-
-variable "s3_revision" {
-  description = "Charm revision for s3-integrator"
-  type        = number
-  default     = null
-}
-
-variable "azure_storage_revision" {
-  description = "Charm revision for azure-storage-integrator"
-  type        = number
-  default     = null
-}
-
-variable "grafana_agent_revision" {
-  description = "Charm revision for grafana-agent-k8s"
-  type        = number
-  default     = null
-}
-
-variable "cos_configuration_revision" {
-  description = "Charm revision for cos-configuration-k8s"
-  type        = number
-  default     = null
-}
-
-variable "pushgateway_revision" {
-  description = "Charm revision for prometheus-pushgateway-k8s"
-  type        = number
-  default     = null
-}
-
-variable "scrape_config_revision" {
-  description = "Charm revision for prometheus-scrape-config-k8s"
-  type        = number
-  default     = null
-}
-
-variable "certificate_common_name" {
-  description = "Common name for the certificate to be used in self-signed"
-  type        = string
-  default     = "charmed-spark"
-}
-
-variable "kyuubi_users_size" {
-  description = "Storage size for the Kyuubi users database"
-  type        = string
-  default     = "1G"
-}
-
-variable "metastore_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-variable "zookeeper_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-
-variable "alertmanager_size" {
-  description = "Storage size for the alertmanager database"
-  type        = string
-  default     = "10G"
-}
-
-variable "grafana_size" {
-  description = "Storage size for the grafana database"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_active_index_directory_size" {
-  description = "Storage size for the active index directory for Loki"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_chunks_size" {
-  description = "Storage size for the Loki chucks storage"
-  type        = string
-  default     = "500G"
-}
-
-variable "prometheus_size" {
-  description = "Storage size for the Prometheus database"
-  type        = string
-  default     = "500G"
-}
-
-variable "traefik_size" {
-  description = "Storage size for the Traefik storage"
-  type        = string
-  default     = "10G"
-}
diff --git a/releases/3.5/terraform/versions.tf b/releases/3.5/terraform/versions.tf
deleted file mode 100644
index d0e50a26..00000000
--- a/releases/3.5/terraform/versions.tf
+++ /dev/null
@@ -1,53 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_providers {
-    juju = {
-      source  = "juju/juju"
-      version = ">=0.20.0"
-    }
-  }
-}
-
-locals {
-  revisions = {
-    history_server  = 96
-    integration_hub = 119
-    kyuubi          = 146
-    kyuubi_users    = 495
-    metastore       = 495
-    zookeeper       = 78
-    data_integrator = 362
-    s3              = 330
-    azure_storage   = 270
-    grafana_agent   = 121
-    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
-    cos_configuration = 65
-    pushgateway       = 27
-    scrape_config     = 67
-  }
-  images = {
-    history_server = {
-      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70"
-    } # rev24,  spark-version: 3.5.7, release date 16/03/2026 
-    integration_hub = {
-      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134"
-    } # rev13, release date 19/03/2026
-    kyuubi = {
-      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:21beea406001d161538b552cef2b47b86423bd899139ada263c1ef665fd89fff"
-    } # rev18, spark-version: 3.5.7, kyuubi-version: 1.10.3, release date 16/03/2026
-    kyuubi_users = {
-      postgresql-image = 165
-    }
-    metastore = {
-      postgresql-image = 165
-    }
-    zookeeper = {
-      zookeeper-image = 34
-    }
-  }
-}
-
diff --git a/releases/3.5/yaml/README.md b/releases/3.5/yaml/README.md
deleted file mode 100644
index 5601fa6d..00000000
--- a/releases/3.5/yaml/README.md
+++ /dev/null
@@ -1,5 +0,0 @@
-# Charmed Apache Spark Kubernetes Bundle
-
-## A fast and fault-tolerant, real-time event streaming platform!
-
-...
\ No newline at end of file
diff --git a/releases/3.5/yaml/bundle-azure-storage.yaml.j2 b/releases/3.5/yaml/bundle-azure-storage.yaml.j2
deleted file mode 100644
index 956ea452..00000000
--- a/releases/3.5/yaml/bundle-azure-storage.yaml.j2
+++ /dev/null
@@ -1,112 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-bundle: kubernetes
-applications:
-  kyuubi:
-    charm: kyuubi-k8s
-    channel: 3.5/stable
-    revision: 142
-    resources:
-      # rev17, spark-3.5.7, kyuubi 1.10.3 release date 2026-02-07
-      kyuubi-image: ghcr.io/canonical/charmed-spark-kyuubi@sha256:cdde52a4f72112ea09de11c6d21f901323619cf41015db89cb9d5742c57da303
-    scale: 3
-    options:
-      namespace: {{ namespace }}
-      service-account: {{ service_account }}
-      expose-external: loadbalancer
-      profile: {{ kyuubi_profile }}
-    constraints: arch=amd64
-    trust: true
-  kyuubi-users:
-    charm: postgresql-k8s
-    channel: 14/stable
-    revision: 495
-    resources:
-      postgresql-image: 165
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pgdata: kubernetes,1,1024M
-    trust: true
-  zookeeper:
-    charm: zookeeper-k8s
-    channel: 3/stable
-    revision: 78
-    scale: {{ zookeeper_units or 3}}
-    constraints: arch=amd64
-    resources:
-      zookeeper-image: 34
-  metastore:
-    charm: postgresql-k8s
-    channel: 14/stable
-    revision: 495
-    resources:
-      postgresql-image: 165
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pgdata: kubernetes,1,1024M
-    trust: true
-  azure-storage:
-    charm: azure-storage-integrator
-    channel: latest/edge
-    revision: 15
-    scale: 1
-    constraints: arch=amd64
-    options:
-      container: {{ container }}
-      storage-account: {{ storage_account }}
-      path: spark-events
-      connection-protocol: abfss
-  history-server:
-    charm: spark-history-server-k8s
-    channel: 3.5/stable
-    revision: 94
-    resources:
-      # rev23,  spark-version: 3.5.7, release date 2026-02-27 
-      spark-history-server-image: ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706
-    scale: 1
-    constraints: arch=amd64
-  integration-hub:
-    charm: spark-integration-hub-k8s
-    channel: 3/stable
-    revision: 113
-    resources:
-      # rev 11, release date 2026-03-02
-      integration-hub-image: ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa
-    scale: 1
-    constraints: arch=amd64
-    trust: true
-  certificates:
-    charm: self-signed-certificates
-    channel: 1/stable
-    revision: 317
-    scale: 1
-    constraints: arch=amd64
-    trust: true
-  data-integrator:
-    charm: data-integrator
-    channel: latest/stable
-    revision: 181
-    scale: 1
-    constraints: arch=amd64
-    options:
-      database-name: integrator
-relations:
-- - integration-hub:azure-storage-credentials
-  - azure-storage:azure-storage-credentials
-- - history-server:azure-storage-credentials
-  - azure-storage:azure-storage-credentials
-- - kyuubi:metastore-db
-  - metastore:database
-- - kyuubi:auth-db
-  - kyuubi-users:database
-- - kyuubi:spark-service-account
-  - integration-hub:spark-service-account
-- - kyuubi:zookeeper
-  - zookeeper:zookeeper
-- - kyuubi:certificates
-  - certificates:certificates
-- - kyuubi:jdbc
-  - data-integrator:kyuubi
diff --git a/releases/3.5/yaml/bundle.yaml.j2 b/releases/3.5/yaml/bundle.yaml.j2
deleted file mode 100644
index 0a4968ad..00000000
--- a/releases/3.5/yaml/bundle.yaml.j2
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-bundle: kubernetes
-applications:
-  kyuubi:
-    charm: kyuubi-k8s
-    channel: 3.5/stable
-    revision: 146
-    resources:
-      # rev18, spark-version: 3.5.7, kyuubi-version: 1.10.3, release date 16/03/2026
-      kyuubi-image: ghcr.io/canonical/charmed-spark-kyuubi@sha256:21beea406001d161538b552cef2b47b86423bd899139ada263c1ef665fd89fff
-    scale: 3
-    options:
-      namespace: {{ namespace }}
-      service-account: {{ service_account }}
-      expose-external: loadbalancer
-      profile: {{ kyuubi_profile }}
-    constraints: arch=amd64
-    trust: true
-  kyuubi-users:
-    charm: postgresql-k8s
-    channel: 14/stable
-    revision: 495
-    resources:
-      postgresql-image: 165
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pgdata: kubernetes,1,{{ kyuubi_user_size }}
-    trust: true 
-  zookeeper:  
-    charm: zookeeper-k8s
-    channel: 3/stable
-    revision: 78
-    scale: {{ zookeeper_units or 3}}
-    constraints: arch=amd64
-    resources:
-      zookeeper-image: 34
-    storage:
-      zookeeper: kubernetes,1,{{ zookeeper_size }}
-  metastore:
-    charm: postgresql-k8s
-    channel: 14/stable
-    revision: 495
-    resources:
-      postgresql-image: 165
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pgdata: kubernetes,1,{{ metastore_size }}
-    trust: true
-  s3:
-    charm: s3-integrator
-    channel: 1/stable
-    revision: 145
-    scale: 1
-    options:
-      bucket: {{ bucket }}
-      endpoint: {{ s3_endpoint }}
-      path: spark-events
-    constraints: arch=amd64
-  history-server:
-    charm: spark-history-server-k8s
-    channel: 3/stable
-    revision: 96
-    resources:
-      # rev24,  spark-version: 3.5.7, release date 16/03/2026 
-      spark-history-server-image: ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70
-    scale: 1
-    constraints: arch=amd64
-  integration-hub:
-    charm: spark-integration-hub-k8s
-    channel: 3/stable
-    revision: 119
-    resources:
-      # rev 13, release date 19/03/2026
-      integration-hub-image: ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134
-    scale: 1
-    constraints: arch=amd64
-    trust: true
-  certificates:
-    charm: self-signed-certificates
-    channel: 1/stable
-    revision: 317
-    scale: 1
-    constraints: arch=amd64
-    trust: true
-  data-integrator:
-    charm: data-integrator
-    channel: latest/stable
-    revision: 179
-    scale: 1
-    constraints: arch=amd64
-    options:
-      database-name: integrator
-relations:
-- - integration-hub:s3-credentials
-  - s3:s3-credentials
-- - history-server:s3-credentials
-  - s3:s3-credentials
-- - kyuubi:metastore-db
-  - metastore:database
-- - kyuubi:auth-db
-  - kyuubi-users:database
-- - kyuubi:spark-service-account
-  - integration-hub:spark-service-account
-- - kyuubi:zookeeper
-  - zookeeper:zookeeper
-- - kyuubi:certificates
-  - certificates:certificates
-- - kyuubi:jdbc
-  - data-integrator:kyuubi
diff --git a/releases/3.5/yaml/charmcraft.yaml b/releases/3.5/yaml/charmcraft.yaml
deleted file mode 100644
index 12cb8fd1..00000000
--- a/releases/3.5/yaml/charmcraft.yaml
+++ /dev/null
@@ -1,4 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-type: bundle
\ No newline at end of file
diff --git a/releases/3.5/yaml/metadata.yaml b/releases/3.5/yaml/metadata.yaml
deleted file mode 100644
index 6a1d78b1..00000000
--- a/releases/3.5/yaml/metadata.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-name: spark-k8s-bundle
-display-name: Charmed Kafka
-description: |
-  Apache Spark is an distributed processing engine, designed to scale out your data 
-  analysis and machine learning tasks. This bundles charms to enable a Apache Spark 
-  deployment on K8s, featuring integration with object storage, monitoring,
-  seamless and integrated configuration.
-summary: Charmed Apache Spark bundle
-docs: https://discourse.charmhub.io/t/charmed-spark-documentation/8963
-source: https://github.com/canonical/spark-k8s-bundle
-issues: https://github.com/canonical/spark-k8s-bundle/issues
-website:
-  - https://ubuntu.com/data/spark
-  - https://canonical.com/data/docs/spark/k8s
-  - https://github.com/canonical/spark-k8s-bundle
-  - https://matrix.to/#/%23charmhub-data-platform%3Aubuntu.com
-maintainers:
-  - Canonical Data Platform <data-platform@lists.launchpad.net>
\ No newline at end of file
diff --git a/releases/3.5/yaml/overlays/cos-integration.yaml.j2 b/releases/3.5/yaml/overlays/cos-integration.yaml.j2
deleted file mode 100644
index fded50c6..00000000
--- a/releases/3.5/yaml/overlays/cos-integration.yaml.j2
+++ /dev/null
@@ -1,94 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-bundle: kubernetes
-saas:
-  grafana-dashboards:
-    url: {{ cos_controller|default('micro', true) }}:admin/{{ cos_model|default('cos', true) }}.grafana-dashboards
-  prometheus-receive-remote-write:
-    url: {{ cos_controller|default('micro', true) }}:admin/{{ cos_model|default('cos', true) }}.prometheus-receive-remote-write
-  traefik:
-    url: {{ cos_controller|default('micro', true) }}:admin/{{ cos_model|default('cos', true) }}.traefik
-  loki-logging:
-    url: {{ cos_controller|default('micro', true) }}:admin/{{ cos_model|default('cos', true) }}.loki-logging
-applications:
-  agent:
-    charm: grafana-agent-k8s
-    channel: 1/stable
-    revision: 121
-    resources:
-      agent-image: 46
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      data: kubernetes,1,1024M
-    trust: true
-  cos-configuration:
-    charm: cos-configuration-k8s
-    channel: 1/stable
-    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
-    revision: 65
-    base: ubuntu@20.04/stable
-    resources:
-      git-sync-image: 35
-    scale: 1
-    options:
-      git_branch: main
-      git_depth: 1
-      git_repo: https://github.com/canonical/spark-k8s-bundle
-      grafana_dashboards_path: releases/3.5/resources/grafana/
-    constraints: arch=amd64
-    storage:
-      content-from-git: kubernetes,1,1024M
-  pushgateway:
-    charm: prometheus-pushgateway-k8s
-    channel: 1/stable
-    revision: 27
-    resources:
-      pushgateway-image: 8
-    scale: 1
-    constraints: arch=amd64
-    storage:
-      pushgateway-store: kubernetes,1,1024M
-  scrape-config:
-    charm: prometheus-scrape-config-k8s
-    channel: 1/stable
-    revision: 67
-    base: ubuntu@20.04/stable
-    scale: 1
-    options:
-      scrape_interval: 10s
-    constraints: arch=amd64
-relations:
-- - cos-configuration:grafana-dashboards
-  - agent:grafana-dashboards-consumer
-- - scrape-config:configurable-scrape-jobs
-  - pushgateway:metrics-endpoint
-- - scrape-config:metrics-endpoint
-  - agent:metrics-endpoint
-- - pushgateway:push-endpoint
-  - integration-hub:cos
-- - agent:grafana-dashboards-provider
-  - grafana-dashboards:grafana-dashboard
-- - agent:send-remote-write
-  - prometheus-receive-remote-write:receive-remote-write
-- - agent:logging-consumer
-  - loki-logging:logging
-- - history-server:ingress
-  - traefik:ingress
-- - history-server:metrics-endpoint
-  - agent:metrics-endpoint
-- - history-server:grafana-dashboard
-  - agent:grafana-dashboards-consumer
-- - history-server:logging
-  - agent:logging-provider
-- - kyuubi:metrics-endpoint
-  - agent:metrics-endpoint
-- - kyuubi:grafana-dashboard
-  - agent:grafana-dashboards-consumer
-- - kyuubi:logging
-  - agent:logging-provider
-- - integration-hub:logging
-  - agent:logging-provider
diff --git a/releases/terraform/components/observability/README.md b/releases/terraform/components/observability/README.md
deleted file mode 100644
index ac296091..00000000
--- a/releases/terraform/components/observability/README.md
+++ /dev/null
@@ -1,67 +0,0 @@
-## Requirements
-
-| Name | Version |
-|------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
-
-## Providers
-
-| Name | Version |
-|------|---------|
-| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
-
-## Modules
-
-No modules.
-
-## Resources
-
-| Name | Type |
-|------|------|
-| [juju_application.cos_configuration](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.pushgateway](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.scrape_config](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_integration.cos_configuration_grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana_agent_grafana_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana_agent_loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana_agent_prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.history_server_grafana_agent_dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.history_server_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.history_server_grafana_agent_metrics](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.integration_hub_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.kyuubi_grafana_agent_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.kyuubi_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.kyuubi_grafana_agent_metrics](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.pushgateway_integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.pushgateway_scrape_config](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.scrape_config_grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
-| [juju_offer.grafana_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
-| [juju_offer.loki_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
-| [juju_offer.prometheus_receive_remote_write](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
-
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| <a name="input_config_grafana_path"></a> [config\_grafana\_path](#input\_config\_grafana\_path) | Grafana dashboard path from configuration repo root. | `string` | `"releases/3.4/resources/grafana/"` | no |
-| <a name="input_config_repo"></a> [config\_repo](#input\_config\_repo) | COS configuration repo URL. | `string` | `"https://github.com/canonical/spark-k8s-bundle"` | no |
-| <a name="input_cos_configuration_revision"></a> [cos\_configuration\_revision](#input\_cos\_configuration\_revision) | Charm revision for cos-configuration-k8s | `number` | n/a | yes |
-| <a name="input_dashboards_offer"></a> [dashboards\_offer](#input\_dashboards\_offer) | URL of the `grafana_dashboard` interface offer. | `string` | n/a | yes |
-| <a name="input_grafana_agent_revision"></a> [grafana\_agent\_revision](#input\_grafana\_agent\_revision) | Charm revision for grafana-agent-k8s | `number` | n/a | yes |
-| <a name="input_logging_offer"></a> [logging\_offer](#input\_logging\_offer) | URL of the `loki_push_api` interface offer. | `string` | n/a | yes |
-| <a name="input_metrics_offer"></a> [metrics\_offer](#input\_metrics\_offer) | URL of the `prometheus_remote_write` interface offer. | `string` | n/a | yes |
-| <a name="input_pushgateway_revision"></a> [pushgateway\_revision](#input\_pushgateway\_revision) | Charm revision for prometheus-pushgateway-k8s | `number` | n/a | yes |
-| <a name="input_pushgateway_size"></a> [pushgateway\_size](#input\_pushgateway\_size) | Storage size for the pushgateway database | `string` | `"10G"` | no |
-| <a name="input_scrape_config_revision"></a> [scrape\_config\_revision](#input\_scrape\_config\_revision) | Charm revision for prometheus-scrape-config-k8s | `number` | n/a | yes |
-| <a name="input_spark_charms"></a> [spark\_charms](#input\_spark\_charms) | Names of the Spark applications in the Spark Juju model. | `map(string)` | n/a | yes |
-| <a name="input_spark_model"></a> [spark\_model](#input\_spark\_model) | Name of the Spark Juju model. | `string` | n/a | yes |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
-| <a name="output_components"></a> [components](#output\_components) | List of the deployed applications for this component module. |
diff --git a/releases/terraform/components/observability/applications.tf b/releases/terraform/components/observability/applications.tf
deleted file mode 100644
index f3188753..00000000
--- a/releases/terraform/components/observability/applications.tf
+++ /dev/null
@@ -1,64 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "grafana_agent" {
-  name  = "grafana-agent"
-  model_uuid = local.target_model_uuid
-  charm {
-    name     = "grafana-agent-k8s"
-    channel  = "1/stable"
-    revision = var.grafana_agent_revision
-  }
-  units       = 1
-  trust       = true
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "cos_configuration" {
-  name  = "cos-configuration"
-  model_uuid = local.target_model_uuid
-  charm {
-    name     = "cos-configuration-k8s"
-    channel  = "1/stable"
-    revision = var.cos_configuration_revision
-  }
-  config = {
-    git_branch              = "main"
-    git_depth               = "1"
-    git_repo                = var.config_repo
-    grafana_dashboards_path = var.config_grafana_path
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "pushgateway" {
-  name  = "pushgateway"
-  model_uuid = local.target_model_uuid
-  charm {
-    name     = "prometheus-pushgateway-k8s"
-    channel  = "1/stable"
-    revision = var.pushgateway_revision
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    pushgateway-store = var.pushgateway_size
-  }
-}
-
-resource "juju_application" "scrape_config" {
-  name  = "scrape-config"
-  model_uuid = local.target_model_uuid
-  charm {
-    name     = "prometheus-scrape-config-k8s"
-    channel  = "1/stable"
-    revision = var.scrape_config_revision
-  }
-  config = {
-    scrape_interval = "10s"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-}
-
diff --git a/releases/terraform/components/observability/integrations.tf b/releases/terraform/components/observability/integrations.tf
deleted file mode 100644
index ea071768..00000000
--- a/releases/terraform/components/observability/integrations.tf
+++ /dev/null
@@ -1,236 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-data "juju_offer" "grafana_dashboards" {
-  url = var.dashboards_offer
-}
-
-data "juju_offer" "prometheus_receive_remote_write" {
-  url = var.metrics_offer
-}
-
-data "juju_offer" "loki_logging" {
-  url = var.logging_offer
-}
-
-resource "juju_integration" "cos_configuration_grafana_agent" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.cos_configuration.name
-    endpoint = "grafana-dashboards"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "pushgateway_scrape_config" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.pushgateway.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.scrape_config.name
-    endpoint = "configurable-scrape-jobs"
-  }
-}
-
-resource "juju_integration" "scrape_config_grafana_agent" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.scrape_config.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "grafana_agent_grafana_dashboards" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-provider"
-  }
-
-  application {
-    offer_url = data.juju_offer.grafana_dashboards.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-
-resource "juju_integration" "grafana_agent_prometheus" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "send-remote-write"
-  }
-
-  application {
-    offer_url = data.juju_offer.prometheus_receive_remote_write.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-
-resource "juju_integration" "grafana_agent_loki" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-consumer"
-  }
-
-  application {
-    offer_url = data.juju_offer.loki_logging.url
-  }
-
-  lifecycle {
-    replace_triggered_by = [
-      juju_application.grafana_agent.name,
-      juju_application.grafana_agent.model,
-      juju_application.grafana_agent.constraints,
-      juju_application.grafana_agent.placement,
-      juju_application.grafana_agent.charm.name,
-    ]
-  }
-}
-resource "juju_integration" "pushgateway_integration_hub" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.pushgateway.name
-    endpoint = "push-endpoint"
-  }
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "cos"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_dashboard" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_logging" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
-
-resource "juju_integration" "history_server_grafana_agent_metrics" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = var.spark_charms.history_server
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_metrics" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_dashboards" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "grafana-dashboards-consumer"
-  }
-}
-
-resource "juju_integration" "kyuubi_grafana_agent_logging" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = var.spark_charms.kyuubi
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
-
-resource "juju_integration" "integration_hub_grafana_agent_logging" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = var.spark_charms.integration_hub
-    endpoint = "logging"
-  }
-
-  application {
-    name     = juju_application.grafana_agent.name
-    endpoint = "logging-provider"
-  }
-}
diff --git a/releases/terraform/components/observability/locals.tf b/releases/terraform/components/observability/locals.tf
deleted file mode 100644
index 44f3a0f6..00000000
--- a/releases/terraform/components/observability/locals.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Local values for $(basename "$moduleDir").
-# Define computed values used in this module.
diff --git a/releases/terraform/components/observability/main.tf b/releases/terraform/components/observability/main.tf
deleted file mode 100644
index 46aca146..00000000
--- a/releases/terraform/components/observability/main.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Main Terraform code for $(basename "$moduleDir").
-# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/components/observability/outputs.tf b/releases/terraform/components/observability/outputs.tf
deleted file mode 100644
index f98fc3a0..00000000
--- a/releases/terraform/components/observability/outputs.tf
+++ /dev/null
@@ -1,21 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    grafana_agent     = juju_application.grafana_agent.name
-    cos_configuration = juju_application.cos_configuration.name
-    pushgateway       = juju_application.pushgateway.name
-    scrape_config     = juju_application.scrape_config.name
-  }
-}
-output "components" {
-  description = "List of the deployed applications for this component module."
-  value = [
-    { name = juju_application.grafana_agent.name, type = "grafana_agent" },
-    { name = juju_application.cos_configuration.name, type = "cos_configuration" },
-    { name = juju_application.pushgateway.name, type = "pushgateway" },
-    { name = juju_application.scrape_config.name, type = "scrape_config" },
-  ]
-}
diff --git a/releases/terraform/components/observability/providers.tf b/releases/terraform/components/observability/providers.tf
deleted file mode 100644
index 0693f27b..00000000
--- a/releases/terraform/components/observability/providers.tf
+++ /dev/null
@@ -1,6 +0,0 @@
-# Provider configuration for module $(basename "$moduleDir").
-# Terraform block moved to terraform.tf and should not be in providers.tf.
-
-# provider "juju" {
-#   # Configure provider here if required by the module.
-# }
diff --git a/releases/terraform/components/observability/resources.tf b/releases/terraform/components/observability/resources.tf
deleted file mode 100644
index 765d64b8..00000000
--- a/releases/terraform/components/observability/resources.tf
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-locals {
-  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
-}
-
-data "juju_model" "spark" {
-  name = var.spark_model
-}
diff --git a/releases/terraform/components/observability/terraform.tf b/releases/terraform/components/observability/terraform.tf
deleted file mode 100644
index 6412125a..00000000
--- a/releases/terraform/components/observability/terraform.tf
+++ /dev/null
@@ -1,10 +0,0 @@
-terraform {
-  required_version = ">=1.0.0"
-
-  required_providers {
-    juju = {
-      source  = "juju/juju"
-      version = ">=1.0.0"
-    }
-  }
-}
diff --git a/releases/terraform/components/observability/variables.tf b/releases/terraform/components/observability/variables.tf
deleted file mode 100644
index 94aaef5b..00000000
--- a/releases/terraform/components/observability/variables.tf
+++ /dev/null
@@ -1,81 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-variable "config_grafana_path" {
-  description = "Grafana dashboard path from configuration repo root."
-  type        = string
-  default     = "releases/3.4/resources/grafana/"
-  nullable    = false
-}
-
-variable "config_repo" {
-  description = "COS configuration repo URL."
-  type        = string
-  default     = "https://github.com/canonical/spark-k8s-bundle"
-  nullable    = false
-}
-
-variable "cos_configuration_revision" {
-  description = "Charm revision for cos-configuration-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "dashboards_offer" {
-  description = "URL of the `grafana_dashboard` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "grafana_agent_revision" {
-  description = "Charm revision for grafana-agent-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "logging_offer" {
-  description = "URL of the `loki_push_api` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "metrics_offer" {
-  description = "URL of the `prometheus_remote_write` interface offer."
-  type        = string
-  nullable    = false
-}
-
-variable "pushgateway_revision" {
-  description = "Charm revision for prometheus-pushgateway-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "pushgateway_size" {
-  description = "Storage size for the pushgateway database"
-  type        = string
-  default     = "10G"
-}
-
-variable "scrape_config_revision" {
-  description = "Charm revision for prometheus-scrape-config-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "spark_charms" {
-  description = "Names of the Spark applications in the Spark Juju model."
-  type        = map(string)
-}
-
-variable "spark_model" {
-  description = "Name of the Spark Juju model."
-  type        = string
-  nullable    = false
-}
-
-variable "model_uuid" {
-  description = "Optional existing Juju model UUID to deploy to."
-  type        = string
-  default     = null
-}
diff --git a/releases/terraform/components/spark-3.5/README.md b/releases/terraform/components/spark-3.5/README.md
deleted file mode 100644
index fe9f167f..00000000
--- a/releases/terraform/components/spark-3.5/README.md
+++ /dev/null
@@ -1,90 +0,0 @@
-## Requirements
-
-| Name | Version |
-|------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
-
-## Providers
-
-| Name | Version |
-|------|---------|
-| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
-
-## Modules
-
-No modules.
-
-## Resources
-
-| Name | Type |
-|------|------|
-| [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
-| [juju_application.data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.kyuubi](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_integration.kyuubi_data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.kyuubi_metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.kyuubi_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.kyuubi_tls](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.kyuubi_zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_offer.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_offer.metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
-| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
-
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
-| <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | n/a | yes |
-| <a name="input_driver_pod_template"></a> [driver\_pod\_template](#input\_driver\_pod\_template) | Define K8s driver pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
-| <a name="input_enable_dynamic_allocation"></a> [enable\_dynamic\_allocation](#input\_enable\_dynamic\_allocation) | Enable dynamic allocation of pods for Spark jobs. | `bool` | `false` | no |
-| <a name="input_executor_pod_template"></a> [executor\_pod\_template](#input\_executor\_pod\_template) | Define K8s executor pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
-| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `map(string)` | n/a | yes |
-| <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | n/a | yes |
-| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | n/a | yes |
-| <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
-| <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | n/a | yes |
-| <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
-| <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
-| <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
-| <a name="input_kyuubi_executor_pod_template"></a> [kyuubi\_executor\_pod\_template](#input\_kyuubi\_executor\_pod\_template) | Define K8s executor pod from a file accessible in the object storage. | `string` | `null` | no |
-| <a name="input_kyuubi_gpu_enable"></a> [kyuubi\_gpu\_enable](#input\_kyuubi\_gpu\_enable) | Enable GPU acceleration for SparkSQLEngine. | `bool` | n/a | yes |
-| <a name="input_kyuubi_gpu_engine_executors_limit"></a> [kyuubi\_gpu\_engine\_executors\_limit](#input\_kyuubi\_gpu\_engine\_executors\_limit) | Limit the number of GPUs an engine can schedule executor pods on. | `number` | n/a | yes |
-| <a name="input_kyuubi_gpu_pinned_memory"></a> [kyuubi\_gpu\_pinned\_memory](#input\_kyuubi\_gpu\_pinned\_memory) | Set the host memory (in GB) per executor reserved for fast data transfer on GPUs. | `number` | n/a | yes |
-| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `map(string)` | n/a | yes |
-| <a name="input_kyuubi_k8s_node_selectors"></a> [kyuubi\_k8s\_node\_selectors](#input\_kyuubi\_k8s\_node\_selectors) | Comma separated label:value selectors for K8s pods in Kyuubi. | `string` | `null` | no |
-| <a name="input_kyuubi_loadbalancer_extra_annotations"></a> [kyuubi\_loadbalancer\_extra\_annotations](#input\_kyuubi\_loadbalancer\_extra\_annotations) | Optional extra annotations to be supplied to the load balancer service in Kyuubi. | `string` | `null` | no |
-| <a name="input_kyuubi_profile"></a> [kyuubi\_profile](#input\_kyuubi\_profile) | The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'. | `string` | `"production"` | no |
-| <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | n/a | yes |
-| <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
-| <a name="input_kyuubi_user"></a> [kyuubi\_user](#input\_kyuubi\_user) | User name to be used for running Kyuubi engines. | `string` | `"kyuubi-spark-engine"` | no |
-| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | n/a | yes |
-| <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | n/a | yes |
-| <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
-| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | n/a | yes |
-| <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | n/a | yes |
-| <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
-| <a name="input_model"></a> [model](#input\_model) | Name of the Juju Model to deploy to. | `string` | n/a | yes |
-| <a name="input_tls_app_name"></a> [tls\_app\_name](#input\_tls\_app\_name) | Name of the application providing the `tls-certificates` interface. | `string` | n/a | yes |
-| <a name="input_tls_certificates_endpoint"></a> [tls\_certificates\_endpoint](#input\_tls\_certificates\_endpoint) | Name of the endpoint providing the `tls-certificates` interface. | `string` | n/a | yes |
-| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
-| <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | n/a | yes |
-| <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | n/a | yes |
-| <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the zookeeper unit | `string` | `"10G"` | no |
-| <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
-| <a name="output_components"></a> [components](#output\_components) | List of the deployed applications for this component module. |
-| <a name="output_offers"></a> [offers](#output\_offers) | n/a |
diff --git a/releases/terraform/components/spark-3.5/applications.tf b/releases/terraform/components/spark-3.5/applications.tf
deleted file mode 100644
index 3943ad1c..00000000
--- a/releases/terraform/components/spark-3.5/applications.tf
+++ /dev/null
@@ -1,190 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "history_server" {
-  name  = "history-server"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "spark-history-server-k8s"
-    channel  = "3/stable"
-    revision = var.history_server_revision
-  }
-
-  resources = var.history_server_image
-  units     = 1
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "kyuubi" {
-  name  = "kyuubi"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "kyuubi-k8s"
-    channel  = "3.5/stable"
-    revision = var.kyuubi_revision
-  }
-
-  resources = var.kyuubi_image
-
-  config = merge(
-    {
-      enable-dynamic-allocation  = var.enable_dynamic_allocation
-      expose-external            = "loadbalancer"
-      gpu-enable                 = var.kyuubi_gpu_enable
-      gpu-engine-executors-limit = var.kyuubi_gpu_engine_executors_limit
-      gpu-pinned-memory          = var.kyuubi_gpu_pinned_memory
-      namespace                  = data.juju_model.spark.name
-      profile                    = var.kyuubi_profile
-      service-account            = var.kyuubi_user
-    },
-    var.kyuubi_k8s_node_selectors == null ? {} : {
-      k8s-node-selectors = var.kyuubi_k8s_node_selectors
-    },
-    var.kyuubi_loadbalancer_extra_annotations == null ? {} : {
-      loadbalancer-extra-annotations = var.kyuubi_loadbalancer_extra_annotations
-    },
-    var.tls_private_key == null ? {} : {
-      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    },
-    var.admin_password == null ? {} : {
-      system-users = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    },
-    var.kyuubi_executor_cores == null ? {} : {
-      executor-cores = var.kyuubi_executor_cores
-    },
-    var.kyuubi_executor_memory == null ? {} : {
-      executor-memory = var.kyuubi_executor_memory
-    },
-    var.kyuubi_driver_pod_template == null ? {} : {
-      driver-pod-template = var.kyuubi_driver_pod_template
-    },
-    var.kyuubi_executor_pod_template == null ? {} : {
-      executor-pod-template = var.kyuubi_executor_pod_template
-    }
-  )
-
-  units = var.kyuubi_units
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "kyuubi_users" {
-  name  = "kyuubi-users"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "postgresql-k8s"
-    channel  = "14/stable"
-    revision = var.kyuubi_users_revision
-  }
-
-  storage_directives = {
-    pgdata = var.kyuubi_users_size
-  }
-
-  resources = var.kyuubi_users_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "metastore" {
-  name  = "metastore"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "postgresql-k8s"
-    channel  = "14/stable"
-    revision = var.metastore_revision
-  }
-
-  storage_directives = {
-    pgdata = var.kyuubi_users_size
-  }
-
-
-  resources = var.metastore_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "integration_hub" {
-  name  = "integration-hub"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "spark-integration-hub-k8s"
-    channel  = "3/stable"
-    revision = var.integration_hub_revision
-  }
-
-  config = merge(
-    {
-      enable-dynamic-allocation = var.enable_dynamic_allocation
-    },
-    var.driver_pod_template == null ? {} : {
-      driver-pod-template = var.driver_pod_template
-    },
-    var.executor_pod_template == null ? {} : {
-      executor-pod-template = var.executor_pod_template
-    },
-    var.integration_hub_monitored_service_accounts == null ? {} : {
-      monitored-service-accounts = var.integration_hub_monitored_service_accounts
-    }
-  )
-
-  resources = var.integration_hub_image
-
-  units = 1
-  trust = true
-
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "zookeeper" {
-  name  = "zookeeper"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "zookeeper-k8s"
-    channel  = "3/stable"
-    revision = var.zookeeper_revision
-  }
-
-  storage_directives = {
-    zookeeper = var.zookeeper_size
-  }
-
-
-  resources = var.zookeeper_image
-
-  units       = var.zookeeper_units
-  constraints = "arch=amd64"
-}
-
-resource "juju_application" "data_integrator" {
-  name  = "data-integrator"
-  model_uuid = data.juju_model.spark.id
-
-  charm {
-    name     = "data-integrator"
-    channel  = "latest/stable"
-    revision = var.data_integrator_revision
-  }
-
-  config = {
-    database-name = "integrator"
-  }
-
-  units       = 1
-  constraints = "arch=amd64"
-}
diff --git a/releases/terraform/components/spark-3.5/integrations.tf b/releases/terraform/components/spark-3.5/integrations.tf
deleted file mode 100644
index 8031e0d5..00000000
--- a/releases/terraform/components/spark-3.5/integrations.tf
+++ /dev/null
@@ -1,86 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_integration" "kyuubi_metastore" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.metastore.name
-    endpoint = "database"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "metastore-db"
-  }
-}
-
-resource "juju_integration" "kyuubi_users" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.kyuubi_users.name
-    endpoint = "database"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "auth-db"
-  }
-}
-
-resource "juju_integration" "kyuubi_service_account" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "spark-service-account"
-  }
-
-  application {
-    name     = juju_application.integration_hub.name
-    endpoint = "spark-service-account"
-  }
-}
-
-resource "juju_integration" "kyuubi_zookeeper" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.zookeeper.name
-    endpoint = "zookeeper"
-  }
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "zookeeper"
-  }
-}
-
-resource "juju_integration" "kyuubi_tls" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "certificates"
-  }
-
-  application {
-    name     = var.tls_app_name
-    endpoint = var.tls_certificates_endpoint
-  }
-}
-
-resource "juju_integration" "kyuubi_data_integrator" {
-  model_uuid = data.juju_model.spark.id
-
-  application {
-    name     = juju_application.kyuubi.name
-    endpoint = "jdbc"
-  }
-
-  application {
-    name     = juju_application.data_integrator.name
-    endpoint = "kyuubi"
-  }
-}
diff --git a/releases/terraform/components/spark-3.5/locals.tf b/releases/terraform/components/spark-3.5/locals.tf
deleted file mode 100644
index 10b756b2..00000000
--- a/releases/terraform/components/spark-3.5/locals.tf
+++ /dev/null
@@ -1,43 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-locals {
-  revisions = {
-    history_server  = 94
-    integration_hub = 113
-    kyuubi          = 142
-    kyuubi_users    = 495
-    metastore       = 495
-    zookeeper       = 78
-    data_integrator = 179
-    s3              = 145
-    azure_storage   = 15
-    grafana_agent   = 121
-    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
-    cos_configuration = 65
-    pushgateway       = 27
-    scrape_config     = 67
-  }
-  images = {
-    history_server = {
-      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
-    }
-    integration_hub = {
-      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
-    }
-    kyuubi = {
-      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:cdde52a4f72112ea09de11c6d21f901323619cf41015db89cb9d5742c57da303"
-    }
-    kyuubi_users = {
-      postgresql-image = 165
-    }
-    metastore = {
-      postgresql-image = 165
-    }
-    zookeeper = {
-      zookeeper-image = 34
-    }
-  }
-}
diff --git a/releases/terraform/components/spark-3.5/main.tf b/releases/terraform/components/spark-3.5/main.tf
deleted file mode 100644
index 46aca146..00000000
--- a/releases/terraform/components/spark-3.5/main.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Main Terraform code for $(basename "$moduleDir").
-# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/components/spark-3.5/offers.tf b/releases/terraform/components/spark-3.5/offers.tf
deleted file mode 100644
index 46f9b8dc..00000000
--- a/releases/terraform/components/spark-3.5/offers.tf
+++ /dev/null
@@ -1,11 +0,0 @@
-resource "juju_offer" "integration_hub" {
-  model_uuid = data.juju_model.spark.id
-  application_name = juju_application.integration_hub.name
-  endpoints        = ["spark-service-account"]
-}
-
-resource "juju_offer" "metastore" {
-  model_uuid = data.juju_model.spark.id
-  application_name = juju_application.metastore.name
-  endpoints        = ["database"]
-}
\ No newline at end of file
diff --git a/releases/terraform/components/spark-3.5/outputs.tf b/releases/terraform/components/spark-3.5/outputs.tf
deleted file mode 100644
index 01714e58..00000000
--- a/releases/terraform/components/spark-3.5/outputs.tf
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    history_server  = juju_application.history_server.name
-    kyuubi          = juju_application.kyuubi.name
-    kyuubi_users    = juju_application.kyuubi_users.name
-    metastore       = juju_application.metastore.name
-    integration_hub = juju_application.integration_hub.name
-    zookeeper       = juju_application.zookeeper.name
-    certificates    = var.tls_app_name
-  }
-}
-
-output "components" {
-  description = "List of the deployed applications for this component module."
-  value = [
-    { name = juju_application.history_server.name, type = "history_server" },
-    { name = juju_application.kyuubi.name, type = "kyuubi" },
-    { name = juju_application.kyuubi_users.name, type = "kyuubi_users" },
-    { name = juju_application.metastore.name, type = "metastore" },
-    { name = juju_application.integration_hub.name, type = "integration_hub" },
-    { name = juju_application.zookeeper.name, type = "zookeeper" },
-  ]
-}
-
-output "offers" {
-  value = {
-    hub_service_account = juju_offer.integration_hub
-    metastore_database  = juju_offer.metastore
-  }
-}
diff --git a/releases/terraform/components/spark-3.5/providers.tf b/releases/terraform/components/spark-3.5/providers.tf
deleted file mode 100644
index 0693f27b..00000000
--- a/releases/terraform/components/spark-3.5/providers.tf
+++ /dev/null
@@ -1,6 +0,0 @@
-# Provider configuration for module $(basename "$moduleDir").
-# Terraform block moved to terraform.tf and should not be in providers.tf.
-
-# provider "juju" {
-#   # Configure provider here if required by the module.
-# }
diff --git a/releases/terraform/components/spark-3.5/resources.tf b/releases/terraform/components/spark-3.5/resources.tf
deleted file mode 100644
index e25d743f..00000000
--- a/releases/terraform/components/spark-3.5/resources.tf
+++ /dev/null
@@ -1,36 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-locals {
-  target_model_uuid = var.model_uuid != null ? var.model_uuid : data.juju_model.spark.id
-}
-
-data "juju_model" "spark" {
-  name = var.model
-}
-
-resource "juju_secret" "system_users_and_private_key_secret" {
-  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model_uuid = local.target_model_uuid
-  name  = "system_users_and_private_key_secret"
-  value = merge(
-    var.admin_password == null ? {} : {
-      admin = var.admin_password
-    },
-    var.tls_private_key == null ? {} : {
-      private-key = var.tls_private_key
-    }
-  )
-  info = "This secret contains password for admin user and the TLS private key."
-}
-
-resource "juju_access_secret" "system_users_and_private_key_secret_access" {
-  count = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model_uuid = local.target_model_uuid
-  applications = [
-    juju_application.kyuubi.name
-  ]
-  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
-}
diff --git a/releases/terraform/components/spark-3.5/terraform.tf b/releases/terraform/components/spark-3.5/terraform.tf
deleted file mode 100644
index 6412125a..00000000
--- a/releases/terraform/components/spark-3.5/terraform.tf
+++ /dev/null
@@ -1,10 +0,0 @@
-terraform {
-  required_version = ">=1.0.0"
-
-  required_providers {
-    juju = {
-      source  = "juju/juju"
-      version = ">=1.0.0"
-    }
-  }
-}
diff --git a/releases/terraform/components/spark-3.5/variables.tf b/releases/terraform/components/spark-3.5/variables.tf
deleted file mode 100644
index 673a1de8..00000000
--- a/releases/terraform/components/spark-3.5/variables.tf
+++ /dev/null
@@ -1,244 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define spark-specific variables
-
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "data_integrator_revision" {
-  description = "Charm revision for data-integrator"
-  type        = number
-  nullable    = false
-}
-
-variable "driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "enable_dynamic_allocation" {
-  description = "Enable dynamic allocation of pods for Spark jobs."
-  type        = bool
-  default     = false
-}
-
-variable "executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "history_server_image" {
-  description = "Image for spark-history-server-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "history_server_revision" {
-  description = "Charm revision for spark-history-server-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "integration_hub_image" {
-  description = "Image for spark-integration-hub-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "integration_hub_monitored_service_accounts" {
-  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-  type        = string
-  default     = null
-}
-
-variable "integration_hub_revision" {
-  description = "Charm revision for spark-integration-hub-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_cores" {
-  description = "Set kyuubi executor pods cpu cores."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_memory" {
-  description = "Set kyuubi executor pods memory (in GB)."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_gpu_enable" {
-  description = "Enable GPU acceleration for SparkSQLEngine."
-  type        = bool
-  nullable    = false
-}
-
-variable "kyuubi_gpu_engine_executors_limit" {
-  description = "Limit the number of GPUs an engine can schedule executor pods on."
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_gpu_pinned_memory" {
-  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_image" {
-  description = "Image for kyuubi-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_k8s_node_selectors" {
-  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_loadbalancer_extra_annotations" {
-  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_profile" {
-  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-  type        = string
-  nullable    = false
-  default     = "production"
-}
-
-variable "kyuubi_revision" {
-  description = "Charm revision for kyuubi-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_units" {
-  description = "Number of Kyuubi units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "kyuubi_user" {
-  description = "User name to be used for running Kyuubi engines."
-  type        = string
-  nullable    = false
-  default     = "kyuubi-spark-engine"
-}
-
-variable "kyuubi_users_image" {
-  description = "Image for postgresql-k8s (auth-db)"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "kyuubi_users_revision" {
-  description = "Charm revision for postgresql-k8s (auth-db)"
-  type        = number
-  nullable    = false
-}
-
-variable "kyuubi_users_size" {
-  description = "Storage size for the Kyuubi users database"
-  type        = string
-  default     = "1G"
-}
-
-variable "metastore_image" {
-  description = "Image for postgresql-k8s (metastore)"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "metastore_revision" {
-  description = "Charm revision for postgresql-k8s (metastore)"
-  type        = number
-  nullable    = false
-}
-
-variable "metastore_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-variable "model" {
-  description = "Name of the Juju Model to deploy to (fallback if model_uuid is not provided)."
-  type        = string
-  nullable    = false
-}
-
-variable "model_uuid" {
-  description = "Optional existing Juju model UUID to deploy to."
-  type        = string
-  default     = null
-}
-
-variable "tls_app_name" {
-  description = "Name of the application providing the `tls-certificates` interface."
-  type        = string
-  nullable    = false
-}
-
-variable "tls_certificates_endpoint" {
-  description = "Name of the endpoint providing the `tls-certificates` interface."
-  type        = string
-  nullable    = false
-}
-
-variable "tls_private_key" {
-  description = "The private key to be used for TLS certificates."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "zookeeper_image" {
-  description = "Image for zookeeper-k8s"
-  type        = map(string)
-  nullable    = false
-}
-
-variable "zookeeper_revision" {
-  description = "Charm revision for zookeeper-k8s"
-  type        = number
-  nullable    = false
-}
-
-variable "zookeeper_size" {
-  description = "Storage size for the zookeeper unit"
-  type        = string
-  default     = "10G"
-}
-
-variable "zookeeper_units" {
-  description = "Number of zookeeper units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
diff --git a/releases/terraform/products/charmed-spark-3.5/README.md b/releases/terraform/products/charmed-spark-3.5/README.md
deleted file mode 100644
index b58335b5..00000000
--- a/releases/terraform/products/charmed-spark-3.5/README.md
+++ /dev/null
@@ -1,100 +0,0 @@
-## Requirements
-
-| Name | Version |
-|------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
-
-## Providers
-
-| Name | Version |
-|------|---------|
-| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
-
-## Modules
-
-| Name | Source | Version |
-|------|--------|---------|
-| <a name="module_azure_storage"></a> [azure\_storage](#module\_azure\_storage) | ./charms/azure-storage-integrator | n/a |
-| <a name="module_bundled_cos"></a> [bundled\_cos](#module\_bundled\_cos) | ./products/cos | n/a |
-| <a name="module_observability"></a> [observability](#module\_observability) | ./components/observability | n/a |
-| <a name="module_s3"></a> [s3](#module\_s3) | ./charms/s3-integrator | n/a |
-| <a name="module_spark"></a> [spark](#module\_spark) | ./components/spark | n/a |
-| <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev326 |
-
-## Resources
-
-| Name | Type |
-|------|------|
-| [juju_model.cos](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
-| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
-
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| <a name="input_K8S_CLOUD"></a> [K8S\_CLOUD](#input\_K8S\_CLOUD) | The kubernetes juju cloud name. | `string` | `"microk8s"` | no |
-| <a name="input_K8S_CREDENTIAL"></a> [K8S\_CREDENTIAL](#input\_K8S\_CREDENTIAL) | The name of the kubernetes juju credential. | `string` | `"microk8s"` | no |
-| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
-| <a name="input_alertmanager_size"></a> [alertmanager\_size](#input\_alertmanager\_size) | Storage size for the alertmanager database | `string` | `"10G"` | no |
-| <a name="input_azure_storage"></a> [azure\_storage](#input\_azure\_storage) | Azure Object storage information | <pre>object({<br/>    container       = optional(string, "azurecontainer")<br/>    storage_account = optional(string, "azurestorageaccount")<br/>    secret_key      = optional(string, "azurestoragesecret")<br/>    protocol        = optional(string, "abfss")<br/>  })</pre> | `{}` | no |
-| <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | `null` | no |
-| <a name="input_certificate_common_name"></a> [certificate\_common\_name](#input\_certificate\_common\_name) | Common name for the certificate to be used in self-signed | `string` | `"charmed-spark"` | no |
-| <a name="input_cos"></a> [cos](#input\_cos) | Observability settings | <pre>object({<br/>    model    = optional(string, "cos")<br/>    deployed = optional(string, "bundled")<br/>    offers = optional(object({<br/>      dashboard = optional(string, null),<br/>      metrics   = optional(string, null),<br/>      logging   = optional(string, null)<br/>    }), {}),<br/>    tls = optional(object({<br/>      cert = optional(string, "")<br/>      key  = optional(string, "")<br/>      ca   = optional(string, "")<br/>    }), {})<br/>  })</pre> | <pre>{<br/>  "deployed": "bundled",<br/>  "model": "cos",<br/>  "offers": {},<br/>  "tls": {}<br/>}</pre> | no |
-| <a name="input_cos_configuration_revision"></a> [cos\_configuration\_revision](#input\_cos\_configuration\_revision) | Charm revision for cos-configuration-k8s | `number` | `null` | no |
-| <a name="input_create_model"></a> [create\_model](#input\_create\_model) | Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism. | `bool` | `true` | no |
-| <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | `null` | no |
-| <a name="input_driver_pod_template"></a> [driver\_pod\_template](#input\_driver\_pod\_template) | Define K8s driver pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
-| <a name="input_enable_dynamic_allocation"></a> [enable\_dynamic\_allocation](#input\_enable\_dynamic\_allocation) | Enable dynamic allocation of pods for Spark jobs. | `bool` | `false` | no |
-| <a name="input_executor_pod_template"></a> [executor\_pod\_template](#input\_executor\_pod\_template) | Define K8s executor pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
-| <a name="input_grafana_agent_revision"></a> [grafana\_agent\_revision](#input\_grafana\_agent\_revision) | Charm revision for grafana-agent-k8s | `number` | `null` | no |
-| <a name="input_grafana_size"></a> [grafana\_size](#input\_grafana\_size) | Storage size for the grafana database | `string` | `"10G"` | no |
-| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `map(string)` | `null` | no |
-| <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | `null` | no |
-| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | `null` | no |
-| <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
-| <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | `null` | no |
-| <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
-| <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
-| <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
-| <a name="input_kyuubi_executor_pod_template"></a> [kyuubi\_executor\_pod\_template](#input\_kyuubi\_executor\_pod\_template) | Define K8s executor pod from a file accessible in the object storage. | `string` | `null` | no |
-| <a name="input_kyuubi_gpu_enable"></a> [kyuubi\_gpu\_enable](#input\_kyuubi\_gpu\_enable) | Enable GPU acceleration for SparkSQLEngine. | `bool` | `false` | no |
-| <a name="input_kyuubi_gpu_engine_executors_limit"></a> [kyuubi\_gpu\_engine\_executors\_limit](#input\_kyuubi\_gpu\_engine\_executors\_limit) | Limit the number of GPUs an engine can schedule executor pods on. | `number` | `1` | no |
-| <a name="input_kyuubi_gpu_pinned_memory"></a> [kyuubi\_gpu\_pinned\_memory](#input\_kyuubi\_gpu\_pinned\_memory) | Set the host memory (in GB) per executor reserved for fast data transfer on GPUs. | `number` | `1` | no |
-| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `map(string)` | `null` | no |
-| <a name="input_kyuubi_k8s_node_selectors"></a> [kyuubi\_k8s\_node\_selectors](#input\_kyuubi\_k8s\_node\_selectors) | Comma separated label:value selectors for K8s pods in Kyuubi. | `string` | `null` | no |
-| <a name="input_kyuubi_loadbalancer_extra_annotations"></a> [kyuubi\_loadbalancer\_extra\_annotations](#input\_kyuubi\_loadbalancer\_extra\_annotations) | Optional extra annotations to be supplied to the load balancer service in Kyuubi. | `string` | `null` | no |
-| <a name="input_kyuubi_profile"></a> [kyuubi\_profile](#input\_kyuubi\_profile) | The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'. | `string` | `"production"` | no |
-| <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | `null` | no |
-| <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
-| <a name="input_kyuubi_user"></a> [kyuubi\_user](#input\_kyuubi\_user) | Define the user to be used for running Kyuubi enginers | `string` | `"kyuubi-spark-engine"` | no |
-| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | `null` | no |
-| <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | `null` | no |
-| <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
-| <a name="input_loki_active_index_directory_size"></a> [loki\_active\_index\_directory\_size](#input\_loki\_active\_index\_directory\_size) | Storage size for the active index directory for Loki | `string` | `"10G"` | no |
-| <a name="input_loki_chunks_size"></a> [loki\_chunks\_size](#input\_loki\_chunks\_size) | Storage size for the Loki chucks storage | `string` | `"500G"` | no |
-| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | `null` | no |
-| <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | `null` | no |
-| <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
-| <a name="input_model"></a> [model](#input\_model) | The name of the juju model to deploy Spark to | `string` | `"spark"` | no |
-| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules. | `string` | `null` | no |
-| <a name="input_prometheus_size"></a> [prometheus\_size](#input\_prometheus\_size) | Storage size for the Prometheus database | `string` | `"500G"` | no |
-| <a name="input_pushgateway_revision"></a> [pushgateway\_revision](#input\_pushgateway\_revision) | Charm revision for prometheus-pushgateway-k8s | `number` | `null` | no |
-| <a name="input_s3"></a> [s3](#input\_s3) | S3 Bucket information | <pre>object({<br/>    bucket   = optional(string, "spark-test")<br/>    endpoint = optional(string, "https://s3.amazonaws.com")<br/>    region   = optional(string, "us-east-1")<br/>  })</pre> | `{}` | no |
-| <a name="input_s3_revision"></a> [s3\_revision](#input\_s3\_revision) | Charm revision for s3-integrator | `number` | `null` | no |
-| <a name="input_scrape_config_revision"></a> [scrape\_config\_revision](#input\_scrape\_config\_revision) | Charm revision for prometheus-scrape-config-k8s | `number` | `null` | no |
-| <a name="input_storage_backend"></a> [storage\_backend](#input\_storage\_backend) | Storage backend to be used | `string` | `"s3"` | no |
-| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The file path of the private key to use for TLS certificates. | `string` | `null` | no |
-| <a name="input_traefik_size"></a> [traefik\_size](#input\_traefik\_size) | Storage size for the Traefik storage | `string` | `"10G"` | no |
-| <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | `null` | no |
-| <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | `null` | no |
-| <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
-| <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Define the number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
-| <a name="output_components"></a> [components](#output\_components) | List of component modules (spark and optional addons) and their deployed charms/offers. |
-| <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
diff --git a/releases/terraform/products/charmed-spark-3.5/locals.tf b/releases/terraform/products/charmed-spark-3.5/locals.tf
deleted file mode 100644
index 10b756b2..00000000
--- a/releases/terraform/products/charmed-spark-3.5/locals.tf
+++ /dev/null
@@ -1,43 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-locals {
-  revisions = {
-    history_server  = 94
-    integration_hub = 113
-    kyuubi          = 142
-    kyuubi_users    = 495
-    metastore       = 495
-    zookeeper       = 78
-    data_integrator = 179
-    s3              = 145
-    azure_storage   = 15
-    grafana_agent   = 121
-    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128
-    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
-    cos_configuration = 65
-    pushgateway       = 27
-    scrape_config     = 67
-  }
-  images = {
-    history_server = {
-      spark-history-server-image = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
-    }
-    integration_hub = {
-      integration-hub-image = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
-    }
-    kyuubi = {
-      kyuubi-image = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:cdde52a4f72112ea09de11c6d21f901323619cf41015db89cb9d5742c57da303"
-    }
-    kyuubi_users = {
-      postgresql-image = 165
-    }
-    metastore = {
-      postgresql-image = 165
-    }
-    zookeeper = {
-      zookeeper-image = 34
-    }
-  }
-}
diff --git a/releases/terraform/products/charmed-spark-3.5/main.tf b/releases/terraform/products/charmed-spark-3.5/main.tf
deleted file mode 100644
index c7c79671..00000000
--- a/releases/terraform/products/charmed-spark-3.5/main.tf
+++ /dev/null
@@ -1,146 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_model" "spark" {
-  count      = (var.model_uuid == null && var.create_model == true) ? 1 : 0
-  name       = var.model
-  credential = var.K8S_CREDENTIAL
-  cloud {
-    name = var.K8S_CLOUD
-  }
-}
-
-resource "juju_model" "cos" {
-  count      = var.cos.deployed == "bundled" && var.model_uuid == null && var.create_model == true ? 1 : 0
-  name       = var.cos.model
-  credential = var.K8S_CREDENTIAL
-  cloud {
-    name = var.K8S_CLOUD
-  }
-}
-
-locals {
-  active_model = coalesce(var.model_uuid, var.model)
-}
-
-data "juju_model" "spark" {
-  name = var.model
-}
-
-module "ssc" {
-  depends_on  = [juju_model.spark]
-  source      = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev586"
-  model_uuid  = coalesce(var.model_uuid, data.juju_model.spark.id)
-  app_name    = "certificates"
-  channel     = "1/stable"
-  revision    = 317
-  constraints = "arch=amd64"
-  base        = "ubuntu@24.04"
-  units       = 1
-  config = {
-    ca-common-name = var.certificate_common_name
-  }
-}
-
-module "spark" {
-  depends_on                                 = [juju_model.spark, module.ssc]
-  source                                     = "../../components/spark-3.5"
-  model                                      = local.active_model
-  kyuubi_user                                = var.kyuubi_user
-  kyuubi_profile                             = var.kyuubi_profile
-  admin_password                             = var.admin_password
-  tls_private_key                            = var.tls_private_key
-  enable_dynamic_allocation                  = var.enable_dynamic_allocation
-  kyuubi_k8s_node_selectors                  = var.kyuubi_k8s_node_selectors
-  kyuubi_loadbalancer_extra_annotations      = var.kyuubi_loadbalancer_extra_annotations
-  kyuubi_gpu_enable                          = var.kyuubi_gpu_enable
-  kyuubi_gpu_engine_executors_limit          = var.kyuubi_gpu_engine_executors_limit
-  kyuubi_gpu_pinned_memory                   = var.kyuubi_gpu_pinned_memory
-  kyuubi_driver_pod_template                 = var.kyuubi_driver_pod_template
-  kyuubi_executor_pod_template               = var.kyuubi_executor_pod_template
-  kyuubi_executor_cores                      = var.kyuubi_executor_cores
-  kyuubi_executor_memory                     = var.kyuubi_executor_memory
-  driver_pod_template                        = var.driver_pod_template
-  executor_pod_template                      = var.executor_pod_template
-  integration_hub_monitored_service_accounts = var.integration_hub_monitored_service_accounts
-
-  tls_app_name              = module.ssc.app_name
-  tls_certificates_endpoint = module.ssc.provides.certificates
-  zookeeper_units           = var.zookeeper_units
-  kyuubi_units              = var.kyuubi_units
-
-  history_server_revision  = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server
-  history_server_image     = var.history_server_image != null ? var.history_server_image : local.images.history_server
-  integration_hub_revision = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub
-  integration_hub_image    = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub
-  kyuubi_revision          = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi
-  kyuubi_image             = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi
-  kyuubi_users_revision    = var.kyuubi_users_revision != null ? var.kyuubi_users_revision : local.revisions.kyuubi_users
-  kyuubi_users_image       = var.kyuubi_users_image != null ? var.kyuubi_users_image : local.images.kyuubi_users
-  metastore_revision       = var.metastore_revision != null ? var.metastore_revision : local.revisions.metastore
-  metastore_image          = var.metastore_image != null ? var.metastore_image : local.images.metastore
-  zookeeper_revision       = var.zookeeper_revision != null ? var.zookeeper_revision : local.revisions.zookeeper
-  zookeeper_image          = var.zookeeper_image != null ? var.zookeeper_image : local.images.zookeeper
-  data_integrator_revision = var.data_integrator_revision != null ? var.data_integrator_revision : local.revisions.data_integrator
-
-  kyuubi_users_size = var.kyuubi_users_size
-  metastore_size    = var.metastore_size
-  zookeeper_size    = var.zookeeper_size
-}
-
-module "azure_storage" {
-  depends_on    = [module.spark]
-  count         = var.storage_backend == "azure_storage" ? 1 : 0
-  source = "../../charms/azure-storage-integrator"
-  model_uuid    = local.active_model
-  spark_charms  = module.spark.charms
-  azure_storage = var.azure_storage
-
-  azure_storage_revision = var.azure_storage_revision != null ? var.azure_storage_revision : local.revisions.azure_storage
-}
-
-module "s3" {
-  depends_on   = [module.spark]
-  count        = var.storage_backend == "s3" ? 1 : 0
-  source = "../../charms/s3-integrator"
-  model_uuid   = local.active_model
-  spark_charms = module.spark.charms
-  s3           = var.s3
-
-  s3_revision = var.s3_revision != null ? var.s3_revision : local.revisions.s3
-}
-
-module "bundled_cos" {
-  depends_on   = [juju_model.cos]
-  count        = var.cos.deployed == "bundled" ? 1 : 0
-  source = "../cos"
-  model        = var.cos.model
-  cos_tls_ca   = var.cos.tls.ca
-  cos_tls_cert = var.cos.tls.cert
-  cos_tls_key  = var.cos.tls.key
-
-  alertmanager_size                = var.alertmanager_size
-  grafana_size                     = var.grafana_size
-  loki_active_index_directory_size = var.loki_active_index_directory_size
-  loki_chunks_size                 = var.loki_chunks_size
-  prometheus_size                  = var.prometheus_size
-  traefik_size                     = var.traefik_size
-}
-
-
-module "observability" {
-  depends_on       = [module.spark, module.bundled_cos]
-  count            = var.cos.deployed == "no" ? 0 : 1
-  source = "../../components/observability"
-  dashboards_offer = var.cos.deployed == "external" ? var.cos.offers.dashboard : one(module.bundled_cos[*].dashboards_offer)
-  logging_offer    = var.cos.deployed == "external" ? var.cos.offers.logging : one(module.bundled_cos[*].logging_offer)
-  metrics_offer    = var.cos.deployed == "external" ? var.cos.offers.metrics : one(module.bundled_cos[*].metrics_offer)
-  spark_model      = var.model
-  model_uuid       = local.active_model
-  spark_charms     = module.spark.charms
-
-  grafana_agent_revision     = var.grafana_agent_revision != null ? var.grafana_agent_revision : local.revisions.grafana_agent
-  cos_configuration_revision = var.cos_configuration_revision != null ? var.cos_configuration_revision : local.revisions.cos_configuration
-  pushgateway_revision       = var.pushgateway_revision != null ? var.pushgateway_revision : local.revisions.pushgateway
-  scrape_config_revision     = var.scrape_config_revision != null ? var.scrape_config_revision : local.revisions.scrape_config
-}
diff --git a/releases/terraform/products/charmed-spark-3.5/outputs.tf b/releases/terraform/products/charmed-spark-3.5/outputs.tf
deleted file mode 100644
index f1a2b99f..00000000
--- a/releases/terraform/products/charmed-spark-3.5/outputs.tf
+++ /dev/null
@@ -1,51 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = merge(
-    concat(
-      module.spark[*].charms, module.azure_storage[*].charms, module.s3[*].charms, module.bundled_cos[*].charms, module.observability[*].charms
-    )...
-  )
-}
-
-output "components" {
-  description = "List of component modules (spark and optional addons) and their deployed charms/offers."
-  value = [
-    {
-      name   = "spark"
-      charms = module.spark.charms
-      offers = module.spark.offers
-    },
-    {
-      name   = "azure_storage"
-      charms = try(module.azure_storage.charms, [])
-      offers = try(module.azure_storage.offers, {})
-    },
-    {
-      name   = "s3"
-      charms = try(module.s3.charms, [])
-      offers = try(module.s3.offers, {})
-    },
-    {
-      name   = "bundled_cos"
-      charms = try(module.bundled_cos.charms, [])
-      offers = try(module.bundled_cos.offers, {})
-    },
-    {
-      name   = "observability"
-      charms = try(module.observability.charms, [])
-      offers = try(module.observability.offers, {})
-    }
-  ]
-}
-
-output "offers" {
-  description = "The name and url of the various offers being exposed"
-  value = merge(
-    concat(
-      module.spark[*].offers
-    )...
-  )
-}
diff --git a/releases/terraform/products/charmed-spark-3.5/providers.tf b/releases/terraform/products/charmed-spark-3.5/providers.tf
deleted file mode 100644
index b60772fa..00000000
--- a/releases/terraform/products/charmed-spark-3.5/providers.tf
+++ /dev/null
@@ -1,6 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-provider "juju" {
-  # Configure the Juju provider if needed (e.g., `controller`, `model`, `username`, etc.).
-}
diff --git a/releases/terraform/products/charmed-spark-3.5/terraform.tf b/releases/terraform/products/charmed-spark-3.5/terraform.tf
deleted file mode 100644
index 975cf66e..00000000
--- a/releases/terraform/products/charmed-spark-3.5/terraform.tf
+++ /dev/null
@@ -1,13 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-terraform {
-  required_version = ">=1.0.0"
-
-  required_providers {
-    juju = {
-      source  = "juju/juju"
-      version = ">=1.0.0"
-    }
-  }
-}
diff --git a/releases/terraform/products/charmed-spark-3.5/variables.tf b/releases/terraform/products/charmed-spark-3.5/variables.tf
deleted file mode 100644
index 57c95ffe..00000000
--- a/releases/terraform/products/charmed-spark-3.5/variables.tf
+++ /dev/null
@@ -1,395 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Juju
-
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "alertmanager_size" {
-  description = "Storage size for the alertmanager database"
-  type        = string
-  default     = "10G"
-}
-
-variable "azure_storage" {
-  description = "Azure Object storage information"
-  type = object({
-    container       = optional(string, "azurecontainer")
-    storage_account = optional(string, "azurestorageaccount")
-    secret_key      = optional(string, "azurestoragesecret")
-    protocol        = optional(string, "abfss")
-  })
-  default = {}
-}
-
-variable "azure_storage_revision" {
-  description = "Charm revision for azure-storage-integrator"
-  type        = number
-  default     = null
-}
-
-variable "certificate_common_name" {
-  description = "Common name for the certificate to be used in self-signed"
-  type        = string
-  default     = "charmed-spark"
-}
-
-variable "cos" {
-  description = "Observability settings"
-  type = object({
-    model    = optional(string, "cos")
-    deployed = optional(string, "bundled")
-    offers = optional(object({
-      dashboard = optional(string, null),
-      metrics   = optional(string, null),
-      logging   = optional(string, null)
-    }), {}),
-    tls = optional(object({
-      cert = optional(string, "")
-      key  = optional(string, "")
-      ca   = optional(string, "")
-    }), {})
-  })
-  default = { model = "cos", deployed = "bundled", offers = {}, tls = {} }
-
-  validation {
-    condition     = contains(["external", "bundled", "no"], var.cos.deployed)
-    error_message = "Valid values for var: cos.deployed are (external, bundled, no)"
-  }
-
-  validation {
-    condition = var.cos.deployed != "external" || alltrue([
-      var.cos.offers.dashboard != null,
-      var.cos.offers.metrics != null,
-      var.cos.offers.logging != null,
-    ])
-    error_message = "When using external cos, please define all offers variables"
-  }
-
-}
-
-variable "cos_configuration_revision" {
-  description = "Charm revision for cos-configuration-k8s"
-  type        = number
-  default     = null
-}
-
-variable "create_model" {
-  description = "Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism."
-  type        = bool
-  default     = true
-  nullable    = false
-}
-
-variable "data_integrator_revision" {
-  description = "Charm revision for data-integrator"
-  type        = number
-  default     = null
-}
-
-variable "driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "enable_dynamic_allocation" {
-  description = "Enable dynamic allocation of pods for Spark jobs."
-  type        = bool
-  default     = false
-}
-
-variable "executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "grafana_agent_revision" {
-  description = "Charm revision for grafana-agent-k8s"
-  type        = number
-  default     = null
-}
-
-variable "grafana_size" {
-  description = "Storage size for the grafana database"
-  type        = string
-  default     = "10G"
-}
-
-variable "history_server_image" {
-  description = "Image for spark-history-server-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "history_server_revision" {
-  description = "Charm revision for spark-history-server-k8s"
-  type        = number
-  default     = null
-}
-
-variable "integration_hub_image" {
-  description = "Image for spark-integration-hub-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "integration_hub_monitored_service_accounts" {
-  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-  type        = string
-  default     = null
-}
-
-variable "integration_hub_revision" {
-  description = "Charm revision for spark-integration-hub-k8s"
-  type        = number
-  default     = null
-}
-
-variable "K8S_CLOUD" {
-  type        = string
-  description = "The kubernetes juju cloud name."
-  default     = "microk8s"
-}
-
-variable "K8S_CREDENTIAL" {
-  type        = string
-  description = "The name of the kubernetes juju credential."
-  default     = "microk8s"
-}
-
-variable "kyuubi_driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_cores" {
-  description = "Set kyuubi executor pods cpu cores."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_memory" {
-  description = "Set kyuubi executor pods memory (in GB)."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_gpu_enable" {
-  description = "Enable GPU acceleration for SparkSQLEngine."
-  type        = bool
-  default     = false
-}
-
-variable "kyuubi_gpu_engine_executors_limit" {
-  description = "Limit the number of GPUs an engine can schedule executor pods on."
-  type        = number
-  default     = 1
-}
-
-variable "kyuubi_gpu_pinned_memory" {
-  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-  type        = number
-  default     = 1
-}
-
-variable "kyuubi_image" {
-  description = "Image for kyuubi-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "kyuubi_k8s_node_selectors" {
-  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_loadbalancer_extra_annotations" {
-  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_profile" {
-  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-  type        = string
-  nullable    = false
-  default     = "production"
-}
-
-variable "kyuubi_revision" {
-  description = "Charm revision for kyuubi-k8s"
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_units" {
-  description = "Number of Kyuubi units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
-
-variable "kyuubi_user" {
-  description = "Define the user to be used for running Kyuubi enginers"
-  type        = string
-  default     = "kyuubi-spark-engine"
-}
-
-variable "kyuubi_users_image" {
-  description = "Image for postgresql-k8s (auth-db)"
-  type        = map(string)
-  default     = null
-}
-
-variable "kyuubi_users_revision" {
-  description = "Charm revision for postgresql-k8s (auth-db)"
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_users_size" {
-  description = "Storage size for the Kyuubi users database"
-  type        = string
-  default     = "1G"
-}
-
-variable "loki_active_index_directory_size" {
-  description = "Storage size for the active index directory for Loki"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_chunks_size" {
-  description = "Storage size for the Loki chucks storage"
-  type        = string
-  default     = "500G"
-}
-
-variable "metastore_image" {
-  description = "Image for postgresql-k8s (metastore)"
-  type        = map(string)
-  default     = null
-}
-
-variable "metastore_revision" {
-  description = "Charm revision for postgresql-k8s (metastore)"
-  type        = number
-  default     = null
-}
-
-variable "metastore_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-variable "model" {
-  description = "The name of the juju model to deploy Spark to"
-  type        = string
-  default     = "spark"
-}
-
-variable "model_uuid" {
-  description = "Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules."
-  type        = string
-  default     = null
-}
-
-variable "prometheus_size" {
-  description = "Storage size for the Prometheus database"
-  type        = string
-  default     = "500G"
-}
-
-variable "pushgateway_revision" {
-  description = "Charm revision for prometheus-pushgateway-k8s"
-  type        = number
-  default     = null
-}
-
-variable "s3" {
-  description = "S3 Bucket information"
-  type = object({
-    bucket   = optional(string, "spark-test")
-    endpoint = optional(string, "https://s3.amazonaws.com")
-    region   = optional(string, "us-east-1")
-  })
-  default = {}
-}
-
-variable "s3_revision" {
-  description = "Charm revision for s3-integrator"
-  type        = number
-  default     = null
-}
-
-variable "scrape_config_revision" {
-  description = "Charm revision for prometheus-scrape-config-k8s"
-  type        = number
-  default     = null
-}
-
-variable "storage_backend" {
-  type        = string
-  description = "Storage backend to be used"
-
-  validation {
-    condition     = contains(["azure_storage", "s3"], var.storage_backend)
-    error_message = "Valid values for var: test_variable are (s3, azure_storage)."
-  }
-
-  default = "s3"
-}
-
-variable "tls_private_key" {
-  description = "The file path of the private key to use for TLS certificates."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
-variable "traefik_size" {
-  description = "Storage size for the Traefik storage"
-  type        = string
-  default     = "10G"
-}
-
-variable "zookeeper_image" {
-  description = "Image for zookeeper-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "zookeeper_revision" {
-  description = "Charm revision for zookeeper-k8s"
-  type        = number
-  default     = null
-}
-
-variable "zookeeper_size" {
-  description = "Storage size for the metastore database"
-  type        = string
-  default     = "10G"
-}
-
-variable "zookeeper_units" {
-  description = "Define the number of zookeeper units. 3 units are recommended for high availability."
-  type        = number
-  default     = 3
-  nullable    = false
-}
diff --git a/releases/terraform/products/cos/README.md b/releases/terraform/products/cos/README.md
deleted file mode 100644
index 8c609af3..00000000
--- a/releases/terraform/products/cos/README.md
+++ /dev/null
@@ -1,77 +0,0 @@
-## Requirements
-
-| Name | Version |
-|------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
-| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
-
-## Providers
-
-| Name | Version |
-|------|---------|
-| <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
-
-## Modules
-
-No modules.
-
-## Resources
-
-| Name | Type |
-|------|------|
-| [juju_application.alertmanager](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.catalogue](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.grafana](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_application.traefik](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
-| [juju_integration.catalogue-alertmanager](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.catalogue-grafana](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.catalogue-prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.catalogue-traefik](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana-alertmanager-dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana-alertmanager-source](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana-loki-dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana-loki-source](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana-prometheus-dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.grafana-prometheus-source](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.loki-alertmanager](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.prometheus-alertmanager-alerting](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.prometheus-alertmanager-metrics](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.prometheus-grafana](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.prometheus-loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.prometheus-traefik](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.traefik-alertmanager](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.traefik-grafana](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.traefik-loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_integration.traefik-prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
-| [juju_offer.grafana_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_offer.loki_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_offer.prometheus_receive_remote_write](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_model.cos](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/model) | data source |
-
-## Inputs
-
-| Name | Description | Type | Default | Required |
-|------|-------------|------|---------|:--------:|
-| <a name="input_alertmanager_size"></a> [alertmanager\_size](#input\_alertmanager\_size) | Storage size for the alertmanager database | `string` | `"10G"` | no |
-| <a name="input_cos_tls_ca"></a> [cos\_tls\_ca](#input\_cos\_tls\_ca) | COS CA certificate | `string` | `""` | no |
-| <a name="input_cos_tls_cert"></a> [cos\_tls\_cert](#input\_cos\_tls\_cert) | COS certificate | `string` | `""` | no |
-| <a name="input_cos_tls_key"></a> [cos\_tls\_key](#input\_cos\_tls\_key) | COS certificate key | `string` | `""` | no |
-| <a name="input_cos_user"></a> [cos\_user](#input\_cos\_user) | The name of the Juju user of the COS deployment. | `string` | `"admin"` | no |
-| <a name="input_grafana_size"></a> [grafana\_size](#input\_grafana\_size) | Storage size for the grafana database | `string` | `"10G"` | no |
-| <a name="input_loki_active_index_directory_size"></a> [loki\_active\_index\_directory\_size](#input\_loki\_active\_index\_directory\_size) | Storage size for the active index directory for Loki | `string` | `"10G"` | no |
-| <a name="input_loki_chunks_size"></a> [loki\_chunks\_size](#input\_loki\_chunks\_size) | Storage size for the Loki chucks storage | `string` | `"500G"` | no |
-| <a name="input_model"></a> [model](#input\_model) | The name of the Juju model to deploy to | `string` | n/a | yes |
-| <a name="input_prometheus_size"></a> [prometheus\_size](#input\_prometheus\_size) | Storage size for the Prometheus database | `string` | `"500G"` | no |
-| <a name="input_traefik_size"></a> [traefik\_size](#input\_traefik\_size) | Storage size for the Traefik storage | `string` | `"10G"` | no |
-
-## Outputs
-
-| Name | Description |
-|------|-------------|
-| <a name="output_charms"></a> [charms](#output\_charms) | The name of the charms which are part of the deployment. |
-| <a name="output_dashboards_offer"></a> [dashboards\_offer](#output\_dashboards\_offer) | n/a |
-| <a name="output_logging_offer"></a> [logging\_offer](#output\_logging\_offer) | n/a |
-| <a name="output_metrics_offer"></a> [metrics\_offer](#output\_metrics\_offer) | n/a |
-| <a name="output_user"></a> [user](#output\_user) | The name of the Juju user of the COS deployment. |
diff --git a/releases/terraform/products/cos/applications.tf b/releases/terraform/products/cos/applications.tf
deleted file mode 100644
index bfc8f7dc..00000000
--- a/releases/terraform/products/cos/applications.tf
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-resource "juju_application" "alertmanager" {
-  name  = "alertmanager"
-  trust = true
-  model_uuid = data.juju_model.cos.id
-  charm {
-    name    = "alertmanager-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    data = var.alertmanager_size
-  }
-}
-
-resource "juju_application" "catalogue" {
-  name  = "catalogue"
-  trust = true
-  model_uuid = data.juju_model.cos.id
-  charm {
-    name    = "catalogue-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  config = {
-    "description" : "Canonical Observability Stack Lite"
-  }
-}
-
-resource "juju_application" "grafana" {
-  name  = "grafana"
-  trust = true
-  model_uuid = data.juju_model.cos.id
-  charm {
-    name    = "grafana-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    database = var.grafana_size
-  }
-}
-
-resource "juju_application" "loki" {
-  name  = "loki"
-  trust = true
-  model_uuid = data.juju_model.cos.id
-  charm {
-    name    = "loki-k8s"
-    channel = "1/stable"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    active-index-directory = var.loki_active_index_directory_size
-    loki-chunks            = var.loki_chunks_size
-  }
-}
-
-resource "juju_application" "prometheus" {
-  name  = "prometheus"
-  trust = true
-  model_uuid = data.juju_model.cos.id
-  charm {
-    name    = "prometheus-k8s"
-    channel = "1/stable"
-  }
-  config = {
-    "metrics_retention_time" : "90d"
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    database = var.prometheus_size
-  }
-}
-
-resource "juju_application" "traefik" {
-  name  = "traefik"
-  trust = true
-  model_uuid = data.juju_model.cos.id
-  charm {
-    name = "traefik-k8s"
-    # FIXME(stable): Use different track when available
-    channel = "latest/stable"
-  }
-  config = {
-    "tls-cert" : var.cos_tls_cert,
-    "tls-key" : var.cos_tls_key,
-    "tls-ca" : var.cos_tls_ca
-  }
-  units       = 1
-  constraints = "arch=amd64"
-  storage_directives = {
-    configurations = var.traefik_size
-  }
-}
-
diff --git a/releases/terraform/products/cos/integrations.tf b/releases/terraform/products/cos/integrations.tf
deleted file mode 100644
index 7ecfd638..00000000
--- a/releases/terraform/products/cos/integrations.tf
+++ /dev/null
@@ -1,300 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-
-
-
-resource "juju_integration" "traefik-grafana" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "traefik-route"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-prometheus" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress-per-unit"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-loki" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress-per-unit"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "traefik-alertmanager" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "ingress"
-  }
-}
-
-resource "juju_integration" "prometheus-alertmanager-alerting" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "alertmanager"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "alerting"
-  }
-}
-
-
-resource "juju_integration" "grafana-prometheus-source" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "grafana-loki-source" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "grafana-alertmanager-source" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-source"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "grafana-source"
-  }
-}
-
-
-resource "juju_integration" "loki-alertmanager" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "alertmanager"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "alerting"
-  }
-}
-
-
-resource "juju_integration" "prometheus-traefik" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-alertmanager-metrics" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "self-metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-loki" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "prometheus-grafana" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "metrics-endpoint"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "metrics-endpoint"
-  }
-}
-
-
-resource "juju_integration" "grafana-loki-dashboard" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.loki.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "grafana-prometheus-dashboard" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "grafana-alertmanager-dashboard" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "grafana-dashboard"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "grafana-dashboard"
-  }
-}
-
-
-resource "juju_integration" "catalogue-traefik" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "ingress"
-  }
-
-  application {
-    name     = juju_application.traefik.name
-    endpoint = "ingress"
-  }
-}
-
-
-resource "juju_integration" "catalogue-grafana" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.grafana.name
-    endpoint = "catalogue"
-  }
-}
-
-
-resource "juju_integration" "catalogue-prometheus" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.prometheus.name
-    endpoint = "catalogue"
-  }
-}
-
-
-resource "juju_integration" "catalogue-alertmanager" {
-  model_uuid = data.juju_model.cos.id
-
-  application {
-    name     = juju_application.catalogue.name
-    endpoint = "catalogue"
-  }
-
-  application {
-    name     = juju_application.alertmanager.name
-    endpoint = "catalogue"
-  }
-}
diff --git a/releases/terraform/products/cos/locals.tf b/releases/terraform/products/cos/locals.tf
deleted file mode 100644
index 44f3a0f6..00000000
--- a/releases/terraform/products/cos/locals.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Local values for $(basename "$moduleDir").
-# Define computed values used in this module.
diff --git a/releases/terraform/products/cos/main.tf b/releases/terraform/products/cos/main.tf
deleted file mode 100644
index 46aca146..00000000
--- a/releases/terraform/products/cos/main.tf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Main Terraform code for $(basename "$moduleDir").
-# Module content is structured across *.tf files in this directory.
diff --git a/releases/terraform/products/cos/outputs.tf b/releases/terraform/products/cos/outputs.tf
deleted file mode 100644
index e206ae26..00000000
--- a/releases/terraform/products/cos/outputs.tf
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-output "charms" {
-  description = "The name of the charms which are part of the deployment."
-  value = {
-    alertmanager = juju_application.alertmanager.name
-    catalogue    = juju_application.catalogue.name
-    grafana      = juju_application.grafana.name
-    loki         = juju_application.loki.name
-    prometheus   = juju_application.prometheus.name
-  }
-}
-
-output "dashboards_offer" {
-  value = juju_offer.grafana_dashboards.url
-}
-
-output "logging_offer" {
-  value = juju_offer.loki_logging.url
-}
-
-output "metrics_offer" {
-  value = juju_offer.prometheus_receive_remote_write.url
-}
-
-output "user" {
-  description = "The name of the Juju user of the COS deployment."
-  value       = var.cos_user
-}
diff --git a/releases/terraform/products/cos/providers.tf b/releases/terraform/products/cos/providers.tf
deleted file mode 100644
index 0693f27b..00000000
--- a/releases/terraform/products/cos/providers.tf
+++ /dev/null
@@ -1,6 +0,0 @@
-# Provider configuration for module $(basename "$moduleDir").
-# Terraform block moved to terraform.tf and should not be in providers.tf.
-
-# provider "juju" {
-#   # Configure provider here if required by the module.
-# }
diff --git a/releases/terraform/products/cos/resources.tf b/releases/terraform/products/cos/resources.tf
deleted file mode 100644
index c48001a0..00000000
--- a/releases/terraform/products/cos/resources.tf
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define juju resources (model, secrets, storage)
-
-data "juju_model" "cos" {
-  name = var.model
-}
-
-resource "juju_offer" "prometheus_receive_remote_write" {
-  model_uuid = data.juju_model.cos.id
-  application_name = juju_application.prometheus.name
-  endpoints        = ["receive-remote-write"]
-}
-
-resource "juju_offer" "grafana_dashboards" {
-  model_uuid = data.juju_model.cos.id
-  application_name = juju_application.grafana.name
-  endpoints        = ["grafana-dashboard"]
-}
-
-resource "juju_offer" "loki_logging" {
-  model_uuid = data.juju_model.cos.id
-  application_name = juju_application.loki.name
-  endpoints        = ["logging"]
-}
-
diff --git a/releases/terraform/products/cos/terraform.tf b/releases/terraform/products/cos/terraform.tf
deleted file mode 100644
index 6412125a..00000000
--- a/releases/terraform/products/cos/terraform.tf
+++ /dev/null
@@ -1,10 +0,0 @@
-terraform {
-  required_version = ">=1.0.0"
-
-  required_providers {
-    juju = {
-      source  = "juju/juju"
-      version = ">=1.0.0"
-    }
-  }
-}
diff --git a/releases/terraform/products/cos/variables.tf b/releases/terraform/products/cos/variables.tf
deleted file mode 100644
index 16a0b362..00000000
--- a/releases/terraform/products/cos/variables.tf
+++ /dev/null
@@ -1,69 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Define cos-specific variables
-
-variable "alertmanager_size" {
-  description = "Storage size for the alertmanager database"
-  type        = string
-  default     = "10G"
-}
-
-variable "cos_tls_ca" {
-  type        = string
-  default     = ""
-  description = "COS CA certificate"
-}
-
-variable "cos_tls_cert" {
-  type        = string
-  default     = ""
-  description = "COS certificate"
-}
-
-variable "cos_tls_key" {
-  type        = string
-  default     = ""
-  description = "COS certificate key"
-}
-
-variable "cos_user" {
-  description = "The name of the Juju user of the COS deployment."
-  type        = string
-  default     = "admin"
-}
-
-variable "grafana_size" {
-  description = "Storage size for the grafana database"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_active_index_directory_size" {
-  description = "Storage size for the active index directory for Loki"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_chunks_size" {
-  description = "Storage size for the Loki chucks storage"
-  type        = string
-  default     = "500G"
-}
-
-variable "model" {
-  description = "The name of the Juju model to deploy to"
-  type        = string
-}
-
-variable "prometheus_size" {
-  description = "Storage size for the Prometheus database"
-  type        = string
-  default     = "500G"
-}
-
-variable "traefik_size" {
-  description = "Storage size for the Traefik storage"
-  type        = string
-  default     = "10G"
-}

From 9f99119735e1da23b8730735a042f940579f9e83 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 1 Apr 2026 15:16:16 +0200
Subject: [PATCH 09/62] Streamline initial module

---
 .../components/spark-3.4/applications.tf      | 102 +++------
 terraform/components/spark-3.4/resources.tf   |  12 +-
 terraform/components/spark-3.4/variables.tf   | 141 ------------
 terraform/products/charmed-spark-3.4/main.tf  |  90 +++-----
 .../products/charmed-spark-3.4/outputs.tf     |   8 +-
 .../products/charmed-spark-3.4/variables.tf   | 204 ++----------------
 6 files changed, 78 insertions(+), 479 deletions(-)

diff --git a/terraform/components/spark-3.4/applications.tf b/terraform/components/spark-3.4/applications.tf
index d9419543..a1409b03 100644
--- a/terraform/components/spark-3.4/applications.tf
+++ b/terraform/components/spark-3.4/applications.tf
@@ -21,66 +21,6 @@ resource "juju_application" "history_server" {
   units = var.history_server.units
 }
 
-resource "juju_application" "kyuubi" {
-  name       = var.kyuubi.app_name
-  model_uuid = var.model_uuid
-
-  charm {
-    name     = "kyuubi-k8s"
-    channel  = var.kyuubi.channel
-    revision = var.kyuubi.revision != null ? var.kyuubi.revision : local.revisions.kyuubi
-  }
-
-  config      = var.kyuubi.config # TODO: parametrize
-  constraints = var.kyuubi.constraints
-  resources = merge({
-    kyuubi-image = local.images.kyuubi
-    },
-    var.kyuubi.resources == null ? {} : var.kyuubi.resources
-  )
-  trust = true
-  units = var.kyuubi.units
-
-  # config = merge(
-  #   {
-  #     enable-dynamic-allocation  = var.enable_dynamic_allocation
-  #     expose-external            = "loadbalancer"
-  #     gpu-enable                 = var.kyuubi_gpu_enable
-  #     gpu-engine-executors-limit = var.kyuubi_gpu_engine_executors_limit
-  #     gpu-pinned-memory          = var.kyuubi_gpu_pinned_memory
-  #     namespace                  = var.model_uuid # TODO: parametrize
-  #     profile                    = var.kyuubi_profile
-  #     service-account            = var.kyuubi_user
-  #   },
-  #   var.kyuubi_k8s_node_selectors == null ? {} : {
-  #     k8s-node-selectors = var.kyuubi_k8s_node_selectors
-  #   },
-  #   var.kyuubi_loadbalancer_extra_annotations == null ? {} : {
-  #     loadbalancer-extra-annotations = var.kyuubi_loadbalancer_extra_annotations
-  #   },
-  #   var.tls_private_key == null ? {} : {
-  #     tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-  #   },
-  #   var.admin_password == null ? {} : {
-  #     system-users = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-  #   },
-  #   var.kyuubi_executor_cores == null ? {} : {
-  #     executor-cores = var.kyuubi_executor_cores
-  #   },
-  #   var.kyuubi_executor_memory == null ? {} : {
-  #     executor-memory = var.kyuubi_executor_memory
-  #   },
-  #   var.kyuubi_driver_pod_template == null ? {} : {
-  #     driver-pod-template = var.kyuubi_driver_pod_template
-  #   },
-  #   var.kyuubi_executor_pod_template == null ? {} : {
-  #     executor-pod-template = var.kyuubi_executor_pod_template
-  #   }
-  # )
-
-}
-
-
 resource "juju_application" "integration_hub" {
   name       = var.integration_hub.app_name
   model_uuid = var.model_uuid
@@ -100,20 +40,34 @@ resource "juju_application" "integration_hub" {
   )
   trust = true
   units = var.integration_hub.units
-  # config = merge(
-  #   {
-  #     enable-dynamic-allocation = var.enable_dynamic_allocation
-  #   },
-  #   var.driver_pod_template == null ? {} : {
-  #     driver-pod-template = var.driver_pod_template
-  #   },
-  #   var.executor_pod_template == null ? {} : {
-  #     executor-pod-template = var.executor_pod_template
-  #   },
-  #   var.integration_hub_monitored_service_accounts == null ? {} : {
-  #     monitored-service-accounts = var.integration_hub_monitored_service_accounts
-  #   }
-  # )
 }
 
+resource "juju_application" "kyuubi" {
+  name       = var.kyuubi.app_name
+  model_uuid = var.model_uuid
 
+  charm {
+    name     = "kyuubi-k8s"
+    channel  = var.kyuubi.channel
+    revision = var.kyuubi.revision != null ? var.kyuubi.revision : local.revisions.kyuubi
+  }
+
+  config = merge(
+    {
+      expose-external = "loadbalancer",
+    },
+    length(juju_secret.system_users_and_private_key_secret) > 0 ? {
+      system-users           = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}",
+      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+    } : {},
+    var.kyuubi.config
+  )
+  constraints = var.kyuubi.constraints
+  resources = merge({
+    kyuubi-image = local.images.kyuubi
+    },
+    var.kyuubi.resources == null ? {} : var.kyuubi.resources
+  )
+  trust = true
+  units = var.kyuubi.units
+}
diff --git a/terraform/components/spark-3.4/resources.tf b/terraform/components/spark-3.4/resources.tf
index 90a96fc7..6ae375a9 100644
--- a/terraform/components/spark-3.4/resources.tf
+++ b/terraform/components/spark-3.4/resources.tf
@@ -3,17 +3,9 @@
 
 # Define juju resources (model, secrets, storage)
 
-locals {
-  target_model_uuid = var.model_uuid
-}
-
-# data "juju_model" "spark" {
-#   name = var.model
-# }
-
 resource "juju_secret" "system_users_and_private_key_secret" {
   count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model_uuid = local.target_model_uuid
+  model_uuid = var.model_uuid
   name       = "system_users_and_private_key_secret"
   value = merge(
     var.admin_password == null ? {} : {
@@ -28,7 +20,7 @@ resource "juju_secret" "system_users_and_private_key_secret" {
 
 resource "juju_access_secret" "system_users_and_private_key_secret_access" {
   count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model_uuid = local.target_model_uuid
+  model_uuid = var.model_uuid
   applications = [
     juju_application.kyuubi.name
   ]
diff --git a/terraform/components/spark-3.4/variables.tf b/terraform/components/spark-3.4/variables.tf
index d10839d9..8c42656b 100644
--- a/terraform/components/spark-3.4/variables.tf
+++ b/terraform/components/spark-3.4/variables.tf
@@ -65,7 +65,6 @@ variable "kyuubi" {
   }
 }
 
-
 variable "admin_password" {
   description = "The password for the admin user."
   type        = string
@@ -73,146 +72,6 @@ variable "admin_password" {
   default     = null
 }
 
-
-# variable "driver_pod_template" {
-#   description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-#   type        = string
-#   default     = null
-# }
-
-# variable "enable_dynamic_allocation" {
-#   description = "Enable dynamic allocation of pods for Spark jobs."
-#   type        = bool
-#   default     = false
-# }
-
-# variable "executor_pod_template" {
-#   description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-#   type        = string
-#   default     = null
-# }
-
-# variable "history_server_image" {
-#   description = "Image for spark-history-server-k8s"
-#   type        = map(string)
-#   nullable    = false
-# }
-
-# variable "history_server_revision" {
-#   description = "Charm revision for spark-history-server-k8s"
-#   type        = number
-#   nullable    = false
-# }
-
-# variable "integration_hub_image" {
-#   description = "Image for spark-integration-hub-k8s"
-#   type        = map(string)
-#   nullable    = false
-# }
-
-# variable "integration_hub_monitored_service_accounts" {
-#   description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-#   type        = string
-#   default     = null
-# }
-
-# variable "integration_hub_revision" {
-#   description = "Charm revision for spark-integration-hub-k8s"
-#   type        = number
-#   nullable    = false
-# }
-
-# variable "kyuubi_driver_pod_template" {
-#   description = "Define K8s driver pod from a file accessible in the object storage."
-#   type        = string
-#   default     = null
-# }
-
-# variable "kyuubi_executor_cores" {
-#   description = "Set kyuubi executor pods cpu cores."
-#   type        = number
-#   default     = null
-# }
-
-# variable "kyuubi_executor_memory" {
-#   description = "Set kyuubi executor pods memory (in GB)."
-#   type        = number
-#   default     = null
-# }
-
-# variable "kyuubi_executor_pod_template" {
-#   description = "Define K8s executor pod from a file accessible in the object storage."
-#   type        = string
-#   default     = null
-# }
-
-# variable "kyuubi_gpu_enable" {
-#   description = "Enable GPU acceleration for SparkSQLEngine."
-#   type        = bool
-#   nullable    = false
-# }
-
-# variable "kyuubi_gpu_engine_executors_limit" {
-#   description = "Limit the number of GPUs an engine can schedule executor pods on."
-#   type        = number
-#   nullable    = false
-# }
-
-# variable "kyuubi_gpu_pinned_memory" {
-#   description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-#   type        = number
-#   nullable    = false
-# }
-
-# variable "kyuubi_image" {
-#   description = "Image for kyuubi-k8s"
-#   type        = map(string)
-#   nullable    = false
-# }
-
-# variable "kyuubi_k8s_node_selectors" {
-#   description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-#   type        = string
-#   default     = null
-# }
-
-# variable "kyuubi_loadbalancer_extra_annotations" {
-#   description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
-#   type        = string
-#   default     = null
-# }
-
-# variable "kyuubi_profile" {
-#   description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-#   type        = string
-#   nullable    = false
-#   default     = "production"
-# }
-
-# variable "kyuubi_revision" {
-#   description = "Charm revision for kyuubi-k8s"
-#   type        = number
-#   nullable    = false
-# }
-
-# variable "kyuubi_units" {
-#   description = "Number of Kyuubi units. 3 units are recommended for high availability."
-#   type        = number
-#   default     = 3
-#   nullable    = false
-# }
-
-# variable "kyuubi_user" {
-#   description = "User name to be used for running Kyuubi engines."
-#   type        = string
-#   nullable    = false
-#   default     = "kyuubi-spark-engine"
-# }
-
-
-
-
-
 variable "data_integrator_endpoint" {
   description = "In-model endpoint for the data-integrator application."
   type = object({
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index fa00f6d1..c2d8e2b2 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -11,16 +11,6 @@ resource "juju_model" "spark" {
   }
 }
 
-# resource "juju_model" "cos" {
-#   count = var.cos.deployed == "bundled" && var.model_uuid == null && var.create_model == true ? 1 : 0
-
-#   name       = var.cos.model
-#   credential = var.K8S_CREDENTIAL
-#   cloud {
-#     name = var.K8S_CLOUD
-#   }
-# }
-
 module "ssc" {
   depends_on = [juju_model.spark]
   source     = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev586"
@@ -108,13 +98,13 @@ module "azure_storage" {
   source     = "../../charms/azure-storage-integrator"
   model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
 
-  azure_storage_secret_key = var.azure_storage.secret_key
+  azure_storage_secret_key = var.azure_storage_config.secret_key
   base                     = "ubuntu@22.04"
   channel                  = "latest/edge"
   config = {
-    connection-protocol = var.azure_storage.protocol
-    container           = var.azure_storage.container
-    storage_account     = var.azure_storage.storage_account
+    connection-protocol = var.azure_storage_config.protocol
+    container           = var.azure_storage_config.container
+    storage_account     = var.azure_storage_config.storage_account
   }
   constraints = "arch=amd64"
   revision    = local.revisions.azure_storage
@@ -129,30 +119,14 @@ module "s3" {
   base    = "ubuntu@22.04"
   channel = "1/stable"
   config = {
-    bucket   = var.s3.bucket
-    endpoint = var.s3.endpoint
-    region   = var.s3.region
+    bucket   = var.s3_config.bucket
+    endpoint = var.s3_config.endpoint
+    region   = var.s3_config.region
   }
   constraints = "arch=amd64"
   revision    = local.revisions.s3
 }
 
-# module "bundled_cos" {
-#   depends_on   = [juju_model.cos]
-#   count        = var.cos.deployed == "bundled" ? 1 : 0
-#   source       = "../cos"
-#   model_uuid   = juju_model.cos.uuid
-#   cos_tls_ca   = var.cos.tls.ca
-#   cos_tls_cert = var.cos.tls.cert
-#   cos_tls_key  = var.cos.tls.key
-
-#   alertmanager_size                = var.alertmanager_size
-#   grafana_size                     = var.grafana_size
-#   loki_active_index_directory_size = var.loki_active_index_directory_size
-#   loki_chunks_size                 = var.loki_chunks_size
-#   prometheus_size                  = var.prometheus_size
-#   traefik_size                     = var.traefik_size
-# }
 
 module "spark" {
   depends_on = [
@@ -168,27 +142,25 @@ module "spark" {
   source     = "../../components/spark-3.4"
   model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
 
-  history_server  = {}
-  integration_hub = {}
-  kyuubi          = {}
-
-  # kyuubi_user                                = var.kyuubi_user
-  # kyuubi_profile                             = var.kyuubi_profile
-  # admin_password                             = var.admin_password
-  # tls_private_key                            = var.tls_private_key
-  # enable_dynamic_allocation                  = var.enable_dynamic_allocation
-  # kyuubi_k8s_node_selectors                  = var.kyuubi_k8s_node_selectors
-  # kyuubi_loadbalancer_extra_annotations      = var.kyuubi_loadbalancer_extra_annotations
-  # kyuubi_gpu_enable                          = var.kyuubi_gpu_enable
-  # kyuubi_gpu_engine_executors_limit          = var.kyuubi_gpu_engine_executors_limit
-  # kyuubi_gpu_pinned_memory                   = var.kyuubi_gpu_pinned_memory
-  # kyuubi_driver_pod_template                 = var.kyuubi_driver_pod_template
-  # kyuubi_executor_pod_template               = var.kyuubi_executor_pod_template
-  # kyuubi_executor_cores                      = var.kyuubi_executor_cores
-  # kyuubi_executor_memory                     = var.kyuubi_executor_memory
-  # driver_pod_template                        = var.driver_pod_template
-  # executor_pod_template                      = var.executor_pod_template
-  # integration_hub_monitored_service_accounts = var.integration_hub_monitored_service_accounts
+  history_server = {
+    config      = var.history_server_config,
+    constraints = "arch=amd64",
+    revision    = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server,
+    resources   = { spark-history-server-image = var.history_server_image != null ? var.history_server_image : local.images.history_server }
+  }
+  integration_hub = {
+    config      = var.integration_hub_config,
+    constraints = "arch=amd64",
+    revision    = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub,
+    resources   = { integration-hub-image = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub }
+  }
+  kyuubi = {
+    config      = var.kyuubi_config,
+    constraints = "arch=amd64",
+    revision    = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi,
+    resources   = { kyuubi-image = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi }
+    units       = var.kyuubi_units
+  }
 
   tls_endpoint = {
     name     = module.ssc.app_name
@@ -219,16 +191,6 @@ module "spark" {
     name     = module.s3 != null ? module.s3[0].application.name : module.azure_storage[0].application.name
     endpoint = module.s3 != null ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
   }
-
-  # kyuubi_units = var.kyuubi_units
-
-  # history_server_revision  = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server
-  # history_server_image     = var.history_server_image != null ? var.history_server_image : local.images.history_server
-  # integration_hub_revision = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub
-  # integration_hub_image    = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub
-  # kyuubi_revision          = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi
-  # kyuubi_image             = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi
-
 }
 
 
diff --git a/terraform/products/charmed-spark-3.4/outputs.tf b/terraform/products/charmed-spark-3.4/outputs.tf
index 089e4cfb..3b8a70c7 100644
--- a/terraform/products/charmed-spark-3.4/outputs.tf
+++ b/terraform/products/charmed-spark-3.4/outputs.tf
@@ -24,7 +24,7 @@ output "models" {
   description = "Maps of the models and the components deployed in each model."
   value = [{
     model_uuid = var.model_uuid
-    components = merge(concat(
+    components = merge(
       module.spark.components,
       {
         zookeeper = module.zookeeper.application
@@ -41,13 +41,13 @@ output "models" {
       {
         kyuubi_users = module.kyuubi_users.app_name # TODO: expose application
       },
-      module.azure_storage == null ? {} : {
+      module.azure_storage == [] ? {} : {
         azure_storage = module.azure_storage[0].application
       },
-      module.s3 == null ? {} : {
+      module.s3 == [] ? {} : {
         s3 = module.s3[0].application
       }
-    ))
+    )
     }
   ]
 }
diff --git a/terraform/products/charmed-spark-3.4/variables.tf b/terraform/products/charmed-spark-3.4/variables.tf
index 8c97ee74..072a2775 100644
--- a/terraform/products/charmed-spark-3.4/variables.tf
+++ b/terraform/products/charmed-spark-3.4/variables.tf
@@ -1,8 +1,6 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-# Juju
-
 variable "admin_password" {
   description = "The password for the admin user."
   type        = string
@@ -10,13 +8,8 @@ variable "admin_password" {
   default     = null
 }
 
-variable "alertmanager_size" {
-  description = "Storage size for the alertmanager database"
-  type        = string
-  default     = "10G"
-}
 
-variable "azure_storage" {
+variable "azure_storage_config" {
   description = "Azure Object storage information"
   type = object({
     container       = optional(string, "azurecontainer")
@@ -39,46 +32,6 @@ variable "certificate_common_name" {
   default     = "charmed-spark"
 }
 
-variable "cos" {
-  description = "Observability settings"
-  type = object({
-    model    = optional(string, "cos")
-    deployed = optional(string, "bundled")
-    offers = optional(object({
-      dashboard = optional(string, null),
-      metrics   = optional(string, null),
-      logging   = optional(string, null)
-    }), {}),
-    tls = optional(object({
-      cert = optional(string, "")
-      key  = optional(string, "")
-      ca   = optional(string, "")
-    }), {})
-  })
-  default = { model = "cos", deployed = "bundled", offers = {}, tls = {} }
-
-  validation {
-    condition     = contains(["external", "bundled", "no"], var.cos.deployed)
-    error_message = "Valid values for var: cos.deployed are (external, bundled, no)"
-  }
-
-  validation {
-    condition = var.cos.deployed != "external" || alltrue([
-      var.cos.offers.dashboard != null,
-      var.cos.offers.metrics != null,
-      var.cos.offers.logging != null,
-    ])
-    error_message = "When using external cos, please define all offers variables"
-  }
-
-}
-
-variable "cos_configuration_revision" {
-  description = "Charm revision for cos-configuration-k8s"
-  type        = number
-  default     = null
-}
-
 variable "create_model" {
   description = "Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism."
   type        = bool
@@ -92,39 +45,21 @@ variable "data_integrator_revision" {
   default     = null
 }
 
-variable "driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
-variable "enable_dynamic_allocation" {
-  description = "Enable dynamic allocation of pods for Spark jobs."
-  type        = bool
-  default     = false
-}
-
-variable "executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible to the `spark-submit` process."
-  type        = string
-  default     = null
-}
-
 variable "grafana_agent_revision" {
   description = "Charm revision for grafana-agent-k8s"
   type        = number
   default     = null
 }
 
-variable "grafana_size" {
-  description = "Storage size for the grafana database"
-  type        = string
-  default     = "10G"
+variable "history_server_config" {
+  description = "History Server configuration options"
+  type        = map(any)
+  default     = {}
 }
 
 variable "history_server_image" {
   description = "Image for spark-history-server-k8s"
-  type        = map(string)
+  type        = string
   default     = null
 }
 
@@ -136,14 +71,14 @@ variable "history_server_revision" {
 
 variable "integration_hub_image" {
   description = "Image for spark-integration-hub-k8s"
-  type        = map(string)
+  type        = string
   default     = null
 }
 
-variable "integration_hub_monitored_service_accounts" {
-  description = "Comma-separated patterns for namespaces and service accounts to monitor and update"
-  type        = string
-  default     = null
+variable "integration_hub_config" {
+  description = "Integration Hub configuration options."
+  type        = map(any)
+  default     = {}
 }
 
 variable "integration_hub_revision" {
@@ -175,73 +110,18 @@ variable "juju_controller" {
   default = null
 }
 
-variable "kyuubi_driver_pod_template" {
-  description = "Define K8s driver pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_executor_cores" {
-  description = "Set kyuubi executor pods cpu cores."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_memory" {
-  description = "Set kyuubi executor pods memory (in GB)."
-  type        = number
-  default     = null
-}
-
-variable "kyuubi_executor_pod_template" {
-  description = "Define K8s executor pod from a file accessible in the object storage."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_gpu_enable" {
-  description = "Enable GPU acceleration for SparkSQLEngine."
-  type        = bool
-  default     = false
-}
-
-variable "kyuubi_gpu_engine_executors_limit" {
-  description = "Limit the number of GPUs an engine can schedule executor pods on."
-  type        = number
-  default     = 1
-}
-
-variable "kyuubi_gpu_pinned_memory" {
-  description = "Set the host memory (in GB) per executor reserved for fast data transfer on GPUs."
-  type        = number
-  default     = 1
+variable "kyuubi_config" {
+  description = "Kyuubi configuration options."
+  type        = map(any)
+  default     = {}
 }
 
 variable "kyuubi_image" {
   description = "Image for kyuubi-k8s"
-  type        = map(string)
-  default     = null
-}
-
-variable "kyuubi_k8s_node_selectors" {
-  description = "Comma separated label:value selectors for K8s pods in Kyuubi."
-  type        = string
-  default     = null
-}
-
-variable "kyuubi_loadbalancer_extra_annotations" {
-  description = "Optional extra annotations to be supplied to the load balancer service in Kyuubi."
   type        = string
   default     = null
 }
 
-variable "kyuubi_profile" {
-  description = "The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'."
-  type        = string
-  nullable    = false
-  default     = "production"
-}
-
 variable "kyuubi_revision" {
   description = "Charm revision for kyuubi-k8s"
   type        = number
@@ -255,15 +135,9 @@ variable "kyuubi_units" {
   nullable    = false
 }
 
-variable "kyuubi_user" {
-  description = "Define the user to be used for running Kyuubi enginers"
-  type        = string
-  default     = "kyuubi-spark-engine"
-}
-
 variable "kyuubi_users_image" {
   description = "Image for postgresql-k8s (auth-db)"
-  type        = map(string)
+  type        = string
   default     = null
 }
 
@@ -279,27 +153,9 @@ variable "kyuubi_users_size" {
   default     = "1G"
 }
 
-variable "loki_active_index_directory_size" {
-  description = "Storage size for the active index directory for Loki"
-  type        = string
-  default     = "10G"
-}
-
-variable "loki_chunks_size" {
-  description = "Storage size for the Loki chucks storage"
-  type        = string
-  default     = "500G"
-}
-
-variable "logging_config" {
-  description = "Logging configuration to be used."
-  type        = string
-  default     = ""
-}
-
 variable "metastore_image" {
   description = "Image for postgresql-k8s (metastore)"
-  type        = map(string)
+  type        = string
   default     = null
 }
 
@@ -327,18 +183,6 @@ variable "model_uuid" {
   default     = null
 }
 
-variable "prometheus_size" {
-  description = "Storage size for the Prometheus database"
-  type        = string
-  default     = "500G"
-}
-
-variable "pushgateway_revision" {
-  description = "Charm revision for prometheus-pushgateway-k8s"
-  type        = number
-  default     = null
-}
-
 variable "proxy" {
   description = "Proxy information for the deployment."
   type = object({
@@ -349,7 +193,7 @@ variable "proxy" {
   default = {}
 }
 
-variable "s3" {
+variable "s3_config" {
   description = "S3 Bucket information"
   type = object({
     bucket   = optional(string, "spark-test")
@@ -365,12 +209,6 @@ variable "s3_revision" {
   default     = null
 }
 
-variable "scrape_config_revision" {
-  description = "Charm revision for prometheus-scrape-config-k8s"
-  type        = number
-  default     = null
-}
-
 variable "storage_backend" {
   type        = string
   description = "Storage backend to be used"
@@ -390,12 +228,6 @@ variable "tls_private_key" {
   default     = null
 }
 
-variable "traefik_size" {
-  description = "Storage size for the Traefik storage"
-  type        = string
-  default     = "10G"
-}
-
 variable "zookeeper_image" {
   description = "Image for zookeeper-k8s"
   type        = map(string)

From a9a41e5fc492ccd2eb894b4c20e1c0b6ee56279e Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 1 Apr 2026 19:13:22 +0200
Subject: [PATCH 10/62] Yaml bundle helpers removal

---
 python/poetry.lock                         |  14 +-
 python/pyproject.toml                      |   3 +-
 python/spark_test/core/bundle/__init__.py  |   1 -
 python/spark_test/core/bundle/terraform.py |  15 +-
 python/spark_test/core/bundle/yaml.py      |  81 --------
 python/tests/__init__.py                   |   3 +-
 python/tests/integration/conftest.py       | 227 +++++----------------
 python/tests/integration/helpers.py        |  96 +--------
 python/tests/integration/resources/main.tf |  83 ++++++++
 python/tox.ini                             |   2 +-
 10 files changed, 157 insertions(+), 368 deletions(-)
 delete mode 100644 python/spark_test/core/bundle/yaml.py
 create mode 100644 python/tests/integration/resources/main.tf

diff --git a/python/poetry.lock b/python/poetry.lock
index a26f7e87..0eff57b5 100644
--- a/python/poetry.lock
+++ b/python/poetry.lock
@@ -1006,7 +1006,7 @@ version = "3.1.6"
 description = "A very fast and expressive template engine."
 optional = false
 python-versions = ">=3.7"
-groups = ["main", "integration"]
+groups = ["main"]
 files = [
     {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"},
     {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"},
@@ -1032,14 +1032,14 @@ files = [
 
 [[package]]
 name = "jubilant"
-version = "1.3.0"
-description = "Juju CLI wrapper for charm integration testing"
+version = "1.8.0"
+description = "Juju CLI wrapper, primarily for charm integration testing"
 optional = false
 python-versions = ">=3.8"
 groups = ["integration"]
 files = [
-    {file = "jubilant-1.3.0-py3-none-any.whl", hash = "sha256:a5ea4a3bf487ab0286eaad0de9df145761657c08beb834931340b9ebb1f41292"},
-    {file = "jubilant-1.3.0.tar.gz", hash = "sha256:ff43d6eb67a986958db6317d7ff3df1c8c160d0c56736628919ac1f7319d444e"},
+    {file = "jubilant-1.8.0-py3-none-any.whl", hash = "sha256:e0495ee645de5f2df81d044a3b6e2827b5a6277de02c6d30935b9632bd868a98"},
+    {file = "jubilant-1.8.0.tar.gz", hash = "sha256:a7cea68299dca94fac3e121a8c8ed92021d20aa2f4461e16822abbcd134d0b8b"},
 ]
 
 [package.dependencies]
@@ -1083,7 +1083,7 @@ version = "3.0.2"
 description = "Safely add untrusted strings to HTML/XML markup."
 optional = false
 python-versions = ">=3.9"
-groups = ["main", "integration"]
+groups = ["main"]
 files = [
     {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"},
     {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"},
@@ -2605,4 +2605,4 @@ type = ["pytest-mypy"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<4.0"
-content-hash = "35f57a76b190df99d7bf187f5beb933bae4d9897471a87717917d09b7ccaa77d"
+content-hash = "9a6a11e0a4e73593d4c7f5e18e9c7cdfbee3cdb35df6e21eee95e0d10f58f9e9"
diff --git a/python/pyproject.toml b/python/pyproject.toml
index 51e5ec00..89aaf076 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -58,8 +58,7 @@ pytest-mock = ">=3.10"
 psycopg2-binary = "^2.9.9"
 tenacity = "^9.0.0"
 httpx = "^0.28.1"
-jubilant = "^1.1.0"
-jinja2 = "*"
+jubilant = "^1.7.0"
 python-dotenv = "^1.2.1"
 
 [tool.poetry.group.benchmark]
diff --git a/python/spark_test/core/bundle/__init__.py b/python/spark_test/core/bundle/__init__.py
index f13286f1..67908406 100644
--- a/python/spark_test/core/bundle/__init__.py
+++ b/python/spark_test/core/bundle/__init__.py
@@ -15,7 +15,6 @@
 class BundleBackendEnum(str, enum.Enum):
     """The various backends to deploy the bundle."""
 
-    YAML = "yaml"
     TERRAFORM = "terraform"
 
 
diff --git a/python/spark_test/core/bundle/terraform.py b/python/spark_test/core/bundle/terraform.py
index 7dadafff..08bce2a0 100644
--- a/python/spark_test/core/bundle/terraform.py
+++ b/python/spark_test/core/bundle/terraform.py
@@ -9,20 +9,25 @@
 import tempfile
 from pathlib import Path
 
-from . import BundleBackend, BundleBackendEnum
+from . import BundleBackend, BundleBackendEnum, DEPLOYED
 
 
 class TerraformBackend(BundleBackend):
     """Terraform bundle backend."""
 
-    def __init__(self, tempdir, module_path: str | Path):
-        self.module_path = module_path
+    def __init__(self, tempdir, terraform_root: Path, entrypoint_content: str) -> None:
+        self.terraform_root = terraform_root
+        self.entrypoint_content = entrypoint_content
+
         super().__init__(backend=BundleBackendEnum.TERRAFORM, tempdir=tempdir)
         self.init()
 
     def init(self) -> None:
         """Initialize terraform."""
-        shutil.copytree(self.module_path, self.tempdir, dirs_exist_ok=True)
+        shutil.copytree(self.terraform_root, self.tempdir, dirs_exist_ok=True)
+        with (self.tempdir / "main.tf").open("w", encoding="utf-8") as f:
+            f.write(self.entrypoint_content)
+
         self.execute(["terraform", "init"])
 
     def apply(self, vars: dict[str, str] | None = None) -> list[str]:
@@ -39,7 +44,7 @@ def apply(self, vars: dict[str, str] | None = None) -> list[str]:
             self.execute(
                 ["terraform", "apply", "-auto-approve", "-var-file", tfvars_file.name]
             )
-        self.deployed_applications = list(self.outputs["charms"]["value"].values())
+        self.deployed_applications = []  # FIXME: Parse components output
         return self.deployed_applications
 
     def destroy(self):
diff --git a/python/spark_test/core/bundle/yaml.py b/python/spark_test/core/bundle/yaml.py
deleted file mode 100644
index 7e309486..00000000
--- a/python/spark_test/core/bundle/yaml.py
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-"""YAML backend to apply the bundle."""
-
-import os
-from pathlib import Path
-
-import jinja2
-import yaml
-
-from . import BundleBackend, BundleBackendEnum
-
-
-class YamlBackend(BundleBackend):
-    """The YAML bundle backend."""
-
-    def __init__(
-        self,
-        model: str,
-        tempdir,
-        bundle_file: str | Path,
-        overlays: list[str | Path],
-    ):
-        super().__init__(backend=BundleBackendEnum.YAML, tempdir=tempdir)
-        self.model = model
-        self.bundle_file = bundle_file
-        self.overlays = overlays
-
-    def render_template(
-        self, template_file: str | Path, vars: dict[str, str] | None
-    ) -> str:
-        """Render the yaml.j2 template file with given template vars."""
-        if vars is None:
-            vars = {}
-        with open(template_file) as source:
-            template = jinja2.Template(source.read())
-            return template.render(**vars)
-
-    def apply(self, vars: dict[str, str] | None = None) -> list[str]:
-        """Apply the YAML bundle."""
-        rendered_bundle_file = self.tempdir / os.path.basename(self.bundle_file)
-        deployed_applications = []
-        with open(rendered_bundle_file, mode="w") as destination:
-            content = self.render_template(self.bundle_file, vars=vars)
-            destination.write(content)
-            destination.flush()
-
-            content_yaml = yaml.safe_load(content)
-            deployed_applications.extend(list(content_yaml["applications"].keys()))
-
-        command = [
-            "juju",
-            "deploy",
-            "--model",
-            self.model,
-            "--trust",
-            str(rendered_bundle_file),
-        ]
-
-        for overlay in self.overlays:
-            rendered_overlay_file = self.tempdir / os.path.basename(overlay)
-            with open(rendered_overlay_file, mode="w") as destination:
-                content = self.render_template(overlay, vars=vars)
-                destination.write(content)
-                destination.flush()
-                deployed_applications.extend(
-                    list(yaml.safe_load(content)["applications"].keys())
-                )
-            command.extend(["--overlay", str(rendered_overlay_file)])
-
-        self.execute(command)
-        self.deployed_applications = deployed_applications
-        return deployed_applications
-
-    def destroy(self):
-        """Destroy the bundle."""
-        for app in self.deployed_applications:
-            command = ["juju", "remove-application", "--force", "--no-prompt", app]
-            self.execute(command)
diff --git a/python/tests/__init__.py b/python/tests/__init__.py
index 0e2f5f43..9aba2f70 100644
--- a/python/tests/__init__.py
+++ b/python/tests/__init__.py
@@ -7,4 +7,5 @@
 
 IE_TEST_DIR = Path(os.path.dirname(__file__))
 
-RELEASE_DIR = IE_TEST_DIR / ".." / ".." / "releases"
+TERRAFORM_DIR = IE_TEST_DIR / ".." / ".." / "terraform"
+PRODUCTS_DIR = TERRAFORM_DIR / "products"
diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index 80f38668..71840d85 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -4,8 +4,6 @@
 from __future__ import annotations
 
 import base64
-import contextlib
-import json
 import logging
 import os
 import shutil
@@ -17,9 +15,8 @@
 from contextlib import contextmanager
 from pathlib import Path
 from subprocess import PIPE, Popen, TimeoutExpired, check_output
-from typing import Generator, cast
+from typing import cast
 
-import httpx
 import jubilant
 import pytest
 from dotenv import load_dotenv
@@ -27,19 +24,15 @@
 from tenacity import retry, stop_after_attempt
 
 from spark_test.core.azure_storage import Container
-from spark_test.core.bundle import BundleBackendEnum
 from spark_test.core.bundle.terraform import TerraformBackend
-from spark_test.core.bundle.yaml import YamlBackend
 from spark_test.core.s3 import Bucket
 from spark_test.fixtures.azure_storage import azure_credentials, container  # noqa
 from spark_test.fixtures.pod import spark_image  # noqa
 from spark_test.fixtures.s3 import bucket, credentials  # noqa
 from spark_test.fixtures.service_account import service_account  # noqa
-from tests import RELEASE_DIR
+from tests import IE_TEST_DIR, PRODUCTS_DIR, TERRAFORM_DIR
 
 from .helpers import (
-    Bundle,
-    deploy_bundle,
     local_tmp_folder,
     set_s3_credentials,
 )
@@ -74,18 +67,6 @@ def pytest_addoption(parser):
         required=False,
         help="Path to the release to be used in integration tests",
     )
-    parser.addoption(
-        "--bundle",
-        required=False,
-        help="Path to a particular bundle. Using single files for YAML bundles "
-        "and directories for terraforms.",
-    )
-    parser.addoption(
-        "--overlay",
-        action="append",
-        type=str,
-        help="Path to the overlay to be used with the bundle.",
-    )
     parser.addoption(
         "--cos-model",
         required=False,
@@ -97,16 +78,6 @@ def pytest_addoption(parser):
         "If the model already exists, we assume COS had already been "
         "deployed.",
     )
-    parser.addoption(
-        "--backend",
-        nargs="?",
-        choices=["yaml", "terraform"],
-        const="yaml",
-        default="yaml",
-        type=str,
-        help="Which backend to use for bundle. Supported values are either "
-        "yaml (default) or terraform.",
-    )
     parser.addoption(
         "--spark-version",
         nargs="?",
@@ -214,12 +185,6 @@ def cos_model(request) -> None | str:
     return request.config.getoption("--cos-model")
 
 
-@pytest.fixture(scope="module")
-def backend(request) -> None | str:
-    """The backend which is to be used to deploy the bundle."""
-    return request.config.getoption("--backend")
-
-
 @pytest.fixture(scope="module")
 def spark_version(request) -> str:
     """The backend which is to be used to deploy the bundle."""
@@ -314,37 +279,8 @@ def integration_test(request):
         )
 
 
-@contextlib.contextmanager
-def switch_model(juju: jubilant.Juju, model: str) -> Generator[jubilant.Juju]:
-    """Context manager to switch to a specific Juju model temporarily.
-
-    When the context manager exists, juju is switched back to the old model (if there was any).
-
-    Provides a :class:`Juju` instance to operate on.
-
-    Args:
-        juju: The instance of `jubilant.Juju` class to operate on.
-        model: Name of the model that juju needs to be switched to.
-
-    Returns:
-        The same `juju` instance that was passed to it.
-    """
-    try:
-        status = juju.cli("status", "--format", "json", include_model=False)
-        old_model = json.loads(status)["model"]["name"]
-    except jubilant.CLIError:
-        old_model = None
-
-    juju.cli("switch", model, include_model=False)
-
-    yield juju
-
-    if old_model:
-        juju.cli("switch", old_model, include_model=False)
-
-
 @pytest.fixture(scope=determine_scope)
-def cos(cos_model: str, backend: str, request: pytest.FixtureRequest):
+def cos(cos_model: str, request: pytest.FixtureRequest):
     """
     Deploy COS bundle depending upon the value of cos_model fixture, and yield its value.
     """
@@ -361,34 +297,6 @@ def cos(cos_model: str, backend: str, request: pytest.FixtureRequest):
     except jubilant.CLIError:
         cos.add_model(cos_model)
 
-        if backend == BundleBackendEnum.YAML.value:
-            base_url = "https://raw.githubusercontent.com/canonical/cos-lite-bundle/main/overlays"
-
-            overlays = ["offers-overlay.yaml", "testing-overlay.yaml"]
-
-            def create_file(path: Path, response: httpx.Response):
-                path.write_text(response.content.decode("utf-8"))
-                return path
-
-            with local_tmp_folder("tmp-cos") as tmp_folder:
-                logger.info(tmp_folder)
-
-                cos_bundle = Bundle[str | Path](
-                    main="cos-lite",
-                    overlays=[
-                        create_file(tmp_folder / overlay, response)
-                        for overlay in overlays
-                        if (response := httpx.get(f"{base_url}/{overlay}"))
-                        if response.status_code == 200
-                    ],
-                )
-
-                deploy_bundle(cos, cos_bundle)
-
-                with switch_model(cos, cos_model) as switched_cos:
-                    switched_cos.offer("traefik", endpoint="ingress")
-                cos.wait(lambda status: jubilant.all_active(status, "traefik"))
-
         yield cos_model
     finally:
         debug_log = cos.debug_log(limit=50)
@@ -534,7 +442,6 @@ def private_key(tempdir: Path) -> str:
 @pytest.fixture(scope=determine_scope)
 def spark_bundle(
     request,
-    backend,
     tempdir,
     spark_version,
     juju,
@@ -545,97 +452,59 @@ def spark_bundle(
     private_key,
     storage_sizes,
 ):
-    """Deploy the Spark K8s bundle, with appropriate backend and object storage."""
+    """Deploy the Spark K8s bundle using Terraform."""
     short_version = ".".join(spark_version.split(".")[:2])
-    release_path = RELEASE_DIR / short_version / backend
-
-    if backend == BundleBackendEnum.TERRAFORM.value:
-        bundle = TerraformBackend(tempdir=tempdir, module_path=release_path)
-        base_vars = {
-            "kyuubi_user": "kyuubi-test-user",
-            "kyuubi_profile": "testing",
-            "model": cast(str, juju.model),
-            "storage_backend": storage_backend,
-            "create_model": False,
-            "zookeeper_units": 1,
-            "admin_password": admin_password,
-            "tls_private_key": private_key,
-        }
-        cos_vars = (
-            {
-                "cos": {
-                    "model": cos,
-                    "deployed": "bundled",
-                }
-            }
-            if cos
-            else {"cos": {"deployed": "no"}}
-        )
-        if storage_backend == "azure_storage":
-            storage_unit = cast(Container, object_storage)
-            storage_vars = {
-                "azure_storage": {
-                    "storage_account": storage_unit.credentials.storage_account,
-                    "container": storage_unit.container_name,
-                    "secret_key": storage_unit.credentials.secret_key,
-                }
-            }
-        else:
-            storage_unit = cast(Bucket, object_storage)
-
-            storage_vars = {
-                "s3": {
-                    "bucket": storage_unit.bucket_name,
-                    "endpoint": storage_unit.s3.meta.endpoint_url,
-                }
+    # module_path = PRODUCTS_DIR / f"charmed-spark-{short_version}"
+
+    # TODO: Customize entry point based on test setup
+    with (IE_TEST_DIR / "integration" / "resources" / "main.tf").open(
+        "r", encoding="utf-8"
+    ) as f:
+        entrypoint_content = f.read()
+
+    bundle = TerraformBackend(
+        tempdir=tempdir,
+        terraform_root=TERRAFORM_DIR,
+        entrypoint_content=entrypoint_content,
+    )
+    base_vars = {
+        "kyuubi_config": {"service-account": "kyuubi-test-user", "profile": "testing"},
+        "model_uuid": cast(str, juju.show_model().model_uuid),
+        "storage_backend": storage_backend,
+        "create_model": False,
+        "zookeeper_units": 1,
+        "admin_password": admin_password,
+        "tls_private_key": private_key,
+    }
+    cos_vars = (
+        {
+            "cos": {
+                "model": cos,
+                "deployed": "bundled",
             }
-
-    elif backend == BundleBackendEnum.YAML.value:
-        bundle_yaml_filename = (
-            "bundle-azure-storage.yaml.j2"
-            if storage_backend == "azure_storage"
-            else "bundle.yaml.j2"
-        )
-        bundle_file = release_path / bundle_yaml_filename
-        overlays = (
-            [release_path / "overlays" / "cos-integration.yaml.j2"] if cos else []
-        )
-        bundle = YamlBackend(
-            model=juju.model,
-            tempdir=tempdir,
-            bundle_file=bundle_file,
-            overlays=overlays,
-        )
-
-        base_vars = {
-            "namespace": cast(str, juju.model),
-            "service_account": "kyuubi-test-user",
-            "zookeeper_units": 1,
-            "kyuubi_profile": "testing",
         }
-        cos_vars = (
-            {"cos_controller": juju.status().model.controller, "cos_model": cos}
-            if cos
-            else {}
-        )
-        if storage_backend == "azure_storage":
-            storage_unit = cast(Container, object_storage)
-            storage_vars = {
+        if cos
+        else {"cos": {"deployed": "no"}}
+    )
+    if storage_backend == "azure_storage":
+        storage_unit = cast(Container, object_storage)
+        storage_vars = {
+            "azure_storage": {
                 "storage_account": storage_unit.credentials.storage_account,
                 "container": storage_unit.container_name,
+                "secret_key": storage_unit.credentials.secret_key,
             }
-        else:
-            storage_unit = cast(Bucket, object_storage)
+        }
+    else:
+        storage_unit = cast(Bucket, object_storage)
 
-            storage_vars = {
-                "s3_endpoint": storage_unit.s3.meta.endpoint_url,
+        storage_vars = {
+            "s3_config": {
                 "bucket": storage_unit.bucket_name,
+                "endpoint": storage_unit.s3.meta.endpoint_url,
+                "path": "spark-events/",
             }
-
-    else:
-        raise NotImplementedError(
-            f"The backend {backend} is not supported for deploying bundle."
-        )
+        }
 
     vars = base_vars | cos_vars | storage_vars | storage_sizes
 
@@ -649,7 +518,7 @@ def spark_bundle(
         juju.config("azure-storage", {"credentials": secret_uri})
     else:
         credentials = request.getfixturevalue("credentials")
-        juju.wait(lambda status: jubilant.all_agents_idle(status, "s3"))
+        juju.wait(lambda status: jubilant.all_agents_idle(status, "s3-integrator"))
         set_s3_credentials(juju, credentials=credentials)
 
     logger.info("Waiting for spark deployment to settle down")
diff --git a/python/tests/integration/helpers.py b/python/tests/integration/helpers.py
index bf37b2f2..8107d8ab 100644
--- a/python/tests/integration/helpers.py
+++ b/python/tests/integration/helpers.py
@@ -5,29 +5,20 @@
 import ast
 import json
 import logging
-import os
 import re
 import shutil
 import subprocess
-import textwrap
 import urllib.parse
 from contextlib import contextmanager
-from dataclasses import dataclass
 from pathlib import Path
-from random import choices
-from string import hexdigits
-from typing import Callable, Generic, TypeVar, cast
+from typing import cast
 
 import httpx
-import jinja2
 import jubilant
-import yaml
 
 from spark_test.core.s3 import Credentials
 from tests.integration.types import KyuubiCredentials
 
-T = TypeVar("T")
-S = TypeVar("S")
 SECRET_NAME_PREFIX = "integrator-hub-conf-"
 COS_ALIAS = "cos"
 JMX_EXPORTER_PORT = 9101
@@ -39,17 +30,11 @@
 logger = logging.getLogger(__name__)
 
 
-@dataclass
-class Bundle(Generic[T]):
-    main: T
-    overlays: list[T]
-
-    def map(self, f: Callable[[T], S]) -> "Bundle[S]":
-        return Bundle(main=f(self.main), overlays=[f(item) for item in self.overlays])
-
-
 def set_s3_credentials(
-    juju: jubilant.Juju, credentials: Credentials, application_name="s3", num_unit=0
+    juju: jubilant.Juju,
+    credentials: Credentials,
+    application_name="s3-integrator",
+    num_unit=0,
 ) -> None:
     """Use the charm action to start a password rotation."""
     params = {
@@ -175,70 +160,6 @@ def get_postgresql_credentials(
     return {"username": "operator", "password": results["password"], "host": address}
 
 
-def render_bundle(bundle, context=None, **kwcontext) -> Path:
-    """Render a templated bundle using Jinja2.
-
-    This can be used to populate built charm paths or config values.
-
-    :param bundle (str or Path): Path to bundle file or YAML content.
-    :param context (dict): Optional context mapping.
-    :param **kwcontext: Additional optional context as keyword args.
-
-    Returns the Path for the rendered bundle.
-    Adapted from pytest_operator.
-    """
-    bundles_dst_dir = Path(".build") / "bundles"
-    bundles_dst_dir.mkdir(exist_ok=True, parents=True)
-    if context is None:
-        context = {}
-    context.update(kwcontext)
-    if re.search(r".yaml(.j2)?$", str(bundle)):
-        bundle_path = Path(bundle)
-        bundle_text = bundle_path.read_text()
-        if bundle_path.suffix == ".j2":
-            bundle_name = bundle_path.stem
-        else:
-            bundle_name = bundle_path.name
-    else:
-        bundle_text = textwrap.dedent(bundle).strip()
-        infix = "".join(choices(hexdigits, k=4))
-        bundle_name = f"spark-{infix}.yaml"
-    logger.info(f"Rendering bundle {bundle_name}")
-    rendered = jinja2.Template(bundle_text).render(**context)
-    dst = bundles_dst_dir / bundle_name
-    dst.write_text(rendered)
-    return dst
-
-
-def render_yaml(file: Path, data: dict) -> dict:
-    if os.path.splitext(file)[1] == ".j2":
-        bundle_file = render_bundle(file, **{str(k): str(v) for k, v in data.items()})
-    else:
-        bundle_file = file
-
-    return yaml.safe_load(bundle_file.read_text())
-
-
-def _local(pointer) -> str:
-    return (
-        f"./{pointer.relative_to(Path.cwd())}"
-        if isinstance(pointer, Path)
-        else str(pointer)
-    )
-
-
-def deploy_bundle(juju: jubilant.Juju, bundle: Bundle) -> str:
-    deploy_command = [
-        "deploy",
-        "--trust",
-        _local(bundle.main),
-    ] + sum([["--overlay", _local(overlay)] for overlay in bundle.overlays], [])
-
-    stdout = juju.cli(*deploy_command)
-
-    return stdout
-
-
 @contextmanager
 def local_tmp_folder(name: str = "tmp"):
     if (tmp_folder := Path.cwd() / name).exists():
@@ -250,13 +171,6 @@ def local_tmp_folder(name: str = "tmp"):
     shutil.rmtree(tmp_folder)
 
 
-def generate_tmp_file(data: dict, tmp_folder: Path) -> Path:
-    import uuid
-
-    (file := tmp_folder / f"{uuid.uuid4().hex}.yaml").write_text(yaml.dump(data))
-    return file
-
-
 def get_secret_data(namespace: str, service_account: str):
     """Retrieve secret data for a given namespace and secret."""
     command = ["kubectl", "get", "secret", "-n", namespace, "--output", "json"]
diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
new file mode 100644
index 00000000..31734ea2
--- /dev/null
+++ b/python/tests/integration/resources/main.tf
@@ -0,0 +1,83 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
+
+variable "create_model" {
+  description = "Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism."
+  type        = bool
+  default     = true
+  nullable    = false
+}
+
+variable "K8S_CLOUD" {
+  type    = string
+  default = "microk8s"
+}
+
+variable "K8S_CREDENTIAL" {
+  type    = string
+  default = "microk8s"
+}
+
+variable "spark_model_name" {
+  type    = string
+  default = "spark-test-model"
+}
+
+variable "model_uuid" {
+  type    = string
+  default = null
+}
+variable "storage_backend" {
+  type = string
+}
+
+variable "admin_password" {
+  type      = string
+  sensitive = true
+}
+
+variable "kyuubi_config" {
+  type = map(any)
+}
+
+variable "s3_config" {
+  type = map(any)
+}
+
+variable "tls_private_key" {
+  type      = string
+  sensitive = true
+  default   = null
+}
+
+resource "juju_model" "spark" {
+  count = (var.model_uuid == null && var.create_model == true) ? 1 : 0
+
+  name       = var.spark_model_name
+  credential = var.K8S_CREDENTIAL
+  cloud {
+    name = var.K8S_CLOUD
+  }
+}
+
+module "spark" {
+  depends_on = [juju_model.spark]
+  source     = "./products/charmed-spark-3.4"
+
+  model_uuid      = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+  create_model    = false
+
+  admin_password  = var.admin_password
+  kyuubi_config   = var.kyuubi_config
+  s3_config       = var.s3_config
+  storage_backend = var.storage_backend
+  tls_private_key = var.tls_private_key
+}
diff --git a/python/tox.ini b/python/tox.ini
index 815744c6..ef75c693 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -21,7 +21,7 @@ tests_path = {tox_root}/tests/
 all_path = {[vars]src_path} {[vars]tests_path}
 github_scripts_path = {tox_root}/../.github/scripts/
 scripts_path = {tox_root}/scripts/
-tf_modules_path = {tox_root}/../releases/
+tf_modules_path = {tox_root}/../terraform/
 
 [testenv]
 allowlist_externals = 

From 68cb538601ee2410ff7b2277400a83d22a2ad9fe Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 1 Apr 2026 19:14:09 +0200
Subject: [PATCH 11/62] Initial working integration tests (simple ones)

---
 python/spark_test/core/bundle/terraform.py    |  2 +-
 python/tests/integration/conftest.py          |  3 +-
 python/tests/integration/resources/main.tf    | 39 +++++++++++++------
 python/tests/integration/test_spark_job.py    |  2 +-
 .../azure-storage-integrator/variables.tf     | 13 +++----
 .../charms/s3-integrator-v1/variables.tf      | 23 ++++++-----
 terraform/products/charmed-spark-3.4/main.tf  | 26 ++++++-------
 .../products/charmed-spark-3.4/providers.tf   |  5 +--
 .../products/charmed-spark-3.4/variables.tf   | 25 +++++++-----
 9 files changed, 77 insertions(+), 61 deletions(-)

diff --git a/python/spark_test/core/bundle/terraform.py b/python/spark_test/core/bundle/terraform.py
index 08bce2a0..556ac6bf 100644
--- a/python/spark_test/core/bundle/terraform.py
+++ b/python/spark_test/core/bundle/terraform.py
@@ -9,7 +9,7 @@
 import tempfile
 from pathlib import Path
 
-from . import BundleBackend, BundleBackendEnum, DEPLOYED
+from . import BundleBackend, BundleBackendEnum
 
 
 class TerraformBackend(BundleBackend):
diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index 71840d85..e3b68c35 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -502,11 +502,12 @@ def spark_bundle(
             "s3_config": {
                 "bucket": storage_unit.bucket_name,
                 "endpoint": storage_unit.s3.meta.endpoint_url,
-                "path": "spark-events/",
+                "path": "spark-events",
             }
         }
 
     vars = base_vars | cos_vars | storage_vars | storage_sizes
+    logger.info(f"Applying vars: {vars}")
 
     deployed_applications = bundle.apply(vars=vars)
     if storage_backend == "azure_storage":
diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index 31734ea2..af051f7e 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -40,8 +40,7 @@ variable "storage_backend" {
 }
 
 variable "admin_password" {
-  type      = string
-  sensitive = true
+  type = string
 }
 
 variable "kyuubi_config" {
@@ -53,9 +52,21 @@ variable "s3_config" {
 }
 
 variable "tls_private_key" {
-  type      = string
-  sensitive = true
-  default   = null
+  type    = string
+  default = null
+}
+
+variable "zookeeper_units" {
+  type = number
+}
+variable "kyuubi_users_size" {
+  type = string
+}
+variable "metastore_size" {
+  type = string
+}
+variable "zookeeper_size" {
+  type = string
 }
 
 resource "juju_model" "spark" {
@@ -72,12 +83,16 @@ module "spark" {
   depends_on = [juju_model.spark]
   source     = "./products/charmed-spark-3.4"
 
-  model_uuid      = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
-  create_model    = false
+  model_uuid   = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+  create_model = false
 
-  admin_password  = var.admin_password
-  kyuubi_config   = var.kyuubi_config
-  s3_config       = var.s3_config
-  storage_backend = var.storage_backend
-  tls_private_key = var.tls_private_key
+  admin_password    = var.admin_password
+  kyuubi_config     = var.kyuubi_config
+  kyuubi_users_size = var.kyuubi_users_size
+  metastore_size    = var.metastore_size
+  s3_config         = var.s3_config
+  storage_backend   = var.storage_backend
+  tls_private_key   = var.tls_private_key
+  zookeeper_size    = var.zookeeper_size
+  zookeeper_units   = var.zookeeper_units
 }
diff --git a/python/tests/integration/test_spark_job.py b/python/tests/integration/test_spark_job.py
index cb082e35..6f34b87a 100644
--- a/python/tests/integration/test_spark_job.py
+++ b/python/tests/integration/test_spark_job.py
@@ -160,7 +160,7 @@ def test_job_logs_are_persisted(
         access_key = confs.props["spark.hadoop.fs.s3a.access.key"]
         secret_key = confs.props["spark.hadoop.fs.s3a.secret.key"]
         endpoint = confs.props["spark.hadoop.fs.s3a.endpoint"]
-        log_match = re.match(r"s3a://(?P<bucket>.*)/(?P<path>.*)", log_folder)
+        log_match = re.match(r"s3a://(?P<bucket>.+)/(?P<path>.+)", log_folder)
         assert log_match is not None
         bucket_name = log_match.group("bucket")
         session = boto3.session.Session(
diff --git a/terraform/charms/azure-storage-integrator/variables.tf b/terraform/charms/azure-storage-integrator/variables.tf
index 5f6d933e..1fab31d6 100644
--- a/terraform/charms/azure-storage-integrator/variables.tf
+++ b/terraform/charms/azure-storage-integrator/variables.tf
@@ -26,14 +26,13 @@ variable "config" {
   description = "Map for configuration options."
   type = object({
     connection-protocol = optional(string, "abfss")
-    container           = optional(string, "azurecontainer")
-    endpoint            = optional(string, "")
-    path                = optional(string, "spark-events")
-    resource-group      = optional(string, "azurerg")
-    storage_account     = optional(string, "azurestorageaccount")
+    container           = optional(string)
+    endpoint            = optional(string)
+    path                = optional(string)
+    resource-group      = optional(string)
+    storage_account     = optional(string)
   })
-  default = {
-  }
+  default = {}
 }
 
 
diff --git a/terraform/charms/s3-integrator-v1/variables.tf b/terraform/charms/s3-integrator-v1/variables.tf
index d9b7e0f6..82be35cb 100644
--- a/terraform/charms/s3-integrator-v1/variables.tf
+++ b/terraform/charms/s3-integrator-v1/variables.tf
@@ -25,19 +25,18 @@ variable "channel" {
 variable "config" {
   description = "Map for configuration options."
   type = object({
-    attributes                          = optional(string, "")
-    bucket                              = optional(string, "spark-bucket")
-    endpoint                            = optional(string, "https://s3.amazonaws.com")
-    experimental-delete-older-than-days = optional(number, 14)
-    path                                = optional(string, "path/")
-    region                              = optional(string, "us-east-1")
-    s3-api-version                      = optional(string, "2")
-    s3-uri-style                        = optional(string, "2")
-    storage-class                       = optional(string, "")
-    tls-ca-chain                        = optional(string, "")
+    attributes                          = optional(string)
+    bucket                              = optional(string)
+    endpoint                            = optional(string)
+    experimental-delete-older-than-days = optional(number)
+    path                                = optional(string)
+    region                              = optional(string)
+    s3-api-version                      = optional(string)
+    s3-uri-style                        = optional(string)
+    storage-class                       = optional(string)
+    tls-ca-chain                        = optional(string)
   })
-  default = {
-  }
+  default = {}
 }
 
 
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index c2d8e2b2..bdbfb4d7 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -14,7 +14,7 @@ resource "juju_model" "spark" {
 module "ssc" {
   depends_on = [juju_model.spark]
   source     = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev586"
-  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
   base    = "ubuntu@24.04"
   channel = "1/stable"
@@ -30,7 +30,7 @@ module "ssc" {
 module "kyuubi_users" {
   depends_on = [juju_model.spark]
   source     = "git::https://github.com/canonical/postgresql-k8s-operator//terraform?ref=rev774"
-  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
   app_name    = "kyuubi-users"
   base        = "ubuntu@22.04"
@@ -49,7 +49,7 @@ module "kyuubi_users" {
 module "metastore" {
   depends_on = [juju_model.spark]
   source     = "git::https://github.com/canonical/postgresql-k8s-operator//terraform?ref=rev774"
-  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
   app_name    = "metastore"
   base        = "ubuntu@22.04"
@@ -69,7 +69,7 @@ module "metastore" {
 module "zookeeper" {
   depends_on = [juju_model.spark]
   source     = "../../charms/zookeeper"
-  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
   base        = "ubuntu@22.04"
   channel     = "3/stable"
@@ -84,7 +84,7 @@ module "zookeeper" {
 module "data_integrator" {
   depends_on = [juju_model.spark]
   source     = "../../charms/data-integrator"
-  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
   base        = "ubuntu@24.04"
   channel     = "latest/stable"
@@ -96,7 +96,7 @@ module "azure_storage" {
   depends_on = [juju_model.spark]
   count      = var.storage_backend == "azure_storage" ? 1 : 0
   source     = "../../charms/azure-storage-integrator"
-  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
   azure_storage_secret_key = var.azure_storage_config.secret_key
   base                     = "ubuntu@22.04"
@@ -114,15 +114,11 @@ module "s3" {
   depends_on = [juju_model.spark]
   count      = var.storage_backend == "s3" ? 1 : 0
   source     = "../../charms/s3-integrator-v1"
-  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
-  base    = "ubuntu@22.04"
-  channel = "1/stable"
-  config = {
-    bucket   = var.s3_config.bucket
-    endpoint = var.s3_config.endpoint
-    region   = var.s3_config.region
-  }
+  base        = "ubuntu@22.04"
+  channel     = "1/stable"
+  config      = var.s3_config
   constraints = "arch=amd64"
   revision    = local.revisions.s3
 }
@@ -140,7 +136,7 @@ module "spark" {
     module.zookeeper
   ]
   source     = "../../components/spark-3.4"
-  model_uuid = juju_model.spark != null ? juju_model.spark[0].uuid : var.model_uuid
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
   history_server = {
     config      = var.history_server_config,
diff --git a/terraform/products/charmed-spark-3.4/providers.tf b/terraform/products/charmed-spark-3.4/providers.tf
index b60772fa..712e728a 100644
--- a/terraform/products/charmed-spark-3.4/providers.tf
+++ b/terraform/products/charmed-spark-3.4/providers.tf
@@ -1,6 +1,5 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-provider "juju" {
-  # Configure the Juju provider if needed (e.g., `controller`, `model`, `username`, etc.).
-}
+# Provider configuration for charmed-spark-3.4.
+# Provider blocks, if any, should be defined here.
diff --git a/terraform/products/charmed-spark-3.4/variables.tf b/terraform/products/charmed-spark-3.4/variables.tf
index 072a2775..4bf10a22 100644
--- a/terraform/products/charmed-spark-3.4/variables.tf
+++ b/terraform/products/charmed-spark-3.4/variables.tf
@@ -2,7 +2,7 @@
 # See LICENSE file for licensing details.
 
 variable "admin_password" {
-  description = "The password for the admin user."
+  description = "The password for the Kyuubi admin user."
   type        = string
   sensitive   = true
   default     = null
@@ -12,9 +12,9 @@ variable "admin_password" {
 variable "azure_storage_config" {
   description = "Azure Object storage information"
   type = object({
-    container       = optional(string, "azurecontainer")
-    storage_account = optional(string, "azurestorageaccount")
-    secret_key      = optional(string, "azurestoragesecret")
+    container       = optional(string)
+    storage_account = optional(string)
+    secret_key      = optional(string)
     protocol        = optional(string, "abfss")
   })
   default = {}
@@ -174,7 +174,7 @@ variable "metastore_size" {
 variable "spark_model_name" {
   description = "The name of the juju model to deploy Spark to"
   type        = string
-  default     = "spark-test"
+  default     = "spark"
 }
 
 variable "model_uuid" {
@@ -194,11 +194,18 @@ variable "proxy" {
 }
 
 variable "s3_config" {
-  description = "S3 Bucket information"
+  description = "S3 integrator configuration"
   type = object({
-    bucket   = optional(string, "spark-test")
-    endpoint = optional(string, "https://s3.amazonaws.com")
-    region   = optional(string, "us-east-1")
+    attributes                          = optional(string)
+    bucket                              = optional(string)
+    endpoint                            = optional(string)
+    experimental-delete-older-than-days = optional(number)
+    path                                = optional(string)
+    region                              = optional(string)
+    s3-api-version                      = optional(string)
+    s3-uri-style                        = optional(string)
+    storage-class                       = optional(string)
+    tls-ca-chain                        = optional(string)
   })
   default = {}
 }

From 7f76a89ffd9d93cec852886d8271a6b6f7a4d502 Mon Sep 17 00:00:00 2001
From: deusebio <enrico.deusebio@canonical.com>
Date: Mon, 30 Mar 2026 17:31:02 +0200
Subject: [PATCH 12/62] [PRA-232] Pinning Juju 3.6.14 in CI (#205)

---
 .github/workflows/on_pull_request.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index eb2f20a4..cd953934 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -8,6 +8,8 @@ concurrency:
   cancel-in-progress: true
 
 on:
+  workflow_dispatch:
+
   pull_request:
     paths-ignore:
       - 'docs/**'

From 0982927e13e8eab95c1d1c8a33b558c18b674bc5 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 11:08:37 +0200
Subject: [PATCH 13/62] WIP: Test on CI

---
 .github/workflows/ci-tests-minimal.yaml | 69 +++++++++++--------------
 .github/workflows/ci-uat.yaml           |  2 +-
 2 files changed, 30 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/ci-tests-minimal.yaml b/.github/workflows/ci-tests-minimal.yaml
index e23e517d..4bca8b42 100644
--- a/.github/workflows/ci-tests-minimal.yaml
+++ b/.github/workflows/ci-tests-minimal.yaml
@@ -24,7 +24,7 @@ jobs:
           - integration-kyuubi
           - integration-backup-restore
         bundle-backend: [ "terraform" ]
-        spark-version: [ "3.4.4", "3.5.7" ]
+        spark-version: [ "3.4.4" ] #, "3.5.7" ]
         storage-backend: [ "s3", "azure_storage" ]
         juju-snap-channel: [ "3.6/stable" ]
         juju-agent-version: [ "3.6.14" ]
@@ -43,48 +43,37 @@ jobs:
             k8s-distribution: "k8s"
             k8s-version: "1.32"
 
-          # Test integration-basic, with Spark 3.5
-          - tox-env: integration-basic
-            spark-version: 3.5.7
-            juju-snap-channel: "3.6/stable"
-            juju-agent-version: "3.6.14"
-            k8s-distribution: "k8s"
-            k8s-version: "1.32"
+          # # Test integration-basic, with Spark 3.5
+          # - tox-env: integration-basic
+          #   spark-version: 3.5.7
+          #   juju-snap-channel: "3.6/stable"
+          #   juju-agent-version: "3.6.14"
+          #   k8s-distribution: "k8s"
+          #   k8s-version: "1.32"
 
-          # Test bundle + COS with YAML backend just once
-          - tox-env: integration-bundle
-            bundle-backend: yaml
-            cos-model: cos
-            storage-backend: s3
-            spark-version: 3.4.4
-            juju-snap-channel: "3.6/stable"
-            juju-agent-version: "3.6.14"
-            k8s-distribution: "k8s"
-            k8s-version: "1.32"
-
-          # Test COS just once with integration-sparkjob
-          - tox-env: integration-sparkjob
-            bundle-backend: terraform
-            storage-backend: s3
-            juju-snap-channel: "3.6/stable"
-            juju-agent-version: "3.6.14"
-            cos-model: cos
+          # # Test COS just once with integration-sparkjob
+          # - tox-env: integration-sparkjob
+          #   bundle-backend: terraform
+          #   storage-backend: s3
+          #   juju-snap-channel: "3.6/stable"
+          #   juju-agent-version: "3.6.14"
+          #   cos-model: cos
 
-          # Test COS just once with integration-kyuubi
-          - tox-env: integration-kyuubi
-            bundle-backend: terraform
-            storage-backend: s3
-            juju-snap-channel: "3.6/stable"
-            juju-agent-version: "3.6.14"
-            cos-model: cos
+          # # Test COS just once with integration-kyuubi
+          # - tox-env: integration-kyuubi
+          #   bundle-backend: terraform
+          #   storage-backend: s3
+          #   juju-snap-channel: "3.6/stable"
+          #   juju-agent-version: "3.6.14"
+          #   cos-model: cos
 
-          # Test COS just once with integration-backup-restore
-          - tox-env: integration-backup-restore
-            bundle-backend: terraform
-            storage-backend: s3
-            juju-snap-channel: "3.6/stable"
-            juju-agent-version: "3.6.14"
-            cos-model: cos
+          # # Test COS just once with integration-backup-restore
+          # - tox-env: integration-backup-restore
+          #   bundle-backend: terraform
+          #   storage-backend: s3
+          #   juju-snap-channel: "3.6/stable"
+          #   juju-agent-version: "3.6.14"
+          #   cos-model: cos
 
     needs:
       - checks
diff --git a/.github/workflows/ci-uat.yaml b/.github/workflows/ci-uat.yaml
index fd379722..391493d0 100644
--- a/.github/workflows/ci-uat.yaml
+++ b/.github/workflows/ci-uat.yaml
@@ -5,7 +5,7 @@ name: Mimic UAT Tests
 
 on:
   workflow_dispatch:
-  pull_request:
+  # pull_request:
     paths-ignore:
       - "docs/**"
       - "**.md"

From 345bf4e899d3bebc82616c6b153ddbd7e4c385ea Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 11:18:04 +0200
Subject: [PATCH 14/62] chore: Lint

---
 python/tests/integration/conftest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index e3b68c35..34cb7211 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -30,7 +30,7 @@
 from spark_test.fixtures.pod import spark_image  # noqa
 from spark_test.fixtures.s3 import bucket, credentials  # noqa
 from spark_test.fixtures.service_account import service_account  # noqa
-from tests import IE_TEST_DIR, PRODUCTS_DIR, TERRAFORM_DIR
+from tests import IE_TEST_DIR, TERRAFORM_DIR
 
 from .helpers import (
     local_tmp_folder,
@@ -453,7 +453,7 @@ def spark_bundle(
     storage_sizes,
 ):
     """Deploy the Spark K8s bundle using Terraform."""
-    short_version = ".".join(spark_version.split(".")[:2])
+    # short_version = ".".join(spark_version.split(".")[:2])
     # module_path = PRODUCTS_DIR / f"charmed-spark-{short_version}"
 
     # TODO: Customize entry point based on test setup

From 2e96f6bb4c9b1567fe65c3a0f66368634b4d43fb Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 11:18:55 +0200
Subject: [PATCH 15/62] Remove TF documentation script

---
 python/scripts/tf_inputs.py | 67 -------------------------------------
 python/tox.ini              | 15 ++-------
 2 files changed, 3 insertions(+), 79 deletions(-)
 delete mode 100644 python/scripts/tf_inputs.py

diff --git a/python/scripts/tf_inputs.py b/python/scripts/tf_inputs.py
deleted file mode 100644
index b71239f0..00000000
--- a/python/scripts/tf_inputs.py
+++ /dev/null
@@ -1,67 +0,0 @@
-"""TF input parser script.
-
-This script provides an easy way to transform the 'variables.tf' file into
-a compact table representation for a readme.
-"""
-
-from textwrap import shorten
-from typing import TypedDict
-
-import hcl2
-
-
-class TFInput(TypedDict):
-    """Table fields."""
-
-    name: str
-    description: str
-    type: str
-    nullable: bool
-    default: str
-
-
-def _main(release: str) -> None:
-    with open(f"../releases/{release}/terraform/variables.tf", "r") as file:
-        variables = hcl2.load(file)["variable"]
-
-    parsed_inputs: list[TFInput] = []
-    for variable in variables:
-        name, definition = next(iter(variable.items()))
-
-        new_entry = TFInput(
-            name=name,
-            description=shorten(definition.get("description", ""), 79),
-            type=(
-                str(content)
-                if len(content := definition.get("type", "")) < 10
-                else "map"
-            ),
-            nullable=definition.get("nullable", True),
-            default=(
-                str(content).replace("None", "null")
-                if isinstance(
-                    (content := definition.get("default", "")),
-                    (str, int, bool, type(None)),
-                )
-                else "See variables.tf"
-            ),
-        )
-        parsed_inputs.append(new_entry)
-
-    data = [["Variable", "Description", "Type", "Nullable", "Default"]] + [
-        list(var.values()) for var in parsed_inputs
-    ]
-    md_table = "\n".join(
-        [
-            "| " + " | ".join(map(str, row)) + " |"
-            for row in [data[0], ["---"] * len(data[0])] + data[1:]
-        ]
-    )
-    print(md_table)
-
-
-if __name__ == "__main__":
-    print("Release 3.4")
-    _main("3.4")
-    print("\n\nRelease 3.5")
-    _main("3.5")
diff --git a/python/tox.ini b/python/tox.ini
index ef75c693..e807902c 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -20,7 +20,6 @@ src_path = {tox_root}/spark_test/
 tests_path = {tox_root}/tests/
 all_path = {[vars]src_path} {[vars]tests_path}
 github_scripts_path = {tox_root}/../.github/scripts/
-scripts_path = {tox_root}/scripts/
 tf_modules_path = {tox_root}/../terraform/
 
 [testenv]
@@ -53,7 +52,7 @@ deps =
 description = Apply coding style standards to code
 commands =
     poetry install --only fmt
-    poetry run ruff format {[vars]all_path} {[vars]github_scripts_path} {[vars]scripts_path}
+    poetry run ruff format {[vars]all_path} {[vars]github_scripts_path}
     /bin/bash -ec "find {[vars]tf_modules_path} -name '*.tf' -print0 | xargs -0 terraform fmt"
 
 
@@ -61,8 +60,8 @@ commands =
 description = Check code against coding style standards
 commands =
     poetry install --only fmt,lint
-    poetry run ruff check --fix {[vars]all_path} {[vars]github_scripts_path} {[vars]scripts_path}
-    poetry run ruff format --check {[vars]all_path} {[vars]github_scripts_path} {[vars]scripts_path}
+    poetry run ruff check --fix {[vars]all_path} {[vars]github_scripts_path}
+    poetry run ruff format --check {[vars]all_path} {[vars]github_scripts_path}
     poetry run codespell {[vars]all_path} pyproject.toml {[vars]github_scripts_path}
     poetry install --all-groups
     poetry run mypy {[vars]src_path}
@@ -98,11 +97,3 @@ pass_env =
 commands =
     poetry install --with integration,benchmark
     poetry run pytest -vv --tb native --log-cli-level=INFO -s {[vars]tests_path}/integration/bench/test_benchmark_kyuubi.py {posargs} -x
-
-[testenv:tfinputs]
-description = Parse TF variables.tf file into a table
-pass_env =
-    {[testenv]pass_env}
-commands =
-    pip install python-hcl2
-    python scripts/tf_inputs.py

From 354e0813a1646403b1e986229fab944d86f0d8d8 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 11:36:56 +0200
Subject: [PATCH 16/62] Remove bundle backend mentions from workflow

---
 .github/actions/run-test/action.yaml    | 11 +++--------
 .github/workflows/ci-tests-minimal.yaml |  7 +------
 2 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/.github/actions/run-test/action.yaml b/.github/actions/run-test/action.yaml
index adab4e6d..dd14b1bf 100644
--- a/.github/actions/run-test/action.yaml
+++ b/.github/actions/run-test/action.yaml
@@ -23,11 +23,6 @@ inputs:
     default: 3.4.4
     required: false
     type: string
-  bundle-backend:
-    description: "The backend to use to deploy the bundle"
-    default: yaml
-    required: false
-    type: string
   storage-backend:
     description: "The object storage backend to be used"
     default: s3
@@ -236,7 +231,7 @@ runs:
       run: |
         juju add-model spark-bundle-test
         juju list-models
-        cd python && tox run -e ${{ inputs.tox-env }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --backend ${{ inputs.bundle-backend }} --cos-model ${{ inputs.cos-model }} --spark-version ${{ inputs.spark-version }} --storage-backend ${{ inputs.storage-backend }} --model spark-bundle-test ${{ inputs.pytest-args }}
+        cd python && tox run -e ${{ inputs.tox-env }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --cos-model ${{ inputs.cos-model }} --spark-version ${{ inputs.spark-version }} --storage-backend ${{ inputs.storage-backend }} --model spark-bundle-test ${{ inputs.pytest-args }}
         echo "TEST_EXIT_CODE=$?" >> $GITHUB_ENV
 
     - id: collect-logs
@@ -293,13 +288,13 @@ runs:
         if [ "${TEST_EXIT_CODE}" != "0" ]; then
           STATUS="failure"
         fi
-        echo '{"tox-env": "${{ inputs.tox-env }}", "k8s-distribution": "${{ inputs.k8s-distribution }}", "spark-version": "${{ inputs.spark-version }}", "bundle-backend": "${{ inputs.bundle-backend }}", "storage-backend": "${{ inputs.storage-backend }}", "cos": "${{ inputs.cos-model != '' }}", "juju-version": "${{ inputs.juju-agent-version }}", "status": "'"$STATUS"'"}' > test-result.json
+        echo '{"tox-env": "${{ inputs.tox-env }}", "k8s-distribution": "${{ inputs.k8s-distribution }}", "spark-version": "${{ inputs.spark-version }}", "storage-backend": "${{ inputs.storage-backend }}", "cos": "${{ inputs.cos-model != '' }}", "juju-version": "${{ inputs.juju-agent-version }}", "status": "'"$STATUS"'"}' > test-result.json
 
     - name: Upload the test result as artifact
       if: always()
       uses: actions/upload-artifact@v6
       with:
-        name: integration-results-${{ inputs.tox-env }}-${{ inputs.k8s-distribution }}-${{ inputs.k8s-version }}-${{ inputs.spark-version }}-${{ inputs.bundle-backend }}-${{ inputs.storage-backend }}-${{ inputs.cos-model }}-${{ inputs.juju-agent-version }}
+        name: integration-results-${{ inputs.tox-env }}-${{ inputs.k8s-distribution }}-${{ inputs.k8s-version }}-${{ inputs.spark-version }}-${{ inputs.storage-backend }}-${{ inputs.cos-model }}-${{ inputs.juju-agent-version }}
         path: test-result.json
 
 
diff --git a/.github/workflows/ci-tests-minimal.yaml b/.github/workflows/ci-tests-minimal.yaml
index 4bca8b42..4167c083 100644
--- a/.github/workflows/ci-tests-minimal.yaml
+++ b/.github/workflows/ci-tests-minimal.yaml
@@ -11,7 +11,7 @@ jobs:
     uses: ./.github/workflows/ci-checks.yaml
 
   integration-tests:
-    name: "${{ matrix.tox-env }} (${{ matrix.k8s-distribution }}:${{ matrix.k8s-version }}): spark-${{ matrix.spark-version }} | ${{ matrix.bundle-backend }} | ${{ matrix.storage-backend }} | juju ${{ matrix.juju-agent-version }} | ${{ matrix.cos-model }}"
+    name: "${{ matrix.tox-env }} (${{ matrix.k8s-distribution }}:${{ matrix.k8s-version }}): spark-${{ matrix.spark-version }} | ${{ matrix.storage-backend }} | juju ${{ matrix.juju-agent-version }} | ${{ matrix.cos-model }}"
     runs-on: ["self-hosted-linux-amd64-jammy-xlarge"]
     # runs-on: ubuntu-22.04
     timeout-minutes: 120
@@ -23,7 +23,6 @@ jobs:
           - integration-sparkjob
           - integration-kyuubi
           - integration-backup-restore
-        bundle-backend: [ "terraform" ]
         spark-version: [ "3.4.4" ] #, "3.5.7" ]
         storage-backend: [ "s3", "azure_storage" ]
         juju-snap-channel: [ "3.6/stable" ]
@@ -53,7 +52,6 @@ jobs:
 
           # # Test COS just once with integration-sparkjob
           # - tox-env: integration-sparkjob
-          #   bundle-backend: terraform
           #   storage-backend: s3
           #   juju-snap-channel: "3.6/stable"
           #   juju-agent-version: "3.6.14"
@@ -61,7 +59,6 @@ jobs:
 
           # # Test COS just once with integration-kyuubi
           # - tox-env: integration-kyuubi
-          #   bundle-backend: terraform
           #   storage-backend: s3
           #   juju-snap-channel: "3.6/stable"
           #   juju-agent-version: "3.6.14"
@@ -69,7 +66,6 @@ jobs:
 
           # # Test COS just once with integration-backup-restore
           # - tox-env: integration-backup-restore
-          #   bundle-backend: terraform
           #   storage-backend: s3
           #   juju-snap-channel: "3.6/stable"
           #   juju-agent-version: "3.6.14"
@@ -87,7 +83,6 @@ jobs:
           k8s-distribution: ${{ matrix.k8s-distribution }}
           k8s-version: ${{ matrix.k8s-version }}
           tox-env: ${{ matrix.tox-env }}
-          bundle-backend: ${{ matrix.bundle-backend || 'terraform'}}
           spark-version: ${{ matrix.spark-version }}
           storage-backend: ${{ matrix.storage-backend || 's3'}}
           cos-model: ${{ matrix.cos-model }}

From f7ad8132ebd5b12b55f770ca30e7d983704f7306 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 14:10:50 +0200
Subject: [PATCH 17/62] Fix self-signed-certificases app name in test

---
 python/tests/integration/helpers.py           |  2 +-
 .../tests/integration/test_backup_restore.py  | 20 ++++++++++++++-----
 python/tests/integration/test_kyuubi.py       |  4 +++-
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/python/tests/integration/helpers.py b/python/tests/integration/helpers.py
index 8107d8ab..fa66847d 100644
--- a/python/tests/integration/helpers.py
+++ b/python/tests/integration/helpers.py
@@ -334,7 +334,7 @@ def assert_logs(loki_address: str) -> None:
 
 
 def get_kyuubi_ca_cert(
-    juju: jubilant.Juju, certificates_app_name: str = "certificates"
+    juju: jubilant.Juju, certificates_app_name: str = "self-signed-certificates"
 ) -> str:
     """Get the CA certificate used by Kyuubi"""
     status = juju.status()
diff --git a/python/tests/integration/test_backup_restore.py b/python/tests/integration/test_backup_restore.py
index be886dfe..e18374e8 100644
--- a/python/tests/integration/test_backup_restore.py
+++ b/python/tests/integration/test_backup_restore.py
@@ -262,7 +262,9 @@ def test_admin_user_connection(
         """Test that the admin user is able to connect."""
         credentials = get_kyuubi_credentials(juju)
         credentials.update({"username": "admin", "password": admin_password})
-        ca_cert = get_kyuubi_ca_cert(juju, certificates_app_name="certificates")
+        ca_cert = get_kyuubi_ca_cert(
+            juju, certificates_app_name="self-signed-certificates"
+        )
         kyuubi_client = KyuubiClient(**credentials, use_ssl=True, ca_cert=ca_cert)
 
         assert "default" in kyuubi_client.databases
@@ -274,7 +276,9 @@ def test_admin_user_connection(
     def test_database_operations(self, juju: jubilant.Juju) -> None:
         """Test some read / write operations on the database."""
         credentials = get_kyuubi_credentials(juju)
-        ca_cert = get_kyuubi_ca_cert(juju, certificates_app_name="certificates")
+        ca_cert = get_kyuubi_ca_cert(
+            juju, certificates_app_name="self-signed-certificates"
+        )
         kyuubi_client = KyuubiClient(**credentials, use_ssl=True, ca_cert=ca_cert)
 
         db = kyuubi_client.get_database(TEST_DB_NAME)
@@ -610,7 +614,9 @@ def test_old_admin_password_restored(
         old_admin_password = context.pop("old_admin_password")
         credentials = get_kyuubi_credentials(juju)
         credentials.update({"username": "admin", "password": old_admin_password})
-        ca_cert = get_kyuubi_ca_cert(juju, certificates_app_name="certificates")
+        ca_cert = get_kyuubi_ca_cert(
+            juju, certificates_app_name="self-signed-certificates"
+        )
         kyuubi_client = KyuubiClient(**credentials, use_ssl=True, ca_cert=ca_cert)
 
         assert "default" in kyuubi_client.databases
@@ -654,7 +660,9 @@ def test_read_previously_written_data(self, juju: jubilant.Juju) -> None:
         """Test some read / write operations on the database."""
         juju.wait(jubilant.all_active, delay=3)
         credentials = get_kyuubi_credentials(juju)
-        ca_cert = get_kyuubi_ca_cert(juju, certificates_app_name="certificates")
+        ca_cert = get_kyuubi_ca_cert(
+            juju, certificates_app_name="self-signed-certificates"
+        )
         kyuubi_client = KyuubiClient(**credentials, use_ssl=True, ca_cert=ca_cert)
 
         assert TEST_DB_NAME in kyuubi_client.databases
@@ -678,7 +686,9 @@ def test_insert_new_rows(self, juju: jubilant.Juju) -> None:
         """Test that the new rows of data are being inserted on top of what's already there."""
         juju.wait(jubilant.all_active, delay=3)
         credentials = get_kyuubi_credentials(juju)
-        ca_cert = get_kyuubi_ca_cert(juju, certificates_app_name="certificates")
+        ca_cert = get_kyuubi_ca_cert(
+            juju, certificates_app_name="self-signed-certificates"
+        )
 
         kyuubi_client = KyuubiClient(**credentials, use_ssl=True, ca_cert=ca_cert)
 
diff --git a/python/tests/integration/test_kyuubi.py b/python/tests/integration/test_kyuubi.py
index 9d975426..b2927699 100644
--- a/python/tests/integration/test_kyuubi.py
+++ b/python/tests/integration/test_kyuubi.py
@@ -70,7 +70,9 @@ def test_jdbc_endpoint(juju: jubilant.Juju) -> None:
 
     logger.info("Get certificate from self-signed-certificates operator")
     status = juju.status()
-    self_signed_certificate_unit = next(iter(status.apps["certificates"].units.keys()))
+    self_signed_certificate_unit = next(
+        iter(status.apps["self-signed-certificates"].units.keys())
+    )
     task = juju.run(self_signed_certificate_unit, "get-issued-certificates")
     assert task.return_code == 0
 

From 7aded55d51fe9f58bf2e452464dd2de068049bb7 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 14:26:38 +0200
Subject: [PATCH 18/62] Add azure_storage_config to test bundle entrypoint
 module

---
 python/tests/integration/conftest.py       |  2 +-
 python/tests/integration/resources/main.tf | 27 ++++++++++++++--------
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index 34cb7211..7f56ae9b 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -489,7 +489,7 @@ def spark_bundle(
     if storage_backend == "azure_storage":
         storage_unit = cast(Container, object_storage)
         storage_vars = {
-            "azure_storage": {
+            "azure_storage_config": {
                 "storage_account": storage_unit.credentials.storage_account,
                 "container": storage_unit.container_name,
                 "secret_key": storage_unit.credentials.secret_key,
diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index af051f7e..c4f63eeb 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -47,8 +47,14 @@ variable "kyuubi_config" {
   type = map(any)
 }
 
+variable "azure_storage_config" {
+  type    = map(any)
+  default = {}
+}
+
 variable "s3_config" {
-  type = map(any)
+  type    = map(any)
+  default = {}
 }
 
 variable "tls_private_key" {
@@ -86,13 +92,14 @@ module "spark" {
   model_uuid   = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
   create_model = false
 
-  admin_password    = var.admin_password
-  kyuubi_config     = var.kyuubi_config
-  kyuubi_users_size = var.kyuubi_users_size
-  metastore_size    = var.metastore_size
-  s3_config         = var.s3_config
-  storage_backend   = var.storage_backend
-  tls_private_key   = var.tls_private_key
-  zookeeper_size    = var.zookeeper_size
-  zookeeper_units   = var.zookeeper_units
+  admin_password       = var.admin_password
+  kyuubi_config        = var.kyuubi_config
+  kyuubi_users_size    = var.kyuubi_users_size
+  metastore_size       = var.metastore_size
+  azure_storage_config = var.azure_storage_config
+  s3_config            = var.s3_config
+  storage_backend      = var.storage_backend
+  tls_private_key      = var.tls_private_key
+  zookeeper_size       = var.zookeeper_size
+  zookeeper_units      = var.zookeeper_units
 }

From b31c0643a4bd0cb652c41c1c1512f92e2f4adc2c Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 14:54:48 +0200
Subject: [PATCH 19/62] Fix empty s3 module condition

---
 terraform/products/charmed-spark-3.4/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index bdbfb4d7..25ca15ec 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -184,8 +184,8 @@ module "spark" {
   }
 
   object_storage_endpoint = {
-    name     = module.s3 != null ? module.s3[0].application.name : module.azure_storage[0].application.name
-    endpoint = module.s3 != null ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
+    name     = module.s3 != [] ? module.s3[0].application.name : module.azure_storage[0].application.name
+    endpoint = module.s3 != [] ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
   }
 }
 

From a8dc2b6eaa793fd868466d747d4e3a8f45910c63 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 15:13:45 +0200
Subject: [PATCH 20/62] Fix azure-storage tf offer

---
 terraform/charms/azure-storage-integrator/main.tf | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/terraform/charms/azure-storage-integrator/main.tf b/terraform/charms/azure-storage-integrator/main.tf
index ebf47196..5c314e80 100644
--- a/terraform/charms/azure-storage-integrator/main.tf
+++ b/terraform/charms/azure-storage-integrator/main.tf
@@ -35,8 +35,7 @@ resource "juju_access_secret" "azure_storage_secret_access" {
 }
 
 resource "juju_offer" "azure_storage_credentials" {
-  name             = "azure-storage-credentials"
   model_uuid       = var.model_uuid
-  application_name = var.app_name
+  application_name = juju_application.azure_storage.name
   endpoints        = ["azure-storage-credentials"]
 }

From 953a93f1f37c1876ee5a03ba349d8aa893548eb6 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 15:27:24 +0200
Subject: [PATCH 21/62] Pass admin_password to bundle

---
 terraform/products/charmed-spark-3.4/main.tf | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index 25ca15ec..bfd54e14 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -187,6 +187,11 @@ module "spark" {
     name     = module.s3 != [] ? module.s3[0].application.name : module.azure_storage[0].application.name
     endpoint = module.s3 != [] ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
   }
+
+  admin_password  = var.admin_password
+  tls_private_key = var.tls_private_key
+
+
 }
 
 

From cba2e6d3d4e678e4a0fec34b8455267e3e73b896 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 15:47:36 +0200
Subject: [PATCH 22/62] Fix azure-storage storage-account hyphen config option

---
 terraform/charms/azure-storage-integrator/variables.tf | 2 +-
 terraform/products/charmed-spark-3.4/main.tf           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform/charms/azure-storage-integrator/variables.tf b/terraform/charms/azure-storage-integrator/variables.tf
index 1fab31d6..9791f049 100644
--- a/terraform/charms/azure-storage-integrator/variables.tf
+++ b/terraform/charms/azure-storage-integrator/variables.tf
@@ -30,7 +30,7 @@ variable "config" {
     endpoint            = optional(string)
     path                = optional(string)
     resource-group      = optional(string)
-    storage_account     = optional(string)
+    storage-account     = optional(string)
   })
   default = {}
 }
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index bfd54e14..ca8dea24 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -104,7 +104,7 @@ module "azure_storage" {
   config = {
     connection-protocol = var.azure_storage_config.protocol
     container           = var.azure_storage_config.container
-    storage_account     = var.azure_storage_config.storage_account
+    storage-account     = var.azure_storage_config.storage_account
   }
   constraints = "arch=amd64"
   revision    = local.revisions.azure_storage

From 130c677b2473fa626528c133f1d5afc613c7925d Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 2 Apr 2026 16:20:18 +0200
Subject: [PATCH 23/62] Remove manual secret granting to az storage

---
 python/tests/integration/conftest.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index 7f56ae9b..bbc4b07e 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -497,7 +497,6 @@ def spark_bundle(
         }
     else:
         storage_unit = cast(Bucket, object_storage)
-
         storage_vars = {
             "s3_config": {
                 "bucket": storage_unit.bucket_name,
@@ -507,17 +506,10 @@ def spark_bundle(
         }
 
     vars = base_vars | cos_vars | storage_vars | storage_sizes
-    logger.info(f"Applying vars: {vars}")
+    logger.info(f"Applying vars: {vars.keys()}")
 
     deployed_applications = bundle.apply(vars=vars)
-    if storage_backend == "azure_storage":
-        credentials = request.getfixturevalue("azure_credentials")
-        secret_uri = juju.add_secret(
-            "iamsecret", {"secret-key": credentials.secret_key}
-        )
-        juju.cli("grant-secret", secret_uri, "azure-storage")
-        juju.config("azure-storage", {"credentials": secret_uri})
-    else:
+    if storage_backend == "s3":
         credentials = request.getfixturevalue("credentials")
         juju.wait(lambda status: jubilant.all_agents_idle(status, "s3-integrator"))
         set_s3_credentials(juju, credentials=credentials)

From 978fff1680369cd24def501279983b11dd7b948d Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 7 Apr 2026 09:40:18 +0200
Subject: [PATCH 24/62] Fix azure-storage missing path

---
 python/tests/integration/conftest.py              | 1 +
 terraform/products/charmed-spark-3.4/main.tf      | 1 +
 terraform/products/charmed-spark-3.4/variables.tf | 5 +++--
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index bbc4b07e..391a3641 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -493,6 +493,7 @@ def spark_bundle(
                 "storage_account": storage_unit.credentials.storage_account,
                 "container": storage_unit.container_name,
                 "secret_key": storage_unit.credentials.secret_key,
+                "path": "spark-events",
             }
         }
     else:
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index ca8dea24..f719e9b5 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -104,6 +104,7 @@ module "azure_storage" {
   config = {
     connection-protocol = var.azure_storage_config.protocol
     container           = var.azure_storage_config.container
+    path                = var.azure_storage_config.path
     storage-account     = var.azure_storage_config.storage_account
   }
   constraints = "arch=amd64"
diff --git a/terraform/products/charmed-spark-3.4/variables.tf b/terraform/products/charmed-spark-3.4/variables.tf
index 4bf10a22..12b5dd3f 100644
--- a/terraform/products/charmed-spark-3.4/variables.tf
+++ b/terraform/products/charmed-spark-3.4/variables.tf
@@ -13,9 +13,10 @@ variable "azure_storage_config" {
   description = "Azure Object storage information"
   type = object({
     container       = optional(string)
-    storage_account = optional(string)
-    secret_key      = optional(string)
+    path            = optional(string)
     protocol        = optional(string, "abfss")
+    secret_key      = optional(string)
+    storage_account = optional(string)
   })
   default = {}
 }

From 62ca62ac9144e41019a29f623d5abddc1469c526 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 7 Apr 2026 14:45:05 +0200
Subject: [PATCH 25/62] Initial re-implementation of COS integration

---
 terraform/components/observability/README.md  |  71 +++++
 .../components/observability/applications.tf  |  64 +++++
 .../components/observability/integrations.tf  | 244 ++++++++++++++++++
 terraform/components/observability/locals.tf  |  11 +
 terraform/components/observability/outputs.tf |  48 ++++
 .../components/observability/terraform.tf     |  10 +
 .../components/observability/variables.tf     | 180 +++++++++++++
 terraform/components/spark-3.4/outputs.tf     |  40 ++-
 terraform/products/charmed-spark-3.4/main.tf  |  38 ++-
 .../products/charmed-spark-3.4/variables.tf   |  10 +
 10 files changed, 692 insertions(+), 24 deletions(-)
 create mode 100644 terraform/components/observability/README.md
 create mode 100644 terraform/components/observability/applications.tf
 create mode 100644 terraform/components/observability/integrations.tf
 create mode 100644 terraform/components/observability/locals.tf
 create mode 100644 terraform/components/observability/outputs.tf
 create mode 100644 terraform/components/observability/terraform.tf
 create mode 100644 terraform/components/observability/variables.tf

diff --git a/terraform/components/observability/README.md b/terraform/components/observability/README.md
new file mode 100644
index 00000000..13837cc7
--- /dev/null
+++ b/terraform/components/observability/README.md
@@ -0,0 +1,71 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_application.cos_configuration](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.pushgateway](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.scrape_config](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.cos_configuration_grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana_agent_grafana_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana_agent_loki](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.grafana_agent_prometheus](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.history_server_grafana_agent_dashboard](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.history_server_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.history_server_grafana_agent_metrics](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.integration_hub_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.integration_hub_pushgateway](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_grafana_agent_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_grafana_agent_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_grafana_agent_metrics](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.pushgateway_scrape_config](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.scrape_config_grafana_agent](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_offer.grafana_dashboards](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
+| [juju_offer.loki_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
+| [juju_offer.prometheus_receive_remote_write](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_cos_configuration"></a> [cos\_configuration](#input\_cos\_configuration) | n/a | <pre>object({<br/>    app_name    = optional(string, "cos-configuration")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "1/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "cos-configuration",<br/>  "base": "ubuntu@22.04",<br/>  "config": {<br/>    "git_branch": "main",<br/>    "git_depth": 1,<br/>    "git_repo": "https://github.com/canonical/spark-k8s-bundle",<br/>    "grafana_dashboards_path": "resources/grafana/"<br/>  },<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_dashboards_offer"></a> [dashboards\_offer](#input\_dashboards\_offer) | URL of the `grafana_dashboard` interface offer. | `string` | n/a | yes |
+| <a name="input_grafana_agent"></a> [grafana\_agent](#input\_grafana\_agent) | n/a | <pre>object({<br/>    app_name    = optional(string, "grafana-agent")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "1/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "grafana-agent",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_history_server_dashboard_endpoint"></a> [history\_server\_dashboard\_endpoint](#input\_history\_server\_dashboard\_endpoint) | In-model endpoint for the History Server dashboard integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_history_server_logging_endpoint"></a> [history\_server\_logging\_endpoint](#input\_history\_server\_logging\_endpoint) | In-model endpoint for the History Server logging integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_history_server_metrics_endpoint"></a> [history\_server\_metrics\_endpoint](#input\_history\_server\_metrics\_endpoint) | In-model endpoint for the History Server metrics integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_integration_hub_cos_endpoint"></a> [integration\_hub\_cos\_endpoint](#input\_integration\_hub\_cos\_endpoint) | In-model endpoint for the Integration Hub COS integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_integration_hub_logging_endpoint"></a> [integration\_hub\_logging\_endpoint](#input\_integration\_hub\_logging\_endpoint) | In-model endpoint for the Integration Hub logging integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_kyuubi_dashboard_endpoint"></a> [kyuubi\_dashboard\_endpoint](#input\_kyuubi\_dashboard\_endpoint) | In-model endpoint for the Kyuubi dashboard integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_kyuubi_logging_endpoint"></a> [kyuubi\_logging\_endpoint](#input\_kyuubi\_logging\_endpoint) | In-model endpoint for the Kyuubi logging integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_kyuubi_metrics_endpoint"></a> [kyuubi\_metrics\_endpoint](#input\_kyuubi\_metrics\_endpoint) | In-model endpoint for the Kyuubi metrics integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_logging_offer"></a> [logging\_offer](#input\_logging\_offer) | URL of the `loki_push_api` interface offer. | `string` | n/a | yes |
+| <a name="input_metrics_offer"></a> [metrics\_offer](#input\_metrics\_offer) | URL of the `prometheus_remote_write` interface offer. | `string` | n/a | yes |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
+| <a name="input_pushgateway"></a> [pushgateway](#input\_pushgateway) | n/a | <pre>object({<br/>    app_name           = optional(string, "pushgateway")<br/>    base               = optional(string, "ubuntu@22.04")<br/>    channel            = optional(string, "1/stable")<br/>    config             = optional(map(string), {})<br/>    constraints        = optional(string, "arch=amd64")<br/>    resources          = optional(map(string), {})<br/>    revision           = optional(number)<br/>    storage_directives = optional(map(string))<br/>    units              = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "pushgateway",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "storage_directives": {<br/>    "pushgateway-store": "10G"<br/>  },<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_scrape_config"></a> [scrape\_config](#input\_scrape\_config) | n/a | <pre>object({<br/>    app_name    = optional(string, "scrape-config")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "1/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "scrape-config",<br/>  "base": "ubuntu@22.04",<br/>  "config": {<br/>    "scrape_interval": "10s"<br/>  },<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_components"></a> [components](#output\_components) | List of the deployed applications for this component module. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provided endpoints. |
+| <a name="output_requires"></a> [requires](#output\_requires) | Map of the required endpoints. |
diff --git a/terraform/components/observability/applications.tf b/terraform/components/observability/applications.tf
new file mode 100644
index 00000000..9ea5298a
--- /dev/null
+++ b/terraform/components/observability/applications.tf
@@ -0,0 +1,64 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "grafana_agent" {
+  name       = var.grafana_agent.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "grafana-agent-k8s"
+    channel  = var.grafana_agent.channel
+    revision = var.grafana_agent.revision != null ? var.grafana_agent.revision : local.revisions.grafana_agent
+  }
+
+  constraints = var.grafana_agent.constraints
+  trust       = true
+  units       = var.grafana_agent.units
+}
+
+resource "juju_application" "cos_configuration" {
+  name       = var.cos_configuration.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "cos-configuration-k8s"
+    channel  = var.cos_configuration.channel
+    revision = var.cos_configuration.revision != null ? var.cos_configuration.revision : local.revisions.cos_configuration
+  }
+
+  config = var.cos_configuration.config
+
+  constraints = var.cos_configuration.constraints
+  units       = var.cos_configuration.units
+}
+
+resource "juju_application" "pushgateway" {
+  name       = var.pushgateway.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "prometheus-pushgateway-k8s"
+    channel  = var.pushgateway.channel
+    revision = var.pushgateway.revision != null ? var.pushgateway.revision : local.revisions.pushgateway
+  }
+
+  constraints        = var.pushgateway.constraints
+  storage_directives = var.pushgateway.storage_directives
+  units              = var.pushgateway.units
+}
+
+resource "juju_application" "scrape_config" {
+  name       = var.scrape_config.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "prometheus-scrape-config-k8s"
+    channel  = var.scrape_config.channel
+    revision = var.scrape_config.revision != null ? var.scrape_config.revision : local.revisions.scrape_config
+  }
+
+  config      = var.scrape_config.config
+  constraints = var.scrape_config.constraints
+  units       = var.scrape_config.units
+}
+
diff --git a/terraform/components/observability/integrations.tf b/terraform/components/observability/integrations.tf
new file mode 100644
index 00000000..349b2164
--- /dev/null
+++ b/terraform/components/observability/integrations.tf
@@ -0,0 +1,244 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+data "juju_offer" "grafana_dashboards" {
+  url = var.dashboards_offer
+}
+
+data "juju_offer" "prometheus_receive_remote_write" {
+  url = var.metrics_offer
+}
+
+data "juju_offer" "loki_logging" {
+  url = var.logging_offer
+}
+
+## Component internal integrations
+
+resource "juju_integration" "cos_configuration_grafana_agent" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.cos_configuration.name
+    endpoint = "grafana-dashboards"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "grafana-dashboards-consumer"
+  }
+}
+
+resource "juju_integration" "pushgateway_scrape_config" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.pushgateway.name
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.scrape_config.name
+    endpoint = "configurable-scrape-jobs"
+  }
+}
+
+resource "juju_integration" "scrape_config_grafana_agent" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.scrape_config.name
+    endpoint = "metrics-endpoint"
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+## Cross-model COS integrations
+
+resource "juju_integration" "grafana_agent_grafana_dashboards" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "grafana-dashboards-provider"
+  }
+
+  application {
+    offer_url = data.juju_offer.grafana_dashboards.url
+  }
+
+  lifecycle {
+    replace_triggered_by = [
+      juju_application.grafana_agent.name,
+      juju_application.grafana_agent.model,
+      juju_application.grafana_agent.constraints,
+      juju_application.grafana_agent.placement,
+      juju_application.grafana_agent.charm.name,
+    ]
+  }
+}
+
+resource "juju_integration" "grafana_agent_prometheus" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "send-remote-write"
+  }
+
+  application {
+    offer_url = data.juju_offer.prometheus_receive_remote_write.url
+  }
+
+  lifecycle {
+    replace_triggered_by = [
+      juju_application.grafana_agent.name,
+      juju_application.grafana_agent.model,
+      juju_application.grafana_agent.constraints,
+      juju_application.grafana_agent.placement,
+      juju_application.grafana_agent.charm.name,
+    ]
+  }
+}
+
+resource "juju_integration" "grafana_agent_loki" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "logging-consumer"
+  }
+
+  application {
+    offer_url = data.juju_offer.loki_logging.url
+  }
+
+  lifecycle {
+    replace_triggered_by = [
+      juju_application.grafana_agent.name,
+      juju_application.grafana_agent.model,
+      juju_application.grafana_agent.constraints,
+      juju_application.grafana_agent.placement,
+      juju_application.grafana_agent.charm.name,
+    ]
+  }
+}
+
+## Spark components integrations
+
+resource "juju_integration" "history_server_grafana_agent_dashboard" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.history_server_dashboard_endpoint.name
+    endpoint = var.history_server_dashboard_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "grafana-dashboards-consumer"
+  }
+}
+
+resource "juju_integration" "history_server_grafana_agent_logging" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.history_server_logging_endpoint.name
+    endpoint = var.history_server_logging_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "logging-provider"
+  }
+}
+
+resource "juju_integration" "history_server_grafana_agent_metrics" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.history_server_metrics_endpoint.name
+    endpoint = var.history_server_metrics_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+resource "juju_integration" "integration_hub_pushgateway" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.integration_hub_cos_endpoint.name
+    endpoint = var.integration_hub_cos_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.pushgateway.name
+    endpoint = "push-endpoint"
+  }
+}
+
+resource "juju_integration" "integration_hub_grafana_agent_logging" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.integration_hub_logging_endpoint.name
+    endpoint = var.integration_hub_logging_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "logging-provider"
+  }
+}
+
+resource "juju_integration" "kyuubi_grafana_agent_metrics" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.kyuubi_metrics_endpoint.name
+    endpoint = var.kyuubi_metrics_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "metrics-endpoint"
+  }
+}
+
+resource "juju_integration" "kyuubi_grafana_agent_dashboards" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.kyuubi_dashboard_endpoint.name
+    endpoint = var.kyuubi_dashboard_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "grafana-dashboards-consumer"
+  }
+}
+
+resource "juju_integration" "kyuubi_grafana_agent_logging" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.kyuubi_logging_endpoint.name
+    endpoint = var.kyuubi_logging_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.grafana_agent.name
+    endpoint = "logging-provider"
+  }
+}
+
diff --git a/terraform/components/observability/locals.tf b/terraform/components/observability/locals.tf
new file mode 100644
index 00000000..0ba8a684
--- /dev/null
+++ b/terraform/components/observability/locals.tf
@@ -0,0 +1,11 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+locals {
+  revisions = {
+    cos_configuration = 65
+    grafana_agent     = 121
+    pushgateway       = 27
+    scrape_config     = 67
+  }
+}
diff --git a/terraform/components/observability/outputs.tf b/terraform/components/observability/outputs.tf
new file mode 100644
index 00000000..cc89038b
--- /dev/null
+++ b/terraform/components/observability/outputs.tf
@@ -0,0 +1,48 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "components" {
+  description = "List of the deployed applications for this component module."
+  value = {
+    grafana_agent     = juju_application.grafana_agent
+    cos_configuration = juju_application.cos_configuration
+    pushgateway       = juju_application.pushgateway
+    scrape_config     = juju_application.scrape_config
+  }
+}
+
+output "provides" {
+  description = "Map of all the provided endpoints."
+  value = {
+    grafana_agent_metrics_endpoint = {
+      name     = juju_application.grafana_agent.name
+      endpoint = "metrics-endpoint"
+    }
+    grafana_agent_logging_provider = {
+      name     = juju_application.grafana_agent.name
+      endpoint = "logging-provider"
+    }
+    grafana_agent_grafana_dashboards_consumer = {
+      name     = juju_application.grafana_agent.name
+      endpoint = "grafana-dashboards-consumer"
+    }
+  }
+}
+
+output "requires" {
+  description = "Map of the required endpoints."
+  value = {
+    grafana_agent_grafana_dashboards_provider = {
+      name     = juju_application.grafana_agent.name
+      endpoint = "grafana-dashboards-provider"
+    }
+    grafana_agent_send_remote_write = {
+      name     = juju_application.grafana_agent.name
+      endpoint = "send-remote-write"
+    }
+    grafana_agent_logging_consumer = {
+      name     = juju_application.grafana_agent.name
+      endpoint = "logging-consumer"
+    }
+  }
+}
diff --git a/terraform/components/observability/terraform.tf b/terraform/components/observability/terraform.tf
new file mode 100644
index 00000000..6412125a
--- /dev/null
+++ b/terraform/components/observability/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/terraform/components/observability/variables.tf b/terraform/components/observability/variables.tf
new file mode 100644
index 00000000..df388c04
--- /dev/null
+++ b/terraform/components/observability/variables.tf
@@ -0,0 +1,180 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "model_uuid" {
+  description = "Reference to an existing model uuid."
+  type        = string
+  nullable    = false
+}
+
+variable "cos_configuration" {
+  type = object({
+    app_name    = optional(string, "cos-configuration")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "1/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {
+    app_name    = "cos-configuration"
+    base        = "ubuntu@22.04"
+    constraints = "arch=amd64"
+    config = {
+      git_branch              = "main"
+      git_depth               = 1
+      git_repo                = "https://github.com/canonical/spark-k8s-bundle"
+      grafana_dashboards_path = "resources/grafana/"
+    }
+    units = 1
+  }
+}
+
+variable "grafana_agent" {
+  type = object({
+    app_name    = optional(string, "grafana-agent")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "1/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {
+    app_name    = "grafana-agent"
+    base        = "ubuntu@22.04"
+    constraints = "arch=amd64"
+    units       = 1
+  }
+}
+
+variable "pushgateway" {
+  type = object({
+    app_name           = optional(string, "pushgateway")
+    base               = optional(string, "ubuntu@22.04")
+    channel            = optional(string, "1/stable")
+    config             = optional(map(string), {})
+    constraints        = optional(string, "arch=amd64")
+    resources          = optional(map(string), {})
+    revision           = optional(number)
+    storage_directives = optional(map(string))
+    units              = optional(number, 1)
+  })
+  default = {
+    app_name    = "pushgateway"
+    base        = "ubuntu@22.04"
+    constraints = "arch=amd64"
+    storage_directives = {
+      pushgateway-store = "10G"
+    }
+    units = 1
+  }
+}
+
+variable "scrape_config" {
+  type = object({
+    app_name    = optional(string, "scrape-config")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "1/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {
+    app_name = "scrape-config"
+    base     = "ubuntu@22.04"
+    config = {
+      scrape_interval = "10s"
+    }
+    constraints = "arch=amd64"
+    units       = 1
+  }
+}
+
+variable "dashboards_offer" {
+  description = "URL of the `grafana_dashboard` interface offer."
+  type        = string
+  nullable    = false
+}
+
+
+variable "logging_offer" {
+  description = "URL of the `loki_push_api` interface offer."
+  type        = string
+  nullable    = false
+}
+
+variable "metrics_offer" {
+  description = "URL of the `prometheus_remote_write` interface offer."
+  type        = string
+  nullable    = false
+}
+
+variable "history_server_logging_endpoint" {
+  description = "In-model endpoint for the History Server logging integration."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "history_server_metrics_endpoint" {
+  description = "In-model endpoint for the History Server metrics integration."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "history_server_dashboard_endpoint" {
+  description = "In-model endpoint for the History Server dashboard integration."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "integration_hub_cos_endpoint" {
+  description = "In-model endpoint for the Integration Hub COS integration."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "integration_hub_logging_endpoint" {
+  description = "In-model endpoint for the Integration Hub logging integration."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "kyuubi_logging_endpoint" {
+  description = "In-model endpoint for the Kyuubi logging integration."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "kyuubi_metrics_endpoint" {
+  description = "In-model endpoint for the Kyuubi metrics integration."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "kyuubi_dashboard_endpoint" {
+  description = "In-model endpoint for the Kyuubi dashboard integration."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
diff --git a/terraform/components/spark-3.4/outputs.tf b/terraform/components/spark-3.4/outputs.tf
index cffe6085..9ec9b073 100644
--- a/terraform/components/spark-3.4/outputs.tf
+++ b/terraform/components/spark-3.4/outputs.tf
@@ -13,17 +13,33 @@ output "components" {
 output "requires" {
   description = "Map of the requires endpoints."
   value = {
+    history_server_logging = {
+      name     = juju_application.history_server.name
+      endpoint = "logging"
+    }
+    integration_hub_cos = {
+      name     = juju_application.integration_hub.name
+      endpoint = "cos"
+    }
+    integration_hub_logging = {
+      name     = juju_application.integration_hub.name
+      endpoint = "logging"
+    }
+    kyuubi_auth_db = {
+      name     = juju_application.kyuubi.name
+      endpoint = "auth-db"
+    }
     kyuubi_certificates = {
       name     = juju_application.kyuubi.name
       endpoint = "certificates"
     }
-    kyuubi_metastore_db = {
+    kyuubi_logging = {
       name     = juju_application.kyuubi.name
-      endpoint = "metastore-db"
+      endpoint = "logging"
     }
-    kyuubi_auth_db = {
+    kyuubi_metastore_db = {
       name     = juju_application.kyuubi.name
-      endpoint = "auth-db"
+      endpoint = "metastore-db"
     }
     kyuubi_zookeeper = {
       name     = juju_application.kyuubi.name
@@ -35,6 +51,14 @@ output "requires" {
 output "provides" {
   description = "Map of all the provides endpoints."
   value = {
+    history_server_dashboard = {
+      name     = juju_application.history_server.name
+      endpoint = "grafana-dashboard"
+    }
+    history_server_metrics = {
+      name     = juju_application.history_server.name
+      endpoint = "metrics-endpoint"
+    }
     integration_hub_service_account = {
       name     = juju_application.integration_hub.name
       endpoint = "spark-service-account"
@@ -43,6 +67,14 @@ output "provides" {
       name     = juju_application.kyuubi.name
       endpoint = "jdbc"
     }
+    kyuubi_metrics = {
+      name     = juju_application.kyuubi.name
+      endpoint = "metrics-endpoint"
+    }
+    kyuubi_dashboard = {
+      name     = juju_application.kyuubi.name
+      endpoint = "grafana-dashboard"
+    }
   }
 }
 
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index f719e9b5..4bbe8a0c 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -191,26 +191,24 @@ module "spark" {
 
   admin_password  = var.admin_password
   tls_private_key = var.tls_private_key
-
-
 }
 
+module "observability" {
+  depends_on = [juju_model.spark, module.spark]
+  count      = var.cos_offers == null ? 0 : 1
+  source     = "../../components/observability"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
-
-
-# module "observability" {
-#   depends_on       = [module.spark, module.bundled_cos]
-#   count            = var.cos.deployed == "no" ? 0 : 1
-#   source           = "../../components/observability"
-#   dashboards_offer = var.cos.deployed == "external" ? var.cos.offers.dashboard : one(module.bundled_cos[*].dashboards_offer)
-#   logging_offer    = var.cos.deployed == "external" ? var.cos.offers.logging : one(module.bundled_cos[*].logging_offer)
-#   metrics_offer    = var.cos.deployed == "external" ? var.cos.offers.metrics : one(module.bundled_cos[*].metrics_offer)
-#   spark_model      = var.model
-#   model_uuid       = local.active_model
-#   spark_charms     = module.spark.charms
-
-#   grafana_agent_revision     = var.grafana_agent_revision != null ? var.grafana_agent_revision : local.revisions.grafana_agent
-#   cos_configuration_revision = var.cos_configuration_revision != null ? var.cos_configuration_revision : local.revisions.cos_configuration
-#   pushgateway_revision       = var.pushgateway_revision != null ? var.pushgateway_revision : local.revisions.pushgateway
-#   scrape_config_revision     = var.scrape_config_revision != null ? var.scrape_config_revision : local.revisions.scrape_config
-# }
+  dashboards_offer = var.cos_offers.dashboard
+  logging_offer    = var.cos_offers.logging
+  metrics_offer    = var.cos_offers.metrics
+
+  history_server_logging_endpoint   = module.spark.requires.history_server_logging
+  history_server_metrics_endpoint   = module.spark.provides.history_server_metrics
+  history_server_dashboard_endpoint = module.spark.provides.history_server_dashboard
+  integration_hub_cos_endpoint      = module.spark.requires.integration_hub_cos
+  integration_hub_logging_endpoint  = module.spark.requires.integration_hub_logging
+  kyuubi_logging_endpoint           = module.spark.requires.kyuubi_logging
+  kyuubi_metrics_endpoint           = module.spark.provides.kyuubi_metrics
+  kyuubi_dashboard_endpoint         = module.spark.provides.kyuubi_dashboard
+}
diff --git a/terraform/products/charmed-spark-3.4/variables.tf b/terraform/products/charmed-spark-3.4/variables.tf
index 12b5dd3f..36b5aadb 100644
--- a/terraform/products/charmed-spark-3.4/variables.tf
+++ b/terraform/products/charmed-spark-3.4/variables.tf
@@ -40,6 +40,16 @@ variable "create_model" {
   nullable    = false
 }
 
+variable "cos_offers" {
+  description = "Observability stack offers."
+  type = object({
+    dashboard = string
+    logging   = string
+    metrics   = string
+  })
+  default = null
+}
+
 variable "data_integrator_revision" {
   description = "Charm revision for data-integrator"
   type        = number

From 742e83a50bc4b422837a9552e0f9b060ead51d8a Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 7 Apr 2026 14:55:43 +0200
Subject: [PATCH 26/62] Rework provides output of charm modules

---
 .../charms/azure-storage-integrator/outputs.tf  |  5 ++++-
 terraform/charms/data-integrator/outputs.tf     |  5 ++++-
 terraform/charms/s3-integrator-v1/outputs.tf    |  5 ++++-
 terraform/charms/zookeeper/outputs.tf           |  5 ++++-
 terraform/products/charmed-spark-3.4/main.tf    | 17 +++--------------
 5 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/terraform/charms/azure-storage-integrator/outputs.tf b/terraform/charms/azure-storage-integrator/outputs.tf
index 88c604b1..d0594bf4 100644
--- a/terraform/charms/azure-storage-integrator/outputs.tf
+++ b/terraform/charms/azure-storage-integrator/outputs.tf
@@ -17,7 +17,10 @@ output "offers" {
 output "provides" {
   description = "Provides endpoints."
   value = {
-    azure_storage_credentials = "azure-storage-credentials"
+    azure_storage_credentials = {
+      name     = juju_application.azure_storage.name
+      endpoint = "azure-storage-credentials"
+    }
   }
 }
 
diff --git a/terraform/charms/data-integrator/outputs.tf b/terraform/charms/data-integrator/outputs.tf
index 425f291c..4d7b84cf 100644
--- a/terraform/charms/data-integrator/outputs.tf
+++ b/terraform/charms/data-integrator/outputs.tf
@@ -14,6 +14,9 @@ output "provides" {
 output "requires" {
   description = "Map of all the required endpoints."
   value = {
-    kyuubi = "kyuubi"
+    kyuubi = {
+      name     = juju_application.data_integrator.name
+      endpoint = "kyuubi"
+    }
   }
 }
diff --git a/terraform/charms/s3-integrator-v1/outputs.tf b/terraform/charms/s3-integrator-v1/outputs.tf
index 37d889bf..20651908 100644
--- a/terraform/charms/s3-integrator-v1/outputs.tf
+++ b/terraform/charms/s3-integrator-v1/outputs.tf
@@ -17,6 +17,9 @@ output "offers" {
 output "provides" {
   description = "Provides endpoints."
   value = {
-    s3_credentials = "s3-credentials"
+    s3_credentials = {
+      name     = juju_application.s3_integrator.name
+      endpoint = "s3-credentials"
+    }
   }
 }
diff --git a/terraform/charms/zookeeper/outputs.tf b/terraform/charms/zookeeper/outputs.tf
index bb25a17f..3fe78895 100644
--- a/terraform/charms/zookeeper/outputs.tf
+++ b/terraform/charms/zookeeper/outputs.tf
@@ -9,7 +9,10 @@ output "application" {
 output "provides" {
   description = "Map of all the provided endpoints."
   value = {
-    zookeeper = "zookeeper"
+    zookeeper = {
+      name     = juju_application.zookeeper.name
+      endpoint = "zookeeper"
+    }
   }
 }
 
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index 4bbe8a0c..04ff7a04 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -174,20 +174,9 @@ module "spark" {
     endpoint = module.kyuubi_users.provides.database
   }
 
-  zookeeper_endpoint = {
-    name     = module.zookeeper.application.name
-    endpoint = module.zookeeper.provides.zookeeper
-  }
-
-  data_integrator_endpoint = {
-    name     = module.data_integrator.application.name
-    endpoint = module.data_integrator.requires.kyuubi
-  }
-
-  object_storage_endpoint = {
-    name     = module.s3 != [] ? module.s3[0].application.name : module.azure_storage[0].application.name
-    endpoint = module.s3 != [] ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
-  }
+  zookeeper_endpoint       = module.zookeeper.provides.zookeeper
+  data_integrator_endpoint = module.data_integrator.requires.kyuubi
+  object_storage_endpoint  = module.s3 != [] ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
 
   admin_password  = var.admin_password
   tls_private_key = var.tls_private_key

From 9b11dbf5771386b50412e62da81e09ee36199b87 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 7 Apr 2026 16:11:54 +0200
Subject: [PATCH 27/62] Add upstream COS lite TF bundle

---
 python/tests/integration/conftest.py       | 23 +++++--------------
 python/tests/integration/resources/main.tf | 26 ++++++++++++++++++----
 2 files changed, 28 insertions(+), 21 deletions(-)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index 391a3641..c2a887c0 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -282,7 +282,7 @@ def integration_test(request):
 @pytest.fixture(scope=determine_scope)
 def cos(cos_model: str, request: pytest.FixtureRequest):
     """
-    Deploy COS bundle depending upon the value of cos_model fixture, and yield its value.
+    Handle COS Juju model lifecycle and yield its uuid.
     """
     if not cos_model:
         yield None
@@ -296,8 +296,7 @@ def cos(cos_model: str, request: pytest.FixtureRequest):
         yield cos_model
     except jubilant.CLIError:
         cos.add_model(cos_model)
-
-        yield cos_model
+        yield cast(str, cos.show_model().model_uuid)
     finally:
         debug_log = cos.debug_log(limit=50)
         status = cos.cli("status")
@@ -453,14 +452,12 @@ def spark_bundle(
     storage_sizes,
 ):
     """Deploy the Spark K8s bundle using Terraform."""
-    # short_version = ".".join(spark_version.split(".")[:2])
-    # module_path = PRODUCTS_DIR / f"charmed-spark-{short_version}"
+    short_version = ".".join(spark_version.split(".")[:2])
 
-    # TODO: Customize entry point based on test setup
     with (IE_TEST_DIR / "integration" / "resources" / "main.tf").open(
         "r", encoding="utf-8"
     ) as f:
-        entrypoint_content = f.read()
+        entrypoint_content = f.read().replace("<spark_flavor>", short_version)
 
     bundle = TerraformBackend(
         tempdir=tempdir,
@@ -476,16 +473,8 @@ def spark_bundle(
         "admin_password": admin_password,
         "tls_private_key": private_key,
     }
-    cos_vars = (
-        {
-            "cos": {
-                "model": cos,
-                "deployed": "bundled",
-            }
-        }
-        if cos
-        else {"cos": {"deployed": "no"}}
-    )
+    cos_vars = {"cos_model_uuid": cos} if cos else {}
+
     if storage_backend == "azure_storage":
         storage_unit = cast(Container, object_storage)
         storage_vars = {
diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index c4f63eeb..795876b8 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -26,6 +26,11 @@ variable "K8S_CREDENTIAL" {
   default = "microk8s"
 }
 
+variable "cos_model_uuid" {
+  type    = string
+  default = null
+}
+
 variable "spark_model_name" {
   type    = string
   default = "spark-test-model"
@@ -75,9 +80,16 @@ variable "zookeeper_size" {
   type = string
 }
 
-resource "juju_model" "spark" {
-  count = (var.model_uuid == null && var.create_model == true) ? 1 : 0
+module "cos" {
+  count = var.cos_model_uuid == null ? 0 : 1
+  # TODO: Pin to tag once available
+  source     = "git::https://github.com/canonical/observability-stack//terraform/cos-lite?ref=cd55b1d"
+  model_uuid = var.cos_model_uuid
+  # channel    = "1/stable"
+}
 
+resource "juju_model" "spark" {
+  count      = (var.model_uuid == null && var.create_model == true) ? 1 : 0
   name       = var.spark_model_name
   credential = var.K8S_CREDENTIAL
   cloud {
@@ -86,8 +98,8 @@ resource "juju_model" "spark" {
 }
 
 module "spark" {
-  depends_on = [juju_model.spark]
-  source     = "./products/charmed-spark-3.4"
+  depends_on = [juju_model.spark, module.cos]
+  source     = "./products/charmed-spark-<spark_flavor>" # filled by test fixture
 
   model_uuid   = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
   create_model = false
@@ -102,4 +114,10 @@ module "spark" {
   tls_private_key      = var.tls_private_key
   zookeeper_size       = var.zookeeper_size
   zookeeper_units      = var.zookeeper_units
+
+  cos_offers = module.cos != [] ? {
+    dashboard = module.cos[0].offers.grafana_dashboards.url
+    logging   = module.cos[0].offers.loki_logging.url
+    metrics   = module.cos[0].offers.prometheus_receive_remote_write.url
+  } : null
 }

From 74282c165d7bd461bc721f4cc0875a481ae31b8b Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 7 Apr 2026 16:13:10 +0200
Subject: [PATCH 28/62] Re-test COS on CI

---
 .github/workflows/ci-tests-minimal.yaml | 36 ++++++++++++-------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/ci-tests-minimal.yaml b/.github/workflows/ci-tests-minimal.yaml
index 4167c083..7bbc2851 100644
--- a/.github/workflows/ci-tests-minimal.yaml
+++ b/.github/workflows/ci-tests-minimal.yaml
@@ -50,26 +50,26 @@ jobs:
           #   k8s-distribution: "k8s"
           #   k8s-version: "1.32"
 
-          # # Test COS just once with integration-sparkjob
-          # - tox-env: integration-sparkjob
-          #   storage-backend: s3
-          #   juju-snap-channel: "3.6/stable"
-          #   juju-agent-version: "3.6.14"
-          #   cos-model: cos
+          # Test COS just once with integration-sparkjob
+          - tox-env: integration-sparkjob
+            storage-backend: s3
+            juju-snap-channel: "3.6/stable"
+            juju-agent-version: "3.6.14"
+            cos-model: cos
 
-          # # Test COS just once with integration-kyuubi
-          # - tox-env: integration-kyuubi
-          #   storage-backend: s3
-          #   juju-snap-channel: "3.6/stable"
-          #   juju-agent-version: "3.6.14"
-          #   cos-model: cos
+          # Test COS just once with integration-kyuubi
+          - tox-env: integration-kyuubi
+            storage-backend: s3
+            juju-snap-channel: "3.6/stable"
+            juju-agent-version: "3.6.14"
+            cos-model: cos
 
-          # # Test COS just once with integration-backup-restore
-          # - tox-env: integration-backup-restore
-          #   storage-backend: s3
-          #   juju-snap-channel: "3.6/stable"
-          #   juju-agent-version: "3.6.14"
-          #   cos-model: cos
+          # Test COS just once with integration-backup-restore
+          - tox-env: integration-backup-restore
+            storage-backend: s3
+            juju-snap-channel: "3.6/stable"
+            juju-agent-version: "3.6.14"
+            cos-model: cos
 
     needs:
       - checks

From aae9e90c1ff8cfe642f2c0012a1a05177e4ee05b Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Tue, 7 Apr 2026 17:39:41 +0200
Subject: [PATCH 29/62] Fix cos model name in test helpers

---
 python/tests/integration/test_backup_restore.py | 12 +++++++-----
 python/tests/integration/test_kyuubi.py         | 10 +++++-----
 python/tests/integration/test_spark_job.py      | 10 +++++-----
 3 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/python/tests/integration/test_backup_restore.py b/python/tests/integration/test_backup_restore.py
index e18374e8..705bc45c 100644
--- a/python/tests/integration/test_backup_restore.py
+++ b/python/tests/integration/test_backup_restore.py
@@ -711,11 +711,13 @@ def test_kyuubi_metrics_in_cos(self, cos: str) -> None:
         # We should leave time for Prometheus data to be published
         for attempt in Retrying(stop=stop_after_attempt(5), wait=wait_fixed(30)):
             with attempt:
-                cos_address = get_cos_address(cos_model_name=cos)
-                assert published_prometheus_data(cos, cos_address, "kyuubi_jvm_uptime")
+                cos_address = get_cos_address(cos_model_name="cos")
+                assert published_prometheus_data(
+                    "cos", cos_address, "kyuubi_jvm_uptime"
+                )
 
                 # Alerts got published to Prometheus
-                alerts_data = published_prometheus_alerts(cos, cos_address)
+                alerts_data = published_prometheus_alerts("cos", cos_address)
                 assert alerts_data is not None
                 logger.info(f"Alerts data: {alerts_data}")
 
@@ -742,14 +744,14 @@ def test_kyuubi_metrics_in_cos(self, cos: str) -> None:
                     )
 
                 # Grafana dashboard got published
-                dashboards_info = published_grafana_dashboards(cos)
+                dashboards_info = published_grafana_dashboards("cos")
                 assert dashboards_info is not None
                 logger.info(f"Dashboard info {dashboards_info}")
                 assert any(board["title"] == "Kyuubi" for board in dashboards_info)
 
                 # Loki logs are ingested
                 logs = published_loki_logs(
-                    cos,
+                    "cos",
                     cos_address,
                     "juju_application",
                     KYUUBI_APP_NAME,
diff --git a/python/tests/integration/test_kyuubi.py b/python/tests/integration/test_kyuubi.py
index b2927699..c8715de2 100644
--- a/python/tests/integration/test_kyuubi.py
+++ b/python/tests/integration/test_kyuubi.py
@@ -160,11 +160,11 @@ def test_kyuubi_metrics_in_cos(cos: str) -> None:
     # We should leave time for Prometheus data to be published
     for attempt in Retrying(stop=stop_after_attempt(10), wait=wait_fixed(30)):
         with attempt:
-            cos_address = get_cos_address(cos_model_name=cos)
-            assert published_prometheus_data(cos, cos_address, "kyuubi_jvm_uptime")
+            cos_address = get_cos_address(cos_model_name="cos")
+            assert published_prometheus_data("cos", cos_address, "kyuubi_jvm_uptime")
 
             # Alerts got published to Prometheus
-            alerts_data = published_prometheus_alerts(cos, cos_address)
+            alerts_data = published_prometheus_alerts("cos", cos_address)
             assert alerts_data is not None
             logger.info(f"Alerts data: {alerts_data}")
 
@@ -191,14 +191,14 @@ def test_kyuubi_metrics_in_cos(cos: str) -> None:
                 )
 
             # Grafana dashboard got published
-            dashboards_info = published_grafana_dashboards(cos)
+            dashboards_info = published_grafana_dashboards("cos")
             assert dashboards_info is not None
             logger.info(f"Dashboard info {dashboards_info}")
             assert any(board["title"] == "Kyuubi" for board in dashboards_info)
 
             # Loki logs are ingested
             logs = published_loki_logs(
-                cos,
+                "cos",
                 cos_address,
                 "juju_application",
                 KYUUBI_APP_NAME,
diff --git a/python/tests/integration/test_spark_job.py b/python/tests/integration/test_spark_job.py
index 6f34b87a..fb975b4e 100644
--- a/python/tests/integration/test_spark_job.py
+++ b/python/tests/integration/test_spark_job.py
@@ -298,7 +298,7 @@ def test_spark_metrics_in_prometheus(
     logger.info(f"Show status: {stdout}")
     logger.info(f"Spark id: {driver_pod.labels['spark-app-selector']}")
     # NOTE: 9090 seems to be commonly in use in some deployments.
-    with port_forward(pod=f"{PROMETHEUS}-0", port=9090, namespace=cos, on_port=19090):
+    with port_forward(pod=f"{PROMETHEUS}-0", port=9090, namespace="cos", on_port=19090):
         for attempt in Retrying(
             stop=stop_after_attempt(5), wait=wait_fixed(30), reraise=True
         ):
@@ -326,7 +326,7 @@ def test_spark_logforwaring_to_loki(
     stdout = juju.cli("status")
     logger.info(f"Show status: {stdout}")
 
-    with port_forward(pod=f"{LOKI}-0", port=3100, namespace=cos):
+    with port_forward(pod=f"{LOKI}-0", port=3100, namespace="cos"):
         assert_logs("127.0.0.1")
 
 
@@ -350,7 +350,7 @@ def test_history_server_metrics_in_cos(
     # We should leave time for Prometheus data to be published
     for attempt in Retrying(stop=stop_after_attempt(5), wait=wait_fixed(30)):
         with attempt:
-            cos_address = get_cos_address(cos_model_name=cos)
+            cos_address = get_cos_address(cos_model_name="cos")
             assert published_prometheus_data(
                 cos, cos_address, "jmx_scrape_duration_seconds"
             )
@@ -377,7 +377,7 @@ def test_history_server_metrics_in_cos(
                 )
 
             # Grafana dashboard got published
-            dashboards_info = published_grafana_dashboards(cos)
+            dashboards_info = published_grafana_dashboards("cos")
             assert dashboards_info is not None
             logger.info(f"Dashboard info {dashboards_info}")
             assert any(
@@ -387,7 +387,7 @@ def test_history_server_metrics_in_cos(
 
             # Loki logs are ingested
             logs = published_loki_logs(
-                cos,
+                "cos",
                 cos_address,
                 "juju_application",
                 HISTORY_SERVER,

From 091bfa49290bf89773a4eda86ec50ad5ff598e72 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 09:57:41 +0200
Subject: [PATCH 30/62] Re-align revisions

---
 terraform/components/observability/locals.tf  |  2 +-
 terraform/components/spark-3.4/locals.tf      | 18 +++++++--------
 .../products/charmed-spark-3.4/locals.tf      | 22 +++++++++----------
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/terraform/components/observability/locals.tf b/terraform/components/observability/locals.tf
index 0ba8a684..560b28db 100644
--- a/terraform/components/observability/locals.tf
+++ b/terraform/components/observability/locals.tf
@@ -4,7 +4,7 @@
 locals {
   revisions = {
     cos_configuration = 65
-    grafana_agent     = 121
+    grafana_agent     = 164
     pushgateway       = 27
     scrape_config     = 67
   }
diff --git a/terraform/components/spark-3.4/locals.tf b/terraform/components/spark-3.4/locals.tf
index 56ec0837..2ed303b9 100644
--- a/terraform/components/spark-3.4/locals.tf
+++ b/terraform/components/spark-3.4/locals.tf
@@ -3,16 +3,16 @@
 
 locals {
   revisions = {
-    history_server  = 94
-    integration_hub = 113
-    kyuubi          = 140
+    history_server  = 96
+    integration_hub = 119
+    kyuubi          = 149
   }
   images = {
-    history_server = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
-    # rev23, spark-version: 3.5.7, release date 2026-02-27
-    integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
-    # rev11, release date 2026-03-02
-    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e"
-    # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
+    history_server = "ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70"
+    # rev24,  spark-version: 3.5.7, release date 16/03/2026 
+    integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134"
+    # rev13, release date 19/03/2026
+    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:60e81dd2e9be50d4a857ce987935341c8518c784c7de36e814fc41dd3451d58f"
+    # rev19, spark-version: 3.4.4, kyuubi-version: 1.10.3, release date 17/03/2026
   }
 }
diff --git a/terraform/products/charmed-spark-3.4/locals.tf b/terraform/products/charmed-spark-3.4/locals.tf
index 2cfcfc42..fa880218 100644
--- a/terraform/products/charmed-spark-3.4/locals.tf
+++ b/terraform/products/charmed-spark-3.4/locals.tf
@@ -3,17 +3,17 @@
 
 locals {
   revisions = {
-    history_server  = 94
-    integration_hub = 113
-    kyuubi          = 140
+    history_server  = 96
+    integration_hub = 119
+    kyuubi          = 149
     kyuubi_users    = 774
     metastore       = 774
     zookeeper       = 78
     data_integrator = 362
     s3              = 330
     ssc             = 586
-    azure_storage   = 15
-    grafana_agent   = 121
+    azure_storage   = 270
+    grafana_agent   = 164
     # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
     # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
     # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
@@ -22,12 +22,12 @@ locals {
     scrape_config     = 67
   }
   images = {
-    history_server = "ghcr.io/canonical/charmed-spark@sha256:c7eef66beaeff463fdb7b707c630c0574912df8b843d8e3169f358adee2ae706"
-    # rev23, spark-version: 3.5.7, release date 2026-02-27
-    integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:c04d2d95874612883f6ead105d5f1dd74230370b23a08ea81175df81ba2e06fa"
-    # rev11, release date 2026-03-02
-    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:761a50ca0597825e5962c071a6575e7275e7eb04cae958d578bd7cc7f54fbb3e"
-    # rev16, spark-3.4.4, kyuubi 1.10.3 release date 2026-02-27
+    history_server = "ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70"
+    # rev24,  spark-version: 3.5.7, release date 16/03/2026 
+    integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134"
+    # rev13, release date 19/03/2026
+    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:60e81dd2e9be50d4a857ce987935341c8518c784c7de36e814fc41dd3451d58f"
+    # rev19, spark-version: 3.4.4, kyuubi-version: 1.10.3, release date 17/03/2026
     kyuubi_users = "184"
     metastore    = "184"
     zookeeper    = "34"

From 9963145aeb8cefd52222c404cd7f04108a4e7af5 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 10:34:49 +0200
Subject: [PATCH 31/62] Use constant for cos namespace in tests

---
 python/tests/integration/helpers.py             |  2 +-
 python/tests/integration/test_backup_restore.py | 13 ++++++-------
 python/tests/integration/test_kyuubi.py         | 11 ++++++-----
 python/tests/integration/test_spark_job.py      | 12 ++++++------
 4 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/python/tests/integration/helpers.py b/python/tests/integration/helpers.py
index fa66847d..5fb6ee01 100644
--- a/python/tests/integration/helpers.py
+++ b/python/tests/integration/helpers.py
@@ -20,7 +20,7 @@
 from tests.integration.types import KyuubiCredentials
 
 SECRET_NAME_PREFIX = "integrator-hub-conf-"
-COS_ALIAS = "cos"
+COS = "cos"
 JMX_EXPORTER_PORT = 9101
 JMX_CC_PORT = 9102
 ZOOKEEPER_PORT = 2181
diff --git a/python/tests/integration/test_backup_restore.py b/python/tests/integration/test_backup_restore.py
index 705bc45c..9487c2c9 100644
--- a/python/tests/integration/test_backup_restore.py
+++ b/python/tests/integration/test_backup_restore.py
@@ -23,6 +23,7 @@
 from spark_test.core.kyuubi import KyuubiClient
 
 from .helpers import (
+    COS,
     get_active_kyuubi_servers_list,
     get_cos_address,
     get_kyuubi_ca_cert,
@@ -711,13 +712,11 @@ def test_kyuubi_metrics_in_cos(self, cos: str) -> None:
         # We should leave time for Prometheus data to be published
         for attempt in Retrying(stop=stop_after_attempt(5), wait=wait_fixed(30)):
             with attempt:
-                cos_address = get_cos_address(cos_model_name="cos")
-                assert published_prometheus_data(
-                    "cos", cos_address, "kyuubi_jvm_uptime"
-                )
+                cos_address = get_cos_address(cos_model_name=COS)
+                assert published_prometheus_data(COS, cos_address, "kyuubi_jvm_uptime")
 
                 # Alerts got published to Prometheus
-                alerts_data = published_prometheus_alerts("cos", cos_address)
+                alerts_data = published_prometheus_alerts(COS, cos_address)
                 assert alerts_data is not None
                 logger.info(f"Alerts data: {alerts_data}")
 
@@ -744,14 +743,14 @@ def test_kyuubi_metrics_in_cos(self, cos: str) -> None:
                     )
 
                 # Grafana dashboard got published
-                dashboards_info = published_grafana_dashboards("cos")
+                dashboards_info = published_grafana_dashboards(COS)
                 assert dashboards_info is not None
                 logger.info(f"Dashboard info {dashboards_info}")
                 assert any(board["title"] == "Kyuubi" for board in dashboards_info)
 
                 # Loki logs are ingested
                 logs = published_loki_logs(
-                    "cos",
+                    COS,
                     cos_address,
                     "juju_application",
                     KYUUBI_APP_NAME,
diff --git a/python/tests/integration/test_kyuubi.py b/python/tests/integration/test_kyuubi.py
index c8715de2..68066b8f 100644
--- a/python/tests/integration/test_kyuubi.py
+++ b/python/tests/integration/test_kyuubi.py
@@ -17,6 +17,7 @@
 from tests.integration.types import PortForwarder
 
 from .helpers import (
+    COS,
     get_active_kyuubi_servers_list,
     get_cos_address,
     get_kyuubi_credentials,
@@ -160,11 +161,11 @@ def test_kyuubi_metrics_in_cos(cos: str) -> None:
     # We should leave time for Prometheus data to be published
     for attempt in Retrying(stop=stop_after_attempt(10), wait=wait_fixed(30)):
         with attempt:
-            cos_address = get_cos_address(cos_model_name="cos")
-            assert published_prometheus_data("cos", cos_address, "kyuubi_jvm_uptime")
+            cos_address = get_cos_address(cos_model_name=COS)
+            assert published_prometheus_data(COS, cos_address, "kyuubi_jvm_uptime")
 
             # Alerts got published to Prometheus
-            alerts_data = published_prometheus_alerts("cos", cos_address)
+            alerts_data = published_prometheus_alerts(COS, cos_address)
             assert alerts_data is not None
             logger.info(f"Alerts data: {alerts_data}")
 
@@ -191,14 +192,14 @@ def test_kyuubi_metrics_in_cos(cos: str) -> None:
                 )
 
             # Grafana dashboard got published
-            dashboards_info = published_grafana_dashboards("cos")
+            dashboards_info = published_grafana_dashboards(COS)
             assert dashboards_info is not None
             logger.info(f"Dashboard info {dashboards_info}")
             assert any(board["title"] == "Kyuubi" for board in dashboards_info)
 
             # Loki logs are ingested
             logs = published_loki_logs(
-                "cos",
+                COS,
                 cos_address,
                 "juju_application",
                 KYUUBI_APP_NAME,
diff --git a/python/tests/integration/test_spark_job.py b/python/tests/integration/test_spark_job.py
index fb975b4e..5b425992 100644
--- a/python/tests/integration/test_spark_job.py
+++ b/python/tests/integration/test_spark_job.py
@@ -26,6 +26,7 @@
 from spark_test.utils import get_spark_drivers
 
 from .helpers import (
+    COS,
     JMX_EXPORTER_PORT,
     assert_logs,
     get_cos_address,
@@ -40,7 +41,6 @@
 
 logger = logging.getLogger(__name__)
 
-COS_ALIAS = "cos"
 HISTORY_SERVER = "history-server"
 PUSHGATEWAY = "pushgateway"
 PROMETHEUS = "prometheus"
@@ -298,7 +298,7 @@ def test_spark_metrics_in_prometheus(
     logger.info(f"Show status: {stdout}")
     logger.info(f"Spark id: {driver_pod.labels['spark-app-selector']}")
     # NOTE: 9090 seems to be commonly in use in some deployments.
-    with port_forward(pod=f"{PROMETHEUS}-0", port=9090, namespace="cos", on_port=19090):
+    with port_forward(pod=f"{PROMETHEUS}-0", port=9090, namespace=COS, on_port=19090):
         for attempt in Retrying(
             stop=stop_after_attempt(5), wait=wait_fixed(30), reraise=True
         ):
@@ -326,7 +326,7 @@ def test_spark_logforwaring_to_loki(
     stdout = juju.cli("status")
     logger.info(f"Show status: {stdout}")
 
-    with port_forward(pod=f"{LOKI}-0", port=3100, namespace="cos"):
+    with port_forward(pod=f"{LOKI}-0", port=3100, namespace=COS):
         assert_logs("127.0.0.1")
 
 
@@ -350,7 +350,7 @@ def test_history_server_metrics_in_cos(
     # We should leave time for Prometheus data to be published
     for attempt in Retrying(stop=stop_after_attempt(5), wait=wait_fixed(30)):
         with attempt:
-            cos_address = get_cos_address(cos_model_name="cos")
+            cos_address = get_cos_address(cos_model_name=COS)
             assert published_prometheus_data(
                 cos, cos_address, "jmx_scrape_duration_seconds"
             )
@@ -377,7 +377,7 @@ def test_history_server_metrics_in_cos(
                 )
 
             # Grafana dashboard got published
-            dashboards_info = published_grafana_dashboards("cos")
+            dashboards_info = published_grafana_dashboards(COS)
             assert dashboards_info is not None
             logger.info(f"Dashboard info {dashboards_info}")
             assert any(
@@ -387,7 +387,7 @@ def test_history_server_metrics_in_cos(
 
             # Loki logs are ingested
             logs = published_loki_logs(
-                "cos",
+                COS,
                 cos_address,
                 "juju_application",
                 HISTORY_SERVER,

From 7af6408bd965b82c3a695f37bc838adaf3a7e61d Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 13:07:14 +0200
Subject: [PATCH 32/62] Cleanup and duplicate folders for 3.5

---
 .../components/observability/variables.tf     |  83 ++----
 terraform/components/spark-3.4/resources.tf   |   2 -
 terraform/components/spark-3.4/variables.tf   |  33 +--
 terraform/components/spark-3.5/README.md      |  62 ++++
 .../components/spark-3.5/applications.tf      |  73 +++++
 .../components/spark-3.5/integrations.tf      | 114 ++++++++
 terraform/components/spark-3.5/locals.tf      |  18 ++
 terraform/components/spark-3.5/offers.tf      |  14 +
 terraform/components/spark-3.5/outputs.tf     |  87 ++++++
 terraform/components/spark-3.5/resources.tf   |  26 ++
 terraform/components/spark-3.5/terraform.tf   |  10 +
 terraform/components/spark-3.5/variables.tf   | 114 ++++++++
 .../products/charmed-spark-3.4/README.md      |  49 +---
 .../products/charmed-spark-3.5/README.md      |  83 ++++++
 .../products/charmed-spark-3.5/locals.tf      |  35 +++
 terraform/products/charmed-spark-3.5/main.tf  | 203 +++++++++++++
 .../products/charmed-spark-3.5/outputs.tf     |  53 ++++
 .../charmed-spark-3.5}/providers.tf           |   2 +-
 .../products/charmed-spark-3.5/terraform.tf   |  13 +
 .../products/charmed-spark-3.5/variables.tf   | 272 ++++++++++++++++++
 20 files changed, 1231 insertions(+), 115 deletions(-)
 create mode 100644 terraform/components/spark-3.5/README.md
 create mode 100644 terraform/components/spark-3.5/applications.tf
 create mode 100644 terraform/components/spark-3.5/integrations.tf
 create mode 100644 terraform/components/spark-3.5/locals.tf
 create mode 100644 terraform/components/spark-3.5/offers.tf
 create mode 100644 terraform/components/spark-3.5/outputs.tf
 create mode 100644 terraform/components/spark-3.5/resources.tf
 create mode 100644 terraform/components/spark-3.5/terraform.tf
 create mode 100644 terraform/components/spark-3.5/variables.tf
 create mode 100644 terraform/products/charmed-spark-3.5/README.md
 create mode 100644 terraform/products/charmed-spark-3.5/locals.tf
 create mode 100644 terraform/products/charmed-spark-3.5/main.tf
 create mode 100644 terraform/products/charmed-spark-3.5/outputs.tf
 rename terraform/{components/spark-3.4 => products/charmed-spark-3.5}/providers.tf (72%)
 create mode 100644 terraform/products/charmed-spark-3.5/terraform.tf
 create mode 100644 terraform/products/charmed-spark-3.5/variables.tf

diff --git a/terraform/components/observability/variables.tf b/terraform/components/observability/variables.tf
index df388c04..d5cf5d5e 100644
--- a/terraform/components/observability/variables.tf
+++ b/terraform/components/observability/variables.tf
@@ -9,27 +9,21 @@ variable "model_uuid" {
 
 variable "cos_configuration" {
   type = object({
-    app_name    = optional(string, "cos-configuration")
-    base        = optional(string, "ubuntu@22.04")
-    channel     = optional(string, "1/stable")
-    config      = optional(map(string), {})
+    app_name = optional(string, "cos-configuration")
+    base     = optional(string, "ubuntu@22.04")
+    channel  = optional(string, "1/stable")
+    config = optional(map(string), {
+      git_branch              = "main"
+      git_depth               = 1
+      git_repo                = "https://github.com/canonical/spark-k8s-bundle"
+      grafana_dashboards_path = "resources/grafana/"
+    })
     constraints = optional(string, "arch=amd64")
     resources   = optional(map(string), {})
     revision    = optional(number)
     units       = optional(number, 1)
   })
-  default = {
-    app_name    = "cos-configuration"
-    base        = "ubuntu@22.04"
-    constraints = "arch=amd64"
-    config = {
-      git_branch              = "main"
-      git_depth               = 1
-      git_repo                = "https://github.com/canonical/spark-k8s-bundle"
-      grafana_dashboards_path = "resources/grafana/"
-    }
-    units = 1
-  }
+  default = {}
 }
 
 variable "grafana_agent" {
@@ -43,57 +37,40 @@ variable "grafana_agent" {
     revision    = optional(number)
     units       = optional(number, 1)
   })
-  default = {
-    app_name    = "grafana-agent"
-    base        = "ubuntu@22.04"
-    constraints = "arch=amd64"
-    units       = 1
-  }
+  default = {}
 }
 
 variable "pushgateway" {
   type = object({
-    app_name           = optional(string, "pushgateway")
-    base               = optional(string, "ubuntu@22.04")
-    channel            = optional(string, "1/stable")
-    config             = optional(map(string), {})
-    constraints        = optional(string, "arch=amd64")
-    resources          = optional(map(string), {})
-    revision           = optional(number)
-    storage_directives = optional(map(string))
-    units              = optional(number, 1)
-  })
-  default = {
-    app_name    = "pushgateway"
-    base        = "ubuntu@22.04"
-    constraints = "arch=amd64"
-    storage_directives = {
+    app_name    = optional(string, "pushgateway")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "1/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    storage_directives = optional(map(string), {
       pushgateway-store = "10G"
-    }
-    units = 1
-  }
+    })
+    units = optional(number, 1)
+  })
+  default = {}
 }
 
 variable "scrape_config" {
   type = object({
-    app_name    = optional(string, "scrape-config")
-    base        = optional(string, "ubuntu@22.04")
-    channel     = optional(string, "1/stable")
-    config      = optional(map(string), {})
+    app_name = optional(string, "scrape-config")
+    base     = optional(string, "ubuntu@22.04")
+    channel  = optional(string, "1/stable")
+    config = optional(map(string), {
+      scrape_interval = "10s"
+    })
     constraints = optional(string, "arch=amd64")
     resources   = optional(map(string), {})
     revision    = optional(number)
     units       = optional(number, 1)
   })
-  default = {
-    app_name = "scrape-config"
-    base     = "ubuntu@22.04"
-    config = {
-      scrape_interval = "10s"
-    }
-    constraints = "arch=amd64"
-    units       = 1
-  }
+  default = {}
 }
 
 variable "dashboards_offer" {
diff --git a/terraform/components/spark-3.4/resources.tf b/terraform/components/spark-3.4/resources.tf
index 6ae375a9..1dc65f8e 100644
--- a/terraform/components/spark-3.4/resources.tf
+++ b/terraform/components/spark-3.4/resources.tf
@@ -1,8 +1,6 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-# Define juju resources (model, secrets, storage)
-
 resource "juju_secret" "system_users_and_private_key_secret" {
   count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
   model_uuid = var.model_uuid
diff --git a/terraform/components/spark-3.4/variables.tf b/terraform/components/spark-3.4/variables.tf
index 8c42656b..86d969ab 100644
--- a/terraform/components/spark-3.4/variables.tf
+++ b/terraform/components/spark-3.4/variables.tf
@@ -18,12 +18,7 @@ variable "history_server" {
     revision    = optional(number)
     units       = optional(number, 1)
   })
-  default = {
-    app_name    = "history-server"
-    base        = "ubuntu@22.04"
-    constraints = "arch=amd64"
-    units       = 1
-  }
+  default = {}
 }
 
 variable "integration_hub" {
@@ -37,12 +32,7 @@ variable "integration_hub" {
     revision    = optional(number)
     units       = optional(number, 1)
   })
-  default = {
-    app_name    = "integration-hub"
-    base        = "ubuntu@22.04"
-    constraints = "arch=amd64"
-    units       = 1
-  }
+  default = {}
 }
 
 
@@ -57,12 +47,7 @@ variable "kyuubi" {
     revision    = optional(number)
     units       = optional(number, 1)
   })
-  default = {
-    app_name    = "kyuubi"
-    base        = "ubuntu@22.04"
-    constraints = "arch=amd64"
-    units       = 1
-  }
+  default = {}
 }
 
 variable "admin_password" {
@@ -76,7 +61,7 @@ variable "data_integrator_endpoint" {
   description = "In-model endpoint for the data-integrator application."
   type = object({
     name     = string
-    endpoint = optional(string, "kyuubi")
+    endpoint = string
   })
 }
 
@@ -84,7 +69,7 @@ variable "metastore_endpoint" {
   description = "In-model endpoint for the metastore (postgresql-k8s) application."
   type = object({
     name     = string
-    endpoint = optional(string, "database")
+    endpoint = string
   })
 }
 
@@ -92,7 +77,7 @@ variable "tls_endpoint" {
   description = "In-model endpoint for the TLS certificates provider."
   type = object({
     name     = string
-    endpoint = optional(string, "certificates")
+    endpoint = string
   })
 }
 
@@ -107,7 +92,7 @@ variable "users_db_endpoint" {
   description = "In-model endpoint for the Kyuubi users database (postgresql-k8s) application."
   type = object({
     name     = string
-    endpoint = optional(string, "database")
+    endpoint = string
   })
 }
 
@@ -115,7 +100,7 @@ variable "zookeeper_endpoint" {
   description = "In-model endpoint for the ZooKeeper application."
   type = object({
     name     = string
-    endpoint = optional(string, "zookeeper")
+    endpoint = string
   })
 }
 
@@ -123,7 +108,7 @@ variable "object_storage_endpoint" {
   description = "In-model endpoint for the object storage integrator application."
   type = object({
     name     = string
-    endpoint = optional(string, "s3_credentials")
+    endpoint = string
   })
 }
 
diff --git a/terraform/components/spark-3.5/README.md b/terraform/components/spark-3.5/README.md
new file mode 100644
index 00000000..3f42f49b
--- /dev/null
+++ b/terraform/components/spark-3.5/README.md
@@ -0,0 +1,62 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
+| [juju_application.history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_application.kyuubi](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
+| [juju_integration.history_server_object_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.integration_hub_object_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_metastore](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_tls](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_users](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_integration.kyuubi_zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
+| [juju_offer.integration_hub_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_offer.kyuubi_jdbc](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
+| [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
+| <a name="input_data_integrator_endpoint"></a> [data\_integrator\_endpoint](#input\_data\_integrator\_endpoint) | In-model endpoint for the data-integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "kyuubi")<br/>  })</pre> | n/a | yes |
+| <a name="input_history_server"></a> [history\_server](#input\_history\_server) | n/a | <pre>object({<br/>    app_name    = optional(string, "history-server")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "history-server",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_integration_hub"></a> [integration\_hub](#input\_integration\_hub) | n/a | <pre>object({<br/>    app_name    = optional(string, "integration-hub")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "integration-hub",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_kyuubi"></a> [kyuubi](#input\_kyuubi) | n/a | <pre>object({<br/>    app_name    = optional(string, "kyuubi")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3.4/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "kyuubi",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_metastore_endpoint"></a> [metastore\_endpoint](#input\_metastore\_endpoint) | In-model endpoint for the metastore (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "database")<br/>  })</pre> | n/a | yes |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
+| <a name="input_object_storage_endpoint"></a> [object\_storage\_endpoint](#input\_object\_storage\_endpoint) | In-model endpoint for the object storage integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "s3_credentials")<br/>  })</pre> | n/a | yes |
+| <a name="input_tls_endpoint"></a> [tls\_endpoint](#input\_tls\_endpoint) | In-model endpoint for the TLS certificates provider. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "certificates")<br/>  })</pre> | n/a | yes |
+| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
+| <a name="input_users_db_endpoint"></a> [users\_db\_endpoint](#input\_users\_db\_endpoint) | In-model endpoint for the Kyuubi users database (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "database")<br/>  })</pre> | n/a | yes |
+| <a name="input_zookeeper_endpoint"></a> [zookeeper\_endpoint](#input\_zookeeper\_endpoint) | In-model endpoint for the ZooKeeper application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "zookeeper")<br/>  })</pre> | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_components"></a> [components](#output\_components) | Map of the deployed applications for this component module. |
+| <a name="output_offers"></a> [offers](#output\_offers) | Map of all offers exposed by the component's charms. |
+| <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provides endpoints. |
+| <a name="output_requires"></a> [requires](#output\_requires) | Map of the requires endpoints. |
diff --git a/terraform/components/spark-3.5/applications.tf b/terraform/components/spark-3.5/applications.tf
new file mode 100644
index 00000000..a1409b03
--- /dev/null
+++ b/terraform/components/spark-3.5/applications.tf
@@ -0,0 +1,73 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_application" "history_server" {
+  name       = var.history_server.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "spark-history-server-k8s"
+    channel  = var.history_server.channel
+    revision = var.history_server.revision != null ? var.history_server.revision : local.revisions.history_server
+  }
+
+  config      = var.history_server.config
+  constraints = var.history_server.constraints
+  resources = merge({
+    spark-history-server-image = local.images.history_server
+    },
+    var.history_server.resources == null ? {} : var.history_server.resources
+  )
+  units = var.history_server.units
+}
+
+resource "juju_application" "integration_hub" {
+  name       = var.integration_hub.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "spark-integration-hub-k8s"
+    channel  = var.integration_hub.channel
+    revision = var.integration_hub.revision != null ? var.integration_hub.revision : local.revisions.integration_hub
+  }
+
+  config      = var.integration_hub.config
+  constraints = var.integration_hub.constraints
+  resources = merge({
+    integration-hub-image = local.images.integration_hub
+    },
+    var.integration_hub.resources == null ? {} : var.integration_hub.resources
+  )
+  trust = true
+  units = var.integration_hub.units
+}
+
+resource "juju_application" "kyuubi" {
+  name       = var.kyuubi.app_name
+  model_uuid = var.model_uuid
+
+  charm {
+    name     = "kyuubi-k8s"
+    channel  = var.kyuubi.channel
+    revision = var.kyuubi.revision != null ? var.kyuubi.revision : local.revisions.kyuubi
+  }
+
+  config = merge(
+    {
+      expose-external = "loadbalancer",
+    },
+    length(juju_secret.system_users_and_private_key_secret) > 0 ? {
+      system-users           = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}",
+      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+    } : {},
+    var.kyuubi.config
+  )
+  constraints = var.kyuubi.constraints
+  resources = merge({
+    kyuubi-image = local.images.kyuubi
+    },
+    var.kyuubi.resources == null ? {} : var.kyuubi.resources
+  )
+  trust = true
+  units = var.kyuubi.units
+}
diff --git a/terraform/components/spark-3.5/integrations.tf b/terraform/components/spark-3.5/integrations.tf
new file mode 100644
index 00000000..4bc8fc38
--- /dev/null
+++ b/terraform/components/spark-3.5/integrations.tf
@@ -0,0 +1,114 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_integration" "kyuubi_metastore" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = var.metastore_endpoint.name
+    endpoint = var.metastore_endpoint.endpoint
+  }
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "metastore-db"
+  }
+}
+
+resource "juju_integration" "kyuubi_users" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "auth-db"
+  }
+
+  application {
+    name     = var.users_db_endpoint.name
+    endpoint = var.users_db_endpoint.endpoint
+  }
+}
+
+resource "juju_integration" "kyuubi_service_account" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "spark-service-account"
+  }
+
+  application {
+    name     = juju_application.integration_hub.name
+    endpoint = "spark-service-account"
+  }
+}
+
+resource "juju_integration" "kyuubi_zookeeper" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "zookeeper"
+  }
+
+  application {
+    name     = var.zookeeper_endpoint.name
+    endpoint = var.zookeeper_endpoint.endpoint
+  }
+}
+
+resource "juju_integration" "kyuubi_tls" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "certificates"
+  }
+
+  application {
+    name     = var.tls_endpoint.name
+    endpoint = var.tls_endpoint.endpoint
+  }
+}
+
+resource "juju_integration" "kyuubi_data_integrator" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.kyuubi.name
+    endpoint = "jdbc"
+  }
+
+  application {
+    name     = var.data_integrator_endpoint.name
+    endpoint = var.data_integrator_endpoint.endpoint
+  }
+}
+
+resource "juju_integration" "integration_hub_object_storage" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.integration_hub.name
+    endpoint = var.object_storage_endpoint.endpoint
+  }
+
+  application {
+    name     = var.object_storage_endpoint.name
+    endpoint = var.object_storage_endpoint.endpoint
+  }
+}
+
+resource "juju_integration" "history_server_object_storage" {
+  model_uuid = var.model_uuid
+
+  application {
+    name     = juju_application.history_server.name
+    endpoint = var.object_storage_endpoint.endpoint
+  }
+
+  application {
+    name     = var.object_storage_endpoint.name
+    endpoint = var.object_storage_endpoint.endpoint
+  }
+}
diff --git a/terraform/components/spark-3.5/locals.tf b/terraform/components/spark-3.5/locals.tf
new file mode 100644
index 00000000..2ed303b9
--- /dev/null
+++ b/terraform/components/spark-3.5/locals.tf
@@ -0,0 +1,18 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+locals {
+  revisions = {
+    history_server  = 96
+    integration_hub = 119
+    kyuubi          = 149
+  }
+  images = {
+    history_server = "ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70"
+    # rev24,  spark-version: 3.5.7, release date 16/03/2026 
+    integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134"
+    # rev13, release date 19/03/2026
+    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:60e81dd2e9be50d4a857ce987935341c8518c784c7de36e814fc41dd3451d58f"
+    # rev19, spark-version: 3.4.4, kyuubi-version: 1.10.3, release date 17/03/2026
+  }
+}
diff --git a/terraform/components/spark-3.5/offers.tf b/terraform/components/spark-3.5/offers.tf
new file mode 100644
index 00000000..21bd6a62
--- /dev/null
+++ b/terraform/components/spark-3.5/offers.tf
@@ -0,0 +1,14 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_offer" "integration_hub_service_account" {
+  model_uuid       = var.model_uuid
+  application_name = juju_application.integration_hub.name
+  endpoints        = ["spark-service-account"]
+}
+
+resource "juju_offer" "kyuubi_jdbc" {
+  model_uuid       = var.model_uuid
+  application_name = juju_application.kyuubi.name
+  endpoints        = ["jdbc"]
+}
diff --git a/terraform/components/spark-3.5/outputs.tf b/terraform/components/spark-3.5/outputs.tf
new file mode 100644
index 00000000..9ec9b073
--- /dev/null
+++ b/terraform/components/spark-3.5/outputs.tf
@@ -0,0 +1,87 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+output "components" {
+  description = "Map of the deployed applications for this component module."
+  value = {
+    history_server  = juju_application.history_server,
+    integration_hub = juju_application.integration_hub,
+    kyuubi          = juju_application.kyuubi,
+  }
+}
+
+output "requires" {
+  description = "Map of the requires endpoints."
+  value = {
+    history_server_logging = {
+      name     = juju_application.history_server.name
+      endpoint = "logging"
+    }
+    integration_hub_cos = {
+      name     = juju_application.integration_hub.name
+      endpoint = "cos"
+    }
+    integration_hub_logging = {
+      name     = juju_application.integration_hub.name
+      endpoint = "logging"
+    }
+    kyuubi_auth_db = {
+      name     = juju_application.kyuubi.name
+      endpoint = "auth-db"
+    }
+    kyuubi_certificates = {
+      name     = juju_application.kyuubi.name
+      endpoint = "certificates"
+    }
+    kyuubi_logging = {
+      name     = juju_application.kyuubi.name
+      endpoint = "logging"
+    }
+    kyuubi_metastore_db = {
+      name     = juju_application.kyuubi.name
+      endpoint = "metastore-db"
+    }
+    kyuubi_zookeeper = {
+      name     = juju_application.kyuubi.name
+      endpoint = "zookeeper"
+    }
+  }
+}
+
+output "provides" {
+  description = "Map of all the provides endpoints."
+  value = {
+    history_server_dashboard = {
+      name     = juju_application.history_server.name
+      endpoint = "grafana-dashboard"
+    }
+    history_server_metrics = {
+      name     = juju_application.history_server.name
+      endpoint = "metrics-endpoint"
+    }
+    integration_hub_service_account = {
+      name     = juju_application.integration_hub.name
+      endpoint = "spark-service-account"
+    }
+    kyuubi_jdbc = {
+      name     = juju_application.kyuubi.name
+      endpoint = "jdbc"
+    }
+    kyuubi_metrics = {
+      name     = juju_application.kyuubi.name
+      endpoint = "metrics-endpoint"
+    }
+    kyuubi_dashboard = {
+      name     = juju_application.kyuubi.name
+      endpoint = "grafana-dashboard"
+    }
+  }
+}
+
+output "offers" {
+  description = "Map of all offers exposed by the component's charms."
+  value = {
+    integration_hub_service_account = juju_offer.integration_hub_service_account
+    kyuubi_jdbc                     = juju_offer.kyuubi_jdbc
+  }
+}
diff --git a/terraform/components/spark-3.5/resources.tf b/terraform/components/spark-3.5/resources.tf
new file mode 100644
index 00000000..1dc65f8e
--- /dev/null
+++ b/terraform/components/spark-3.5/resources.tf
@@ -0,0 +1,26 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_secret" "system_users_and_private_key_secret" {
+  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = var.model_uuid
+  name       = "system_users_and_private_key_secret"
+  value = merge(
+    var.admin_password == null ? {} : {
+      admin = var.admin_password
+    },
+    var.tls_private_key == null ? {} : {
+      private-key = var.tls_private_key
+    }
+  )
+  info = "This secret contains password for admin user and the TLS private key."
+}
+
+resource "juju_access_secret" "system_users_and_private_key_secret_access" {
+  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = var.model_uuid
+  applications = [
+    juju_application.kyuubi.name
+  ]
+  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
+}
diff --git a/terraform/components/spark-3.5/terraform.tf b/terraform/components/spark-3.5/terraform.tf
new file mode 100644
index 00000000..6412125a
--- /dev/null
+++ b/terraform/components/spark-3.5/terraform.tf
@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/terraform/components/spark-3.5/variables.tf b/terraform/components/spark-3.5/variables.tf
new file mode 100644
index 00000000..86d969ab
--- /dev/null
+++ b/terraform/components/spark-3.5/variables.tf
@@ -0,0 +1,114 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "model_uuid" {
+  description = "Reference to an existing model uuid."
+  type        = string
+  nullable    = false
+}
+
+variable "history_server" {
+  type = object({
+    app_name    = optional(string, "history-server")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "3/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {}
+}
+
+variable "integration_hub" {
+  type = object({
+    app_name    = optional(string, "integration-hub")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "3/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {}
+}
+
+
+variable "kyuubi" {
+  type = object({
+    app_name    = optional(string, "kyuubi")
+    base        = optional(string, "ubuntu@22.04")
+    channel     = optional(string, "3.4/stable")
+    config      = optional(map(string), {})
+    constraints = optional(string, "arch=amd64")
+    resources   = optional(map(string), {})
+    revision    = optional(number)
+    units       = optional(number, 1)
+  })
+  default = {}
+}
+
+variable "admin_password" {
+  description = "The password for the admin user."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "data_integrator_endpoint" {
+  description = "In-model endpoint for the data-integrator application."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "metastore_endpoint" {
+  description = "In-model endpoint for the metastore (postgresql-k8s) application."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "tls_endpoint" {
+  description = "In-model endpoint for the TLS certificates provider."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "tls_private_key" {
+  description = "The private key to be used for TLS certificates."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "users_db_endpoint" {
+  description = "In-model endpoint for the Kyuubi users database (postgresql-k8s) application."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "zookeeper_endpoint" {
+  description = "In-model endpoint for the ZooKeeper application."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
+variable "object_storage_endpoint" {
+  description = "In-model endpoint for the object storage integrator application."
+  type = object({
+    name     = string
+    endpoint = string
+  })
+}
+
diff --git a/terraform/products/charmed-spark-3.4/README.md b/terraform/products/charmed-spark-3.4/README.md
index 4eda79c2..5143a3a3 100644
--- a/terraform/products/charmed-spark-3.4/README.md
+++ b/terraform/products/charmed-spark-3.4/README.md
@@ -19,6 +19,7 @@
 | <a name="module_data_integrator"></a> [data\_integrator](#module\_data\_integrator) | ../../charms/data-integrator | n/a |
 | <a name="module_kyuubi_users"></a> [kyuubi\_users](#module\_kyuubi\_users) | git::https://github.com/canonical/postgresql-k8s-operator//terraform | rev774 |
 | <a name="module_metastore"></a> [metastore](#module\_metastore) | git::https://github.com/canonical/postgresql-k8s-operator//terraform | rev774 |
+| <a name="module_observability"></a> [observability](#module\_observability) | ../../components/observability | n/a |
 | <a name="module_s3"></a> [s3](#module\_s3) | ../../charms/s3-integrator-v1 | n/a |
 | <a name="module_spark"></a> [spark](#module\_spark) | ../../components/spark-3.4 | n/a |
 | <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev586 |
@@ -36,60 +37,38 @@
 |------|-------------|------|---------|:--------:|
 | <a name="input_K8S_CLOUD"></a> [K8S\_CLOUD](#input\_K8S\_CLOUD) | The kubernetes juju cloud name. | `string` | `"microk8s"` | no |
 | <a name="input_K8S_CREDENTIAL"></a> [K8S\_CREDENTIAL](#input\_K8S\_CREDENTIAL) | The name of the kubernetes juju credential. | `string` | `"microk8s"` | no |
-| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
-| <a name="input_alertmanager_size"></a> [alertmanager\_size](#input\_alertmanager\_size) | Storage size for the alertmanager database | `string` | `"10G"` | no |
-| <a name="input_azure_storage"></a> [azure\_storage](#input\_azure\_storage) | Azure Object storage information | <pre>object({<br/>    container       = optional(string, "azurecontainer")<br/>    storage_account = optional(string, "azurestorageaccount")<br/>    secret_key      = optional(string, "azurestoragesecret")<br/>    protocol        = optional(string, "abfss")<br/>  })</pre> | `{}` | no |
+| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the Kyuubi admin user. | `string` | `null` | no |
+| <a name="input_azure_storage_config"></a> [azure\_storage\_config](#input\_azure\_storage\_config) | Azure Object storage information | <pre>object({<br/>    container       = optional(string)<br/>    path            = optional(string)<br/>    protocol        = optional(string, "abfss")<br/>    secret_key      = optional(string)<br/>    storage_account = optional(string)<br/>  })</pre> | `{}` | no |
 | <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | `null` | no |
 | <a name="input_certificate_common_name"></a> [certificate\_common\_name](#input\_certificate\_common\_name) | Common name for the certificate to be used in self-signed | `string` | `"charmed-spark"` | no |
-| <a name="input_cos"></a> [cos](#input\_cos) | Observability settings | <pre>object({<br/>    model    = optional(string, "cos")<br/>    deployed = optional(string, "bundled")<br/>    offers = optional(object({<br/>      dashboard = optional(string, null),<br/>      metrics   = optional(string, null),<br/>      logging   = optional(string, null)<br/>    }), {}),<br/>    tls = optional(object({<br/>      cert = optional(string, "")<br/>      key  = optional(string, "")<br/>      ca   = optional(string, "")<br/>    }), {})<br/>  })</pre> | <pre>{<br/>  "deployed": "bundled",<br/>  "model": "cos",<br/>  "offers": {},<br/>  "tls": {}<br/>}</pre> | no |
-| <a name="input_cos_configuration_revision"></a> [cos\_configuration\_revision](#input\_cos\_configuration\_revision) | Charm revision for cos-configuration-k8s | `number` | `null` | no |
+| <a name="input_cos_offers"></a> [cos\_offers](#input\_cos\_offers) | Observability stack offers. | <pre>object({<br/>    dashboard = string<br/>    logging   = string<br/>    metrics   = string<br/>  })</pre> | `null` | no |
 | <a name="input_create_model"></a> [create\_model](#input\_create\_model) | Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism. | `bool` | `true` | no |
 | <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | `null` | no |
-| <a name="input_driver_pod_template"></a> [driver\_pod\_template](#input\_driver\_pod\_template) | Define K8s driver pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
-| <a name="input_enable_dynamic_allocation"></a> [enable\_dynamic\_allocation](#input\_enable\_dynamic\_allocation) | Enable dynamic allocation of pods for Spark jobs. | `bool` | `false` | no |
-| <a name="input_executor_pod_template"></a> [executor\_pod\_template](#input\_executor\_pod\_template) | Define K8s executor pod from a file accessible to the `spark-submit` process. | `string` | `null` | no |
 | <a name="input_grafana_agent_revision"></a> [grafana\_agent\_revision](#input\_grafana\_agent\_revision) | Charm revision for grafana-agent-k8s | `number` | `null` | no |
-| <a name="input_grafana_size"></a> [grafana\_size](#input\_grafana\_size) | Storage size for the grafana database | `string` | `"10G"` | no |
-| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `map(string)` | `null` | no |
+| <a name="input_history_server_config"></a> [history\_server\_config](#input\_history\_server\_config) | History Server configuration options | `map(any)` | `{}` | no |
+| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `string` | `null` | no |
 | <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | `null` | no |
-| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `map(string)` | `null` | no |
-| <a name="input_integration_hub_monitored_service_accounts"></a> [integration\_hub\_monitored\_service\_accounts](#input\_integration\_hub\_monitored\_service\_accounts) | Comma-separated patterns for namespaces and service accounts to monitor and update | `string` | `null` | no |
+| <a name="input_integration_hub_config"></a> [integration\_hub\_config](#input\_integration\_hub\_config) | Integration Hub configuration options. | `map(any)` | `{}` | no |
+| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `string` | `null` | no |
 | <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | `null` | no |
 | <a name="input_juju_controller"></a> [juju\_controller](#input\_juju\_controller) | Controller information: endpoint, username, password and CA certificate. | <pre>object({<br/>    endpoint = string<br/>    username = string<br/>    password = string<br/>    ca       = string<br/>  })</pre> | `null` | no |
-| <a name="input_kyuubi_driver_pod_template"></a> [kyuubi\_driver\_pod\_template](#input\_kyuubi\_driver\_pod\_template) | Define K8s driver pod from a file accessible in the object storage. | `string` | `null` | no |
-| <a name="input_kyuubi_executor_cores"></a> [kyuubi\_executor\_cores](#input\_kyuubi\_executor\_cores) | Set kyuubi executor pods cpu cores. | `number` | `null` | no |
-| <a name="input_kyuubi_executor_memory"></a> [kyuubi\_executor\_memory](#input\_kyuubi\_executor\_memory) | Set kyuubi executor pods memory (in GB). | `number` | `null` | no |
-| <a name="input_kyuubi_executor_pod_template"></a> [kyuubi\_executor\_pod\_template](#input\_kyuubi\_executor\_pod\_template) | Define K8s executor pod from a file accessible in the object storage. | `string` | `null` | no |
-| <a name="input_kyuubi_gpu_enable"></a> [kyuubi\_gpu\_enable](#input\_kyuubi\_gpu\_enable) | Enable GPU acceleration for SparkSQLEngine. | `bool` | `false` | no |
-| <a name="input_kyuubi_gpu_engine_executors_limit"></a> [kyuubi\_gpu\_engine\_executors\_limit](#input\_kyuubi\_gpu\_engine\_executors\_limit) | Limit the number of GPUs an engine can schedule executor pods on. | `number` | `1` | no |
-| <a name="input_kyuubi_gpu_pinned_memory"></a> [kyuubi\_gpu\_pinned\_memory](#input\_kyuubi\_gpu\_pinned\_memory) | Set the host memory (in GB) per executor reserved for fast data transfer on GPUs. | `number` | `1` | no |
-| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `map(string)` | `null` | no |
-| <a name="input_kyuubi_k8s_node_selectors"></a> [kyuubi\_k8s\_node\_selectors](#input\_kyuubi\_k8s\_node\_selectors) | Comma separated label:value selectors for K8s pods in Kyuubi. | `string` | `null` | no |
-| <a name="input_kyuubi_loadbalancer_extra_annotations"></a> [kyuubi\_loadbalancer\_extra\_annotations](#input\_kyuubi\_loadbalancer\_extra\_annotations) | Optional extra annotations to be supplied to the load balancer service in Kyuubi. | `string` | `null` | no |
-| <a name="input_kyuubi_profile"></a> [kyuubi\_profile](#input\_kyuubi\_profile) | The profile to be used for Kyuubi; should be one of 'testing', 'staging' and 'production'. | `string` | `"production"` | no |
+| <a name="input_kyuubi_config"></a> [kyuubi\_config](#input\_kyuubi\_config) | Kyuubi configuration options. | `map(any)` | `{}` | no |
+| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `string` | `null` | no |
 | <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | `null` | no |
 | <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
-| <a name="input_kyuubi_user"></a> [kyuubi\_user](#input\_kyuubi\_user) | Define the user to be used for running Kyuubi enginers | `string` | `"kyuubi-spark-engine"` | no |
-| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `map(string)` | `null` | no |
+| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `string` | `null` | no |
 | <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | `null` | no |
 | <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
-| <a name="input_logging_config"></a> [logging\_config](#input\_logging\_config) | Logging configuration to be used. | `string` | `""` | no |
-| <a name="input_loki_active_index_directory_size"></a> [loki\_active\_index\_directory\_size](#input\_loki\_active\_index\_directory\_size) | Storage size for the active index directory for Loki | `string` | `"10G"` | no |
-| <a name="input_loki_chunks_size"></a> [loki\_chunks\_size](#input\_loki\_chunks\_size) | Storage size for the Loki chucks storage | `string` | `"500G"` | no |
-| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `map(string)` | `null` | no |
+| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `string` | `null` | no |
 | <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | `null` | no |
 | <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules. | `string` | `null` | no |
-| <a name="input_prometheus_size"></a> [prometheus\_size](#input\_prometheus\_size) | Storage size for the Prometheus database | `string` | `"500G"` | no |
 | <a name="input_proxy"></a> [proxy](#input\_proxy) | Proxy information for the deployment. | <pre>object({<br/>    http     = optional(string, "")<br/>    https    = optional(string, "")<br/>    no_proxy = optional(string, "")<br/>  })</pre> | `{}` | no |
-| <a name="input_pushgateway_revision"></a> [pushgateway\_revision](#input\_pushgateway\_revision) | Charm revision for prometheus-pushgateway-k8s | `number` | `null` | no |
-| <a name="input_s3"></a> [s3](#input\_s3) | S3 Bucket information | <pre>object({<br/>    bucket   = optional(string, "spark-test")<br/>    endpoint = optional(string, "https://s3.amazonaws.com")<br/>    region   = optional(string, "us-east-1")<br/>  })</pre> | `{}` | no |
+| <a name="input_s3_config"></a> [s3\_config](#input\_s3\_config) | S3 integrator configuration | <pre>object({<br/>    attributes                          = optional(string)<br/>    bucket                              = optional(string)<br/>    endpoint                            = optional(string)<br/>    experimental-delete-older-than-days = optional(number)<br/>    path                                = optional(string)<br/>    region                              = optional(string)<br/>    s3-api-version                      = optional(string)<br/>    s3-uri-style                        = optional(string)<br/>    storage-class                       = optional(string)<br/>    tls-ca-chain                        = optional(string)<br/>  })</pre> | `{}` | no |
 | <a name="input_s3_revision"></a> [s3\_revision](#input\_s3\_revision) | Charm revision for s3-integrator | `number` | `null` | no |
-| <a name="input_scrape_config_revision"></a> [scrape\_config\_revision](#input\_scrape\_config\_revision) | Charm revision for prometheus-scrape-config-k8s | `number` | `null` | no |
-| <a name="input_spark_model_name"></a> [spark\_model\_name](#input\_spark\_model\_name) | The name of the juju model to deploy Spark to | `string` | `"spark-test"` | no |
+| <a name="input_spark_model_name"></a> [spark\_model\_name](#input\_spark\_model\_name) | The name of the juju model to deploy Spark to | `string` | `"spark"` | no |
 | <a name="input_storage_backend"></a> [storage\_backend](#input\_storage\_backend) | Storage backend to be used | `string` | `"s3"` | no |
 | <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The file path of the private key to use for TLS certificates. | `string` | `null` | no |
-| <a name="input_traefik_size"></a> [traefik\_size](#input\_traefik\_size) | Storage size for the Traefik storage | `string` | `"10G"` | no |
 | <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | `null` | no |
 | <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | `null` | no |
 | <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
diff --git a/terraform/products/charmed-spark-3.5/README.md b/terraform/products/charmed-spark-3.5/README.md
new file mode 100644
index 00000000..5143a3a3
--- /dev/null
+++ b/terraform/products/charmed-spark-3.5/README.md
@@ -0,0 +1,83 @@
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
+| <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_azure_storage"></a> [azure\_storage](#module\_azure\_storage) | ../../charms/azure-storage-integrator | n/a |
+| <a name="module_data_integrator"></a> [data\_integrator](#module\_data\_integrator) | ../../charms/data-integrator | n/a |
+| <a name="module_kyuubi_users"></a> [kyuubi\_users](#module\_kyuubi\_users) | git::https://github.com/canonical/postgresql-k8s-operator//terraform | rev774 |
+| <a name="module_metastore"></a> [metastore](#module\_metastore) | git::https://github.com/canonical/postgresql-k8s-operator//terraform | rev774 |
+| <a name="module_observability"></a> [observability](#module\_observability) | ../../components/observability | n/a |
+| <a name="module_s3"></a> [s3](#module\_s3) | ../../charms/s3-integrator-v1 | n/a |
+| <a name="module_spark"></a> [spark](#module\_spark) | ../../components/spark-3.4 | n/a |
+| <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev586 |
+| <a name="module_zookeeper"></a> [zookeeper](#module\_zookeeper) | ../../charms/zookeeper | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_K8S_CLOUD"></a> [K8S\_CLOUD](#input\_K8S\_CLOUD) | The kubernetes juju cloud name. | `string` | `"microk8s"` | no |
+| <a name="input_K8S_CREDENTIAL"></a> [K8S\_CREDENTIAL](#input\_K8S\_CREDENTIAL) | The name of the kubernetes juju credential. | `string` | `"microk8s"` | no |
+| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the Kyuubi admin user. | `string` | `null` | no |
+| <a name="input_azure_storage_config"></a> [azure\_storage\_config](#input\_azure\_storage\_config) | Azure Object storage information | <pre>object({<br/>    container       = optional(string)<br/>    path            = optional(string)<br/>    protocol        = optional(string, "abfss")<br/>    secret_key      = optional(string)<br/>    storage_account = optional(string)<br/>  })</pre> | `{}` | no |
+| <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | `null` | no |
+| <a name="input_certificate_common_name"></a> [certificate\_common\_name](#input\_certificate\_common\_name) | Common name for the certificate to be used in self-signed | `string` | `"charmed-spark"` | no |
+| <a name="input_cos_offers"></a> [cos\_offers](#input\_cos\_offers) | Observability stack offers. | <pre>object({<br/>    dashboard = string<br/>    logging   = string<br/>    metrics   = string<br/>  })</pre> | `null` | no |
+| <a name="input_create_model"></a> [create\_model](#input\_create\_model) | Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism. | `bool` | `true` | no |
+| <a name="input_data_integrator_revision"></a> [data\_integrator\_revision](#input\_data\_integrator\_revision) | Charm revision for data-integrator | `number` | `null` | no |
+| <a name="input_grafana_agent_revision"></a> [grafana\_agent\_revision](#input\_grafana\_agent\_revision) | Charm revision for grafana-agent-k8s | `number` | `null` | no |
+| <a name="input_history_server_config"></a> [history\_server\_config](#input\_history\_server\_config) | History Server configuration options | `map(any)` | `{}` | no |
+| <a name="input_history_server_image"></a> [history\_server\_image](#input\_history\_server\_image) | Image for spark-history-server-k8s | `string` | `null` | no |
+| <a name="input_history_server_revision"></a> [history\_server\_revision](#input\_history\_server\_revision) | Charm revision for spark-history-server-k8s | `number` | `null` | no |
+| <a name="input_integration_hub_config"></a> [integration\_hub\_config](#input\_integration\_hub\_config) | Integration Hub configuration options. | `map(any)` | `{}` | no |
+| <a name="input_integration_hub_image"></a> [integration\_hub\_image](#input\_integration\_hub\_image) | Image for spark-integration-hub-k8s | `string` | `null` | no |
+| <a name="input_integration_hub_revision"></a> [integration\_hub\_revision](#input\_integration\_hub\_revision) | Charm revision for spark-integration-hub-k8s | `number` | `null` | no |
+| <a name="input_juju_controller"></a> [juju\_controller](#input\_juju\_controller) | Controller information: endpoint, username, password and CA certificate. | <pre>object({<br/>    endpoint = string<br/>    username = string<br/>    password = string<br/>    ca       = string<br/>  })</pre> | `null` | no |
+| <a name="input_kyuubi_config"></a> [kyuubi\_config](#input\_kyuubi\_config) | Kyuubi configuration options. | `map(any)` | `{}` | no |
+| <a name="input_kyuubi_image"></a> [kyuubi\_image](#input\_kyuubi\_image) | Image for kyuubi-k8s | `string` | `null` | no |
+| <a name="input_kyuubi_revision"></a> [kyuubi\_revision](#input\_kyuubi\_revision) | Charm revision for kyuubi-k8s | `number` | `null` | no |
+| <a name="input_kyuubi_units"></a> [kyuubi\_units](#input\_kyuubi\_units) | Number of Kyuubi units. 3 units are recommended for high availability. | `number` | `3` | no |
+| <a name="input_kyuubi_users_image"></a> [kyuubi\_users\_image](#input\_kyuubi\_users\_image) | Image for postgresql-k8s (auth-db) | `string` | `null` | no |
+| <a name="input_kyuubi_users_revision"></a> [kyuubi\_users\_revision](#input\_kyuubi\_users\_revision) | Charm revision for postgresql-k8s (auth-db) | `number` | `null` | no |
+| <a name="input_kyuubi_users_size"></a> [kyuubi\_users\_size](#input\_kyuubi\_users\_size) | Storage size for the Kyuubi users database | `string` | `"1G"` | no |
+| <a name="input_metastore_image"></a> [metastore\_image](#input\_metastore\_image) | Image for postgresql-k8s (metastore) | `string` | `null` | no |
+| <a name="input_metastore_revision"></a> [metastore\_revision](#input\_metastore\_revision) | Charm revision for postgresql-k8s (metastore) | `number` | `null` | no |
+| <a name="input_metastore_size"></a> [metastore\_size](#input\_metastore\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
+| <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules. | `string` | `null` | no |
+| <a name="input_proxy"></a> [proxy](#input\_proxy) | Proxy information for the deployment. | <pre>object({<br/>    http     = optional(string, "")<br/>    https    = optional(string, "")<br/>    no_proxy = optional(string, "")<br/>  })</pre> | `{}` | no |
+| <a name="input_s3_config"></a> [s3\_config](#input\_s3\_config) | S3 integrator configuration | <pre>object({<br/>    attributes                          = optional(string)<br/>    bucket                              = optional(string)<br/>    endpoint                            = optional(string)<br/>    experimental-delete-older-than-days = optional(number)<br/>    path                                = optional(string)<br/>    region                              = optional(string)<br/>    s3-api-version                      = optional(string)<br/>    s3-uri-style                        = optional(string)<br/>    storage-class                       = optional(string)<br/>    tls-ca-chain                        = optional(string)<br/>  })</pre> | `{}` | no |
+| <a name="input_s3_revision"></a> [s3\_revision](#input\_s3\_revision) | Charm revision for s3-integrator | `number` | `null` | no |
+| <a name="input_spark_model_name"></a> [spark\_model\_name](#input\_spark\_model\_name) | The name of the juju model to deploy Spark to | `string` | `"spark"` | no |
+| <a name="input_storage_backend"></a> [storage\_backend](#input\_storage\_backend) | Storage backend to be used | `string` | `"s3"` | no |
+| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The file path of the private key to use for TLS certificates. | `string` | `null` | no |
+| <a name="input_zookeeper_image"></a> [zookeeper\_image](#input\_zookeeper\_image) | Image for zookeeper-k8s | `map(string)` | `null` | no |
+| <a name="input_zookeeper_revision"></a> [zookeeper\_revision](#input\_zookeeper\_revision) | Charm revision for zookeeper-k8s | `number` | `null` | no |
+| <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
+| <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Define the number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_metadata"></a> [metadata](#output\_metadata) | Metadata of the product deployment. |
+| <a name="output_models"></a> [models](#output\_models) | Maps of the models and the components deployed in each model. |
+| <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
diff --git a/terraform/products/charmed-spark-3.5/locals.tf b/terraform/products/charmed-spark-3.5/locals.tf
new file mode 100644
index 00000000..fa880218
--- /dev/null
+++ b/terraform/products/charmed-spark-3.5/locals.tf
@@ -0,0 +1,35 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+locals {
+  revisions = {
+    history_server  = 96
+    integration_hub = 119
+    kyuubi          = 149
+    kyuubi_users    = 774
+    metastore       = 774
+    zookeeper       = 78
+    data_integrator = 362
+    s3              = 330
+    ssc             = 586
+    azure_storage   = 270
+    grafana_agent   = 164
+    # TODO: bump the revision to 1/stable when both of the following issue gets fixed:
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/128 
+    # https://github.com/canonical/cos-configuration-k8s-operator/issues/84
+    cos_configuration = 65
+    pushgateway       = 27
+    scrape_config     = 67
+  }
+  images = {
+    history_server = "ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70"
+    # rev24,  spark-version: 3.5.7, release date 16/03/2026 
+    integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134"
+    # rev13, release date 19/03/2026
+    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:60e81dd2e9be50d4a857ce987935341c8518c784c7de36e814fc41dd3451d58f"
+    # rev19, spark-version: 3.4.4, kyuubi-version: 1.10.3, release date 17/03/2026
+    kyuubi_users = "184"
+    metastore    = "184"
+    zookeeper    = "34"
+  }
+}
diff --git a/terraform/products/charmed-spark-3.5/main.tf b/terraform/products/charmed-spark-3.5/main.tf
new file mode 100644
index 00000000..04ff7a04
--- /dev/null
+++ b/terraform/products/charmed-spark-3.5/main.tf
@@ -0,0 +1,203 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+resource "juju_model" "spark" {
+  count = (var.model_uuid == null && var.create_model == true) ? 1 : 0
+
+  name       = var.spark_model_name
+  credential = var.K8S_CREDENTIAL
+  cloud {
+    name = var.K8S_CLOUD
+  }
+}
+
+module "ssc" {
+  depends_on = [juju_model.spark]
+  source     = "git::https://github.com/canonical/self-signed-certificates-operator//terraform?ref=rev586"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  base    = "ubuntu@24.04"
+  channel = "1/stable"
+  config = {
+    ca-common-name = var.certificate_common_name
+  }
+  constraints = "arch=amd64"
+  revision    = local.revisions.ssc
+  units       = 1
+
+}
+
+module "kyuubi_users" {
+  depends_on = [juju_model.spark]
+  source     = "git::https://github.com/canonical/postgresql-k8s-operator//terraform?ref=rev774"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  app_name    = "kyuubi-users"
+  base        = "ubuntu@22.04"
+  channel     = "14/stable"
+  constraints = "arch=amd64"
+  revision    = local.revisions.kyuubi_users
+  resources = {
+    postgresql-image = local.images.kyuubi_users
+  }
+  storage_directives = {
+    pgdata = var.kyuubi_users_size
+  }
+  units = 1
+}
+
+module "metastore" {
+  depends_on = [juju_model.spark]
+  source     = "git::https://github.com/canonical/postgresql-k8s-operator//terraform?ref=rev774"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  app_name    = "metastore"
+  base        = "ubuntu@22.04"
+  channel     = "14/stable"
+  constraints = "arch=amd64"
+  revision    = local.revisions.metastore
+  resources = {
+    postgresql-image = local.images.metastore
+  }
+  storage_directives = {
+    pgdata = var.metastore_size
+  }
+  units = 1
+}
+
+
+module "zookeeper" {
+  depends_on = [juju_model.spark]
+  source     = "../../charms/zookeeper"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  base        = "ubuntu@22.04"
+  channel     = "3/stable"
+  constraints = "arch=amd64"
+  revision    = local.revisions.zookeeper
+  resources = {
+    zookeeper-image = local.images.zookeeper
+  }
+  units = var.zookeeper_units
+}
+
+module "data_integrator" {
+  depends_on = [juju_model.spark]
+  source     = "../../charms/data-integrator"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  base        = "ubuntu@24.04"
+  channel     = "latest/stable"
+  constraints = "arch=amd64"
+  revision    = local.revisions.data_integrator
+}
+
+module "azure_storage" {
+  depends_on = [juju_model.spark]
+  count      = var.storage_backend == "azure_storage" ? 1 : 0
+  source     = "../../charms/azure-storage-integrator"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  azure_storage_secret_key = var.azure_storage_config.secret_key
+  base                     = "ubuntu@22.04"
+  channel                  = "latest/edge"
+  config = {
+    connection-protocol = var.azure_storage_config.protocol
+    container           = var.azure_storage_config.container
+    path                = var.azure_storage_config.path
+    storage-account     = var.azure_storage_config.storage_account
+  }
+  constraints = "arch=amd64"
+  revision    = local.revisions.azure_storage
+}
+
+module "s3" {
+  depends_on = [juju_model.spark]
+  count      = var.storage_backend == "s3" ? 1 : 0
+  source     = "../../charms/s3-integrator-v1"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  base        = "ubuntu@22.04"
+  channel     = "1/stable"
+  config      = var.s3_config
+  constraints = "arch=amd64"
+  revision    = local.revisions.s3
+}
+
+
+module "spark" {
+  depends_on = [
+    juju_model.spark,
+    module.azure_storage,
+    module.data_integrator,
+    module.kyuubi_users,
+    module.metastore,
+    module.s3,
+    module.ssc,
+    module.zookeeper
+  ]
+  source     = "../../components/spark-3.4"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  history_server = {
+    config      = var.history_server_config,
+    constraints = "arch=amd64",
+    revision    = var.history_server_revision != null ? var.history_server_revision : local.revisions.history_server,
+    resources   = { spark-history-server-image = var.history_server_image != null ? var.history_server_image : local.images.history_server }
+  }
+  integration_hub = {
+    config      = var.integration_hub_config,
+    constraints = "arch=amd64",
+    revision    = var.integration_hub_revision != null ? var.integration_hub_revision : local.revisions.integration_hub,
+    resources   = { integration-hub-image = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub }
+  }
+  kyuubi = {
+    config      = var.kyuubi_config,
+    constraints = "arch=amd64",
+    revision    = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi,
+    resources   = { kyuubi-image = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi }
+    units       = var.kyuubi_units
+  }
+
+  tls_endpoint = {
+    name     = module.ssc.app_name
+    endpoint = module.ssc.provides.certificates
+  }
+
+  metastore_endpoint = {
+    name     = module.metastore.app_name
+    endpoint = module.metastore.provides.database
+  }
+
+  users_db_endpoint = {
+    name     = module.kyuubi_users.app_name
+    endpoint = module.kyuubi_users.provides.database
+  }
+
+  zookeeper_endpoint       = module.zookeeper.provides.zookeeper
+  data_integrator_endpoint = module.data_integrator.requires.kyuubi
+  object_storage_endpoint  = module.s3 != [] ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
+
+  admin_password  = var.admin_password
+  tls_private_key = var.tls_private_key
+}
+
+module "observability" {
+  depends_on = [juju_model.spark, module.spark]
+  count      = var.cos_offers == null ? 0 : 1
+  source     = "../../components/observability"
+  model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
+
+  dashboards_offer = var.cos_offers.dashboard
+  logging_offer    = var.cos_offers.logging
+  metrics_offer    = var.cos_offers.metrics
+
+  history_server_logging_endpoint   = module.spark.requires.history_server_logging
+  history_server_metrics_endpoint   = module.spark.provides.history_server_metrics
+  history_server_dashboard_endpoint = module.spark.provides.history_server_dashboard
+  integration_hub_cos_endpoint      = module.spark.requires.integration_hub_cos
+  integration_hub_logging_endpoint  = module.spark.requires.integration_hub_logging
+  kyuubi_logging_endpoint           = module.spark.requires.kyuubi_logging
+  kyuubi_metrics_endpoint           = module.spark.provides.kyuubi_metrics
+  kyuubi_dashboard_endpoint         = module.spark.provides.kyuubi_dashboard
+}
diff --git a/terraform/products/charmed-spark-3.5/outputs.tf b/terraform/products/charmed-spark-3.5/outputs.tf
new file mode 100644
index 00000000..3b8a70c7
--- /dev/null
+++ b/terraform/products/charmed-spark-3.5/outputs.tf
@@ -0,0 +1,53 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+
+output "offers" {
+  description = "The name and url of the various offers being exposed"
+  value = merge(
+    concat(
+      module.spark[*].offers
+    )...
+  )
+}
+
+output "metadata" {
+  description = "Metadata of the product deployment."
+  value = {
+    version     = "3.4"
+    deployed_at = timestamp()
+    updated_at  = timestamp()
+  }
+}
+
+output "models" {
+  description = "Maps of the models and the components deployed in each model."
+  value = [{
+    model_uuid = var.model_uuid
+    components = merge(
+      module.spark.components,
+      {
+        zookeeper = module.zookeeper.application
+      },
+      {
+        data_integrator = module.data_integrator.application
+      },
+      {
+        self_signed_certificates = module.ssc.app_name # TODO: expose application
+      },
+      {
+        metastore = module.metastore.app_name # TODO: expose application
+      },
+      {
+        kyuubi_users = module.kyuubi_users.app_name # TODO: expose application
+      },
+      module.azure_storage == [] ? {} : {
+        azure_storage = module.azure_storage[0].application
+      },
+      module.s3 == [] ? {} : {
+        s3 = module.s3[0].application
+      }
+    )
+    }
+  ]
+}
diff --git a/terraform/components/spark-3.4/providers.tf b/terraform/products/charmed-spark-3.5/providers.tf
similarity index 72%
rename from terraform/components/spark-3.4/providers.tf
rename to terraform/products/charmed-spark-3.5/providers.tf
index 1b0ee6a1..712e728a 100644
--- a/terraform/components/spark-3.4/providers.tf
+++ b/terraform/products/charmed-spark-3.5/providers.tf
@@ -1,5 +1,5 @@
 # Copyright 2024 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-# Provider configuration for spark-3.4.
+# Provider configuration for charmed-spark-3.4.
 # Provider blocks, if any, should be defined here.
diff --git a/terraform/products/charmed-spark-3.5/terraform.tf b/terraform/products/charmed-spark-3.5/terraform.tf
new file mode 100644
index 00000000..975cf66e
--- /dev/null
+++ b/terraform/products/charmed-spark-3.5/terraform.tf
@@ -0,0 +1,13 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+terraform {
+  required_version = ">=1.0.0"
+
+  required_providers {
+    juju = {
+      source  = "juju/juju"
+      version = ">=1.0.0"
+    }
+  }
+}
diff --git a/terraform/products/charmed-spark-3.5/variables.tf b/terraform/products/charmed-spark-3.5/variables.tf
new file mode 100644
index 00000000..36b5aadb
--- /dev/null
+++ b/terraform/products/charmed-spark-3.5/variables.tf
@@ -0,0 +1,272 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+variable "admin_password" {
+  description = "The password for the Kyuubi admin user."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+
+variable "azure_storage_config" {
+  description = "Azure Object storage information"
+  type = object({
+    container       = optional(string)
+    path            = optional(string)
+    protocol        = optional(string, "abfss")
+    secret_key      = optional(string)
+    storage_account = optional(string)
+  })
+  default = {}
+}
+
+variable "azure_storage_revision" {
+  description = "Charm revision for azure-storage-integrator"
+  type        = number
+  default     = null
+}
+
+variable "certificate_common_name" {
+  description = "Common name for the certificate to be used in self-signed"
+  type        = string
+  default     = "charmed-spark"
+}
+
+variable "create_model" {
+  description = "Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism."
+  type        = bool
+  default     = true
+  nullable    = false
+}
+
+variable "cos_offers" {
+  description = "Observability stack offers."
+  type = object({
+    dashboard = string
+    logging   = string
+    metrics   = string
+  })
+  default = null
+}
+
+variable "data_integrator_revision" {
+  description = "Charm revision for data-integrator"
+  type        = number
+  default     = null
+}
+
+variable "grafana_agent_revision" {
+  description = "Charm revision for grafana-agent-k8s"
+  type        = number
+  default     = null
+}
+
+variable "history_server_config" {
+  description = "History Server configuration options"
+  type        = map(any)
+  default     = {}
+}
+
+variable "history_server_image" {
+  description = "Image for spark-history-server-k8s"
+  type        = string
+  default     = null
+}
+
+variable "history_server_revision" {
+  description = "Charm revision for spark-history-server-k8s"
+  type        = number
+  default     = null
+}
+
+variable "integration_hub_image" {
+  description = "Image for spark-integration-hub-k8s"
+  type        = string
+  default     = null
+}
+
+variable "integration_hub_config" {
+  description = "Integration Hub configuration options."
+  type        = map(any)
+  default     = {}
+}
+
+variable "integration_hub_revision" {
+  description = "Charm revision for spark-integration-hub-k8s"
+  type        = number
+  default     = null
+}
+
+variable "K8S_CLOUD" {
+  type        = string
+  description = "The kubernetes juju cloud name."
+  default     = "microk8s"
+}
+
+variable "K8S_CREDENTIAL" {
+  type        = string
+  description = "The name of the kubernetes juju credential."
+  default     = "microk8s"
+}
+
+variable "juju_controller" {
+  description = "Controller information: endpoint, username, password and CA certificate."
+  type = object({
+    endpoint = string
+    username = string
+    password = string
+    ca       = string
+  })
+  default = null
+}
+
+variable "kyuubi_config" {
+  description = "Kyuubi configuration options."
+  type        = map(any)
+  default     = {}
+}
+
+variable "kyuubi_image" {
+  description = "Image for kyuubi-k8s"
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_revision" {
+  description = "Charm revision for kyuubi-k8s"
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_units" {
+  description = "Number of Kyuubi units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}
+
+variable "kyuubi_users_image" {
+  description = "Image for postgresql-k8s (auth-db)"
+  type        = string
+  default     = null
+}
+
+variable "kyuubi_users_revision" {
+  description = "Charm revision for postgresql-k8s (auth-db)"
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_users_size" {
+  description = "Storage size for the Kyuubi users database"
+  type        = string
+  default     = "1G"
+}
+
+variable "metastore_image" {
+  description = "Image for postgresql-k8s (metastore)"
+  type        = string
+  default     = null
+}
+
+variable "metastore_revision" {
+  description = "Charm revision for postgresql-k8s (metastore)"
+  type        = number
+  default     = null
+}
+
+variable "metastore_size" {
+  description = "Storage size for the metastore database"
+  type        = string
+  default     = "10G"
+}
+
+variable "spark_model_name" {
+  description = "The name of the juju model to deploy Spark to"
+  type        = string
+  default     = "spark"
+}
+
+variable "model_uuid" {
+  description = "Optional existing Juju model UUID to deploy Spark to. If provided, model creation is skipped in higher-level modules."
+  type        = string
+  default     = null
+}
+
+variable "proxy" {
+  description = "Proxy information for the deployment."
+  type = object({
+    http     = optional(string, "")
+    https    = optional(string, "")
+    no_proxy = optional(string, "")
+  })
+  default = {}
+}
+
+variable "s3_config" {
+  description = "S3 integrator configuration"
+  type = object({
+    attributes                          = optional(string)
+    bucket                              = optional(string)
+    endpoint                            = optional(string)
+    experimental-delete-older-than-days = optional(number)
+    path                                = optional(string)
+    region                              = optional(string)
+    s3-api-version                      = optional(string)
+    s3-uri-style                        = optional(string)
+    storage-class                       = optional(string)
+    tls-ca-chain                        = optional(string)
+  })
+  default = {}
+}
+
+variable "s3_revision" {
+  description = "Charm revision for s3-integrator"
+  type        = number
+  default     = null
+}
+
+variable "storage_backend" {
+  type        = string
+  description = "Storage backend to be used"
+
+  validation {
+    condition     = contains(["azure_storage", "s3"], var.storage_backend)
+    error_message = "Valid values for var: test_variable are (s3, azure_storage)."
+  }
+
+  default = "s3"
+}
+
+variable "tls_private_key" {
+  description = "The file path of the private key to use for TLS certificates."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
+variable "zookeeper_image" {
+  description = "Image for zookeeper-k8s"
+  type        = map(string)
+  default     = null
+}
+
+variable "zookeeper_revision" {
+  description = "Charm revision for zookeeper-k8s"
+  type        = number
+  default     = null
+}
+
+variable "zookeeper_size" {
+  description = "Storage size for the metastore database"
+  type        = string
+  default     = "10G"
+}
+
+variable "zookeeper_units" {
+  description = "Define the number of zookeeper units. 3 units are recommended for high availability."
+  type        = number
+  default     = 3
+  nullable    = false
+}

From 080096fe11323e68d227896132d0f7b26eb6b160 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 15:12:37 +0200
Subject: [PATCH 33/62] Disable internal tls for COS testing

---
 python/tests/integration/resources/main.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index 795876b8..64ad1fb5 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -83,9 +83,9 @@ variable "zookeeper_size" {
 module "cos" {
   count = var.cos_model_uuid == null ? 0 : 1
   # TODO: Pin to tag once available
-  source     = "git::https://github.com/canonical/observability-stack//terraform/cos-lite?ref=cd55b1d"
-  model_uuid = var.cos_model_uuid
-  # channel    = "1/stable"
+  source       = "git::https://github.com/canonical/observability-stack//terraform/cos-lite?ref=cd55b1d"
+  model_uuid   = var.cos_model_uuid
+  internal_tls = false
 }
 
 resource "juju_model" "spark" {

From 23ccd6ff765d0e4be05b527efcbc9f9b09825075 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 15:21:11 +0200
Subject: [PATCH 34/62] Re-align 3.5 revisions

---
 terraform/components/spark-3.5/locals.tf       | 6 +++---
 terraform/products/charmed-spark-3.5/locals.tf | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/terraform/components/spark-3.5/locals.tf b/terraform/components/spark-3.5/locals.tf
index 2ed303b9..e13bd58b 100644
--- a/terraform/components/spark-3.5/locals.tf
+++ b/terraform/components/spark-3.5/locals.tf
@@ -5,14 +5,14 @@ locals {
   revisions = {
     history_server  = 96
     integration_hub = 119
-    kyuubi          = 149
+    kyuubi          = 146
   }
   images = {
     history_server = "ghcr.io/canonical/charmed-spark@sha256:f7f387a76ba2f3b6cfc34a35f36c0f4ae85bfa78c8c9d4bfa06ba86401b95d70"
     # rev24,  spark-version: 3.5.7, release date 16/03/2026 
     integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134"
     # rev13, release date 19/03/2026
-    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:60e81dd2e9be50d4a857ce987935341c8518c784c7de36e814fc41dd3451d58f"
-    # rev19, spark-version: 3.4.4, kyuubi-version: 1.10.3, release date 17/03/2026
+    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:21beea406001d161538b552cef2b47b86423bd899139ada263c1ef665fd89fff"
+    # rev19, spark-version: 3.5.7, kyuubi-version: 1.10.3, release date 16/03/2026
   }
 }
diff --git a/terraform/products/charmed-spark-3.5/locals.tf b/terraform/products/charmed-spark-3.5/locals.tf
index fa880218..341119be 100644
--- a/terraform/products/charmed-spark-3.5/locals.tf
+++ b/terraform/products/charmed-spark-3.5/locals.tf
@@ -5,7 +5,7 @@ locals {
   revisions = {
     history_server  = 96
     integration_hub = 119
-    kyuubi          = 149
+    kyuubi          = 146
     kyuubi_users    = 774
     metastore       = 774
     zookeeper       = 78
@@ -26,8 +26,8 @@ locals {
     # rev24,  spark-version: 3.5.7, release date 16/03/2026 
     integration_hub = "ghcr.io/canonical/spark-integration-hub@sha256:a0439da3e6a9433a0db9501c8ea7b28ba3652af9a9ec22fcc2ad1b0197d48134"
     # rev13, release date 19/03/2026
-    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:60e81dd2e9be50d4a857ce987935341c8518c784c7de36e814fc41dd3451d58f"
-    # rev19, spark-version: 3.4.4, kyuubi-version: 1.10.3, release date 17/03/2026
+    kyuubi = "ghcr.io/canonical/charmed-spark-kyuubi@sha256:21beea406001d161538b552cef2b47b86423bd899139ada263c1ef665fd89fff"
+    # rev19, spark-version: 3.5.7, kyuubi-version: 1.10.3, release date 16/03/2026
     kyuubi_users = "184"
     metastore    = "184"
     zookeeper    = "34"

From 6ad4d771aa994bd2bca1ec39e908e195cabcac63 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 15:22:12 +0200
Subject: [PATCH 35/62] Re-enable full testing

---
 .github/workflows/ci-tests-full.yaml    |  9 +--------
 .github/workflows/ci-tests-minimal.yaml |  2 +-
 .github/workflows/ci-uat.yaml           | 18 ++++++++----------
 3 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/ci-tests-full.yaml b/.github/workflows/ci-tests-full.yaml
index a42d526b..d219a1ee 100644
--- a/.github/workflows/ci-tests-full.yaml
+++ b/.github/workflows/ci-tests-full.yaml
@@ -14,7 +14,7 @@ jobs:
     uses: ./.github/workflows/ci-checks.yaml
 
   integration-tests:
-    name: "${{ matrix.tox-env }} (${{ matrix.k8s-distribution }}:${{ matrix.k8s-version }}): spark-${{ matrix.spark-version }} | ${{ matrix.bundle-backend }} | ${{ matrix.storage-backend }} | juju ${{ matrix.juju.agent }} | ${{ matrix.cos-model }}"
+    name: "${{ matrix.tox-env }} (${{ matrix.k8s-distribution }}:${{ matrix.k8s-version }}): spark-${{ matrix.spark-version }} | ${{ matrix.storage-backend }} | juju ${{ matrix.juju.agent }} | ${{ matrix.cos-model }}"
     runs-on: ["self-hosted-linux-amd64-jammy-xlarge"]
     timeout-minutes: 120
     strategy:
@@ -25,7 +25,6 @@ jobs:
           - integration-sparkjob
           - integration-kyuubi
           - integration-backup-restore
-        bundle-backend: ["yaml", "terraform"]
         spark-version: ["3.4.4", "3.5.7"]
         storage-backend: ["s3", "azure_storage"]
         cos-model: ["cos"]
@@ -60,11 +59,6 @@ jobs:
             juju-snap-channel: "3.6/stable"
             juju-agent-version: "3.6.14"
 
-        exclude:
-          # Backup / Restore tests only supported on Terraform
-          - tox-env: integration-backup-restore
-            bundle-backend: yaml
-
     needs:
       - checks
     steps:
@@ -77,7 +71,6 @@ jobs:
           k8s-distribution: ${{ matrix.k8s-distribution }}
           k8s-version: ${{ matrix.k8s-version }}
           tox-env: ${{ matrix.tox-env }}
-          bundle-backend: ${{ matrix.bundle-backend }}
           spark-version: ${{ matrix.spark-version }}
           storage-backend: ${{ matrix.storage-backend }}
           cos-model: ${{ matrix.cos-model }}
diff --git a/.github/workflows/ci-tests-minimal.yaml b/.github/workflows/ci-tests-minimal.yaml
index 7bbc2851..bbe5f943 100644
--- a/.github/workflows/ci-tests-minimal.yaml
+++ b/.github/workflows/ci-tests-minimal.yaml
@@ -23,7 +23,7 @@ jobs:
           - integration-sparkjob
           - integration-kyuubi
           - integration-backup-restore
-        spark-version: [ "3.4.4" ] #, "3.5.7" ]
+        spark-version: [ "3.4.4", "3.5.7" ]
         storage-backend: [ "s3", "azure_storage" ]
         juju-snap-channel: [ "3.6/stable" ]
         juju-agent-version: [ "3.6.14" ]
diff --git a/.github/workflows/ci-uat.yaml b/.github/workflows/ci-uat.yaml
index 391493d0..b7e9a96b 100644
--- a/.github/workflows/ci-uat.yaml
+++ b/.github/workflows/ci-uat.yaml
@@ -5,14 +5,14 @@ name: Mimic UAT Tests
 
 on:
   workflow_dispatch:
-  # pull_request:
+  pull_request:
     paths-ignore:
       - "docs/**"
       - "**.md"
 
 jobs:
   integration-tests:
-    name: "UATs (MicroK8s:${{ matrix.k8s-version }}): spark-${{ matrix.spark-version }} | ${{ matrix.bundle-backend }} | ${{ matrix.storage-backend }} | juju ${{ matrix.juju.agent }} "
+    name: "UATs (MicroK8s:${{ matrix.k8s-version }}): spark-${{ matrix.spark-version }} | ${{ matrix.storage-backend }} | juju ${{ matrix.juju.agent }} "
     runs-on: ["self-hosted-linux-amd64-jammy-xlarge"]
     timeout-minutes: 120
     strategy:
@@ -20,8 +20,6 @@ jobs:
       matrix:
         spark-version:
           - "3.4.4"
-        backend:
-          - terraform
         storage-backend:
           - s3
           - azure_storage
@@ -137,19 +135,19 @@ jobs:
           sleep 60
 
           # The first tests deploy the bundle
-          tox run -e integration-bundle -- --backend ${{ matrix.backend }} --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --keep-models --model test-uat --cos-model cos --uuid $UUID
+          tox run -e integration-bundle -- --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --keep-models --model test-uat --cos-model cos --uuid $UUID
 
           # Next we only use --no-deploy flag
 
-          tox run -e integration-kyuubi -- --backend ${{ matrix.backend }} --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --no-deploy --keep-models --model test-uat --cos-model cos --uuid $UUID
-          tox run -e integration-sparkjob -- --backend ${{ matrix.backend }} --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --no-deploy --keep-models --model test-uat --cos-model cos --uuid $UUID
+          tox run -e integration-kyuubi -- --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --no-deploy --keep-models --model test-uat --cos-model cos --uuid $UUID
+          tox run -e integration-sparkjob -- --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --no-deploy --keep-models --model test-uat --cos-model cos --uuid $UUID
 
           # We re-run the last tests to make sure that the tests are idempotent
-          tox run -e integration-kyuubi -- --backend ${{ matrix.backend }} --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --no-deploy --keep-models --model test-uat --cos-model cos --uuid $UUID
-          tox run -e integration-sparkjob -- --backend ${{ matrix.backend }} --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --no-deploy --model test-uat --cos-model cos --uuid $UUID
+          tox run -e integration-kyuubi -- --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --no-deploy --keep-models --model test-uat --cos-model cos --uuid $UUID
+          tox run -e integration-sparkjob -- --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --no-deploy --model test-uat --cos-model cos --uuid $UUID
 
           # # Run backup / restore tests separately without --no-deploy and --keep-models
-          # tox run -e integration-backup-restore -- --backend ${{ matrix.backend }} --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --model test-uat --cos-model cos --uuid $UUID
+          # tox run -e integration-backup-restore -- --spark-version ${{ matrix.version }} --storage-backend ${{ matrix.storage-backend }} --model test-uat --cos-model cos --uuid $UUID
       - id: collect-logs
         name: Collect logs if job failed
         shell: bash

From 7e75ad962ffd5413b6c0c8a49694ff39831a377e Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 16:35:55 +0200
Subject: [PATCH 36/62] Add reference doc automation

---
 python/scripts/module_ref.sh                  | 29 +++++++++++++
 python/tox.ini                                |  6 +++
 .../charms/azure-storage-integrator/README.md | 22 ++++++----
 terraform/charms/data-integrator/README.md    | 20 ++++++---
 terraform/charms/s3-integrator-v1/README.md   | 22 ++++++----
 terraform/charms/zookeeper/README.md          | 20 ++++++---
 terraform/components/observability/README.md  | 35 +++++++++++-----
 terraform/components/spark-3.4/README.md      | 41 ++++++++++++-------
 terraform/components/spark-3.5/README.md      | 41 ++++++++++++-------
 .../products/charmed-spark-3.4/README.md      | 18 +++++---
 .../products/charmed-spark-3.5/README.md      | 18 +++++---
 11 files changed, 194 insertions(+), 78 deletions(-)
 create mode 100644 python/scripts/module_ref.sh

diff --git a/python/scripts/module_ref.sh b/python/scripts/module_ref.sh
new file mode 100644
index 00000000..761d74d2
--- /dev/null
+++ b/python/scripts/module_ref.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+if ! command -v terraform-docs &> /dev/null; then
+  echo "Error: 'terraform-docs' is not installed or not in your PATH."
+  echo "Please install it with 'sudo snap install terraform-docs' before running this script."
+  exit 1
+fi
+
+TERRAFORM_DIR=$(pwd)/../terraform
+
+echo "Starting terraform-docs update from $TERRAFORM_DIR"
+
+find "$TERRAFORM_DIR" -maxdepth 2 -mindepth 2 -type d | while read -r MODULE_DIR; do
+  echo "Processing $MODULE_DIR"
+  if [ -d "$MODULE_DIR" ]; then
+    cd "$MODULE_DIR"
+
+    if [ -f "README.md" ]; then
+      terraform-docs markdown table . \
+          --output-mode inject \
+          --output-file README.md \
+          --indent 3
+    else
+      echo "Skipping $MODULE_DIR"
+    fi
+  fi
+done
+
+echo "Finished updating documentation."
diff --git a/python/tox.ini b/python/tox.ini
index e807902c..4a90f172 100644
--- a/python/tox.ini
+++ b/python/tox.ini
@@ -27,6 +27,7 @@ allowlist_externals =
     poetry
     sg
     /bin/bash
+    terraform-docs
 set_env =
     PYTHONPATH = {tox_root}:{[vars]src_path}
     PYTHONBREAKPOINT=ipdb.set_trace
@@ -66,6 +67,11 @@ commands =
     poetry install --all-groups
     poetry run mypy {[vars]src_path}
 
+[testenv:generate-docs]
+description = Generate reference docs for terraform modules
+commands =
+    /bin/bash scripts/module_ref.sh
+
 [testenv:unit]
 description = Run unit tests
 commands =
diff --git a/terraform/charms/azure-storage-integrator/README.md b/terraform/charms/azure-storage-integrator/README.md
index 48cfeeee..cb92ef71 100644
--- a/terraform/charms/azure-storage-integrator/README.md
+++ b/terraform/charms/azure-storage-integrator/README.md
@@ -1,21 +1,28 @@
-## Requirements
+# Azure storage integrator charm terraform module
+
+To be contributed upstream.
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
-## Modules
+### Modules
 
 No modules.
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
@@ -24,7 +31,7 @@ No modules.
 | [juju_offer.azure_storage_credentials](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
 | [juju_secret.azure_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
@@ -32,16 +39,17 @@ No modules.
 | <a name="input_azure_storage_secret_key"></a> [azure\_storage\_secret\_key](#input\_azure\_storage\_secret\_key) | Secret key to the Azure Storage account. | `string` | n/a | yes |
 | <a name="input_base"></a> [base](#input\_base) | The operating system on which to deploy | `string` | `"ubuntu@22.04"` | no |
 | <a name="input_channel"></a> [channel](#input\_channel) | Channel of the charm. | `string` | `"latest/edge"` | no |
-| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    connection-protocol = optional(string, "abfss")<br/>    container           = optional(string, "azurecontainer")<br/>    endpoint            = optional(string, "")<br/>    path                = optional(string, "spark-events")<br/>    resource-group      = optional(string, "azurerg")<br/>    storage_account     = optional(string, "azurestorageaccount")<br/>  })</pre> | `{}` | no |
+| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    connection-protocol = optional(string, "abfss")<br/>    container           = optional(string)<br/>    endpoint            = optional(string)<br/>    path                = optional(string)<br/>    resource-group      = optional(string)<br/>    storage-account     = optional(string)<br/>  })</pre> | `{}` | no |
 | <a name="input_constraints"></a> [constraints](#input\_constraints) | String listing constraints for this application. | `string` | `null` | no |
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
 | <a name="input_revision"></a> [revision](#input\_revision) | Revision number of the charm. | `number` | `null` | no |
 | <a name="input_units"></a> [units](#input\_units) | Unit count. | `number` | `1` | no |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_application"></a> [application](#output\_application) | Object representing the deployed application. |
 | <a name="output_offers"></a> [offers](#output\_offers) | Map of all offers exposed by the single charm. |
 | <a name="output_provides"></a> [provides](#output\_provides) | Provides endpoints. |
+<!-- END_TF_DOCS -->
diff --git a/terraform/charms/data-integrator/README.md b/terraform/charms/data-integrator/README.md
index 40a2b260..96f91495 100644
--- a/terraform/charms/data-integrator/README.md
+++ b/terraform/charms/data-integrator/README.md
@@ -1,27 +1,34 @@
-## Requirements
+# Data-integrator charm terraform module
+
+To be contributed upstream.
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | >=1.0.0 |
 
-## Modules
+### Modules
 
 No modules.
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
 | [juju_application.data_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
@@ -34,10 +41,11 @@ No modules.
 | <a name="input_revision"></a> [revision](#input\_revision) | Revision number of the charm. | `number` | `null` | no |
 | <a name="input_units"></a> [units](#input\_units) | Unit count. | `number` | `1` | no |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_application"></a> [application](#output\_application) | Object representing the deployed application. |
 | <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provided endpoints. |
 | <a name="output_requires"></a> [requires](#output\_requires) | Map of all the required endpoints. |
+<!-- END_TF_DOCS -->
diff --git a/terraform/charms/s3-integrator-v1/README.md b/terraform/charms/s3-integrator-v1/README.md
index 725b258b..f1b281e0 100644
--- a/terraform/charms/s3-integrator-v1/README.md
+++ b/terraform/charms/s3-integrator-v1/README.md
@@ -1,44 +1,52 @@
-## Requirements
+# S3 integrator (v1) charm terraform module
+
+To be contributed upstream.
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
-## Modules
+### Modules
 
 No modules.
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
 | [juju_application.s3_integrator](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_offer.s3_credentials](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_app_name"></a> [app\_name](#input\_app\_name) | Name to give the deployed application. | `string` | `"s3-integrator"` | no |
 | <a name="input_base"></a> [base](#input\_base) | The operating system on which to deploy | `string` | `"ubuntu@22.04"` | no |
 | <a name="input_channel"></a> [channel](#input\_channel) | Channel of the charm. | `string` | `"1/stable"` | no |
-| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    attributes                          = optional(string, "")<br/>    bucket                              = optional(string, "spark-bucket")<br/>    endpoint                            = optional(string, "https://s3.amazonaws.com")<br/>    experimental-delete-older-than-days = optional(number, 14)<br/>    path                                = optional(string, "path/")<br/>    region                              = optional(string, "us-east-1")<br/>    s3-api-version                      = optional(string, "2")<br/>    s3-uri-style                        = optional(string, "2")<br/>    storage-class                       = optional(string, "")<br/>    tls-ca-chain                        = optional(string, "")<br/>  })</pre> | `{}` | no |
+| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    attributes                          = optional(string)<br/>    bucket                              = optional(string)<br/>    endpoint                            = optional(string)<br/>    experimental-delete-older-than-days = optional(number)<br/>    path                                = optional(string)<br/>    region                              = optional(string)<br/>    s3-api-version                      = optional(string)<br/>    s3-uri-style                        = optional(string)<br/>    storage-class                       = optional(string)<br/>    tls-ca-chain                        = optional(string)<br/>  })</pre> | `{}` | no |
 | <a name="input_constraints"></a> [constraints](#input\_constraints) | String listing constraints for this application. | `string` | `null` | no |
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
 | <a name="input_revision"></a> [revision](#input\_revision) | Revision number of the charm. | `number` | `null` | no |
 | <a name="input_units"></a> [units](#input\_units) | Unit count. | `number` | `1` | no |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_application"></a> [application](#output\_application) | Object representing the deployed application. |
 | <a name="output_offers"></a> [offers](#output\_offers) | Map of all offers exposed by the single charm. |
 | <a name="output_provides"></a> [provides](#output\_provides) | Provides endpoints. |
+<!-- END_TF_DOCS -->
diff --git a/terraform/charms/zookeeper/README.md b/terraform/charms/zookeeper/README.md
index 20e53f4b..f72fa222 100644
--- a/terraform/charms/zookeeper/README.md
+++ b/terraform/charms/zookeeper/README.md
@@ -1,27 +1,34 @@
-## Requirements
+# ZooKeeper charm terraform module
+
+To be contributed upstream.
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
-## Modules
+### Modules
 
 No modules.
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
 | [juju_application.zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
@@ -36,10 +43,11 @@ No modules.
 | <a name="input_storage_size"></a> [storage\_size](#input\_storage\_size) | Storage size for the zookeeper data. | `string` | `"10G"` | no |
 | <a name="input_units"></a> [units](#input\_units) | Unit count. 3 units are recommended for high availability. | `number` | `3` | no |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_application"></a> [application](#output\_application) | Object representing the deployed application. |
 | <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provided endpoints. |
 | <a name="output_requires"></a> [requires](#output\_requires) | Map of all the required endpoints. |
+<!-- END_TF_DOCS -->
diff --git a/terraform/components/observability/README.md b/terraform/components/observability/README.md
index 13837cc7..bc1fb504 100644
--- a/terraform/components/observability/README.md
+++ b/terraform/components/observability/README.md
@@ -1,21 +1,35 @@
-## Requirements
+# Observability component terraform module
+
+Bundles the following charms:
+- Grafana-agent
+- COS-configuration
+- Prometheus Pushgateway
+- Prometheus-scrape-config
+
+While we do not maintain those charms, we have specific requirements for them.
+As such, it is easier to define our own component module for now.
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
-## Modules
+### Modules
 
 No modules.
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
@@ -41,13 +55,13 @@ No modules.
 | [juju_offer.loki_logging](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
 | [juju_offer.prometheus_receive_remote_write](https://registry.terraform.io/providers/juju/juju/latest/docs/data-sources/offer) | data source |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_cos_configuration"></a> [cos\_configuration](#input\_cos\_configuration) | n/a | <pre>object({<br/>    app_name    = optional(string, "cos-configuration")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "1/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "cos-configuration",<br/>  "base": "ubuntu@22.04",<br/>  "config": {<br/>    "git_branch": "main",<br/>    "git_depth": 1,<br/>    "git_repo": "https://github.com/canonical/spark-k8s-bundle",<br/>    "grafana_dashboards_path": "resources/grafana/"<br/>  },<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_cos_configuration"></a> [cos\_configuration](#input\_cos\_configuration) | n/a | <pre>object({<br/>    app_name = optional(string, "cos-configuration")<br/>    base     = optional(string, "ubuntu@22.04")<br/>    channel  = optional(string, "1/stable")<br/>    config = optional(map(string), {<br/>      git_branch              = "main"<br/>      git_depth               = 1<br/>      git_repo                = "https://github.com/canonical/spark-k8s-bundle"<br/>      grafana_dashboards_path = "resources/grafana/"<br/>    })<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
 | <a name="input_dashboards_offer"></a> [dashboards\_offer](#input\_dashboards\_offer) | URL of the `grafana_dashboard` interface offer. | `string` | n/a | yes |
-| <a name="input_grafana_agent"></a> [grafana\_agent](#input\_grafana\_agent) | n/a | <pre>object({<br/>    app_name    = optional(string, "grafana-agent")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "1/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "grafana-agent",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_grafana_agent"></a> [grafana\_agent](#input\_grafana\_agent) | n/a | <pre>object({<br/>    app_name    = optional(string, "grafana-agent")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "1/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
 | <a name="input_history_server_dashboard_endpoint"></a> [history\_server\_dashboard\_endpoint](#input\_history\_server\_dashboard\_endpoint) | In-model endpoint for the History Server dashboard integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_history_server_logging_endpoint"></a> [history\_server\_logging\_endpoint](#input\_history\_server\_logging\_endpoint) | In-model endpoint for the History Server logging integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_history_server_metrics_endpoint"></a> [history\_server\_metrics\_endpoint](#input\_history\_server\_metrics\_endpoint) | In-model endpoint for the History Server metrics integration. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
@@ -59,13 +73,14 @@ No modules.
 | <a name="input_logging_offer"></a> [logging\_offer](#input\_logging\_offer) | URL of the `loki_push_api` interface offer. | `string` | n/a | yes |
 | <a name="input_metrics_offer"></a> [metrics\_offer](#input\_metrics\_offer) | URL of the `prometheus_remote_write` interface offer. | `string` | n/a | yes |
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
-| <a name="input_pushgateway"></a> [pushgateway](#input\_pushgateway) | n/a | <pre>object({<br/>    app_name           = optional(string, "pushgateway")<br/>    base               = optional(string, "ubuntu@22.04")<br/>    channel            = optional(string, "1/stable")<br/>    config             = optional(map(string), {})<br/>    constraints        = optional(string, "arch=amd64")<br/>    resources          = optional(map(string), {})<br/>    revision           = optional(number)<br/>    storage_directives = optional(map(string))<br/>    units              = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "pushgateway",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "storage_directives": {<br/>    "pushgateway-store": "10G"<br/>  },<br/>  "units": 1<br/>}</pre> | no |
-| <a name="input_scrape_config"></a> [scrape\_config](#input\_scrape\_config) | n/a | <pre>object({<br/>    app_name    = optional(string, "scrape-config")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "1/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "scrape-config",<br/>  "base": "ubuntu@22.04",<br/>  "config": {<br/>    "scrape_interval": "10s"<br/>  },<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
+| <a name="input_pushgateway"></a> [pushgateway](#input\_pushgateway) | n/a | <pre>object({<br/>    app_name    = optional(string, "pushgateway")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "1/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    storage_directives = optional(map(string), {<br/>      pushgateway-store = "10G"<br/>    })<br/>    units = optional(number, 1)<br/>  })</pre> | `{}` | no |
+| <a name="input_scrape_config"></a> [scrape\_config](#input\_scrape\_config) | n/a | <pre>object({<br/>    app_name = optional(string, "scrape-config")<br/>    base     = optional(string, "ubuntu@22.04")<br/>    channel  = optional(string, "1/stable")<br/>    config = optional(map(string), {<br/>      scrape_interval = "10s"<br/>    })<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_components"></a> [components](#output\_components) | List of the deployed applications for this component module. |
 | <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provided endpoints. |
 | <a name="output_requires"></a> [requires](#output\_requires) | Map of the required endpoints. |
+<!-- END_TF_DOCS -->
diff --git a/terraform/components/spark-3.4/README.md b/terraform/components/spark-3.4/README.md
index 3f42f49b..62b92274 100644
--- a/terraform/components/spark-3.4/README.md
+++ b/terraform/components/spark-3.4/README.md
@@ -1,21 +1,31 @@
-## Requirements
+# Spark (3.4) component terraform module
+
+Bundles the following charms:
+- Spark History Server
+- Spark Integration Hub
+- Kyuubi
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
-## Modules
+### Modules
 
 No modules.
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
@@ -35,24 +45,24 @@ No modules.
 | [juju_offer.kyuubi_jdbc](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
 | [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
-| <a name="input_data_integrator_endpoint"></a> [data\_integrator\_endpoint](#input\_data\_integrator\_endpoint) | In-model endpoint for the data-integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "kyuubi")<br/>  })</pre> | n/a | yes |
-| <a name="input_history_server"></a> [history\_server](#input\_history\_server) | n/a | <pre>object({<br/>    app_name    = optional(string, "history-server")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "history-server",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
-| <a name="input_integration_hub"></a> [integration\_hub](#input\_integration\_hub) | n/a | <pre>object({<br/>    app_name    = optional(string, "integration-hub")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "integration-hub",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
-| <a name="input_kyuubi"></a> [kyuubi](#input\_kyuubi) | n/a | <pre>object({<br/>    app_name    = optional(string, "kyuubi")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3.4/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "kyuubi",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
-| <a name="input_metastore_endpoint"></a> [metastore\_endpoint](#input\_metastore\_endpoint) | In-model endpoint for the metastore (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "database")<br/>  })</pre> | n/a | yes |
+| <a name="input_data_integrator_endpoint"></a> [data\_integrator\_endpoint](#input\_data\_integrator\_endpoint) | In-model endpoint for the data-integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_history_server"></a> [history\_server](#input\_history\_server) | n/a | <pre>object({<br/>    app_name    = optional(string, "history-server")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
+| <a name="input_integration_hub"></a> [integration\_hub](#input\_integration\_hub) | n/a | <pre>object({<br/>    app_name    = optional(string, "integration-hub")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
+| <a name="input_kyuubi"></a> [kyuubi](#input\_kyuubi) | n/a | <pre>object({<br/>    app_name    = optional(string, "kyuubi")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3.4/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
+| <a name="input_metastore_endpoint"></a> [metastore\_endpoint](#input\_metastore\_endpoint) | In-model endpoint for the metastore (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
-| <a name="input_object_storage_endpoint"></a> [object\_storage\_endpoint](#input\_object\_storage\_endpoint) | In-model endpoint for the object storage integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "s3_credentials")<br/>  })</pre> | n/a | yes |
-| <a name="input_tls_endpoint"></a> [tls\_endpoint](#input\_tls\_endpoint) | In-model endpoint for the TLS certificates provider. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "certificates")<br/>  })</pre> | n/a | yes |
+| <a name="input_object_storage_endpoint"></a> [object\_storage\_endpoint](#input\_object\_storage\_endpoint) | In-model endpoint for the object storage integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_tls_endpoint"></a> [tls\_endpoint](#input\_tls\_endpoint) | In-model endpoint for the TLS certificates provider. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
-| <a name="input_users_db_endpoint"></a> [users\_db\_endpoint](#input\_users\_db\_endpoint) | In-model endpoint for the Kyuubi users database (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "database")<br/>  })</pre> | n/a | yes |
-| <a name="input_zookeeper_endpoint"></a> [zookeeper\_endpoint](#input\_zookeeper\_endpoint) | In-model endpoint for the ZooKeeper application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "zookeeper")<br/>  })</pre> | n/a | yes |
+| <a name="input_users_db_endpoint"></a> [users\_db\_endpoint](#input\_users\_db\_endpoint) | In-model endpoint for the Kyuubi users database (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_zookeeper_endpoint"></a> [zookeeper\_endpoint](#input\_zookeeper\_endpoint) | In-model endpoint for the ZooKeeper application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
@@ -60,3 +70,4 @@ No modules.
 | <a name="output_offers"></a> [offers](#output\_offers) | Map of all offers exposed by the component's charms. |
 | <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provides endpoints. |
 | <a name="output_requires"></a> [requires](#output\_requires) | Map of the requires endpoints. |
+<!-- END_TF_DOCS -->
diff --git a/terraform/components/spark-3.5/README.md b/terraform/components/spark-3.5/README.md
index 3f42f49b..21d5481f 100644
--- a/terraform/components/spark-3.5/README.md
+++ b/terraform/components/spark-3.5/README.md
@@ -1,21 +1,31 @@
-## Requirements
+# Spark (3.5) component terraform module
+
+Bundles the following charms:
+- Spark History Server
+- Spark Integration Hub
+- Kyuubi
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
-## Modules
+### Modules
 
 No modules.
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
@@ -35,24 +45,24 @@ No modules.
 | [juju_offer.kyuubi_jdbc](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
 | [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
-| <a name="input_data_integrator_endpoint"></a> [data\_integrator\_endpoint](#input\_data\_integrator\_endpoint) | In-model endpoint for the data-integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "kyuubi")<br/>  })</pre> | n/a | yes |
-| <a name="input_history_server"></a> [history\_server](#input\_history\_server) | n/a | <pre>object({<br/>    app_name    = optional(string, "history-server")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "history-server",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
-| <a name="input_integration_hub"></a> [integration\_hub](#input\_integration\_hub) | n/a | <pre>object({<br/>    app_name    = optional(string, "integration-hub")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "integration-hub",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
-| <a name="input_kyuubi"></a> [kyuubi](#input\_kyuubi) | n/a | <pre>object({<br/>    app_name    = optional(string, "kyuubi")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3.4/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | <pre>{<br/>  "app_name": "kyuubi",<br/>  "base": "ubuntu@22.04",<br/>  "constraints": "arch=amd64",<br/>  "units": 1<br/>}</pre> | no |
-| <a name="input_metastore_endpoint"></a> [metastore\_endpoint](#input\_metastore\_endpoint) | In-model endpoint for the metastore (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "database")<br/>  })</pre> | n/a | yes |
+| <a name="input_data_integrator_endpoint"></a> [data\_integrator\_endpoint](#input\_data\_integrator\_endpoint) | In-model endpoint for the data-integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_history_server"></a> [history\_server](#input\_history\_server) | n/a | <pre>object({<br/>    app_name    = optional(string, "history-server")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
+| <a name="input_integration_hub"></a> [integration\_hub](#input\_integration\_hub) | n/a | <pre>object({<br/>    app_name    = optional(string, "integration-hub")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
+| <a name="input_kyuubi"></a> [kyuubi](#input\_kyuubi) | n/a | <pre>object({<br/>    app_name    = optional(string, "kyuubi")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3.4/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
+| <a name="input_metastore_endpoint"></a> [metastore\_endpoint](#input\_metastore\_endpoint) | In-model endpoint for the metastore (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
-| <a name="input_object_storage_endpoint"></a> [object\_storage\_endpoint](#input\_object\_storage\_endpoint) | In-model endpoint for the object storage integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "s3_credentials")<br/>  })</pre> | n/a | yes |
-| <a name="input_tls_endpoint"></a> [tls\_endpoint](#input\_tls\_endpoint) | In-model endpoint for the TLS certificates provider. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "certificates")<br/>  })</pre> | n/a | yes |
+| <a name="input_object_storage_endpoint"></a> [object\_storage\_endpoint](#input\_object\_storage\_endpoint) | In-model endpoint for the object storage integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_tls_endpoint"></a> [tls\_endpoint](#input\_tls\_endpoint) | In-model endpoint for the TLS certificates provider. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
-| <a name="input_users_db_endpoint"></a> [users\_db\_endpoint](#input\_users\_db\_endpoint) | In-model endpoint for the Kyuubi users database (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "database")<br/>  })</pre> | n/a | yes |
-| <a name="input_zookeeper_endpoint"></a> [zookeeper\_endpoint](#input\_zookeeper\_endpoint) | In-model endpoint for the ZooKeeper application. | <pre>object({<br/>    name     = string<br/>    endpoint = optional(string, "zookeeper")<br/>  })</pre> | n/a | yes |
+| <a name="input_users_db_endpoint"></a> [users\_db\_endpoint](#input\_users\_db\_endpoint) | In-model endpoint for the Kyuubi users database (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
+| <a name="input_zookeeper_endpoint"></a> [zookeeper\_endpoint](#input\_zookeeper\_endpoint) | In-model endpoint for the ZooKeeper application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
@@ -60,3 +70,4 @@ No modules.
 | <a name="output_offers"></a> [offers](#output\_offers) | Map of all offers exposed by the component's charms. |
 | <a name="output_provides"></a> [provides](#output\_provides) | Map of all the provides endpoints. |
 | <a name="output_requires"></a> [requires](#output\_requires) | Map of the requires endpoints. |
+<!-- END_TF_DOCS -->
diff --git a/terraform/products/charmed-spark-3.4/README.md b/terraform/products/charmed-spark-3.4/README.md
index 5143a3a3..336c9ebc 100644
--- a/terraform/products/charmed-spark-3.4/README.md
+++ b/terraform/products/charmed-spark-3.4/README.md
@@ -1,17 +1,22 @@
-## Requirements
+# Charmed Apache Spark (3.4) product terraform module
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
-## Modules
+### Modules
 
 | Name | Source | Version |
 |------|--------|---------|
@@ -25,13 +30,13 @@
 | <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev586 |
 | <a name="module_zookeeper"></a> [zookeeper](#module\_zookeeper) | ../../charms/zookeeper | n/a |
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
 | [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
@@ -74,10 +79,11 @@
 | <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
 | <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Define the number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_metadata"></a> [metadata](#output\_metadata) | Metadata of the product deployment. |
 | <a name="output_models"></a> [models](#output\_models) | Maps of the models and the components deployed in each model. |
 | <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
+<!-- END_TF_DOCS -->
diff --git a/terraform/products/charmed-spark-3.5/README.md b/terraform/products/charmed-spark-3.5/README.md
index 5143a3a3..5cc031f9 100644
--- a/terraform/products/charmed-spark-3.5/README.md
+++ b/terraform/products/charmed-spark-3.5/README.md
@@ -1,17 +1,22 @@
-## Requirements
+# Charmed Apache Spark (3.5) product terraform module
+
+## Module reference
+
+<!-- BEGIN_TF_DOCS -->
+### Requirements
 
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >=1.0.0 |
 | <a name="requirement_juju"></a> [juju](#requirement\_juju) | >=1.0.0 |
 
-## Providers
+### Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_juju"></a> [juju](#provider\_juju) | 1.3.1 |
 
-## Modules
+### Modules
 
 | Name | Source | Version |
 |------|--------|---------|
@@ -25,13 +30,13 @@
 | <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev586 |
 | <a name="module_zookeeper"></a> [zookeeper](#module\_zookeeper) | ../../charms/zookeeper | n/a |
 
-## Resources
+### Resources
 
 | Name | Type |
 |------|------|
 | [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
 
-## Inputs
+### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
@@ -74,10 +79,11 @@
 | <a name="input_zookeeper_size"></a> [zookeeper\_size](#input\_zookeeper\_size) | Storage size for the metastore database | `string` | `"10G"` | no |
 | <a name="input_zookeeper_units"></a> [zookeeper\_units](#input\_zookeeper\_units) | Define the number of zookeeper units. 3 units are recommended for high availability. | `number` | `3` | no |
 
-## Outputs
+### Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_metadata"></a> [metadata](#output\_metadata) | Metadata of the product deployment. |
 | <a name="output_models"></a> [models](#output\_models) | Maps of the models and the components deployed in each model. |
 | <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
+<!-- END_TF_DOCS -->

From b6e9c72ee95adcfd4821b3b2a4e5b8a485bfdd11 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 16:38:26 +0200
Subject: [PATCH 37/62] Fix a couple of wrong COS namespaces

---
 python/tests/integration/test_spark_job.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/tests/integration/test_spark_job.py b/python/tests/integration/test_spark_job.py
index 5b425992..80879cbf 100644
--- a/python/tests/integration/test_spark_job.py
+++ b/python/tests/integration/test_spark_job.py
@@ -352,11 +352,11 @@ def test_history_server_metrics_in_cos(
         with attempt:
             cos_address = get_cos_address(cos_model_name=COS)
             assert published_prometheus_data(
-                cos, cos_address, "jmx_scrape_duration_seconds"
+                COS, cos_address, "jmx_scrape_duration_seconds"
             )
 
             # Alerts got published to Prometheus
-            alerts_data = published_prometheus_alerts(cos, cos_address)
+            alerts_data = published_prometheus_alerts(COS, cos_address)
             assert alerts_data is not None
             logger.info(f"Alerts data: {alerts_data}")
 

From 404d802b5d41282cf0893f454a0d303bc53a1ccf Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 18:07:57 +0200
Subject: [PATCH 38/62] Fix module output

---
 python/spark_test/core/bundle/terraform.py      |  4 +++-
 python/tests/integration/resources/main.tf      |  4 ++++
 terraform/products/charmed-spark-3.4/outputs.tf | 12 ++++++------
 terraform/products/charmed-spark-3.5/outputs.tf | 14 +++++++-------
 4 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/python/spark_test/core/bundle/terraform.py b/python/spark_test/core/bundle/terraform.py
index 556ac6bf..c87f3c34 100644
--- a/python/spark_test/core/bundle/terraform.py
+++ b/python/spark_test/core/bundle/terraform.py
@@ -44,7 +44,9 @@ def apply(self, vars: dict[str, str] | None = None) -> list[str]:
             self.execute(
                 ["terraform", "apply", "-auto-approve", "-var-file", tfvars_file.name]
             )
-        self.deployed_applications = []  # FIXME: Parse components output
+        self.deployed_applications = list(
+            self.outputs["model"]["value"]["components"].keys()
+        )
         return self.deployed_applications
 
     def destroy(self):
diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index 64ad1fb5..7c11934a 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -121,3 +121,7 @@ module "spark" {
     metrics   = module.cos[0].offers.prometheus_receive_remote_write.url
   } : null
 }
+
+output "model" {
+  value = module.spark.model
+}
diff --git a/terraform/products/charmed-spark-3.4/outputs.tf b/terraform/products/charmed-spark-3.4/outputs.tf
index 3b8a70c7..603e95de 100644
--- a/terraform/products/charmed-spark-3.4/outputs.tf
+++ b/terraform/products/charmed-spark-3.4/outputs.tf
@@ -20,9 +20,9 @@ output "metadata" {
   }
 }
 
-output "models" {
-  description = "Maps of the models and the components deployed in each model."
-  value = [{
+output "model" {
+  description = "Map of the key of the model and the components deployed in the model."
+  value = {
     model_uuid = var.model_uuid
     components = merge(
       module.spark.components,
@@ -46,8 +46,8 @@ output "models" {
       },
       module.s3 == [] ? {} : {
         s3 = module.s3[0].application
-      }
+      },
+      module.observability == [] ? {} : module.observability[0].components
     )
-    }
-  ]
+  }
 }
diff --git a/terraform/products/charmed-spark-3.5/outputs.tf b/terraform/products/charmed-spark-3.5/outputs.tf
index 3b8a70c7..6d096a7a 100644
--- a/terraform/products/charmed-spark-3.5/outputs.tf
+++ b/terraform/products/charmed-spark-3.5/outputs.tf
@@ -14,15 +14,15 @@ output "offers" {
 output "metadata" {
   description = "Metadata of the product deployment."
   value = {
-    version     = "3.4"
+    version     = "3.5"
     deployed_at = timestamp()
     updated_at  = timestamp()
   }
 }
 
-output "models" {
-  description = "Maps of the models and the components deployed in each model."
-  value = [{
+output "model" {
+  description = "Map of the key of the model and the components deployed in the model."
+  value = {
     model_uuid = var.model_uuid
     components = merge(
       module.spark.components,
@@ -46,8 +46,8 @@ output "models" {
       },
       module.s3 == [] ? {} : {
         s3 = module.s3[0].application
-      }
+      },
+      module.observability == [] ? {} : module.observability[0].components
     )
-    }
-  ]
+  }
 }

From 5e4e3fe1a0277cb1ed0e81bac822e1452f2bb1bc Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 18:17:39 +0200
Subject: [PATCH 39/62] Uncomment basic test with 3.5

---
 .github/workflows/ci-tests-minimal.yaml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci-tests-minimal.yaml b/.github/workflows/ci-tests-minimal.yaml
index bbe5f943..ba143f5d 100644
--- a/.github/workflows/ci-tests-minimal.yaml
+++ b/.github/workflows/ci-tests-minimal.yaml
@@ -42,13 +42,13 @@ jobs:
             k8s-distribution: "k8s"
             k8s-version: "1.32"
 
-          # # Test integration-basic, with Spark 3.5
-          # - tox-env: integration-basic
-          #   spark-version: 3.5.7
-          #   juju-snap-channel: "3.6/stable"
-          #   juju-agent-version: "3.6.14"
-          #   k8s-distribution: "k8s"
-          #   k8s-version: "1.32"
+          # Test integration-basic, with Spark 3.5
+          - tox-env: integration-basic
+            spark-version: 3.5.7
+            juju-snap-channel: "3.6/stable"
+            juju-agent-version: "3.6.14"
+            k8s-distribution: "k8s"
+            k8s-version: "1.32"
 
           # Test COS just once with integration-sparkjob
           - tox-env: integration-sparkjob

From afd52ac1695c3b94dc037ee367562e929594bc95 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 18:18:28 +0200
Subject: [PATCH 40/62] Remove storage size fixture, size hardcoded for the
 tests

---
 python/tests/integration/conftest.py       | 43 +---------------------
 python/tests/integration/resources/main.tf | 23 +++---------
 2 files changed, 6 insertions(+), 60 deletions(-)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index c2a887c0..26266fa7 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -119,15 +119,6 @@ def pytest_addoption(parser):
         help="This, together with the `--model` parameter, ensures that all functions "
         "marked with the` skip_if_deployed` tag are skipped.",
     )
-    parser.addoption(
-        "--storage-sizes",
-        choices=["small", "medium"],
-        nargs="?",
-        const="small",
-        default="small",
-        type=str,
-        help="Setting for the storage.",
-    )
 
 
 def determine_scope(fixture_name, config):
@@ -222,36 +213,6 @@ def container_name(test_uuid):
     return f"spark-container-{test_uuid}"
 
 
-@pytest.fixture(scope="module")
-def storage_sizes(request) -> dict[str, str]:
-    """The name of the model in which COS is either already deployed or is to be deployed."""
-    option = request.config.getoption("--storage-sizes")
-    if option == "small":
-        return {
-            "kyuubi_users_size": "500M",
-            "metastore_size": "500M",
-            "zookeeper_size": "10G",
-            "alertmanager_size": "100M",
-            "grafana_size": "100M",
-            "loki_active_index_directory_size": "100M",
-            "loki_chunks_size": "500M",
-            "prometheus_size": "500M",
-            "traefik_size": "100M",
-        }
-    else:
-        return {
-            "kyuubi_users_size": "1G",
-            "metastore_size": "10G",
-            "zookeeper_size": "10G",
-            "alertmanager_size": "10G",
-            "grafana_size": "10G",
-            "loki_active_index_directory_size": "10G",
-            "loki_chunks_size": "500G",
-            "prometheus_size": "500G",
-            "traefik_size": "10G",
-        }
-
-
 @pytest.fixture
 def pod_name():
     return "my-testpod"
@@ -449,7 +410,6 @@ def spark_bundle(
     object_storage,
     admin_password,
     private_key,
-    storage_sizes,
 ):
     """Deploy the Spark K8s bundle using Terraform."""
     short_version = ".".join(spark_version.split(".")[:2])
@@ -469,7 +429,6 @@ def spark_bundle(
         "model_uuid": cast(str, juju.show_model().model_uuid),
         "storage_backend": storage_backend,
         "create_model": False,
-        "zookeeper_units": 1,
         "admin_password": admin_password,
         "tls_private_key": private_key,
     }
@@ -495,7 +454,7 @@ def spark_bundle(
             }
         }
 
-    vars = base_vars | cos_vars | storage_vars | storage_sizes
+    vars = base_vars | cos_vars | storage_vars
     logger.info(f"Applying vars: {vars.keys()}")
 
     deployed_applications = bundle.apply(vars=vars)
diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index 7c11934a..14f99d3b 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -67,19 +67,6 @@ variable "tls_private_key" {
   default = null
 }
 
-variable "zookeeper_units" {
-  type = number
-}
-variable "kyuubi_users_size" {
-  type = string
-}
-variable "metastore_size" {
-  type = string
-}
-variable "zookeeper_size" {
-  type = string
-}
-
 module "cos" {
   count = var.cos_model_uuid == null ? 0 : 1
   # TODO: Pin to tag once available
@@ -105,15 +92,15 @@ module "spark" {
   create_model = false
 
   admin_password       = var.admin_password
-  kyuubi_config        = var.kyuubi_config
-  kyuubi_users_size    = var.kyuubi_users_size
-  metastore_size       = var.metastore_size
   azure_storage_config = var.azure_storage_config
+  kyuubi_config        = var.kyuubi_config
+  kyuubi_users_size    = "500M"
+  metastore_size       = "500M"
   s3_config            = var.s3_config
   storage_backend      = var.storage_backend
   tls_private_key      = var.tls_private_key
-  zookeeper_size       = var.zookeeper_size
-  zookeeper_units      = var.zookeeper_units
+  zookeeper_size       = "10G"
+  zookeeper_units      = 1
 
   cos_offers = module.cos != [] ? {
     dashboard = module.cos[0].offers.grafana_dashboards.url

From 398fc960286af6b4004343de54b9855c2e54393f Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Wed, 8 Apr 2026 18:19:01 +0200
Subject: [PATCH 41/62] Update output doc

---
 terraform/products/charmed-spark-3.4/README.md | 2 +-
 terraform/products/charmed-spark-3.5/README.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform/products/charmed-spark-3.4/README.md b/terraform/products/charmed-spark-3.4/README.md
index 336c9ebc..b5bf450f 100644
--- a/terraform/products/charmed-spark-3.4/README.md
+++ b/terraform/products/charmed-spark-3.4/README.md
@@ -84,6 +84,6 @@
 | Name | Description |
 |------|-------------|
 | <a name="output_metadata"></a> [metadata](#output\_metadata) | Metadata of the product deployment. |
-| <a name="output_models"></a> [models](#output\_models) | Maps of the models and the components deployed in each model. |
+| <a name="output_model"></a> [model](#output\_model) | Map of the key of the model and the components deployed in the model. |
 | <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
 <!-- END_TF_DOCS -->
diff --git a/terraform/products/charmed-spark-3.5/README.md b/terraform/products/charmed-spark-3.5/README.md
index 5cc031f9..49e2ad3d 100644
--- a/terraform/products/charmed-spark-3.5/README.md
+++ b/terraform/products/charmed-spark-3.5/README.md
@@ -84,6 +84,6 @@
 | Name | Description |
 |------|-------------|
 | <a name="output_metadata"></a> [metadata](#output\_metadata) | Metadata of the product deployment. |
-| <a name="output_models"></a> [models](#output\_models) | Maps of the models and the components deployed in each model. |
+| <a name="output_model"></a> [model](#output\_model) | Map of the key of the model and the components deployed in the model. |
 | <a name="output_offers"></a> [offers](#output\_offers) | The name and url of the various offers being exposed |
 <!-- END_TF_DOCS -->

From e011cae06c5d05641f7e4e227f1675b5786c5e6d Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 9 Apr 2026 16:42:45 +0200
Subject: [PATCH 42/62] Align copyright years

---
 python/tests/integration/resources/main.tf             | 3 +++
 terraform/charms/azure-storage-integrator/outputs.tf   | 4 ++--
 terraform/charms/azure-storage-integrator/providers.tf | 5 -----
 terraform/charms/azure-storage-integrator/terraform.tf | 3 +++
 terraform/charms/azure-storage-integrator/variables.tf | 2 +-
 terraform/charms/data-integrator/providers.tf          | 5 -----
 terraform/charms/s3-integrator-v1/main.tf              | 1 -
 terraform/charms/s3-integrator-v1/outputs.tf           | 2 +-
 terraform/charms/s3-integrator-v1/providers.tf         | 5 -----
 terraform/charms/s3-integrator-v1/terraform.tf         | 3 +++
 terraform/charms/s3-integrator-v1/variables.tf         | 2 +-
 terraform/charms/zookeeper/providers.tf                | 5 -----
 terraform/components/observability/applications.tf     | 2 +-
 terraform/components/observability/integrations.tf     | 2 +-
 terraform/components/observability/locals.tf           | 2 +-
 terraform/components/observability/outputs.tf          | 2 +-
 terraform/components/observability/terraform.tf        | 3 +++
 terraform/components/observability/variables.tf        | 2 +-
 terraform/components/spark-3.4/applications.tf         | 2 +-
 terraform/components/spark-3.4/integrations.tf         | 2 +-
 terraform/components/spark-3.4/locals.tf               | 2 +-
 terraform/components/spark-3.4/outputs.tf              | 2 +-
 terraform/components/spark-3.4/resources.tf            | 2 +-
 terraform/components/spark-3.4/terraform.tf            | 3 +++
 terraform/components/spark-3.4/variables.tf            | 2 +-
 terraform/components/spark-3.5/applications.tf         | 2 +-
 terraform/components/spark-3.5/integrations.tf         | 2 +-
 terraform/components/spark-3.5/locals.tf               | 2 +-
 terraform/components/spark-3.5/outputs.tf              | 2 +-
 terraform/components/spark-3.5/resources.tf            | 2 +-
 terraform/components/spark-3.5/terraform.tf            | 3 +++
 terraform/components/spark-3.5/variables.tf            | 2 +-
 32 files changed, 40 insertions(+), 43 deletions(-)
 delete mode 100644 terraform/charms/azure-storage-integrator/providers.tf
 delete mode 100644 terraform/charms/data-integrator/providers.tf
 delete mode 100644 terraform/charms/s3-integrator-v1/providers.tf
 delete mode 100644 terraform/charms/zookeeper/providers.tf

diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index 14f99d3b..3f8999c1 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -1,3 +1,6 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
 terraform {
   required_version = ">=1.0.0"
 
diff --git a/terraform/charms/azure-storage-integrator/outputs.tf b/terraform/charms/azure-storage-integrator/outputs.tf
index d0594bf4..c92bed73 100644
--- a/terraform/charms/azure-storage-integrator/outputs.tf
+++ b/terraform/charms/azure-storage-integrator/outputs.tf
@@ -1,6 +1,6 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
-# 
+
 output "application" {
   description = "Object representing the deployed application."
   value       = juju_application.azure_storage
diff --git a/terraform/charms/azure-storage-integrator/providers.tf b/terraform/charms/azure-storage-integrator/providers.tf
deleted file mode 100644
index 58f9c66b..00000000
--- a/terraform/charms/azure-storage-integrator/providers.tf
+++ /dev/null
@@ -1,5 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Provider configuration for azure-storage-integrator.
-# Provider blocks, if any, should be defined here.
diff --git a/terraform/charms/azure-storage-integrator/terraform.tf b/terraform/charms/azure-storage-integrator/terraform.tf
index 6412125a..63a76877 100644
--- a/terraform/charms/azure-storage-integrator/terraform.tf
+++ b/terraform/charms/azure-storage-integrator/terraform.tf
@@ -1,3 +1,6 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
 terraform {
   required_version = ">=1.0.0"
 
diff --git a/terraform/charms/azure-storage-integrator/variables.tf b/terraform/charms/azure-storage-integrator/variables.tf
index 9791f049..45dc77b0 100644
--- a/terraform/charms/azure-storage-integrator/variables.tf
+++ b/terraform/charms/azure-storage-integrator/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 variable "app_name" {
diff --git a/terraform/charms/data-integrator/providers.tf b/terraform/charms/data-integrator/providers.tf
deleted file mode 100644
index bb5bbccd..00000000
--- a/terraform/charms/data-integrator/providers.tf
+++ /dev/null
@@ -1,5 +0,0 @@
-# Copyright 2026  Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Provider configuration for data-integrator.
-# Provider blocks, if any, should be defined here.
diff --git a/terraform/charms/s3-integrator-v1/main.tf b/terraform/charms/s3-integrator-v1/main.tf
index 5b07ace9..e3414d4e 100644
--- a/terraform/charms/s3-integrator-v1/main.tf
+++ b/terraform/charms/s3-integrator-v1/main.tf
@@ -1,7 +1,6 @@
 # Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
-
 resource "juju_application" "s3_integrator" {
   name       = var.app_name
   model_uuid = var.model_uuid
diff --git a/terraform/charms/s3-integrator-v1/outputs.tf b/terraform/charms/s3-integrator-v1/outputs.tf
index 20651908..9e257a15 100644
--- a/terraform/charms/s3-integrator-v1/outputs.tf
+++ b/terraform/charms/s3-integrator-v1/outputs.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 output "application" {
diff --git a/terraform/charms/s3-integrator-v1/providers.tf b/terraform/charms/s3-integrator-v1/providers.tf
deleted file mode 100644
index efca3ca1..00000000
--- a/terraform/charms/s3-integrator-v1/providers.tf
+++ /dev/null
@@ -1,5 +0,0 @@
-# Copyright 2024 Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Provider configuration for s3-integrator-v1.
-# Provider blocks, if any, should be defined here.
diff --git a/terraform/charms/s3-integrator-v1/terraform.tf b/terraform/charms/s3-integrator-v1/terraform.tf
index 6412125a..63a76877 100644
--- a/terraform/charms/s3-integrator-v1/terraform.tf
+++ b/terraform/charms/s3-integrator-v1/terraform.tf
@@ -1,3 +1,6 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
 terraform {
   required_version = ">=1.0.0"
 
diff --git a/terraform/charms/s3-integrator-v1/variables.tf b/terraform/charms/s3-integrator-v1/variables.tf
index 82be35cb..90812f1a 100644
--- a/terraform/charms/s3-integrator-v1/variables.tf
+++ b/terraform/charms/s3-integrator-v1/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 variable "app_name" {
diff --git a/terraform/charms/zookeeper/providers.tf b/terraform/charms/zookeeper/providers.tf
deleted file mode 100644
index 528856dd..00000000
--- a/terraform/charms/zookeeper/providers.tf
+++ /dev/null
@@ -1,5 +0,0 @@
-# Copyright 2026  Canonical Ltd.
-# See LICENSE file for licensing details.
-
-# Provider configuration for zookeeper.
-# Provider blocks, if any, should be defined here.
diff --git a/terraform/components/observability/applications.tf b/terraform/components/observability/applications.tf
index 9ea5298a..d15284c4 100644
--- a/terraform/components/observability/applications.tf
+++ b/terraform/components/observability/applications.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 resource "juju_application" "grafana_agent" {
diff --git a/terraform/components/observability/integrations.tf b/terraform/components/observability/integrations.tf
index 349b2164..276e0bd2 100644
--- a/terraform/components/observability/integrations.tf
+++ b/terraform/components/observability/integrations.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 data "juju_offer" "grafana_dashboards" {
diff --git a/terraform/components/observability/locals.tf b/terraform/components/observability/locals.tf
index 560b28db..b3310a8c 100644
--- a/terraform/components/observability/locals.tf
+++ b/terraform/components/observability/locals.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 locals {
diff --git a/terraform/components/observability/outputs.tf b/terraform/components/observability/outputs.tf
index cc89038b..8f082b95 100644
--- a/terraform/components/observability/outputs.tf
+++ b/terraform/components/observability/outputs.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 output "components" {
diff --git a/terraform/components/observability/terraform.tf b/terraform/components/observability/terraform.tf
index 6412125a..63a76877 100644
--- a/terraform/components/observability/terraform.tf
+++ b/terraform/components/observability/terraform.tf
@@ -1,3 +1,6 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
 terraform {
   required_version = ">=1.0.0"
 
diff --git a/terraform/components/observability/variables.tf b/terraform/components/observability/variables.tf
index d5cf5d5e..e546e30f 100644
--- a/terraform/components/observability/variables.tf
+++ b/terraform/components/observability/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 variable "model_uuid" {
diff --git a/terraform/components/spark-3.4/applications.tf b/terraform/components/spark-3.4/applications.tf
index a1409b03..e564bb73 100644
--- a/terraform/components/spark-3.4/applications.tf
+++ b/terraform/components/spark-3.4/applications.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 resource "juju_application" "history_server" {
diff --git a/terraform/components/spark-3.4/integrations.tf b/terraform/components/spark-3.4/integrations.tf
index 4bc8fc38..937b1072 100644
--- a/terraform/components/spark-3.4/integrations.tf
+++ b/terraform/components/spark-3.4/integrations.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 resource "juju_integration" "kyuubi_metastore" {
diff --git a/terraform/components/spark-3.4/locals.tf b/terraform/components/spark-3.4/locals.tf
index 2ed303b9..2ea094b2 100644
--- a/terraform/components/spark-3.4/locals.tf
+++ b/terraform/components/spark-3.4/locals.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 locals {
diff --git a/terraform/components/spark-3.4/outputs.tf b/terraform/components/spark-3.4/outputs.tf
index 9ec9b073..0a9139fb 100644
--- a/terraform/components/spark-3.4/outputs.tf
+++ b/terraform/components/spark-3.4/outputs.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 output "components" {
diff --git a/terraform/components/spark-3.4/resources.tf b/terraform/components/spark-3.4/resources.tf
index 1dc65f8e..fe9e3333 100644
--- a/terraform/components/spark-3.4/resources.tf
+++ b/terraform/components/spark-3.4/resources.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 resource "juju_secret" "system_users_and_private_key_secret" {
diff --git a/terraform/components/spark-3.4/terraform.tf b/terraform/components/spark-3.4/terraform.tf
index 6412125a..63a76877 100644
--- a/terraform/components/spark-3.4/terraform.tf
+++ b/terraform/components/spark-3.4/terraform.tf
@@ -1,3 +1,6 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
 terraform {
   required_version = ">=1.0.0"
 
diff --git a/terraform/components/spark-3.4/variables.tf b/terraform/components/spark-3.4/variables.tf
index 86d969ab..e1832aec 100644
--- a/terraform/components/spark-3.4/variables.tf
+++ b/terraform/components/spark-3.4/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 variable "model_uuid" {
diff --git a/terraform/components/spark-3.5/applications.tf b/terraform/components/spark-3.5/applications.tf
index a1409b03..e564bb73 100644
--- a/terraform/components/spark-3.5/applications.tf
+++ b/terraform/components/spark-3.5/applications.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 resource "juju_application" "history_server" {
diff --git a/terraform/components/spark-3.5/integrations.tf b/terraform/components/spark-3.5/integrations.tf
index 4bc8fc38..937b1072 100644
--- a/terraform/components/spark-3.5/integrations.tf
+++ b/terraform/components/spark-3.5/integrations.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 resource "juju_integration" "kyuubi_metastore" {
diff --git a/terraform/components/spark-3.5/locals.tf b/terraform/components/spark-3.5/locals.tf
index e13bd58b..707de05f 100644
--- a/terraform/components/spark-3.5/locals.tf
+++ b/terraform/components/spark-3.5/locals.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 locals {
diff --git a/terraform/components/spark-3.5/outputs.tf b/terraform/components/spark-3.5/outputs.tf
index 9ec9b073..0a9139fb 100644
--- a/terraform/components/spark-3.5/outputs.tf
+++ b/terraform/components/spark-3.5/outputs.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 output "components" {
diff --git a/terraform/components/spark-3.5/resources.tf b/terraform/components/spark-3.5/resources.tf
index 1dc65f8e..fe9e3333 100644
--- a/terraform/components/spark-3.5/resources.tf
+++ b/terraform/components/spark-3.5/resources.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 resource "juju_secret" "system_users_and_private_key_secret" {
diff --git a/terraform/components/spark-3.5/terraform.tf b/terraform/components/spark-3.5/terraform.tf
index 6412125a..63a76877 100644
--- a/terraform/components/spark-3.5/terraform.tf
+++ b/terraform/components/spark-3.5/terraform.tf
@@ -1,3 +1,6 @@
+# Copyright 2026 Canonical Ltd.
+# See LICENSE file for licensing details.
+
 terraform {
   required_version = ">=1.0.0"
 
diff --git a/terraform/components/spark-3.5/variables.tf b/terraform/components/spark-3.5/variables.tf
index 86d969ab..e1832aec 100644
--- a/terraform/components/spark-3.5/variables.tf
+++ b/terraform/components/spark-3.5/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2024 Canonical Ltd.
+# Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
 
 variable "model_uuid" {

From 135cf1d7b2305c4b0881d0eac70a5c5a55e8e3a6 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 9 Apr 2026 17:08:20 +0200
Subject: [PATCH 43/62] Move Azure Storage secret handling to product module

---
 python/tests/integration/conftest.py          |  4 +-
 python/tests/integration/resources/main.tf    |  7 ++++
 .../charms/azure-storage-integrator/README.md |  5 +--
 .../charms/azure-storage-integrator/main.tf   | 22 +---------
 .../azure-storage-integrator/variables.tf     |  9 +---
 .../products/charmed-spark-3.4/README.md      |  5 ++-
 terraform/products/charmed-spark-3.4/main.tf  | 40 +++++++++++++-----
 .../products/charmed-spark-3.4/variables.tf   |  9 +++-
 .../products/charmed-spark-3.5/README.md      |  5 ++-
 terraform/products/charmed-spark-3.5/main.tf  | 42 ++++++++++++++-----
 .../products/charmed-spark-3.5/variables.tf   |  9 +++-
 11 files changed, 97 insertions(+), 60 deletions(-)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index 26266fa7..4471e1d5 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -440,9 +440,9 @@ def spark_bundle(
             "azure_storage_config": {
                 "storage_account": storage_unit.credentials.storage_account,
                 "container": storage_unit.container_name,
-                "secret_key": storage_unit.credentials.secret_key,
                 "path": "spark-events",
-            }
+            },
+            "azure_storage_secret_key": storage_unit.credentials.secret_key,
         }
     else:
         storage_unit = cast(Bucket, object_storage)
diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index 3f8999c1..114cd7ef 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -60,6 +60,13 @@ variable "azure_storage_config" {
   default = {}
 }
 
+variable "azure_storage_secret_key" {
+  description = "Secret key to the Azure Storage account."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
 variable "s3_config" {
   type    = map(any)
   default = {}
diff --git a/terraform/charms/azure-storage-integrator/README.md b/terraform/charms/azure-storage-integrator/README.md
index cb92ef71..4b2789c5 100644
--- a/terraform/charms/azure-storage-integrator/README.md
+++ b/terraform/charms/azure-storage-integrator/README.md
@@ -26,20 +26,17 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [juju_access_secret.azure_storage_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
 | [juju_application.azure_storage](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_offer.azure_storage_credentials](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_secret.azure_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
 ### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_app_name"></a> [app\_name](#input\_app\_name) | Name to give the deployed application. | `string` | `"azure-storage-integrator"` | no |
-| <a name="input_azure_storage_secret_key"></a> [azure\_storage\_secret\_key](#input\_azure\_storage\_secret\_key) | Secret key to the Azure Storage account. | `string` | n/a | yes |
 | <a name="input_base"></a> [base](#input\_base) | The operating system on which to deploy | `string` | `"ubuntu@22.04"` | no |
 | <a name="input_channel"></a> [channel](#input\_channel) | Channel of the charm. | `string` | `"latest/edge"` | no |
-| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    connection-protocol = optional(string, "abfss")<br/>    container           = optional(string)<br/>    endpoint            = optional(string)<br/>    path                = optional(string)<br/>    resource-group      = optional(string)<br/>    storage-account     = optional(string)<br/>  })</pre> | `{}` | no |
+| <a name="input_config"></a> [config](#input\_config) | Map for configuration options. | <pre>object({<br/>    connection-protocol = optional(string, "abfss")<br/>    container           = optional(string)<br/>    credentials         = optional(string)<br/>    endpoint            = optional(string)<br/>    path                = optional(string)<br/>    resource-group      = optional(string)<br/>    storage-account     = optional(string)<br/>  })</pre> | `{}` | no |
 | <a name="input_constraints"></a> [constraints](#input\_constraints) | String listing constraints for this application. | `string` | `null` | no |
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
 | <a name="input_revision"></a> [revision](#input\_revision) | Revision number of the charm. | `number` | `null` | no |
diff --git a/terraform/charms/azure-storage-integrator/main.tf b/terraform/charms/azure-storage-integrator/main.tf
index 5c314e80..cf67f448 100644
--- a/terraform/charms/azure-storage-integrator/main.tf
+++ b/terraform/charms/azure-storage-integrator/main.tf
@@ -9,31 +9,11 @@ resource "juju_application" "azure_storage" {
     channel  = var.channel
     revision = var.revision
   }
-  config = merge(var.config, {
-    credentials = "secret:${juju_secret.azure_storage_secret.secret_id}"
-  })
+  config      = var.config
   units       = var.units
   constraints = var.constraints
 }
 
-# TODO: Secret creation must be handled at the product level
-resource "juju_secret" "azure_storage_secret" {
-  model_uuid = var.model_uuid
-  name       = "azure_storage_secret"
-  value = {
-    secret-key = var.azure_storage_secret_key
-  }
-  info = "This is the secret key for the Azure storage account"
-}
-
-resource "juju_access_secret" "azure_storage_secret_access" {
-  model_uuid = var.model_uuid
-  applications = [
-    juju_application.azure_storage.name
-  ]
-  secret_id = juju_secret.azure_storage_secret.secret_id
-}
-
 resource "juju_offer" "azure_storage_credentials" {
   model_uuid       = var.model_uuid
   application_name = juju_application.azure_storage.name
diff --git a/terraform/charms/azure-storage-integrator/variables.tf b/terraform/charms/azure-storage-integrator/variables.tf
index 45dc77b0..8fd6bd45 100644
--- a/terraform/charms/azure-storage-integrator/variables.tf
+++ b/terraform/charms/azure-storage-integrator/variables.tf
@@ -27,6 +27,7 @@ variable "config" {
   type = object({
     connection-protocol = optional(string, "abfss")
     container           = optional(string)
+    credentials         = optional(string)
     endpoint            = optional(string)
     path                = optional(string)
     resource-group      = optional(string)
@@ -60,11 +61,3 @@ variable "units" {
   type        = number
   default     = 1
 }
-
-# We should eventually handle this differently 
-variable "azure_storage_secret_key" {
-  description = "Secret key to the Azure Storage account."
-  type        = string
-  nullable    = false
-  sensitive   = true
-}
diff --git a/terraform/products/charmed-spark-3.4/README.md b/terraform/products/charmed-spark-3.4/README.md
index b5bf450f..db32a9b3 100644
--- a/terraform/products/charmed-spark-3.4/README.md
+++ b/terraform/products/charmed-spark-3.4/README.md
@@ -34,7 +34,9 @@
 
 | Name | Type |
 |------|------|
+| [juju_access_secret.azure_storage_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
 | [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
+| [juju_secret.azure_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
 ### Inputs
 
@@ -43,8 +45,9 @@
 | <a name="input_K8S_CLOUD"></a> [K8S\_CLOUD](#input\_K8S\_CLOUD) | The kubernetes juju cloud name. | `string` | `"microk8s"` | no |
 | <a name="input_K8S_CREDENTIAL"></a> [K8S\_CREDENTIAL](#input\_K8S\_CREDENTIAL) | The name of the kubernetes juju credential. | `string` | `"microk8s"` | no |
 | <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the Kyuubi admin user. | `string` | `null` | no |
-| <a name="input_azure_storage_config"></a> [azure\_storage\_config](#input\_azure\_storage\_config) | Azure Object storage information | <pre>object({<br/>    container       = optional(string)<br/>    path            = optional(string)<br/>    protocol        = optional(string, "abfss")<br/>    secret_key      = optional(string)<br/>    storage_account = optional(string)<br/>  })</pre> | `{}` | no |
+| <a name="input_azure_storage_config"></a> [azure\_storage\_config](#input\_azure\_storage\_config) | Azure Object storage information | <pre>object({<br/>    container       = optional(string)<br/>    credentials     = optional(string)<br/>    path            = optional(string)<br/>    protocol        = optional(string, "abfss")<br/>    storage_account = optional(string)<br/>  })</pre> | `{}` | no |
 | <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | `null` | no |
+| <a name="input_azure_storage_secret_key"></a> [azure\_storage\_secret\_key](#input\_azure\_storage\_secret\_key) | Secret key to the Azure Storage account. | `string` | `null` | no |
 | <a name="input_certificate_common_name"></a> [certificate\_common\_name](#input\_certificate\_common\_name) | Common name for the certificate to be used in self-signed | `string` | `"charmed-spark"` | no |
 | <a name="input_cos_offers"></a> [cos\_offers](#input\_cos\_offers) | Observability stack offers. | <pre>object({<br/>    dashboard = string<br/>    logging   = string<br/>    metrics   = string<br/>  })</pre> | `null` | no |
 | <a name="input_create_model"></a> [create\_model](#input\_create\_model) | Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism. | `bool` | `true` | no |
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index 04ff7a04..470d9f87 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -92,25 +92,45 @@ module "data_integrator" {
   revision    = local.revisions.data_integrator
 }
 
-module "azure_storage" {
+resource "juju_secret" "azure_storage_secret" {
   depends_on = [juju_model.spark]
   count      = var.storage_backend == "azure_storage" ? 1 : 0
+  model_uuid = var.model_uuid
+  name       = "azure_storage_secret"
+  value = {
+    secret-key = var.azure_storage_secret_key
+  }
+  info = "This is the secret key for the Azure storage account"
+}
+
+module "azure_storage" {
+  depends_on = [juju_model.spark, juju_secret.azure_storage_secret]
+  count      = var.storage_backend == "azure_storage" ? 1 : 0
   source     = "../../charms/azure-storage-integrator"
   model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
-  azure_storage_secret_key = var.azure_storage_config.secret_key
-  base                     = "ubuntu@22.04"
-  channel                  = "latest/edge"
-  config = {
-    connection-protocol = var.azure_storage_config.protocol
-    container           = var.azure_storage_config.container
-    path                = var.azure_storage_config.path
-    storage-account     = var.azure_storage_config.storage_account
-  }
+  base    = "ubuntu@22.04"
+  channel = "latest/edge"
+  config = merge(
+    {
+      credentials = "secret:${juju_secret.azure_storage_secret[0].secret_id}"
+    },
+    var.azure_storage_config
+  )
   constraints = "arch=amd64"
   revision    = local.revisions.azure_storage
 }
 
+resource "juju_access_secret" "azure_storage_secret_access" {
+  depends_on = [juju_model.spark, juju_secret.azure_storage_secret, module.azure_storage]
+  count      = var.storage_backend == "azure_storage" ? 1 : 0
+  model_uuid = var.model_uuid
+  applications = [
+    module.azure_storage[0].application.name
+  ]
+  secret_id = juju_secret.azure_storage_secret[0].secret_id
+}
+
 module "s3" {
   depends_on = [juju_model.spark]
   count      = var.storage_backend == "s3" ? 1 : 0
diff --git a/terraform/products/charmed-spark-3.4/variables.tf b/terraform/products/charmed-spark-3.4/variables.tf
index 36b5aadb..4d3b79c2 100644
--- a/terraform/products/charmed-spark-3.4/variables.tf
+++ b/terraform/products/charmed-spark-3.4/variables.tf
@@ -13,9 +13,9 @@ variable "azure_storage_config" {
   description = "Azure Object storage information"
   type = object({
     container       = optional(string)
+    credentials     = optional(string)
     path            = optional(string)
     protocol        = optional(string, "abfss")
-    secret_key      = optional(string)
     storage_account = optional(string)
   })
   default = {}
@@ -27,6 +27,13 @@ variable "azure_storage_revision" {
   default     = null
 }
 
+variable "azure_storage_secret_key" {
+  description = "Secret key to the Azure Storage account."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
 variable "certificate_common_name" {
   description = "Common name for the certificate to be used in self-signed"
   type        = string
diff --git a/terraform/products/charmed-spark-3.5/README.md b/terraform/products/charmed-spark-3.5/README.md
index 49e2ad3d..b93846bd 100644
--- a/terraform/products/charmed-spark-3.5/README.md
+++ b/terraform/products/charmed-spark-3.5/README.md
@@ -34,7 +34,9 @@
 
 | Name | Type |
 |------|------|
+| [juju_access_secret.azure_storage_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
 | [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
+| [juju_secret.azure_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
 ### Inputs
 
@@ -43,8 +45,9 @@
 | <a name="input_K8S_CLOUD"></a> [K8S\_CLOUD](#input\_K8S\_CLOUD) | The kubernetes juju cloud name. | `string` | `"microk8s"` | no |
 | <a name="input_K8S_CREDENTIAL"></a> [K8S\_CREDENTIAL](#input\_K8S\_CREDENTIAL) | The name of the kubernetes juju credential. | `string` | `"microk8s"` | no |
 | <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the Kyuubi admin user. | `string` | `null` | no |
-| <a name="input_azure_storage_config"></a> [azure\_storage\_config](#input\_azure\_storage\_config) | Azure Object storage information | <pre>object({<br/>    container       = optional(string)<br/>    path            = optional(string)<br/>    protocol        = optional(string, "abfss")<br/>    secret_key      = optional(string)<br/>    storage_account = optional(string)<br/>  })</pre> | `{}` | no |
+| <a name="input_azure_storage_config"></a> [azure\_storage\_config](#input\_azure\_storage\_config) | Azure Object storage information | <pre>object({<br/>    container       = optional(string)<br/>    credentials     = optional(string)<br/>    path            = optional(string)<br/>    protocol        = optional(string, "abfss")<br/>    storage_account = optional(string)<br/>  })</pre> | `{}` | no |
 | <a name="input_azure_storage_revision"></a> [azure\_storage\_revision](#input\_azure\_storage\_revision) | Charm revision for azure-storage-integrator | `number` | `null` | no |
+| <a name="input_azure_storage_secret_key"></a> [azure\_storage\_secret\_key](#input\_azure\_storage\_secret\_key) | Secret key to the Azure Storage account. | `string` | `null` | no |
 | <a name="input_certificate_common_name"></a> [certificate\_common\_name](#input\_certificate\_common\_name) | Common name for the certificate to be used in self-signed | `string` | `"charmed-spark"` | no |
 | <a name="input_cos_offers"></a> [cos\_offers](#input\_cos\_offers) | Observability stack offers. | <pre>object({<br/>    dashboard = string<br/>    logging   = string<br/>    metrics   = string<br/>  })</pre> | `null` | no |
 | <a name="input_create_model"></a> [create\_model](#input\_create\_model) | Should terraform create the Juju models? If set to false, assume the models are created by a different mechanism. | `bool` | `true` | no |
diff --git a/terraform/products/charmed-spark-3.5/main.tf b/terraform/products/charmed-spark-3.5/main.tf
index 04ff7a04..02676eb8 100644
--- a/terraform/products/charmed-spark-3.5/main.tf
+++ b/terraform/products/charmed-spark-3.5/main.tf
@@ -92,25 +92,45 @@ module "data_integrator" {
   revision    = local.revisions.data_integrator
 }
 
-module "azure_storage" {
+resource "juju_secret" "azure_storage_secret" {
   depends_on = [juju_model.spark]
   count      = var.storage_backend == "azure_storage" ? 1 : 0
+  model_uuid = var.model_uuid
+  name       = "azure_storage_secret"
+  value = {
+    secret-key = var.azure_storage_secret_key
+  }
+  info = "This is the secret key for the Azure storage account"
+}
+
+module "azure_storage" {
+  depends_on = [juju_model.spark, juju_secret.azure_storage_secret]
+  count      = var.storage_backend == "azure_storage" ? 1 : 0
   source     = "../../charms/azure-storage-integrator"
   model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
-  azure_storage_secret_key = var.azure_storage_config.secret_key
-  base                     = "ubuntu@22.04"
-  channel                  = "latest/edge"
-  config = {
-    connection-protocol = var.azure_storage_config.protocol
-    container           = var.azure_storage_config.container
-    path                = var.azure_storage_config.path
-    storage-account     = var.azure_storage_config.storage_account
-  }
+  base    = "ubuntu@22.04"
+  channel = "latest/edge"
+  config = merge(
+    {
+      credentials = "secret:${juju_secret.azure_storage_secret[0].secret_id}"
+    },
+    var.azure_storage_config
+  )
   constraints = "arch=amd64"
   revision    = local.revisions.azure_storage
 }
 
+resource "juju_access_secret" "azure_storage_secret_access" {
+  depends_on = [juju_model.spark, juju_secret.azure_storage_secret, module.azure_storage]
+  count      = var.storage_backend == "azure_storage" ? 1 : 0
+  model_uuid = var.model_uuid
+  applications = [
+    module.azure_storage[0].application.name
+  ]
+  secret_id = juju_secret.azure_storage_secret[0].secret_id
+}
+
 module "s3" {
   depends_on = [juju_model.spark]
   count      = var.storage_backend == "s3" ? 1 : 0
@@ -136,7 +156,7 @@ module "spark" {
     module.ssc,
     module.zookeeper
   ]
-  source     = "../../components/spark-3.4"
+  source     = "../../components/spark-3.5"
   model_uuid = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
 
   history_server = {
diff --git a/terraform/products/charmed-spark-3.5/variables.tf b/terraform/products/charmed-spark-3.5/variables.tf
index 36b5aadb..4d3b79c2 100644
--- a/terraform/products/charmed-spark-3.5/variables.tf
+++ b/terraform/products/charmed-spark-3.5/variables.tf
@@ -13,9 +13,9 @@ variable "azure_storage_config" {
   description = "Azure Object storage information"
   type = object({
     container       = optional(string)
+    credentials     = optional(string)
     path            = optional(string)
     protocol        = optional(string, "abfss")
-    secret_key      = optional(string)
     storage_account = optional(string)
   })
   default = {}
@@ -27,6 +27,13 @@ variable "azure_storage_revision" {
   default     = null
 }
 
+variable "azure_storage_secret_key" {
+  description = "Secret key to the Azure Storage account."
+  type        = string
+  sensitive   = true
+  default     = null
+}
+
 variable "certificate_common_name" {
   description = "Common name for the certificate to be used in self-signed"
   type        = string

From c1ee0e3875fd5c9200bfa04a31cd00fc87c8ef3e Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 9 Apr 2026 17:27:41 +0200
Subject: [PATCH 44/62] Move Kyuubi secret handling to product

---
 terraform/components/spark-3.4/README.md      |  4 --
 .../components/spark-3.4/applications.tf      | 11 +----
 terraform/components/spark-3.4/resources.tf   | 24 -----------
 terraform/components/spark-3.4/variables.tf   | 14 -------
 terraform/components/spark-3.5/README.md      |  4 --
 .../components/spark-3.5/applications.tf      | 11 +----
 terraform/components/spark-3.5/resources.tf   | 24 -----------
 terraform/components/spark-3.5/variables.tf   | 14 -------
 .../products/charmed-spark-3.4/README.md      |  2 +
 terraform/products/charmed-spark-3.4/main.tf  | 42 +++++++++++++++++--
 .../products/charmed-spark-3.5/README.md      |  4 +-
 terraform/products/charmed-spark-3.5/main.tf  | 42 +++++++++++++++++--
 12 files changed, 85 insertions(+), 111 deletions(-)

diff --git a/terraform/components/spark-3.4/README.md b/terraform/components/spark-3.4/README.md
index 62b92274..d590f1c5 100644
--- a/terraform/components/spark-3.4/README.md
+++ b/terraform/components/spark-3.4/README.md
@@ -29,7 +29,6 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
 | [juju_application.history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_application.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_application.kyuubi](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
@@ -43,13 +42,11 @@ No modules.
 | [juju_integration.kyuubi_zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
 | [juju_offer.integration_hub_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
 | [juju_offer.kyuubi_jdbc](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
 ### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
 | <a name="input_data_integrator_endpoint"></a> [data\_integrator\_endpoint](#input\_data\_integrator\_endpoint) | In-model endpoint for the data-integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_history_server"></a> [history\_server](#input\_history\_server) | n/a | <pre>object({<br/>    app_name    = optional(string, "history-server")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
 | <a name="input_integration_hub"></a> [integration\_hub](#input\_integration\_hub) | n/a | <pre>object({<br/>    app_name    = optional(string, "integration-hub")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
@@ -58,7 +55,6 @@ No modules.
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
 | <a name="input_object_storage_endpoint"></a> [object\_storage\_endpoint](#input\_object\_storage\_endpoint) | In-model endpoint for the object storage integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_tls_endpoint"></a> [tls\_endpoint](#input\_tls\_endpoint) | In-model endpoint for the TLS certificates provider. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
-| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
 | <a name="input_users_db_endpoint"></a> [users\_db\_endpoint](#input\_users\_db\_endpoint) | In-model endpoint for the Kyuubi users database (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_zookeeper_endpoint"></a> [zookeeper\_endpoint](#input\_zookeeper\_endpoint) | In-model endpoint for the ZooKeeper application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 
diff --git a/terraform/components/spark-3.4/applications.tf b/terraform/components/spark-3.4/applications.tf
index e564bb73..791ee63c 100644
--- a/terraform/components/spark-3.4/applications.tf
+++ b/terraform/components/spark-3.4/applications.tf
@@ -52,16 +52,7 @@ resource "juju_application" "kyuubi" {
     revision = var.kyuubi.revision != null ? var.kyuubi.revision : local.revisions.kyuubi
   }
 
-  config = merge(
-    {
-      expose-external = "loadbalancer",
-    },
-    length(juju_secret.system_users_and_private_key_secret) > 0 ? {
-      system-users           = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}",
-      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    } : {},
-    var.kyuubi.config
-  )
+  config      = var.kyuubi.config
   constraints = var.kyuubi.constraints
   resources = merge({
     kyuubi-image = local.images.kyuubi
diff --git a/terraform/components/spark-3.4/resources.tf b/terraform/components/spark-3.4/resources.tf
index fe9e3333..fa89e9d7 100644
--- a/terraform/components/spark-3.4/resources.tf
+++ b/terraform/components/spark-3.4/resources.tf
@@ -1,26 +1,2 @@
 # Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
-
-resource "juju_secret" "system_users_and_private_key_secret" {
-  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model_uuid = var.model_uuid
-  name       = "system_users_and_private_key_secret"
-  value = merge(
-    var.admin_password == null ? {} : {
-      admin = var.admin_password
-    },
-    var.tls_private_key == null ? {} : {
-      private-key = var.tls_private_key
-    }
-  )
-  info = "This secret contains password for admin user and the TLS private key."
-}
-
-resource "juju_access_secret" "system_users_and_private_key_secret_access" {
-  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model_uuid = var.model_uuid
-  applications = [
-    juju_application.kyuubi.name
-  ]
-  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
-}
diff --git a/terraform/components/spark-3.4/variables.tf b/terraform/components/spark-3.4/variables.tf
index e1832aec..de28d0be 100644
--- a/terraform/components/spark-3.4/variables.tf
+++ b/terraform/components/spark-3.4/variables.tf
@@ -50,13 +50,6 @@ variable "kyuubi" {
   default = {}
 }
 
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
 variable "data_integrator_endpoint" {
   description = "In-model endpoint for the data-integrator application."
   type = object({
@@ -81,13 +74,6 @@ variable "tls_endpoint" {
   })
 }
 
-variable "tls_private_key" {
-  description = "The private key to be used for TLS certificates."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
 variable "users_db_endpoint" {
   description = "In-model endpoint for the Kyuubi users database (postgresql-k8s) application."
   type = object({
diff --git a/terraform/components/spark-3.5/README.md b/terraform/components/spark-3.5/README.md
index 21d5481f..88fb05c9 100644
--- a/terraform/components/spark-3.5/README.md
+++ b/terraform/components/spark-3.5/README.md
@@ -29,7 +29,6 @@ No modules.
 
 | Name | Type |
 |------|------|
-| [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
 | [juju_application.history_server](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_application.integration_hub](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
 | [juju_application.kyuubi](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/application) | resource |
@@ -43,13 +42,11 @@ No modules.
 | [juju_integration.kyuubi_zookeeper](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/integration) | resource |
 | [juju_offer.integration_hub_service_account](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
 | [juju_offer.kyuubi_jdbc](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/offer) | resource |
-| [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
 ### Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_admin_password"></a> [admin\_password](#input\_admin\_password) | The password for the admin user. | `string` | `null` | no |
 | <a name="input_data_integrator_endpoint"></a> [data\_integrator\_endpoint](#input\_data\_integrator\_endpoint) | In-model endpoint for the data-integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_history_server"></a> [history\_server](#input\_history\_server) | n/a | <pre>object({<br/>    app_name    = optional(string, "history-server")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
 | <a name="input_integration_hub"></a> [integration\_hub](#input\_integration\_hub) | n/a | <pre>object({<br/>    app_name    = optional(string, "integration-hub")<br/>    base        = optional(string, "ubuntu@22.04")<br/>    channel     = optional(string, "3/stable")<br/>    config      = optional(map(string), {})<br/>    constraints = optional(string, "arch=amd64")<br/>    resources   = optional(map(string), {})<br/>    revision    = optional(number)<br/>    units       = optional(number, 1)<br/>  })</pre> | `{}` | no |
@@ -58,7 +55,6 @@ No modules.
 | <a name="input_model_uuid"></a> [model\_uuid](#input\_model\_uuid) | Reference to an existing model uuid. | `string` | n/a | yes |
 | <a name="input_object_storage_endpoint"></a> [object\_storage\_endpoint](#input\_object\_storage\_endpoint) | In-model endpoint for the object storage integrator application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_tls_endpoint"></a> [tls\_endpoint](#input\_tls\_endpoint) | In-model endpoint for the TLS certificates provider. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
-| <a name="input_tls_private_key"></a> [tls\_private\_key](#input\_tls\_private\_key) | The private key to be used for TLS certificates. | `string` | `null` | no |
 | <a name="input_users_db_endpoint"></a> [users\_db\_endpoint](#input\_users\_db\_endpoint) | In-model endpoint for the Kyuubi users database (postgresql-k8s) application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 | <a name="input_zookeeper_endpoint"></a> [zookeeper\_endpoint](#input\_zookeeper\_endpoint) | In-model endpoint for the ZooKeeper application. | <pre>object({<br/>    name     = string<br/>    endpoint = string<br/>  })</pre> | n/a | yes |
 
diff --git a/terraform/components/spark-3.5/applications.tf b/terraform/components/spark-3.5/applications.tf
index e564bb73..791ee63c 100644
--- a/terraform/components/spark-3.5/applications.tf
+++ b/terraform/components/spark-3.5/applications.tf
@@ -52,16 +52,7 @@ resource "juju_application" "kyuubi" {
     revision = var.kyuubi.revision != null ? var.kyuubi.revision : local.revisions.kyuubi
   }
 
-  config = merge(
-    {
-      expose-external = "loadbalancer",
-    },
-    length(juju_secret.system_users_and_private_key_secret) > 0 ? {
-      system-users           = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}",
-      tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
-    } : {},
-    var.kyuubi.config
-  )
+  config      = var.kyuubi.config
   constraints = var.kyuubi.constraints
   resources = merge({
     kyuubi-image = local.images.kyuubi
diff --git a/terraform/components/spark-3.5/resources.tf b/terraform/components/spark-3.5/resources.tf
index fe9e3333..fa89e9d7 100644
--- a/terraform/components/spark-3.5/resources.tf
+++ b/terraform/components/spark-3.5/resources.tf
@@ -1,26 +1,2 @@
 # Copyright 2026 Canonical Ltd.
 # See LICENSE file for licensing details.
-
-resource "juju_secret" "system_users_and_private_key_secret" {
-  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model_uuid = var.model_uuid
-  name       = "system_users_and_private_key_secret"
-  value = merge(
-    var.admin_password == null ? {} : {
-      admin = var.admin_password
-    },
-    var.tls_private_key == null ? {} : {
-      private-key = var.tls_private_key
-    }
-  )
-  info = "This secret contains password for admin user and the TLS private key."
-}
-
-resource "juju_access_secret" "system_users_and_private_key_secret_access" {
-  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
-  model_uuid = var.model_uuid
-  applications = [
-    juju_application.kyuubi.name
-  ]
-  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
-}
diff --git a/terraform/components/spark-3.5/variables.tf b/terraform/components/spark-3.5/variables.tf
index e1832aec..de28d0be 100644
--- a/terraform/components/spark-3.5/variables.tf
+++ b/terraform/components/spark-3.5/variables.tf
@@ -50,13 +50,6 @@ variable "kyuubi" {
   default = {}
 }
 
-variable "admin_password" {
-  description = "The password for the admin user."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
 variable "data_integrator_endpoint" {
   description = "In-model endpoint for the data-integrator application."
   type = object({
@@ -81,13 +74,6 @@ variable "tls_endpoint" {
   })
 }
 
-variable "tls_private_key" {
-  description = "The private key to be used for TLS certificates."
-  type        = string
-  sensitive   = true
-  default     = null
-}
-
 variable "users_db_endpoint" {
   description = "In-model endpoint for the Kyuubi users database (postgresql-k8s) application."
   type = object({
diff --git a/terraform/products/charmed-spark-3.4/README.md b/terraform/products/charmed-spark-3.4/README.md
index db32a9b3..07f13789 100644
--- a/terraform/products/charmed-spark-3.4/README.md
+++ b/terraform/products/charmed-spark-3.4/README.md
@@ -35,8 +35,10 @@
 | Name | Type |
 |------|------|
 | [juju_access_secret.azure_storage_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
+| [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
 | [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
 | [juju_secret.azure_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
+| [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
 ### Inputs
 
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index 470d9f87..3c3802e0 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -144,10 +144,26 @@ module "s3" {
   revision    = local.revisions.s3
 }
 
+resource "juju_secret" "system_users_and_private_key_secret" {
+  depends_on = [juju_model.spark]
+  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = var.model_uuid
+  name       = "system_users_and_private_key_secret"
+  value = merge(
+    var.admin_password == null ? {} : {
+      admin = var.admin_password
+    },
+    var.tls_private_key == null ? {} : {
+      private-key = var.tls_private_key
+    }
+  )
+  info = "This secret contains password for admin user and the TLS private key."
+}
 
 module "spark" {
   depends_on = [
     juju_model.spark,
+    juju_secret.system_users_and_private_key_secret,
     module.azure_storage,
     module.data_integrator,
     module.kyuubi_users,
@@ -172,7 +188,16 @@ module "spark" {
     resources   = { integration-hub-image = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub }
   }
   kyuubi = {
-    config      = var.kyuubi_config,
+    config = merge(
+      {
+        expose-external = "loadbalancer",
+      },
+      length(juju_secret.system_users_and_private_key_secret) > 0 ? {
+        system-users           = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}",
+        tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+      } : {},
+      var.kyuubi_config
+    )
     constraints = "arch=amd64",
     revision    = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi,
     resources   = { kyuubi-image = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi }
@@ -197,9 +222,20 @@ module "spark" {
   zookeeper_endpoint       = module.zookeeper.provides.zookeeper
   data_integrator_endpoint = module.data_integrator.requires.kyuubi
   object_storage_endpoint  = module.s3 != [] ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
+}
 
-  admin_password  = var.admin_password
-  tls_private_key = var.tls_private_key
+resource "juju_access_secret" "system_users_and_private_key_secret_access" {
+  depends_on = [
+    juju_model.spark,
+    juju_secret.system_users_and_private_key_secret,
+    module.spark
+  ]
+  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = var.model_uuid
+  applications = [
+    module.spark.components.kyuubi.name
+  ]
+  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
 }
 
 module "observability" {
diff --git a/terraform/products/charmed-spark-3.5/README.md b/terraform/products/charmed-spark-3.5/README.md
index b93846bd..a596545d 100644
--- a/terraform/products/charmed-spark-3.5/README.md
+++ b/terraform/products/charmed-spark-3.5/README.md
@@ -26,7 +26,7 @@
 | <a name="module_metastore"></a> [metastore](#module\_metastore) | git::https://github.com/canonical/postgresql-k8s-operator//terraform | rev774 |
 | <a name="module_observability"></a> [observability](#module\_observability) | ../../components/observability | n/a |
 | <a name="module_s3"></a> [s3](#module\_s3) | ../../charms/s3-integrator-v1 | n/a |
-| <a name="module_spark"></a> [spark](#module\_spark) | ../../components/spark-3.4 | n/a |
+| <a name="module_spark"></a> [spark](#module\_spark) | ../../components/spark-3.5 | n/a |
 | <a name="module_ssc"></a> [ssc](#module\_ssc) | git::https://github.com/canonical/self-signed-certificates-operator//terraform | rev586 |
 | <a name="module_zookeeper"></a> [zookeeper](#module\_zookeeper) | ../../charms/zookeeper | n/a |
 
@@ -35,8 +35,10 @@
 | Name | Type |
 |------|------|
 | [juju_access_secret.azure_storage_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
+| [juju_access_secret.system_users_and_private_key_secret_access](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/access_secret) | resource |
 | [juju_model.spark](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/model) | resource |
 | [juju_secret.azure_storage_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
+| [juju_secret.system_users_and_private_key_secret](https://registry.terraform.io/providers/juju/juju/latest/docs/resources/secret) | resource |
 
 ### Inputs
 
diff --git a/terraform/products/charmed-spark-3.5/main.tf b/terraform/products/charmed-spark-3.5/main.tf
index 02676eb8..1b0cd7f0 100644
--- a/terraform/products/charmed-spark-3.5/main.tf
+++ b/terraform/products/charmed-spark-3.5/main.tf
@@ -144,10 +144,26 @@ module "s3" {
   revision    = local.revisions.s3
 }
 
+resource "juju_secret" "system_users_and_private_key_secret" {
+  depends_on = [juju_model.spark]
+  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = var.model_uuid
+  name       = "system_users_and_private_key_secret"
+  value = merge(
+    var.admin_password == null ? {} : {
+      admin = var.admin_password
+    },
+    var.tls_private_key == null ? {} : {
+      private-key = var.tls_private_key
+    }
+  )
+  info = "This secret contains password for admin user and the TLS private key."
+}
 
 module "spark" {
   depends_on = [
     juju_model.spark,
+    juju_secret.system_users_and_private_key_secret,
     module.azure_storage,
     module.data_integrator,
     module.kyuubi_users,
@@ -172,7 +188,16 @@ module "spark" {
     resources   = { integration-hub-image = var.integration_hub_image != null ? var.integration_hub_image : local.images.integration_hub }
   }
   kyuubi = {
-    config      = var.kyuubi_config,
+    config = merge(
+      {
+        expose-external = "loadbalancer",
+      },
+      length(juju_secret.system_users_and_private_key_secret) > 0 ? {
+        system-users           = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}",
+        tls-client-private-key = "secret:${juju_secret.system_users_and_private_key_secret[0].secret_id}"
+      } : {},
+      var.kyuubi_config
+    )
     constraints = "arch=amd64",
     revision    = var.kyuubi_revision != null ? var.kyuubi_revision : local.revisions.kyuubi,
     resources   = { kyuubi-image = var.kyuubi_image != null ? var.kyuubi_image : local.images.kyuubi }
@@ -197,9 +222,20 @@ module "spark" {
   zookeeper_endpoint       = module.zookeeper.provides.zookeeper
   data_integrator_endpoint = module.data_integrator.requires.kyuubi
   object_storage_endpoint  = module.s3 != [] ? module.s3[0].provides.s3_credentials : module.azure_storage[0].provides.azure_storage_credentials
+}
 
-  admin_password  = var.admin_password
-  tls_private_key = var.tls_private_key
+resource "juju_access_secret" "system_users_and_private_key_secret_access" {
+  depends_on = [
+    juju_model.spark,
+    juju_secret.system_users_and_private_key_secret,
+    module.spark
+  ]
+  count      = var.tls_private_key == null && var.admin_password == null ? 0 : 1
+  model_uuid = var.model_uuid
+  applications = [
+    module.spark.components.kyuubi.name
+  ]
+  secret_id = juju_secret.system_users_and_private_key_secret[0].secret_id
 }
 
 module "observability" {

From 265290133f38293baaba313bf7b393b200d29ee1 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 9 Apr 2026 18:02:58 +0200
Subject: [PATCH 45/62] Add missing variable in test entrypoint

---
 python/tests/integration/resources/main.tf | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index 114cd7ef..b0ef8646 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -101,16 +101,17 @@ module "spark" {
   model_uuid   = juju_model.spark != [] ? juju_model.spark[0].uuid : var.model_uuid
   create_model = false
 
-  admin_password       = var.admin_password
-  azure_storage_config = var.azure_storage_config
-  kyuubi_config        = var.kyuubi_config
-  kyuubi_users_size    = "500M"
-  metastore_size       = "500M"
-  s3_config            = var.s3_config
-  storage_backend      = var.storage_backend
-  tls_private_key      = var.tls_private_key
-  zookeeper_size       = "10G"
-  zookeeper_units      = 1
+  admin_password           = var.admin_password
+  azure_storage_config     = var.azure_storage_config
+  azure_storage_secret_key = var.azure_storage_secret_key
+  kyuubi_config            = var.kyuubi_config
+  kyuubi_users_size        = "500M"
+  metastore_size           = "500M"
+  s3_config                = var.s3_config
+  storage_backend          = var.storage_backend
+  tls_private_key          = var.tls_private_key
+  zookeeper_size           = "10G"
+  zookeeper_units          = 1
 
   cos_offers = module.cos != [] ? {
     dashboard = module.cos[0].offers.grafana_dashboards.url

From 73174b2c4bcc853e2551c9a774f51c20d52a9d77 Mon Sep 17 00:00:00 2001
From: Alex Batisse <alex.batisse@canonical.com>
Date: Thu, 9 Apr 2026 22:35:34 +0200
Subject: [PATCH 46/62] Fix storage-account config option

---
 python/tests/integration/conftest.py              | 2 +-
 terraform/products/charmed-spark-3.4/main.tf      | 4 ++--
 terraform/products/charmed-spark-3.4/variables.tf | 2 +-
 terraform/products/charmed-spark-3.5/main.tf      | 4 ++--
 terraform/products/charmed-spark-3.5/variables.tf | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index 4471e1d5..3717c100 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -438,7 +438,7 @@ def spark_bundle(
         storage_unit = cast(Container, object_storage)
         storage_vars = {
             "azure_storage_config": {
-                "storage_account": storage_unit.credentials.storage_account,
+                "storage-account": storage_unit.credentials.storage_account,
                 "container": storage_unit.container_name,
                 "path": "spark-events",
             },
diff --git a/terraform/products/charmed-spark-3.4/main.tf b/terraform/products/charmed-spark-3.4/main.tf
index 3c3802e0..0f318b3c 100644
--- a/terraform/products/charmed-spark-3.4/main.tf
+++ b/terraform/products/charmed-spark-3.4/main.tf
@@ -112,10 +112,10 @@ module "azure_storage" {
   base    = "ubuntu@22.04"
   channel = "latest/edge"
   config = merge(
+    var.azure_storage_config,
     {
       credentials = "secret:${juju_secret.azure_storage_secret[0].secret_id}"
-    },
-    var.azure_storage_config
+    }
   )
   constraints = "arch=amd64"
   revision    = local.revisions.azure_storage
diff --git a/terraform/products/charmed-spark-3.4/variables.tf b/terraform/products/charmed-spark-3.4/variables.tf
index 4d3b79c2..69afbda2 100644
--- a/terraform/products/charmed-spark-3.4/variables.tf
+++ b/terraform/products/charmed-spark-3.4/variables.tf
@@ -16,7 +16,7 @@ variable "azure_storage_config" {
     credentials     = optional(string)
     path            = optional(string)
     protocol        = optional(string, "abfss")
-    storage_account = optional(string)
+    storage-account = optional(string)
   })
   default = {}
 }
diff --git a/terraform/products/charmed-spark-3.5/main.tf b/terraform/products/charmed-spark-3.5/main.tf
index 1b0cd7f0..a562e359 100644
--- a/terraform/products/charmed-spark-3.5/main.tf
+++ b/terraform/products/charmed-spark-3.5/main.tf
@@ -112,10 +112,10 @@ module "azure_storage" {
   base    = "ubuntu@22.04"
   channel = "latest/edge"
   config = merge(
+    var.azure_storage_config,
     {
       credentials = "secret:${juju_secret.azure_storage_secret[0].secret_id}"
-    },
-    var.azure_storage_config
+    }
   )
   constraints = "arch=amd64"
   revision    = local.revisions.azure_storage
diff --git a/terraform/products/charmed-spark-3.5/variables.tf b/terraform/products/charmed-spark-3.5/variables.tf
index 4d3b79c2..69afbda2 100644
--- a/terraform/products/charmed-spark-3.5/variables.tf
+++ b/terraform/products/charmed-spark-3.5/variables.tf
@@ -16,7 +16,7 @@ variable "azure_storage_config" {
     credentials     = optional(string)
     path            = optional(string)
     protocol        = optional(string, "abfss")
-    storage_account = optional(string)
+    storage-account = optional(string)
   })
   default = {}
 }

From 04b9d2552854807e305b3065c86ff51ea4ae4de9 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:42:48 +0200
Subject: [PATCH 47/62] adding business logic to fetch revisions

---
 .github/scripts/charms_promotions.py          | 292 ++++++++++++++++++
 .../scripts/generate_revisions_from_specs.py  | 188 +++++++++++
 .../workflows/generate-charm-revisions.yaml   |  58 ++++
 3 files changed, 538 insertions(+)
 create mode 100644 .github/scripts/charms_promotions.py
 create mode 100644 .github/scripts/generate_revisions_from_specs.py
 create mode 100644 .github/workflows/generate-charm-revisions.yaml

diff --git a/.github/scripts/charms_promotions.py b/.github/scripts/charms_promotions.py
new file mode 100644
index 00000000..e2d4cc17
--- /dev/null
+++ b/.github/scripts/charms_promotions.py
@@ -0,0 +1,292 @@
+import json
+import re
+import shlex
+import subprocess
+import sys
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+import yaml
+
+RISKS = {"edge": 1, "beta": 2, "candidate": 3, "stable": 4}
+
+class Charm:
+
+    def __init__(self, name: str, revision: Optional[int]=None, channel: Optional[str]=None, architecture: Optional[str] = None):
+        self.name = name
+        self.revision = revision
+        self.channel = channel
+        self.architecture = architecture
+
+        self._status: Optional[dict] = None
+
+    def __str__(self):
+        return f"{self.name}:{self.channel}({self.revision})"
+
+    def __repr__(self):
+        return self.__str__()
+
+    def get_status(self, channel: Optional[str] = None, revision: Optional[int] = None, architecture: Optional[str] = None):
+
+        if not channel:
+            channel = self.channel
+        if not revision:
+            revision = self.revision
+        if not architecture:
+            architecture = self.architecture
+
+        if not self._status:
+
+            output = subprocess.run(
+                ["charmcraft", "status", self.name, "--format", "json"],
+                capture_output=True
+            )
+
+            if output.returncode != 0:
+                err_output = (output.stderr or b"").decode("utf-8", errors="replace").strip()
+                output = (output.stdout or b"").decode("utf-8", errors="replace").strip()
+                if "permission-required" in output or "permission-required" in err_output:
+                    raise PermissionError(self.name)
+                else:
+                    raise RuntimeError(charm.name, shlex.join(err.cmd), err)
+
+            self._status = json.loads(output.stdout.decode("utf-8"))
+
+        items = [
+            mapping["base"] | release
+            for track in self._status
+            for mapping in track["mappings"]
+            for release in mapping["releases"]
+            if mapping["base"]
+            if (not channel or channel == release["channel"]) and (
+                        not revision or revision == release["revision"]) and (
+                        not architecture or architecture == mapping["base"]["architecture"]
+                        )
+        ]
+
+        return items
+
+
+    def promote_version(self, risk: str, dry_run: bool = True):
+        if risk not in RISKS.keys():
+            raise ValueError("The risk is not recognized")
+
+        if not self.channel:
+            raise ValueError("channel field is not specified, that is required for promotion")
+
+        items = self.get_status(revision=self.revision, channel=self.channel)
+
+        if len(items) > 1:
+            raise ValueError(f"Multiple tracks match the provide revision and channel combination: {items}")
+
+        if len(items) == 0:
+            raise ValueError(f"No match found for charm {self}")
+
+        item = items[0]
+
+        _channel = Path(item["channel"])
+        risk_from = str(_channel.name)
+
+        if risk_from not in RISKS.keys():
+            raise ValueError("The revision does not belong to a risk channel")
+
+        if RISKS[risk] <= RISKS[risk_from]:
+            raise ValueError("A charm revision cannot be promoted to a lower risk.")
+
+        channel_to = str(_channel.parent / risk)
+
+        cmds = (
+                ["charmcraft", "release", self.name, f"--channel={channel_to}",
+                 f"--revision={self.revision}"] +
+                [f"--resource={resource['name']}:{resource['revision']}" for
+                 resource in item["resources"]]
+        )
+
+        if dry_run:
+            return "INFO: (dry-run mode) " + shlex.join(cmds)
+
+        return subprocess.check_output(cmds).decode("utf-8")
+
+
+class Format(str,Enum):
+    TEXT = "text"
+    YAML = "yaml"
+
+@dataclass
+class Bundle:
+
+    charms: list[Charm]
+
+    @classmethod
+    def from_status(cls, content: str, format: Format | str = Format.TEXT):
+        parsers = {
+            Format.TEXT: TextParser,
+            Format.YAML: YAMLParser
+        }
+        normalized_format = Format(format)
+
+        return parsers[normalized_format].parse(content)
+
+
+class YAMLParser:
+
+    @staticmethod
+    def parse(content: str):
+        data = yaml.safe_load(content)
+
+        return Bundle([
+            Charm(app["charm-name"], int(app["charm-rev"]), app.get("charm-channel"))
+            for _, app in data["applications"].items()
+        ])
+
+
+class TextParser:
+    """
+    Parse a juju status output to find the charms deployed and their metadata (channel, revision, etc)
+
+    The `parse` method is used to extract the list of charms, returned as a Bundle class.
+    The reason why simpler parsing methods could not be used, is that the output format of a juju status is
+    made peculiar by some columns header being aligned right instead of left, e.g. see Rev below:
+
+    App                      Version                  Status   Scale  Charm                    Channel         Rev  Address       Exposed  Message
+    ...
+    istio-ingressgateway                              active       1  istio-gateway            1.28/edge      1594  10.0.165.149  no
+    ...
+
+    The algorithm parse the header and finds a first guess for the columns start. When parsing each rows, the algorithm updates the width
+    of each column using the width of the value itself, then treating the remaining chars as the values for the next columns. This algorithm falls short
+    if the values has multiple white spaces (also in headers). However this only happens in the value of the "Message" column, which is accounted for.
+    """
+
+    # The following regex extract the first word that may be preceded by
+    # leading spaces
+    word_with_leading_spaces = re.compile(r"^\s*[^\s]+")
+
+    @staticmethod
+    def extract_first_word(mystring):
+        m = TextParser.word_with_leading_spaces.match(mystring)
+        if m:
+            return mystring[m.start():m.end()]
+        return ""
+
+    @staticmethod
+    def parse_line(line, indices):
+        return [line[start:end].strip() for start, end in indices]
+
+    @staticmethod
+    def parse(content: str):
+        lines = content.splitlines()
+
+        white_spaces = re.compile(r"\s+\s+")
+
+        app_header_index = next(
+            (
+                index
+                for index, line in enumerate(lines)
+                if line.strip().startswith("App")
+                and "Charm" in line
+                and "Channel" in line
+            ),
+            None,
+        )
+
+        if app_header_index is None:
+            raise ValueError("Could not locate applications table in status text")
+
+        table_lines = []
+        for line in lines[app_header_index:]:
+            if table_lines and (not line.strip() or line.strip().startswith("Unit")):
+                break
+            table_lines.append(line)
+
+        if len(table_lines) < 2:
+            raise ValueError("Applications table is empty in status text")
+
+        # Get header
+        header = table_lines[0]
+
+        # First guess of width based on headers
+        ends=[s.end() for s in white_spaces.finditer(header)]
+        indices = list(zip([0]+ends, ends+[-1]))
+
+        # Columns names
+        columns = TextParser.parse_line(header, indices)
+
+        # Second guess of width based on maximum width of values
+        # This is due to the fact that some columns the text extends to before the start
+        # of the columns header (text aligned right)
+        widths = [len(column) for column in columns]
+        for line in table_lines[1:]:
+
+            widths = list(map(max,zip(
+                widths,
+                [len(TextParser.extract_first_word(line[start:end])) for start, end in indices]
+            )))
+
+        # New guess
+        ends = [start+width for (start,_), width in zip(indices, widths)]
+
+        indices = list(zip([0]+ends[:-1], ends[:-1]+[-1]))
+
+        data = [
+            dict(zip(columns, TextParser.parse_line(line, indices)))
+            for line in table_lines[1:]
+            if line.strip()
+        ]
+
+        return Bundle([
+            Charm(item["Charm"], int(item["Rev"]), item["Channel"])
+            for item in data
+            if item["Charm"]
+        ])
+
+
+if __name__ == "__main__":
+
+    import argparse
+
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--file", required=True)
+    action_group = parser.add_mutually_exclusive_group()
+    action_group.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Print release commands without executing them (default)",
+    )
+    action_group.add_argument(
+        "--apply",
+        dest="dry_run",
+        action="store_false",
+        help="Execute charmcraft release commands",
+    )
+    parser.set_defaults(dry_run=True)
+    parser.add_argument("--format", choices=("text", "yaml"), default="text")
+    parser.add_argument("--promote-to", choices=("beta", "candidate", "stable"), default="beta")
+    parser.add_argument("--exclude", nargs="*", default=["mysql-k8s"])
+    args = parser.parse_args()
+
+    with open(args.file) as fid:
+        bundle = Bundle.from_status(fid.read(), args.format)
+
+    for charm in bundle.charms:
+        if not charm.name in args.exclude:
+            try:
+                print(charm.promote_version(args.promote_to, args.dry_run))
+            except ValueError as err:
+                print(f"ERROR: skipping {charm.name} due to failure in promotion. Reason: {type(err).__name__} - Details: {','.join(err.args)}")
+            except PermissionError as err:
+                print(
+                        f"WARNING: skipping '{charm.name}' due to missing permissions. Add it to --exclude to avoid this warning.",
+                        file=sys.stderr,
+                )
+            except RuntimeError as err:
+                print(
+                    f"WARNING: skipping '{err.args[0]}' after command failure: {err.args[1]}",
+                    file=sys.stderr,
+                )
+        else:
+            print(f"WARNING: excluding charm {charm.name} (see --exclude flag)", file=sys.stderr)
diff --git a/.github/scripts/generate_revisions_from_specs.py b/.github/scripts/generate_revisions_from_specs.py
new file mode 100644
index 00000000..61dfe091
--- /dev/null
+++ b/.github/scripts/generate_revisions_from_specs.py
@@ -0,0 +1,188 @@
+import argparse
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+import yaml
+
+from charms_promotions import Charm
+
+
+@dataclass(frozen=True)
+class CharmSpec:
+    name: str
+    channel: str
+
+
+@dataclass(frozen=True)
+class Specs:
+    architectures: list[str]
+    releases: dict[str, list[CharmSpec]]
+    revision_mapping: dict[str, str]
+
+
+def normalize_channel(channel: str, risk: str) -> str:
+    """Expand channel prefixes like '3.4/' to full channels like '3.4/beta'."""
+    if channel.endswith("/"):
+        return f"{channel}{risk}"
+    return channel
+
+
+def parse_specs(specs_file: Path) -> Specs:
+    with specs_file.open("r", encoding="utf-8") as file_obj:
+        data = yaml.safe_load(file_obj)
+
+    if not isinstance(data, dict):
+        raise ValueError("Invalid specs format: root must be a mapping")
+
+    architectures = data.get("architectures")
+    releases = data.get("releases")
+    revision_mapping = data.get("revision_mapping")
+    
+
+    if not isinstance(architectures, list) or not all(isinstance(a, str) for a in architectures):
+        raise ValueError("Invalid specs format: architectures must be a list of strings")
+
+    if not isinstance(releases, dict):
+        raise ValueError("Invalid specs format: releases must be a mapping")
+
+    if not isinstance(revision_mapping, dict):
+        raise ValueError("Invalid specs format: revision_mapping must be a mapping")
+
+    if not all(
+        isinstance(charm_name, str) and isinstance(mapped_name, str)
+        for charm_name, mapped_name in revision_mapping.items()
+    ):
+        raise ValueError(
+            "Invalid specs format: revision_mapping keys and values must be strings"
+        )
+
+    parsed_releases: dict[str, list[CharmSpec]] = {}
+    for release, charms in releases.items():
+        if not isinstance(charms, list):
+            raise ValueError(
+                f"Invalid specs format: releases.{release} must be a list"
+            )
+
+        parsed_entries: list[CharmSpec] = []
+        for entry in charms:
+            if not isinstance(entry, dict):
+                raise ValueError(
+                    f"Invalid specs format: release entry for {release} must be a mapping"
+                )
+
+            charm_name = entry.get("name")
+            channel = entry.get("channel")
+            if not isinstance(charm_name, str) or not isinstance(channel, str):
+                raise ValueError(
+                    f"Invalid specs format: release entry for {release} must have string name and channel"
+                )
+
+            parsed_entries.append(CharmSpec(name=charm_name, channel=channel))
+
+        parsed_releases[str(release)] = parsed_entries
+
+    return Specs(
+        architectures=architectures,
+        releases=parsed_releases,
+        revision_mapping=revision_mapping,
+    )
+
+
+def resolve_revision(charm: Charm) -> int:
+    items = charm.get_status()
+
+    if not items:
+        raise ValueError(
+            f"No revision found for charm={charm.name}, channel={charm.channel}, architecture={charm.architecture}"
+        )
+
+    if len(items) > 1:
+        revisions = sorted(item["revision"] for item in items)
+        raise ValueError(
+            "Multiple revisions found for "
+            f"charm={charm.name}, channel={charm.channel}, architecture={charm.architecture}: {revisions}"
+        )
+
+    revision = items[0]["revision"]
+
+    if not isinstance(revision, int):
+        raise ValueError(
+            f"Unexpected revision type for charm={charm.name}, channel={charm.channel}, architecture={charm.architecture}"
+        )
+
+    return revision
+
+
+def build_output(specs: Specs, risk: str) -> dict:
+    output = {}
+    revision_mapping = specs.revision_mapping
+
+    for architecture in specs.architectures:
+        output[architecture] = {}
+        for release, charms in specs.releases.items():
+            output[architecture][str(release)] = {}
+
+            for entry in charms:
+                charm_name = entry.name
+                channel = normalize_channel(entry.channel, risk)
+                mapped_name = revision_mapping.get(charm_name)
+
+                if not mapped_name:
+                    raise ValueError(
+                        f"Missing revision mapping for charm={charm_name}"
+                    )
+
+                charm = Charm(
+                    name=charm_name,
+                    revision=None,
+                    channel=channel,
+                    architecture=architecture,
+                )
+                revision = resolve_revision(charm)
+                output[architecture][str(release)][mapped_name] = revision
+
+    return output
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Parse specs.yaml and fetch charm revisions by architecture/release/channel "
+            "using charms_promotions.Charm"
+        )
+    )
+    parser.add_argument(
+        "--specs-file",
+        default="scripts/specs.yaml",
+        help="Input specs file (default: scripts/specs.yaml)",
+    )
+    parser.add_argument(
+        "--output-file",
+        default="scripts/revisions.yaml",
+        help="Output YAML file (default: scripts/revisions.yaml)",
+    )
+    parser.add_argument(
+        "--risk",
+        default="beta",
+        choices=("edge", "beta", "candidate", "stable"),
+        help="Risk suffix used when channel in specs ends with '/'.",
+    )
+    args = parser.parse_args()
+
+    try:
+        specs = parse_specs(Path(args.specs_file))
+        output = build_output(specs, risk=args.risk)
+
+        with Path(args.output_file).open("w", encoding="utf-8") as file_obj:
+            yaml.safe_dump(output, file_obj, sort_keys=False)
+
+    except (KeyError, PermissionError, RuntimeError, ValueError) as err:
+        print(f"ERROR: {type(err).__name__}: {err}", file=sys.stderr)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/.github/workflows/generate-charm-revisions.yaml b/.github/workflows/generate-charm-revisions.yaml
new file mode 100644
index 00000000..ba1d70f8
--- /dev/null
+++ b/.github/workflows/generate-charm-revisions.yaml
@@ -0,0 +1,58 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+name: Generate charm revisions
+
+on:
+  workflow_call:
+    inputs:
+      risk:
+        description: "Risk level used to resolve charm revisions"
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      risk:
+        description: "Risk level used to resolve charm revisions"
+        required: true
+        default: beta
+        type: choice
+        options:
+          - edge
+          - beta
+          - candidate
+          - stable
+
+jobs:
+  generate-revisions:
+    name: Generate revisions.yaml
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          architecture: x64
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Generate revisions.yaml
+        env:
+          CHARMCRAFT_AUTH: ${{ secrets.CHARMCRAFT_AUTH }}
+        run: |
+          python3 .github/scripts/generate_revisions_from_specs.py \
+            --specs-file specs.yaml \
+            --output-file revisions.yaml \
+            --risk ${{ inputs.risk }}
+
+      - name: Upload revisions.yaml artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: charms-${{ inputs.risk }}
+          path: revisions.yaml
+          if-no-files-found: error

From 79484e28edc90541b1a73cb5ba048cd2bcbc797d Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:43:44 +0200
Subject: [PATCH 48/62] small fix

---
 .github/workflows/generate-charm-revisions.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/generate-charm-revisions.yaml b/.github/workflows/generate-charm-revisions.yaml
index ba1d70f8..8cfc2cfb 100644
--- a/.github/workflows/generate-charm-revisions.yaml
+++ b/.github/workflows/generate-charm-revisions.yaml
@@ -45,7 +45,8 @@ jobs:
         env:
           CHARMCRAFT_AUTH: ${{ secrets.CHARMCRAFT_AUTH }}
         run: |
-          python3 .github/scripts/generate_revisions_from_specs.py \
+          cd .github/scripts
+          python3 generate_revisions_from_specs.py \
             --specs-file specs.yaml \
             --output-file revisions.yaml \
             --risk ${{ inputs.risk }}

From 1fa1ff85844c7a29fa8fd7d6586ea9d902633c3b Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:46:29 +0200
Subject: [PATCH 49/62] limit ci

---
 .github/workflows/ci-uat.yaml          | 8 ++++----
 .github/workflows/on_pull_request.yaml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci-uat.yaml b/.github/workflows/ci-uat.yaml
index b7e9a96b..9af4e5ec 100644
--- a/.github/workflows/ci-uat.yaml
+++ b/.github/workflows/ci-uat.yaml
@@ -5,10 +5,10 @@ name: Mimic UAT Tests
 
 on:
   workflow_dispatch:
-  pull_request:
-    paths-ignore:
-      - "docs/**"
-      - "**.md"
+  # pull_request:
+  #   paths-ignore:
+  #     - "docs/**"
+  #     - "**.md"
 
 jobs:
   integration-tests:
diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index cd953934..35586b21 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -18,5 +18,5 @@ on:
 jobs:
   tests:
     name: Run CI tests on a PR
-    uses: ./.github/workflows/ci-tests-minimal.yaml
+    uses: ./.github/workflows/generate-charm-revisions.yaml
     secrets: inherit
\ No newline at end of file

From 49c80ae1ae38af98435b7c9a1336d5c1f3953c08 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:47:20 +0200
Subject: [PATCH 50/62] fix

---
 .github/workflows/on_pull_request.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index 35586b21..9e0ffa08 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -19,4 +19,6 @@ jobs:
   tests:
     name: Run CI tests on a PR
     uses: ./.github/workflows/generate-charm-revisions.yaml
-    secrets: inherit
\ No newline at end of file
+    secrets: inherit
+    with:
+      risk: beta
\ No newline at end of file

From 0465bd2d37839d5118b664f54a6bcd345c932df0 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:47:59 +0200
Subject: [PATCH 51/62] fix

---
 specs.yaml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 specs.yaml

diff --git a/specs.yaml b/specs.yaml
new file mode 100644
index 00000000..0f2eb567
--- /dev/null
+++ b/specs.yaml
@@ -0,0 +1,22 @@
+architectures:
+  - amd64
+  - arm64
+releases:
+  3.4:
+    - name: kyuubi-k8s
+      channel: 3.4/
+    - name: spark-history-server-k8s
+      channel: 3/
+    - name: spark-integration-hub-k8s
+      channel: 3/
+  3.5:
+    - name: kyuubi-k8s
+      channel: 3.5/
+    - name: spark-history-server-k8s
+      channel: 3/
+    - name: spark-integration-hub-k8s
+      channel: 3/
+revision_mapping:
+  kyuubi-k8s: kyuubi_revision
+  spark-history-server-k8s: history_server_revision
+  spark-integration-hub-k8s: integration_hub_revision

From 33ceff375a68443272132da2ec494bd9640fe63a Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:49:49 +0200
Subject: [PATCH 52/62] fix

---
 .github/workflows/generate-charm-revisions.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/generate-charm-revisions.yaml b/.github/workflows/generate-charm-revisions.yaml
index 8cfc2cfb..64fd476b 100644
--- a/.github/workflows/generate-charm-revisions.yaml
+++ b/.github/workflows/generate-charm-revisions.yaml
@@ -45,7 +45,7 @@ jobs:
         env:
           CHARMCRAFT_AUTH: ${{ secrets.CHARMCRAFT_AUTH }}
         run: |
-          cd .github/scripts
+          cp specs.yaml .github/scripts && cd .github/scripts 
           python3 generate_revisions_from_specs.py \
             --specs-file specs.yaml \
             --output-file revisions.yaml \

From 61763969a03f86fe49006bb0923a2cda7e7f4de6 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:51:40 +0200
Subject: [PATCH 53/62] adding charmcraft

---
 .github/workflows/generate-charm-revisions.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/generate-charm-revisions.yaml b/.github/workflows/generate-charm-revisions.yaml
index 64fd476b..44178ac6 100644
--- a/.github/workflows/generate-charm-revisions.yaml
+++ b/.github/workflows/generate-charm-revisions.yaml
@@ -38,6 +38,11 @@ jobs:
           python-version: "3.10"
           architecture: x64
 
+      - name: Install charmcraft
+        run: |
+          sudo snap install charmcraft --classic
+          charmcraft version
+
       - name: Install dependencies
         run: pip install pyyaml
 

From 52ceccc38aa2d1e96696f3b5ae297d3bac4b5317 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:54:35 +0200
Subject: [PATCH 54/62] fix

---
 .github/scripts/charms_promotions.py            | 4 ++--
 .github/workflows/generate-charm-revisions.yaml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/charms_promotions.py b/.github/scripts/charms_promotions.py
index e2d4cc17..9854c37d 100644
--- a/.github/scripts/charms_promotions.py
+++ b/.github/scripts/charms_promotions.py
@@ -50,12 +50,12 @@ def get_status(self, channel: Optional[str] = None, revision: Optional[int] = No
                 if "permission-required" in output or "permission-required" in err_output:
                     raise PermissionError(self.name)
                 else:
-                    raise RuntimeError(charm.name, shlex.join(err.cmd), err)
+                    raise RuntimeError(self.name, shlex.join(err.cmd), err)
 
             self._status = json.loads(output.stdout.decode("utf-8"))
 
         items = [
-            mapping["base"] | release
+            {"base": mapping["base"]} | release
             for track in self._status
             for mapping in track["mappings"]
             for release in mapping["releases"]
diff --git a/.github/workflows/generate-charm-revisions.yaml b/.github/workflows/generate-charm-revisions.yaml
index 44178ac6..1c342ac9 100644
--- a/.github/workflows/generate-charm-revisions.yaml
+++ b/.github/workflows/generate-charm-revisions.yaml
@@ -48,7 +48,7 @@ jobs:
 
       - name: Generate revisions.yaml
         env:
-          CHARMCRAFT_AUTH: ${{ secrets.CHARMCRAFT_AUTH }}
+          CHARMCRAFT_AUTH: ${{ secrets.CHARMHUB_TOKEN }}
         run: |
           cp specs.yaml .github/scripts && cd .github/scripts 
           python3 generate_revisions_from_specs.py \

From 61fcbed82b60edaa8d326e3a9dfb87ef7439c4f0 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 16:59:03 +0200
Subject: [PATCH 55/62] fix

---
 .github/workflows/generate-charm-revisions.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/generate-charm-revisions.yaml b/.github/workflows/generate-charm-revisions.yaml
index 1c342ac9..ccdc6899 100644
--- a/.github/workflows/generate-charm-revisions.yaml
+++ b/.github/workflows/generate-charm-revisions.yaml
@@ -55,6 +55,8 @@ jobs:
             --specs-file specs.yaml \
             --output-file revisions.yaml \
             --risk ${{ inputs.risk }}
+          cat revisions.yaml
+          cp revisions.yaml ../../revisions.yaml
 
       - name: Upload revisions.yaml artifact
         uses: actions/upload-artifact@v4

From 8152b476d3db38b19b9dea51140a1ec0edbfc39c Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sat, 11 Apr 2026 23:57:54 +0200
Subject: [PATCH 56/62] adding possibility to feed variables to tests

---
 python/tests/integration/conftest.py       | 40 ++++++++++++++++++++++
 python/tests/integration/resources/main.tf | 21 ++++++++++++
 2 files changed, 61 insertions(+)

diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index 3717c100..c770c935 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -4,6 +4,7 @@
 from __future__ import annotations
 
 import base64
+import json
 import logging
 import os
 import shutil
@@ -19,6 +20,7 @@
 
 import jubilant
 import pytest
+import yaml
 from dotenv import load_dotenv
 from spark8t.domain import PropertyFile
 from tenacity import retry, stop_after_attempt
@@ -119,6 +121,12 @@ def pytest_addoption(parser):
         help="This, together with the `--model` parameter, ensures that all functions "
         "marked with the` skip_if_deployed` tag are skipped.",
     )
+    parser.addoption(
+        "--tfvars-file",
+        type=str,
+        default=None,
+        help="Path to a JSON/YAML file containing Terraform variables to pass to the bundle deployment",
+    )
 
 
 def determine_scope(fixture_name, config):
@@ -194,6 +202,34 @@ def test_uuid(request) -> str:
     return request.config.getoption("--uuid") or str(uuid.uuid4())
 
 
+@pytest.fixture(scope="module")
+def tfvars(request) -> dict:
+    """External Terraform variables loaded from a file."""
+    tfvars_file = request.config.getoption("--tfvars-file")
+    
+    if not tfvars_file:
+        return {}
+    
+    file_path = Path(tfvars_file)
+    if not file_path.exists():
+        raise FileNotFoundError(f"Terraform variables file not found: {tfvars_file}")
+    
+    # Load JSON or YAML file
+    with file_path.open("r", encoding="utf-8") as f:
+        try:
+            if file_path.suffix.lower() == ".json":
+                data = json.load(f)
+            else:
+                data = yaml.safe_load(f)
+        except (json.JSONDecodeError, yaml.YAMLError) as err:
+            raise ValueError(f"Failed to parse Terraform variables file: {err}")
+    
+    if not isinstance(data, dict):
+        raise ValueError("Terraform variables file must contain a JSON/YAML mapping")
+    
+    return data
+
+
 @pytest.fixture(scope="module")
 def image_properties(spark_image):
     return PropertyFile(
@@ -410,6 +446,7 @@ def spark_bundle(
     object_storage,
     admin_password,
     private_key,
+    tfvars,
 ):
     """Deploy the Spark K8s bundle using Terraform."""
     short_version = ".".join(spark_version.split(".")[:2])
@@ -432,6 +469,9 @@ def spark_bundle(
         "admin_password": admin_password,
         "tls_private_key": private_key,
     }
+    # Merge external Terraform variables
+    base_vars.update(tfvars)
+
     cos_vars = {"cos_model_uuid": cos} if cos else {}
 
     if storage_backend == "azure_storage":
diff --git a/python/tests/integration/resources/main.tf b/python/tests/integration/resources/main.tf
index b0ef8646..7c681dd6 100644
--- a/python/tests/integration/resources/main.tf
+++ b/python/tests/integration/resources/main.tf
@@ -77,6 +77,24 @@ variable "tls_private_key" {
   default = null
 }
 
+variable "history_server_revision" {
+  description = "Charm revision for spark-history-server-k8s"
+  type        = number
+  default     = null
+}
+
+variable "kyuubi_revision" {
+  description = "Charm revision for kyuubi-k8s"
+  type        = number
+  default     = null
+}
+
+variable "integration_hub_revision" {
+  description = "Charm revision for spark-integration-hub-k8s"
+  type        = number
+  default     = null
+}
+
 module "cos" {
   count = var.cos_model_uuid == null ? 0 : 1
   # TODO: Pin to tag once available
@@ -104,7 +122,10 @@ module "spark" {
   admin_password           = var.admin_password
   azure_storage_config     = var.azure_storage_config
   azure_storage_secret_key = var.azure_storage_secret_key
+  history_server_revision  = var.history_server_revision
+  integration_hub_revision = var.integration_hub_revision
   kyuubi_config            = var.kyuubi_config
+  kyuubi_revision          = var.kyuubi_revision
   kyuubi_users_size        = "500M"
   metastore_size           = "500M"
   s3_config                = var.s3_config

From a0b1c7461d3a57254a14404d2128fb4980226560 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sun, 12 Apr 2026 00:25:21 +0200
Subject: [PATCH 57/62] test workflow

---
 .github/actions/run-test/action.yaml          | 26 ++++++++++++++++++-
 .github/workflows/ci-tests-minimal.yaml       |  7 +++++
 .../workflows/generate-charm-revisions.yaml   | 11 ++++++++
 .github/workflows/on_pull_request.yaml        | 12 +++++++--
 4 files changed, 53 insertions(+), 3 deletions(-)

diff --git a/.github/actions/run-test/action.yaml b/.github/actions/run-test/action.yaml
index dd14b1bf..9babfcc0 100644
--- a/.github/actions/run-test/action.yaml
+++ b/.github/actions/run-test/action.yaml
@@ -58,6 +58,11 @@ inputs:
     required: false
     default: ""
     type: string
+  tfvars-file:
+    description: "Optional artifact name containing a tfvars file to pass to pytest"
+    required: false
+    default: ""
+    type: string
 
 runs:
   using: "composite"
@@ -222,6 +227,14 @@ runs:
           sudo snap install terraform --classic
         fi
 
+    - id: download-tfvars
+      name: Download tfvars artifact
+      if: ${{ inputs.tfvars-file != '' }}
+      uses: actions/download-artifact@v4
+      with:
+        name: ${{ inputs.tfvars-file }}
+        path: tfvars-artifacts
+
     - id: tests-integration
       name: Run Integration Tests
       shell: bash
@@ -229,9 +242,20 @@ runs:
         AZURE_STORAGE_ACCOUNT: ${{ inputs.azure-storage-account }}
         AZURE_STORAGE_KEY: ${{ inputs.azure-storage-key }}
       run: |
+        TFVARS_ARG=""
+        if [[ -n "${{ inputs.tfvars-file }}" ]]; then
+          TFVARS_PATH="$(find tfvars-artifacts -type f | head -n1)"
+          if [[ -z "$TFVARS_PATH" ]]; then
+            echo "ERROR: no tfvars file found in artifact '${{ inputs.tfvars-file }}'."
+            find tfvars-artifacts -type f || true
+            exit 1
+          fi
+          TFVARS_ARG="--tfvars-file $TFVARS_PATH"
+        fi
+
         juju add-model spark-bundle-test
         juju list-models
-        cd python && tox run -e ${{ inputs.tox-env }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --cos-model ${{ inputs.cos-model }} --spark-version ${{ inputs.spark-version }} --storage-backend ${{ inputs.storage-backend }} --model spark-bundle-test ${{ inputs.pytest-args }}
+        cd python && tox run -e ${{ inputs.tox-env }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' --cos-model ${{ inputs.cos-model }} --spark-version ${{ inputs.spark-version }} --storage-backend ${{ inputs.storage-backend }} --model spark-bundle-test ${{ inputs.pytest-args }} $TFVARS_ARG
         echo "TEST_EXIT_CODE=$?" >> $GITHUB_ENV
 
     - id: collect-logs
diff --git a/.github/workflows/ci-tests-minimal.yaml b/.github/workflows/ci-tests-minimal.yaml
index ba143f5d..7784c116 100644
--- a/.github/workflows/ci-tests-minimal.yaml
+++ b/.github/workflows/ci-tests-minimal.yaml
@@ -5,6 +5,12 @@ name: Run CI tests
 
 on:
   workflow_call:
+    inputs:
+      tfvars-file:
+        description: "Optional artifact name containing tfvars for test runs"
+        required: false
+        type: string
+        default: ""
 
 jobs:
   checks:
@@ -91,6 +97,7 @@ jobs:
           azure-storage-account: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
           azure-storage-key: ${{ secrets.AZURE_STORAGE_KEY }}
           pytest-args: ${{ matrix.tox-env != 'integration-backup-restore' && '--keep-models' || '' }}
+          tfvars-file: ${{ inputs.tfvars-file }}
 
   aggregate-results:
     name: Aggregate Results
diff --git a/.github/workflows/generate-charm-revisions.yaml b/.github/workflows/generate-charm-revisions.yaml
index ccdc6899..85edde9a 100644
--- a/.github/workflows/generate-charm-revisions.yaml
+++ b/.github/workflows/generate-charm-revisions.yaml
@@ -10,6 +10,10 @@ on:
         description: "Risk level used to resolve charm revisions"
         required: true
         type: string
+    outputs:
+      tfvars-artifact-name:
+        description: "Name of the artifact containing generated tfvars"
+        value: ${{ jobs.generate-revisions.outputs.tfvars-artifact-name }}
   workflow_dispatch:
     inputs:
       risk:
@@ -28,6 +32,8 @@ jobs:
     name: Generate revisions.yaml
     runs-on: ubuntu-22.04
     timeout-minutes: 10
+    outputs:
+      tfvars-artifact-name: ${{ steps.artifact-name.outputs.value }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -59,8 +65,13 @@ jobs:
           cp revisions.yaml ../../revisions.yaml
 
       - name: Upload revisions.yaml artifact
+        id: upload-revisions
         uses: actions/upload-artifact@v4
         with:
           name: charms-${{ inputs.risk }}
           path: revisions.yaml
           if-no-files-found: error
+
+      - name: Expose artifact name
+        id: artifact-name
+        run: echo "value=charms-${{ inputs.risk }}" >> "$GITHUB_OUTPUT"
diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index 9e0ffa08..f2d6bf59 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -16,9 +16,17 @@ on:
       - '**.md'
 
 jobs:
+  generate-revisions:
+    name: Generate tfvars revisions
+    uses: ./.github/workflows/generate-charm-revisions.yaml
+    secrets: inherit
+    with:
+      risk: beta
+
   tests:
     name: Run CI tests on a PR
-    uses: ./.github/workflows/generate-charm-revisions.yaml
+    needs: generate-revisions
+    uses: ./.github/workflows/ci-tests-single-kyuubi.yaml
     secrets: inherit
     with:
-      risk: beta
\ No newline at end of file
+      tfvars-file: ${{ needs.generate-revisions.outputs.tfvars-artifact-name }}
\ No newline at end of file

From afd03fc6da63479fa7cdbfebbe0037158f97f1da Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sun, 12 Apr 2026 00:26:19 +0200
Subject: [PATCH 58/62] adding test workflow

---
 .github/workflows/ci-tests-single-kyuubi.yaml | 96 +++++++++++++++++++
 1 file changed, 96 insertions(+)
 create mode 100644 .github/workflows/ci-tests-single-kyuubi.yaml

diff --git a/.github/workflows/ci-tests-single-kyuubi.yaml b/.github/workflows/ci-tests-single-kyuubi.yaml
new file mode 100644
index 00000000..66526d3d
--- /dev/null
+++ b/.github/workflows/ci-tests-single-kyuubi.yaml
@@ -0,0 +1,96 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+name: Run CI single test
+
+on:
+  workflow_call:
+    inputs:
+      tfvars-file:
+        description: "Optional artifact name containing tfvars for test runs"
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  checks:
+    uses: ./.github/workflows/ci-checks.yaml
+
+  integration-tests:
+    name: "${{ matrix.tox-env }} (${{ matrix.k8s-distribution }}:${{ matrix.k8s-version }}): spark-${{ matrix.spark-version }} | ${{ matrix.storage-backend }} | juju ${{ matrix.juju-agent-version }}"
+    runs-on: ["self-hosted-linux-amd64-jammy-xlarge"]
+    timeout-minutes: 120
+    strategy:
+      max-parallel: 1
+      fail-fast: false
+      matrix:
+        tox-env:
+          - integration-kyuubi
+        spark-version:
+          - "3.5.7"
+        storage-backend:
+          - "s3"
+        juju-snap-channel:
+          - "3.6/stable"
+        juju-agent-version:
+          - "3.6.14"
+        k8s-distribution:
+          - "k8s"
+        k8s-version:
+          - "1.32"
+
+    needs:
+      - checks
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+      - name: Run integration test
+        uses: ./.github/actions/run-test
+        with:
+          k8s-distribution: ${{ matrix.k8s-distribution }}
+          k8s-version: ${{ matrix.k8s-version }}
+          tox-env: ${{ matrix.tox-env }}
+          spark-version: ${{ matrix.spark-version }}
+          storage-backend: ${{ matrix.storage-backend }}
+          juju-snap-channel: ${{ matrix.juju-snap-channel }}
+          juju-agent-version: ${{ matrix.juju-agent-version }}
+          azure-storage-account: ${{ secrets.AZURE_STORAGE_ACCOUNT }}
+          azure-storage-key: ${{ secrets.AZURE_STORAGE_KEY }}
+          pytest-args: --keep-models
+          tfvars-file: ${{ inputs.tfvars-file }}
+
+  aggregate-results:
+    name: Aggregate Results
+    runs-on: ubuntu-22.04
+    needs: integration-tests
+    if: ${{ success() || failure() }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v6
+      - name: Download artifacts
+        uses: actions/download-artifact@v7
+        with:
+          path: results
+      - id: setup-python
+        name: Setup Python
+        uses: actions/setup-python@v5.0.0
+        with:
+          python-version: 3.12
+          architecture: x64
+      - name: Install pandas & tabulate
+        run: |
+          pip install pandas
+          pip install tabulate
+
+      - name: Run result aggregation script
+        run: python3 .github/scripts/aggregate_results.py results/integration-results-*/test-result.json
+
+      - name: Upload summary
+        uses: actions/upload-artifact@v6
+        with:
+          name: matrix-summary
+          path: matrix-summary.md
+
+      - name: Add to GitHub Summary
+        run: cat matrix-summary.md >> $GITHUB_STEP_SUMMARY

From 0fd86234beda7d66679f6721c9392269bc72bae4 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sun, 12 Apr 2026 00:38:07 +0200
Subject: [PATCH 59/62] fix linting

---
 .github/scripts/charms_promotions.py          | 181 ++++++++++++------
 .../scripts/generate_revisions_from_specs.py  |  36 ++--
 python/tests/integration/conftest.py          |  14 +-
 3 files changed, 151 insertions(+), 80 deletions(-)

diff --git a/.github/scripts/charms_promotions.py b/.github/scripts/charms_promotions.py
index 9854c37d..509a2989 100644
--- a/.github/scripts/charms_promotions.py
+++ b/.github/scripts/charms_promotions.py
@@ -1,3 +1,5 @@
+"""Utilities for parsing charm status and promoting charm revisions."""
+
 import json
 import re
 import shlex
@@ -12,9 +14,17 @@
 
 RISKS = {"edge": 1, "beta": 2, "candidate": 3, "stable": 4}
 
-class Charm:
 
-    def __init__(self, name: str, revision: Optional[int]=None, channel: Optional[str]=None, architecture: Optional[str] = None):
+class Charm:
+    """Representation of a charm with helpers to query status and promote revisions."""
+
+    def __init__(
+        self,
+        name: str,
+        revision: Optional[int] = None,
+        channel: Optional[str] = None,
+        architecture: Optional[str] = None,
+    ):
         self.name = name
         self.revision = revision
         self.channel = channel
@@ -23,13 +33,20 @@ def __init__(self, name: str, revision: Optional[int]=None, channel: Optional[st
         self._status: Optional[dict] = None
 
     def __str__(self):
+        """Return a readable charm representation."""
         return f"{self.name}:{self.channel}({self.revision})"
 
     def __repr__(self):
+        """Return the canonical debug representation."""
         return self.__str__()
 
-    def get_status(self, channel: Optional[str] = None, revision: Optional[int] = None, architecture: Optional[str] = None):
-
+    def get_status(
+        self,
+        channel: Optional[str] = None,
+        revision: Optional[int] = None,
+        architecture: Optional[str] = None,
+    ):
+        """Return matching status entries for this charm."""
         if not channel:
             channel = self.channel
         if not revision:
@@ -38,21 +55,29 @@ def get_status(self, channel: Optional[str] = None, revision: Optional[int] = No
             architecture = self.architecture
 
         if not self._status:
-
-            output = subprocess.run(
+            result = subprocess.run(
                 ["charmcraft", "status", self.name, "--format", "json"],
-                capture_output=True
+                capture_output=True,
             )
 
-            if output.returncode != 0:
-                err_output = (output.stderr or b"").decode("utf-8", errors="replace").strip()
-                output = (output.stdout or b"").decode("utf-8", errors="replace").strip()
-                if "permission-required" in output or "permission-required" in err_output:
+            if result.returncode != 0:
+                err_output = (
+                    (result.stderr or b"").decode("utf-8", errors="replace").strip()
+                )
+                std_output = (
+                    (result.stdout or b"").decode("utf-8", errors="replace").strip()
+                )
+                if (
+                    "permission-required" in std_output
+                    or "permission-required" in err_output
+                ):
                     raise PermissionError(self.name)
                 else:
-                    raise RuntimeError(self.name, shlex.join(err.cmd), err)
+                    raise RuntimeError(
+                        self.name, shlex.join(result.args), err_output or std_output
+                    )
 
-            self._status = json.loads(output.stdout.decode("utf-8"))
+            self._status = json.loads(result.stdout.decode("utf-8"))
 
         items = [
             {"base": mapping["base"]} | release
@@ -60,26 +85,29 @@ def get_status(self, channel: Optional[str] = None, revision: Optional[int] = No
             for mapping in track["mappings"]
             for release in mapping["releases"]
             if mapping["base"]
-            if (not channel or channel == release["channel"]) and (
-                        not revision or revision == release["revision"]) and (
-                        not architecture or architecture == mapping["base"]["architecture"]
-                        )
+            if (not channel or channel == release["channel"])
+            and (not revision or revision == release["revision"])
+            and (not architecture or architecture == mapping["base"]["architecture"])
         ]
 
         return items
 
-
     def promote_version(self, risk: str, dry_run: bool = True):
+        """Promote a charm revision to a higher-risk channel."""
         if risk not in RISKS.keys():
             raise ValueError("The risk is not recognized")
 
         if not self.channel:
-            raise ValueError("channel field is not specified, that is required for promotion")
+            raise ValueError(
+                "channel field is not specified, that is required for promotion"
+            )
 
         items = self.get_status(revision=self.revision, channel=self.channel)
 
         if len(items) > 1:
-            raise ValueError(f"Multiple tracks match the provide revision and channel combination: {items}")
+            raise ValueError(
+                f"Multiple tracks match the provide revision and channel combination: {items}"
+            )
 
         if len(items) == 0:
             raise ValueError(f"No match found for charm {self}")
@@ -97,12 +125,16 @@ def promote_version(self, risk: str, dry_run: bool = True):
 
         channel_to = str(_channel.parent / risk)
 
-        cmds = (
-                ["charmcraft", "release", self.name, f"--channel={channel_to}",
-                 f"--revision={self.revision}"] +
-                [f"--resource={resource['name']}:{resource['revision']}" for
-                 resource in item["resources"]]
-        )
+        cmds = [
+            "charmcraft",
+            "release",
+            self.name,
+            f"--channel={channel_to}",
+            f"--revision={self.revision}",
+        ] + [
+            f"--resource={resource['name']}:{resource['revision']}"
+            for resource in item["resources"]
+        ]
 
         if dry_run:
             return "INFO: (dry-run mode) " + shlex.join(cmds)
@@ -110,41 +142,49 @@ def promote_version(self, risk: str, dry_run: bool = True):
         return subprocess.check_output(cmds).decode("utf-8")
 
 
-class Format(str,Enum):
+class Format(str, Enum):
+    """Supported status input formats."""
+
     TEXT = "text"
     YAML = "yaml"
 
+
 @dataclass
 class Bundle:
+    """Collection of charms parsed from status content."""
 
     charms: list[Charm]
 
     @classmethod
     def from_status(cls, content: str, format: Format | str = Format.TEXT):
-        parsers = {
-            Format.TEXT: TextParser,
-            Format.YAML: YAMLParser
-        }
+        """Parse status content into a Bundle."""
+        parsers = {Format.TEXT: TextParser, Format.YAML: YAMLParser}
         normalized_format = Format(format)
 
         return parsers[normalized_format].parse(content)
 
 
 class YAMLParser:
+    """Parser for Juju YAML status output."""
 
     @staticmethod
     def parse(content: str):
+        """Parse YAML status content and return a Bundle."""
         data = yaml.safe_load(content)
 
-        return Bundle([
-            Charm(app["charm-name"], int(app["charm-rev"]), app.get("charm-channel"))
-            for _, app in data["applications"].items()
-        ])
+        return Bundle(
+            [
+                Charm(
+                    app["charm-name"], int(app["charm-rev"]), app.get("charm-channel")
+                )
+                for _, app in data["applications"].items()
+            ]
+        )
 
 
 class TextParser:
     """
-    Parse a juju status output to find the charms deployed and their metadata (channel, revision, etc)
+    Parse a juju status output to find the charms deployed and their metadata (channel, revision, etc).
 
     The `parse` method is used to extract the list of charms, returned as a Bundle class.
     The reason why simpler parsing methods could not be used, is that the output format of a juju status is
@@ -166,17 +206,20 @@ class TextParser:
 
     @staticmethod
     def extract_first_word(mystring):
+        """Extract the first token, preserving leading spaces in the match."""
         m = TextParser.word_with_leading_spaces.match(mystring)
         if m:
-            return mystring[m.start():m.end()]
+            return mystring[m.start() : m.end()]
         return ""
 
     @staticmethod
     def parse_line(line, indices):
+        """Split a line using fixed-width index ranges."""
         return [line[start:end].strip() for start, end in indices]
 
     @staticmethod
     def parse(content: str):
+        """Parse text status output and return a Bundle."""
         lines = content.splitlines()
 
         white_spaces = re.compile(r"\s+\s+")
@@ -208,8 +251,8 @@ def parse(content: str):
         header = table_lines[0]
 
         # First guess of width based on headers
-        ends=[s.end() for s in white_spaces.finditer(header)]
-        indices = list(zip([0]+ends, ends+[-1]))
+        ends = [s.end() for s in white_spaces.finditer(header)]
+        indices = list(zip([0] + ends, ends + [-1], strict=False))
 
         # Columns names
         columns = TextParser.parse_line(header, indices)
@@ -219,32 +262,43 @@ def parse(content: str):
         # of the columns header (text aligned right)
         widths = [len(column) for column in columns]
         for line in table_lines[1:]:
-
-            widths = list(map(max,zip(
-                widths,
-                [len(TextParser.extract_first_word(line[start:end])) for start, end in indices]
-            )))
+            widths = list(
+                map(
+                    max,
+                    zip(
+                        widths,
+                        [
+                            len(TextParser.extract_first_word(line[start:end]))
+                            for start, end in indices
+                        ],
+                        strict=False,
+                    ),
+                )
+            )
 
         # New guess
-        ends = [start+width for (start,_), width in zip(indices, widths)]
+        ends = [
+            start + width for (start, _), width in zip(indices, widths, strict=False)
+        ]
 
-        indices = list(zip([0]+ends[:-1], ends[:-1]+[-1]))
+        indices = list(zip([0] + ends[:-1], ends[:-1] + [-1], strict=False))
 
         data = [
-            dict(zip(columns, TextParser.parse_line(line, indices)))
+            dict(zip(columns, TextParser.parse_line(line, indices), strict=False))
             for line in table_lines[1:]
             if line.strip()
         ]
 
-        return Bundle([
-            Charm(item["Charm"], int(item["Rev"]), item["Channel"])
-            for item in data
-            if item["Charm"]
-        ])
+        return Bundle(
+            [
+                Charm(item["Charm"], int(item["Rev"]), item["Channel"])
+                for item in data
+                if item["Charm"]
+            ]
+        )
 
 
 if __name__ == "__main__":
-
     import argparse
 
     parser = argparse.ArgumentParser()
@@ -265,7 +319,9 @@ def parse(content: str):
     )
     parser.set_defaults(dry_run=True)
     parser.add_argument("--format", choices=("text", "yaml"), default="text")
-    parser.add_argument("--promote-to", choices=("beta", "candidate", "stable"), default="beta")
+    parser.add_argument(
+        "--promote-to", choices=("beta", "candidate", "stable"), default="beta"
+    )
     parser.add_argument("--exclude", nargs="*", default=["mysql-k8s"])
     args = parser.parse_args()
 
@@ -273,15 +329,17 @@ def parse(content: str):
         bundle = Bundle.from_status(fid.read(), args.format)
 
     for charm in bundle.charms:
-        if not charm.name in args.exclude:
+        if charm.name not in args.exclude:
             try:
                 print(charm.promote_version(args.promote_to, args.dry_run))
             except ValueError as err:
-                print(f"ERROR: skipping {charm.name} due to failure in promotion. Reason: {type(err).__name__} - Details: {','.join(err.args)}")
-            except PermissionError as err:
                 print(
-                        f"WARNING: skipping '{charm.name}' due to missing permissions. Add it to --exclude to avoid this warning.",
-                        file=sys.stderr,
+                    f"ERROR: skipping {charm.name} due to failure in promotion. Reason: {type(err).__name__} - Details: {','.join(err.args)}"
+                )
+            except PermissionError:
+                print(
+                    f"WARNING: skipping '{charm.name}' due to missing permissions. Add it to --exclude to avoid this warning.",
+                    file=sys.stderr,
                 )
             except RuntimeError as err:
                 print(
@@ -289,4 +347,7 @@ def parse(content: str):
                     file=sys.stderr,
                 )
         else:
-            print(f"WARNING: excluding charm {charm.name} (see --exclude flag)", file=sys.stderr)
+            print(
+                f"WARNING: excluding charm {charm.name} (see --exclude flag)",
+                file=sys.stderr,
+            )
diff --git a/.github/scripts/generate_revisions_from_specs.py b/.github/scripts/generate_revisions_from_specs.py
index 61dfe091..06c8b510 100644
--- a/.github/scripts/generate_revisions_from_specs.py
+++ b/.github/scripts/generate_revisions_from_specs.py
@@ -1,21 +1,26 @@
+"""Generate a revisions mapping file from a specs definition."""
+
 import argparse
 import sys
 from dataclasses import dataclass
 from pathlib import Path
 
 import yaml
-
 from charms_promotions import Charm
 
 
 @dataclass(frozen=True)
 class CharmSpec:
+    """Charm release entry from specs."""
+
     name: str
     channel: str
 
 
 @dataclass(frozen=True)
 class Specs:
+    """Top-level specs model."""
+
     architectures: list[str]
     releases: dict[str, list[CharmSpec]]
     revision_mapping: dict[str, str]
@@ -29,6 +34,7 @@ def normalize_channel(channel: str, risk: str) -> str:
 
 
 def parse_specs(specs_file: Path) -> Specs:
+    """Load and validate specs YAML, returning a typed Specs object."""
     with specs_file.open("r", encoding="utf-8") as file_obj:
         data = yaml.safe_load(file_obj)
 
@@ -38,10 +44,13 @@ def parse_specs(specs_file: Path) -> Specs:
     architectures = data.get("architectures")
     releases = data.get("releases")
     revision_mapping = data.get("revision_mapping")
-    
 
-    if not isinstance(architectures, list) or not all(isinstance(a, str) for a in architectures):
-        raise ValueError("Invalid specs format: architectures must be a list of strings")
+    if not isinstance(architectures, list) or not all(
+        isinstance(a, str) for a in architectures
+    ):
+        raise ValueError(
+            "Invalid specs format: architectures must be a list of strings"
+        )
 
     if not isinstance(releases, dict):
         raise ValueError("Invalid specs format: releases must be a mapping")
@@ -60,9 +69,7 @@ def parse_specs(specs_file: Path) -> Specs:
     parsed_releases: dict[str, list[CharmSpec]] = {}
     for release, charms in releases.items():
         if not isinstance(charms, list):
-            raise ValueError(
-                f"Invalid specs format: releases.{release} must be a list"
-            )
+            raise ValueError(f"Invalid specs format: releases.{release} must be a list")
 
         parsed_entries: list[CharmSpec] = []
         for entry in charms:
@@ -90,6 +97,7 @@ def parse_specs(specs_file: Path) -> Specs:
 
 
 def resolve_revision(charm: Charm) -> int:
+    """Resolve a single revision for a charm/channel/architecture tuple."""
     items = charm.get_status()
 
     if not items:
@@ -115,6 +123,7 @@ def resolve_revision(charm: Charm) -> int:
 
 
 def build_output(specs: Specs, risk: str) -> dict:
+    """Build the output mapping keyed by architecture, release, and mapped revision key."""
     output = {}
     revision_mapping = specs.revision_mapping
 
@@ -129,9 +138,7 @@ def build_output(specs: Specs, risk: str) -> dict:
                 mapped_name = revision_mapping.get(charm_name)
 
                 if not mapped_name:
-                    raise ValueError(
-                        f"Missing revision mapping for charm={charm_name}"
-                    )
+                    raise ValueError(f"Missing revision mapping for charm={charm_name}")
 
                 charm = Charm(
                     name=charm_name,
@@ -146,6 +153,7 @@ def build_output(specs: Specs, risk: str) -> dict:
 
 
 def main() -> int:
+    """CLI entrypoint."""
     parser = argparse.ArgumentParser(
         description=(
             "Parse specs.yaml and fetch charm revisions by architecture/release/channel "
@@ -154,13 +162,13 @@ def main() -> int:
     )
     parser.add_argument(
         "--specs-file",
-        default="scripts/specs.yaml",
-        help="Input specs file (default: scripts/specs.yaml)",
+        default="specs.yaml",
+        help="Input specs file (default: specs.yaml)",
     )
     parser.add_argument(
         "--output-file",
-        default="scripts/revisions.yaml",
-        help="Output YAML file (default: scripts/revisions.yaml)",
+        default="revisions.yaml",
+        help="Output YAML file (default: revisions.yaml)",
     )
     parser.add_argument(
         "--risk",
diff --git a/python/tests/integration/conftest.py b/python/tests/integration/conftest.py
index c770c935..c5108f7e 100644
--- a/python/tests/integration/conftest.py
+++ b/python/tests/integration/conftest.py
@@ -206,14 +206,14 @@ def test_uuid(request) -> str:
 def tfvars(request) -> dict:
     """External Terraform variables loaded from a file."""
     tfvars_file = request.config.getoption("--tfvars-file")
-    
+
     if not tfvars_file:
         return {}
-    
+
     file_path = Path(tfvars_file)
     if not file_path.exists():
         raise FileNotFoundError(f"Terraform variables file not found: {tfvars_file}")
-    
+
     # Load JSON or YAML file
     with file_path.open("r", encoding="utf-8") as f:
         try:
@@ -222,11 +222,13 @@ def tfvars(request) -> dict:
             else:
                 data = yaml.safe_load(f)
         except (json.JSONDecodeError, yaml.YAMLError) as err:
-            raise ValueError(f"Failed to parse Terraform variables file: {err}")
-    
+            raise ValueError(
+                f"Failed to parse Terraform variables file: {err}"
+            ) from err
+
     if not isinstance(data, dict):
         raise ValueError("Terraform variables file must contain a JSON/YAML mapping")
-    
+
     return data
 
 

From 0ea68d02b16649a0b40603aca3afa9132c1f37b4 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sun, 12 Apr 2026 01:02:07 +0200
Subject: [PATCH 60/62] fix file creation and feeding

---
 .github/actions/run-test/action.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/actions/run-test/action.yaml b/.github/actions/run-test/action.yaml
index 9babfcc0..e7264679 100644
--- a/.github/actions/run-test/action.yaml
+++ b/.github/actions/run-test/action.yaml
@@ -63,6 +63,11 @@ inputs:
     required: false
     default: ""
     type: string
+  architecture:
+    description: "Target architecture for the test run"
+    required: false
+    default: "amd64"
+    type: string
 
 runs:
   using: "composite"

From 77a53a76b154536ab9773244c4df5b494b45db19 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sun, 12 Apr 2026 01:16:21 +0200
Subject: [PATCH 61/62] fix

---
 .github/actions/run-test/action.yaml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/actions/run-test/action.yaml b/.github/actions/run-test/action.yaml
index e7264679..a3b8763b 100644
--- a/.github/actions/run-test/action.yaml
+++ b/.github/actions/run-test/action.yaml
@@ -255,7 +255,15 @@ runs:
             find tfvars-artifacts -type f || true
             exit 1
           fi
-          TFVARS_ARG="--tfvars-file $TFVARS_PATH"
+
+          version=${{ inputs.spark-version }}
+          major_minor="${version%.*}"   # -> 3.5
+          arch="${{ inputs.architecture }}"
+
+          # Extract nested mapping under amd64 and "<major>.<minor>"
+          yq ".${arch}.\"${major_minor}\"" $TFVARS_PATH > python/revisions.yaml
+
+          TFVARS_ARG="--tfvars-file revisions.yaml"
         fi
 
         juju add-model spark-bundle-test

From 55ff22ae691bcacf51c3ef3d99548a1284679416 Mon Sep 17 00:00:00 2001
From: Enrico Deusebio <enrico.deusebio@canonical.com>
Date: Sun, 12 Apr 2026 01:48:44 +0200
Subject: [PATCH 62/62] rollout all tests

---
 .github/workflows/on_pull_request.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml
index f2d6bf59..d9d9b38c 100644
--- a/.github/workflows/on_pull_request.yaml
+++ b/.github/workflows/on_pull_request.yaml
@@ -21,12 +21,12 @@ jobs:
     uses: ./.github/workflows/generate-charm-revisions.yaml
     secrets: inherit
     with:
-      risk: beta
+      risk: edge
 
   tests:
     name: Run CI tests on a PR
     needs: generate-revisions
-    uses: ./.github/workflows/ci-tests-single-kyuubi.yaml
+    uses: ./.github/workflows/ci-tests-minimal.yaml
     secrets: inherit
     with:
       tfvars-file: ${{ needs.generate-revisions.outputs.tfvars-artifact-name }}
\ No newline at end of file