Tde update key (#284)

Author: whites11

.github/workflows/e2e.yaml

@@ -147,7 +147,7 @@ jobs:
           region: ${{ steps.credentials.outputs.region }}
           aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
       - name: cleanup
-        if: ${{ always() && matrix.test.cloud == 'aws' && matrix.test.name == 'private_endpoint' }}
+        if: ${{ always() && matrix.test.cloud == 'aws' }}
         uses: ./.github/actions/cleanup-aws
         with:
           service_name: ${{steps.name.outputs.test_name}}
@@ -218,13 +218,15 @@ jobs:
           skip_build: "false"
           region: ${{ steps.credentials.outputs.region }}
           aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
+
       - name: cleanup
-        if: ${{ always() && matrix.test.cloud == 'aws' && matrix.test.name == 'private_endpoint' }}
+        if: ${{ always() && matrix.test.cloud == 'aws' }}
         uses: ./.github/actions/cleanup-aws
         with:
           service_name: ${{steps.name.outputs.test_name}}
           aws_region: ${{ steps.credentials.outputs.region }}
           aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
+
       - name: Mark error
         id: status
         if: failure()

.github/workflows/release.yaml

@@ -136,6 +136,14 @@ jobs:
           region: ${{ steps.credentials.outputs.region }}
           aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
 
+      - name: cleanup
+        if: ${{ always() && matrix.test.cloud == 'aws' }}
+        uses: ./.github/actions/cleanup-aws
+        with:
+          service_name: "[e2e]-${{ matrix.test.name }}-${{ matrix.tf_release }}-${{ matrix.test.cloud }}-${{ needs.token.outputs.token }}"
+          aws_region: ${{ steps.credentials.outputs.region }}
+          aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
+
       - name: Mark error
         id: status
         if: failure()
@@ -190,6 +198,14 @@ jobs:
           region: ${{ steps.credentials.outputs.region }}
           aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
 
+      - name: cleanup
+        if: ${{ always() && matrix.test.cloud == 'aws' }}
+        uses: ./.github/actions/cleanup-aws
+        with:
+          service_name: "[upg]-${{ matrix.test.name }}-${{ matrix.tf_release }}-${{ matrix.test.cloud }}-${{ needs.token.outputs.token }}"
+          aws_region: ${{ steps.credentials.outputs.region }}
+          aws_role_arn: ${{ secrets.AWS_ASSUME_ROLE_ARN }}
+
       - name: Mark error
         id: status
         if: failure()

docs/resources/service.md

@@ -59,12 +59,11 @@ resource "clickhouse_service" "service" {
 - `encryption_assumed_role_identifier` (String) Custom role identifier ARN.
 - `encryption_key` (String) Custom encryption key ARN.
 - `endpoints` (Attributes) Allow to enable and configure additional endpoints (read protocols) to expose on the ClickHouse service. (see [below for nested schema](#nestedatt--endpoints))
-- `has_transparent_data_encryption` (Boolean) If true, the Transparent Data Encryption (TDE) feature is enabled in the service. Only supported in AWS and GCP. Requires an organization with the Enterprise plan.
 - `idle_scaling` (Boolean) When set to true the service is allowed to scale down to zero when idle.
 - `idle_timeout_minutes` (Number) Set minimum idling timeout (in minutes). Must be greater than or equal to 5 minutes. Must be set if idle_scaling is enabled.
-- `max_replica_memory_gb` (Number) Maximum memory of a single replica during auto-scaling in Gb. Must be a multiple of 8. `max_replica_memory_gb` x `num_replicas` (default 3) must be lower than 360 for non paid services or 720 for paid services.
+- `max_replica_memory_gb` (Number) Maximum memory of a single replica during auto-scaling in Gb. Must be a multiple of 4 greater than or equal to 8. `max_replica_memory_gb` x `num_replicas` (default 3) must be lower than 360 for non paid services or 720 for paid services.
 - `max_total_memory_gb` (Number, Deprecated) Maximum total memory of all workers during auto-scaling in Gb. Must be a multiple of 12 and lower than 360 for non paid services or 720 for paid services.
-- `min_replica_memory_gb` (Number) Minimum memory of a single replica during auto-scaling in Gb. Must be a multiple of 8. `min_replica_memory_gb` x `num_replicas` (default 3) must be lower than 360 for non paid services or 720 for paid services.
+- `min_replica_memory_gb` (Number) Minimum memory of a single replica during auto-scaling in Gb. Must be a multiple of 4 greater than or equal to 8. `min_replica_memory_gb` x `num_replicas` (default 3) must be lower than 360 for non paid services or 720 for paid services.
 - `min_total_memory_gb` (Number, Deprecated) Minimum total memory of all workers during auto-scaling in Gb. Must be a multiple of 12 and greater than 24.
 - `num_replicas` (Number) Number of replicas for the service. Must be between 3 and 20. Contact support to enable this feature.
 - `password` (String, Sensitive) Password for the default user. One of either `password` or `password_hash` must be specified.
@@ -73,6 +72,7 @@ resource "clickhouse_service" "service" {
 - `readonly` (Boolean) Indicates if this service should be read only. Only allowed for secondary services, those which share data with another service (i.e. when `warehouse_id` field is set).
 - `release_channel` (String) Release channel to use for this service. Either 'default' or 'fast'. Switching from 'fast' to 'default' release channel is not supported.
 - `tier` (String) Tier of the service: 'development', 'production'. Required for organizations using the Legacy ClickHouse Cloud Tiers, must be omitted for organizations using the new ClickHouse Cloud Tiers.
+- `transparent_data_encryption` (Attributes) Configuration of the Transparent Data Encryption (TDE) feature. Requires an organization with the Enterprise plan. (see [below for nested schema](#nestedatt--transparent_data_encryption))
 - `warehouse_id` (String) ID of the warehouse to share the data with. Must be in the same cloud and region.
 
 ### Read-Only
@@ -155,6 +155,18 @@ Optional:
 - `allowed_origins` (String) Comma separated list of domain names to be allowed cross-origin resource sharing (CORS) access to the query API. Leave this field empty to restrict access to backend servers only
 
 
+<a id="nestedatt--transparent_data_encryption"></a>
+### Nested Schema for `transparent_data_encryption`
+
+Optional:
+
+- `enabled` (Boolean) If true, TDE is enabled for the service.
+
+Read-Only:
+
+- `role_id` (String) ID of Role to be used for granting access to the Encryption Key. This is an ARN for AWS services and a Service Account Identifier for GCP.
+
+
 <a id="nestedatt--private_endpoint_config"></a>
 ### Nested Schema for `private_endpoint_config`
 

docs/resources/service_transparent_data_encryption_key_association.md

@@ -0,0 +1,47 @@
+---
+# generated by https://github.com/hashicorp/terraform-plugin-docs
+page_title: "clickhouse_service_transparent_data_encryption_key_association Resource - clickhouse"
+subcategory: ""
+description: |-
+  You can use the clickhouse_service_transparent_data_encryption_key_association resource to associate your own Encryption Key with a Clickhouse Service with the Transparent Data Encryption (TDE) feature enabled.
+  Please note that this feature requires an organization with the Enterprise plan.
+---
+
+# clickhouse_service_transparent_data_encryption_key_association (Resource)
+
+You can use the *clickhouse_service_transparent_data_encryption_key_association* resource to associate your own Encryption Key with a Clickhouse Service with the Transparent Data Encryption (TDE) feature enabled.
+Please note that this feature requires an organization with the `Enterprise` plan.
+
+## Example Usage
+
+```terraform
+resource "clickhouse_service" "service" {
+  ...
+}
+
+resource "aws_kms_key" "enc" {
+  ...
+}
+
+resource "clickhouse_service_transparent_data_encryption_key_association" "service_key_association" {
+  service_id = clickhouse_service.service.id
+  key_id = aws_kms_key.enc.arn
+}
+```
+
+<!-- schema generated by tfplugindocs -->
+## Schema
+
+### Required
+
+- `key_id` (String) ID of the Encryption key to use for data encryption. Must be an ARN for AWS services or a Key Resource Path for GCP services.
+- `service_id` (String) ClickHouse Service ID
+
+## Import
+
+Import is supported using the following syntax:
+
+```shell
+# Endpoint Attachments can be imported by specifying the clickhouse service UUID
+terraform import clickhouse_service_transparent_data_encryption_key_association.example xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+```

examples/full/tde/aws/aws.tf

@@ -0,0 +1,108 @@
+variable "aws_key" {
+  type = string
+}
+
+variable "aws_secret" {
+  type = string
+}
+
+variable "aws_session_token" {
+  type = string
+  default = ""
+}
+
+locals {
+  tags = {
+    Role = "terraform-e2e-test"
+    ServiceName = replace(var.service_name, "/[^-a-zA-Z0-9_.:/=+@ ]/", "_")
+  }
+}
+
+provider "aws" {
+  region     = var.region
+  access_key = var.aws_key
+  secret_key = var.aws_secret
+  token      = var.aws_session_token
+}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_iam_policy_document" "policy" {
+  # Allow root user on the account all access.
+  statement {
+    sid = "AllowRoot"
+
+    actions   = ["kms:*"]
+    resources = ["*"]
+    principals {
+      type        = "AWS"
+      identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"]
+    }
+  }
+
+  # Allow user that runs terraform to manage the KMS key.
+  statement {
+    sid = "AllowAdmins"
+    actions = [
+      "kms:Create*",
+      "kms:Describe*",
+      "kms:Enable*",
+      "kms:List*",
+      "kms:Put*",
+      "kms:Update*",
+      "kms:Revoke*",
+      "kms:Disable*",
+      "kms:Get*",
+      "kms:Delete*",
+      "kms:TagResource",
+      "kms:UntagResource",
+      "kms:ScheduleKeyDeletion",
+      "kms:CancelKeyDeletion",
+      "kms:RotateKeyOnDemand",
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:DescribeKey",
+      "kms:CreateGrant",
+      "kms:ListGrants",
+      "kms:RevokeGrant"
+    ]
+    resources = ["*"]
+
+    principals {
+      type        = "AWS"
+      identifiers = [data.aws_caller_identity.current.arn]
+    }
+  }
+
+  # Allow clickhouse's accounts to access the KMS key.
+  statement {
+    sid = "AllowClickHouse"
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ReEncrypt*",
+      "kms:DescribeKey",
+    ]
+    resources = ["*"]
+
+    principals {
+      type        = "AWS"
+      identifiers = [clickhouse_service.service.transparent_data_encryption.role_id]
+    }
+  }
+}
+
+resource "aws_kms_key" "enc" {
+  customer_master_key_spec = "SYMMETRIC_DEFAULT"
+  deletion_window_in_days  = 7
+  description              = var.service_name
+  enable_key_rotation      = false
+  is_enabled               = true
+  key_usage                = "ENCRYPT_DECRYPT"
+  multi_region             = false
+
+  policy = data.aws_iam_policy_document.policy.json
+
+  tags = local.tags
+}

examples/full/tde/aws/main.tf

@@ -48,22 +48,6 @@ resource "clickhouse_service" "service" {
     }
   ]
 
-  endpoints = {
-    mysql = {
-      enabled = true
-    }
-  }
-
-  query_api_endpoints = {
-    api_key_ids = [
-      data.clickhouse_api_key_id.self.id,
-    ]
-    roles = [
-      "sql_console_admin"
-    ]
-    allowed_origins = null
-  }
-
   min_replica_memory_gb = 8
   max_replica_memory_gb = 120
 
@@ -73,7 +57,14 @@ resource "clickhouse_service" "service" {
     backup_start_time                = null
   }
 
-  has_transparent_data_encryption = true
+  transparent_data_encryption = {
+    enabled = true
+  }
+}
+
+resource "clickhouse_service_transparent_data_encryption_key_association" "service_key_association" {
+  service_id = clickhouse_service.service.id
+  key_id = aws_kms_key.enc.arn
 }
 
 output "service_endpoints" {

examples/full/tde/aws/variables.tfvars.sample

@@ -2,3 +2,8 @@
 organization_id = "aee076c1-3f83-4637-95b1-ad5a0a825b71"
 token_key       = "avhj1U5QCdWAE9CA9"
 token_secret    = "4b1dROiHQEuSXJHlV8zHFd0S7WQj7CGxz5kGJeJnca"
+
+# AWS
+aws_key                      = "key"
+aws_secret                   = "secret"
+aws_region                   = "us-west-2"

examples/full/tde/gcp/main.tf

@@ -71,7 +71,9 @@ resource "clickhouse_service" "service" {
     backup_retention_period_in_hours = 48
   }
 
-  has_transparent_data_encryption = true
+  transparent_data_encryption = {
+    enabled = true
+  }
 }
 
 output "service_endpoints" {

examples/resources/clickhouse_service_transparent_data_encryption_key_association/import.sh

@@ -0,0 +1,2 @@
+# Endpoint Attachments can be imported by specifying the clickhouse service UUID
+terraform import clickhouse_service_transparent_data_encryption_key_association.example xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx

examples/resources/clickhouse_service_transparent_data_encryption_key_association/resource.tf

@@ -0,0 +1,12 @@
+resource "clickhouse_service" "service" {
+  ...
+}
+
+resource "aws_kms_key" "enc" {
+  ...
+}
+
+resource "clickhouse_service_transparent_data_encryption_key_association" "service_key_association" {
+  service_id = clickhouse_service.service.id
+  key_id = aws_kms_key.enc.arn
+}