ClickPipes reverse private endpoint support

examples/clickpipe/externally_managed_table/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/kafka_azure_eventhub/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/kafka_confluent/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/kafka_msk_iam_role/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/kafka_msk_iam_user/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/kafka_offset_strategy/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/kafka_redpanda_scram/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/kafka_schema_registry/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/multiple_pipes_example/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/object_storage_iam_role/provider.tf

@@ -11,13 +11,9 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/object_storage_iam_user/provider.tf

@@ -11,13 +11,10 @@ terraform {
 variable "organization_id" {}
 variable "token_key" {}
 variable "token_secret" {}
-variable "api_url" {
-  default = "https://api.clickhouse.cloud/v1"
-}
+
 
 provider "clickhouse" {
   organization_id = var.organization_id
   token_key       = var.token_key
   token_secret    = var.token_secret
-  api_url = var.api_url
 }

examples/clickpipe/reverse_private_endpoint_msk/main.tf

@@ -0,0 +1,39 @@
+variable "service_id" {
+  description = "ClickHouse service ID"
+}
+
+variable "msk_cluster_arn" {
+  description = "MSK cluter ARN"
+}
+
+variable "msk_authentication" {
+  description = "MSK authentication"
+  default     = "SASL_SCRAM"
+}
+
+resource "clickhouse_clickpipes_reverse_private_endpoint" "endpoint" {
+  service_id         = var.service_id
+  description        = "Reverse private endpoint for ClickPipes"
+  type               = "MSK_MULTI_VPC"
+  msk_cluster_arn    = var.msk_cluster_arn
+  msk_authentication = var.msk_authentication
+}
+
+output "id" {
+  value = clickhouse_clickpipes_reverse_private_endpoint.endpoint.id
+}
+
+output "msk_vpc_connection_id" {
+  value = clickhouse_clickpipes_reverse_private_endpoint.endpoint.endpoint_id
+}
+
+output "dns_names" {
+  value = concat(
+    clickhouse_clickpipes_reverse_private_endpoint.endpoint.dns_names != null ? clickhouse_clickpipes_reverse_private_endpoint.endpoint.dns_names : [],
+    clickhouse_clickpipes_reverse_private_endpoint.endpoint.private_dns_names != null ? clickhouse_clickpipes_reverse_private_endpoint.endpoint.private_dns_names : []
+  )
+}
+
+output "status" {
+  value = clickhouse_clickpipes_reverse_private_endpoint.endpoint.status
+}

examples/clickpipe/reverse_private_endpoint_msk/provider.tf

@@ -0,0 +1,19 @@
+# This file is generated automatically please do not edit
+terraform {
+  required_providers {
+    clickhouse = {
+      version = "2.0.0-alpha1"
+      source  = "ClickHouse/clickhouse"
+    }
+  }
+}
+
+variable "organization_id" {}
+variable "token_key" {}
+variable "token_secret" {}
+
+provider "clickhouse" {
+  organization_id = var.organization_id
+  token_key       = var.token_key
+  token_secret    = var.token_secret
+}

examples/clickpipe/reverse_private_endpoint_msk/variables.sample.tfvars

@@ -0,0 +1,8 @@
+# these keys are for example only and won't work when pointed to a deployed ClickHouse OpenAPI server
+organization_id = "aee076c1-3f83-4637-95b1-ad5a0a825b71"
+token_key       = "avhj1U5QCdWAE9CA9"
+token_secret    = "4b1dROiHQEuSXJHlV8zHFd0S7WQj7CGxz5kGJeJnca"
+service_id      = "aee076c1-3f83-4637-95b1-ad5a0a825b71"
+
+msk_cluster_arn    = "arn:aws:kafka:us-east-2:123456789012:cluster/ExampleCluster/12345678-1234-1234-1234-123456789012-1"
+msk_authentication = "SASL_SCRAM" # or SASL_IAM

examples/clickpipe/reverse_private_endpoint_msk_pipe/clickpipe.tf

@@ -0,0 +1,68 @@
+locals {
+  // MSK brokers for multi-VPC connectivity ports starting from 14001 incremented by a broker number
+  msk_brokers = [for i, dns in clickhouse_clickpipes_reverse_private_endpoint.endpoint.private_dns_names : "${dns}:${i + 14001}"]
+}
+
+resource "clickhouse_clickpipe" "msk" {
+  name        = "MSK pipe using Reverse Private Endpoint"
+  description = "This pipe is using a secure private endpoint to connect to MSK"
+
+  service_id = var.service_id
+
+  scaling = {
+    replicas = 1
+  }
+
+  state = "Running"
+
+  source = {
+    kafka = {
+      type    = "msk"
+      format  = "JSONEachRow"
+      brokers = join(",", local.msk_brokers)
+      topics  = var.kafka_topic
+
+      authentication = var.msk_authentication
+      credentials = {
+        username = var.msk_scram_user
+        password = var.msk_scram_password
+      }
+
+      reverse_private_endpoint_ids = [
+        clickhouse_clickpipes_reverse_private_endpoint.endpoint.id
+      ]
+    }
+  }
+
+  destination = {
+    table         = "my_table"
+    managed_table = true
+
+    table_definition = {
+      engine = {
+        type = "MergeTree"
+      }
+    }
+
+    columns = [
+      {
+        name = "my_field1"
+        type = "String"
+        }, {
+        name = "my_field2"
+        type = "UInt64"
+      }
+    ]
+  }
+
+  field_mappings = [
+    {
+      source_field      = "my_field"
+      destination_field = "my_field1"
+    }
+  ]
+}
+
+output "clickpipe_id" {
+  value = clickhouse_clickpipe.msk.id
+}

examples/clickpipe/reverse_private_endpoint_msk_pipe/provider.tf

@@ -0,0 +1,19 @@
+# This file is generated automatically please do not edit
+terraform {
+  required_providers {
+    clickhouse = {
+      version = "2.0.0-alpha1"
+      source  = "ClickHouse/clickhouse"
+    }
+  }
+}
+
+variable "organization_id" {}
+variable "token_key" {}
+variable "token_secret" {}
+
+provider "clickhouse" {
+  organization_id = var.organization_id
+  token_key       = var.token_key
+  token_secret    = var.token_secret
+}

examples/clickpipe/reverse_private_endpoint_msk_pipe/reverse_private_endpoint.tf

@@ -0,0 +1,27 @@
+locals {
+  rpe_msk_authentication_mapping = {
+    "SCRAM-SHA-512" = "SASL_SCRAM"
+    "IAM_ROLE"      = "SASL_IAM"
+    "IAM_USER"     = "SASL_IAM"
+  }
+
+  // In this example we map from a ClickPipe authentication method
+  // into a MSK multi-VPC authentication method.
+  rpe_msk_authentication = local.rpe_msk_authentication_mapping[var.msk_authentication]
+}
+
+resource "clickhouse_clickpipes_reverse_private_endpoint" "endpoint" {
+  service_id                = var.service_id
+  description               = "Reverse private endpoint for my ClickPipe"
+  type                      = "MSK_MULTI_VPC"
+  msk_cluster_arn = var.msk_cluster_arn
+  msk_authentication = local.rpe_msk_authentication
+}
+
+output "reverse_private_endpoint_id" {
+  value = clickhouse_clickpipes_reverse_private_endpoint.endpoint.id
+}
+
+output "msk_vpc_connection_id" {
+  value = clickhouse_clickpipes_reverse_private_endpoint.endpoint.endpoint_id
+}

examples/clickpipe/reverse_private_endpoint_msk_pipe/variables.sample.tfvars

@@ -0,0 +1,11 @@
+# these keys are for example only and won't work when pointed to a deployed ClickHouse OpenAPI server
+organization_id = "aee076c1-3f83-4637-95b1-ad5a0a825b71"
+token_key       = "avhj1U5QCdWAE9CA9"
+token_secret    = "4b1dROiHQEuSXJHlV8zHFd0S7WQj7CGxz5kGJeJnca"
+service_id      = "aee076c1-3f83-4637-95b1-ad5a0a825b71"
+
+msk_cluster_arn = "arn:aws:kafka:us-east-2:123456789012:cluster/ExampleCluster/12345678-1234-1234-1234-123456789012-1"
+msk_authentication = "SASL_SCRAM" # or SASL_IAM
+msk_scram_user = "example_user"
+msk_scram_password = "example_password"
+kafka_topic = "example_topic"

examples/clickpipe/reverse_private_endpoint_msk_pipe/vars.tf

@@ -0,0 +1,27 @@
+variable "service_id" {
+  description = "ClickHouse service ID"
+}
+
+variable "msk_cluster_arn" {
+  description = "MSK cluter ARN"
+}
+
+variable "msk_authentication" {
+  description = "MSK authentication"
+  default     = "SCRAM-SHA-512" # or IAM_USER or IAM_ROLE
+}
+
+variable "msk_scram_user" {
+  description = "MSK scram user"
+  default     = "scram_user"
+}
+
+variable "msk_scram_password" {
+  description = "MSK scram password"
+  default     = "scram_password"
+}
+
+variable "kafka_topic" {
+  description = "Kafka topic"
+  default     = "my_topic"
+}