Merge pull request #13 from santosh-philips/25681-Kafka-Update-Config

Added kafka message related property configuration

Author: krishna-prasad-s

README.md

@@ -3,7 +3,7 @@
 # HSDP Kafka module
 
 Module to create an Apache kafka cluster deployed
-on the HSDP Container Host infrastructure. This module serves as a 
+on the HSDP Container Host infrastructure. This module serves as a
 blueprint for future HSDP Container Host modules. Example usage
 
 ```hcl
@@ -31,15 +31,16 @@ module "kafka" {
   zoo_key_store     = {
     keystore   = "./zookeystore.jks"
     password   = "somepass"
-  }    
+  }
+  max_request_size = 1048576
 }
 ```
 
 __IMPORTANT SECURITY INFORMATION__
 > This module currently **enables** only mTLS-SSL
 > between Kafka, Zookeeper or any connecting client apps.
 > Operating and maintaining applications on Container Host is always
-> your responsibility. This includes ensuring any security 
+> your responsibility. This includes ensuring any security
 > measures are in place in case you need them.
 
 ## Requirements
@@ -84,6 +85,8 @@ __IMPORTANT SECURITY INFORMATION__
 | zoo\_key\_store | Zookeeper Key store for SSL | <pre>object(<br>    { keystore = string,<br>    password = string }<br>  )</pre> | n/a | yes |
 | zoo\_trust\_store | Zookeeper Trust store for SSL | <pre>object(<br>    { truststore = string,<br>    password = string }<br>  )</pre> | n/a | yes |
 | zookeeper\_connect | Zookeeper connect string to use | `string` | n/a | yes |
+| max\_request\_size | Maximum request size of a message supported in broker | `number` | n/a | no |
+| max\_partition\_fetch\_bytes | The maximum amount of data per-partition the broker will return | `number` | n/a | no |
 
 Incase you are wondering why we need zookeeper key store, its required by bitnami please refer to bitnami documentation.
 
@@ -128,7 +131,7 @@ openssl pkcs12 -in $inputJksFilename -out $outputPublicKeyFilename -clcerts -nok
 
 #Export the private key with the PEM format
 keytool -importkeystore -srckeystore $inputJksFilename -destkeystore tempKeyStore.p12 -deststoretype PKCS12 -srcstorepass $inputKeyStorePassword -deststorepass $inputKeyStorePassword -noprompt
-openssl pkcs12 -in tempKeyStore.p12 -nodes -nocerts -out $outputPrivateKeyFilename  -passin pass:$inputKeyStorePassword -passout pass:$inputKeyStorePassword 
+openssl pkcs12 -in tempKeyStore.p12 -nodes -nocerts -out $outputPrivateKeyFilename  -passin pass:$inputKeyStorePassword -passout pass:$inputKeyStorePassword
 ```
 
 

main.tf

@@ -102,6 +102,6 @@ resource "hsdp_container_host_exec" "cluster" {
   commands = [
     "chmod +x /home/${var.user}/bootstrap-cluster.sh",
     "chmod 755 /home/${var.user}/jmxconfig.yml.tmpl",
-    "/home/${var.user}/bootstrap-cluster.sh -n ${join(",", hsdp_container_host.kafka.*.private_ip)} -c ${random_id.id.hex} -d ${var.image} -i ${count.index + 1} -z ${var.zookeeper_connect} -x ${element(hsdp_container_host.kafka.*.private_ip, count.index)} -r \"${var.retention_hours}\" -p ${var.kafka_key_store.password} -t ${var.zoo_trust_store.password} -k ${var.zoo_key_store.password} -R ${var.default_replication_factor} -a ${var.auto_create_topics_enable} -e ${var.enable_exporters}"
+    "/home/${var.user}/bootstrap-cluster.sh -n ${join(",", hsdp_container_host.kafka.*.private_ip)} -c ${random_id.id.hex} -d ${var.image} -i ${count.index + 1} -z ${var.zookeeper_connect} -x ${element(hsdp_container_host.kafka.*.private_ip, count.index)} -r \"${var.retention_hours}\" -p ${var.kafka_key_store.password} -t ${var.zoo_trust_store.password} -k ${var.zoo_key_store.password} -R ${var.default_replication_factor} -a ${var.auto_create_topics_enable} -e ${var.enable_exporters} -m ${var.max_request_size} -f ${var.max_partition_fetch_bytes}"
   ]
-}
+}

scripts/bootstrap-cluster.sh

@@ -16,6 +16,8 @@ usage: bootstrap-cluster.sh
       -R default_replication_factor
       -a auto_create_topics_enable
       -e enable_exporters
+      -m max-request-size
+      -f max-partition-fetch-bytes
 EOF
 }
 
@@ -80,6 +82,8 @@ start_kafka() {
   local zoo_trust_pass="$9"
   local default_replication_factor="${10}"
   local auto_create_topics_enable="${11}"
+  local max-request-size="${12}"
+  local max-partition-fetch-bytes="${13}"
 
   servers="$(kafka_servers "$index" "$nodes")"
   echo KAFKA_SERVERS="$servers"
@@ -104,6 +108,8 @@ start_kafka() {
     --env JMX_PORT=5555 \
     --env KAFKA_CFG_DEFAULT_REPLICATION_FACTOR=$default_replication_factor \
     --env KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=$auto_create_topics_enable \
+    --env KAFKA_CFG_MAX_REQUEST_SIZE=$max-request-size \
+    --env KAFKA_CFG_MAX_PARTITION_FETCH_BYTES=$max-partition-fetch-bytes \
     --network $kafka_network \
     -v $kafka_broker_name:/bitnami/kafka \
     -v 'kafkacert:/bitnami/kafka/config/certs/' \
@@ -133,7 +139,7 @@ start_jmx_exporter(){
   # Substitute container name in jmx config and move it
   export container_name=$kafka_broker_name
   envsubst < jmxconfig.yml.tmpl > ./jmx/config.yml
-  
+
   # create jmx volume mapping the jmx config file
   docker volume create --driver local --name jmx_config_volume --opt type=none --opt device=`pwd`/jmx --opt o=uid=root,gid=root --opt o=bind
 
@@ -187,6 +193,8 @@ zoo_trust_store_pass=
 default_replication_factor=
 auto_create_topics_enable=
 enable_exporters=
+max-request-size=
+max-partition-fetch-bytes=
 
 while [ "$1" != "" ]; do
     case $1 in
@@ -226,6 +234,12 @@ while [ "$1" != "" ]; do
         -a | --auto-create-topics ) shift
                                 auto_create_topics_enable=$1
                                 ;;
+        -m | --max-request-size ) shift
+                                max-request-size=$1
+                                ;;
+        -f | --max-partition-fetch-bytes ) shift
+                                max-partition-fetch-bytes=$1
+                                ;;
         -e | --enable-exporters ) shift
                                 enable_exporters=$1
                                 ;;
@@ -246,7 +260,7 @@ kill_monitoring
 kill_kafka
 create_volume
 create_network
-start_kafka "$index" "$nodes" "$image" "$zookeeper_connect" "$external_ip" "$retention_hours" "$kafka_cert_pass" "$zoo_key_store_pass" "$zoo_trust_store_pass" "$default_replication_factor" "$auto_create_topics_enable"
+start_kafka "$index" "$nodes" "$image" "$zookeeper_connect" "$external_ip" "$retention_hours" "$kafka_cert_pass" "$zoo_key_store_pass" "$zoo_trust_store_pass" "$default_replication_factor" "$auto_create_topics_enable" "$max-request-size" "$max-partition-fetch-bytes"
 load_certificates_and_restart
 
 if [ "$enable_exporters" == true ]; then
@@ -258,4 +272,4 @@ sleep 30 # wait for 5 seconds to print out docker status
 docker ps -a
 docker start kafka_prometheus_exporter
 sleep 15
-docker ps -a
+docker ps -a

variables.tf

@@ -130,12 +130,24 @@ variable "kafka_public_key" {
 
 variable "kafka_private_key" {
   description = "Private Key for SSL (only applicable when exporter is required, so only when 'enable_exporters==true')"
-  default     = "" 
+  default     = ""
   type        = string
 }
 
 variable "enable_exporters" {
   description = "Indicates whether jmx exporter and kafka exporter is enabled or not"
   default     = false
-  type        = bool 
-}
+  type        = bool
+}
+
+variable "max_request_size" {
+  description = "The maximum size of a request in bytes"
+  default     = 1048576
+  type        = number
+}
+
+variable "max_partition_fetch_bytes" {
+  description = "The maximum amount of data per-partition the server will return"
+  default     = 1048576
+  type        = number
+}