Merge pull request #29 from arruzk/kafka_config

Kafka configuration

Author: AlexeySetevoi

README.md

@@ -138,6 +138,22 @@ F: Flag for remove clickhouse from host(disabled by default)
 clickhouse_remove: no
 ```
 
+F: You can manage [Kafka configuration](https://clickhouse.yandex/docs/en/operations/table_engines/kafka/#configuration)
+```yaml
+# global configuration
+clickhouse_kafka_config:
+  auto_offset_reset: smallest
+  debug: cgrp
+# topic-level configuration
+clickhouse_kafka_topics_config:
+  topic1:
+    retry_backoff_ms: 250
+    fetch_min_bytes: 100000
+  topic2:
+    retry_backoff_ms: 300
+    fetch_min_bytes: 120000
+```
+
 Example Playbook
 ----------------
 

defaults/main.yml

@@ -88,4 +88,8 @@ clickhouse_path_user_files: "{{ clickhouse_path_base }}/clickhouse/user_files/"
 
 clickhouse_mlock_status: false
 
-max_partitions_per_insert_block: 100
+max_partitions_per_insert_block: 100
+
+clickhouse_kafka_config: []
+
+clickhouse_kafka_topics_config: []

templates/config.j2

@@ -1,3 +1,4 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
 <?xml version="1.0"?>
 <!-- {{ ansible_managed }} -->
 <yandex>
@@ -359,4 +360,22 @@
 
     <!-- Uncomment to disable ClickHouse internal DNS caching. -->
     <!-- <disable_internal_dns_cache>1</disable_internal_dns_cache> -->
+
+{% if clickhouse_kafka_config is defined %}
+    <kafka>
+        {% for config in clickhouse_kafka_config %}
+        <{{ config }}>{{ clickhouse_kafka_config[config] }}</{{ config }}>
+        {% endfor %}
+    </kafka>
+{% endif %}
+
+{% for kafka_topic in clickhouse_kafka_topics_config %}
+    <kafka_{{ kafka_topic }}>
+    {% for item in clickhouse_kafka_topics_config[kafka_topic] %}
+        <{{ item }}>{{ clickhouse_kafka_topics_config[kafka_topic][item] }}</{{ item }}>
+    {% endfor %}
+    </kafka_{{ kafka_topic }}>
+
+{% endfor %}
+
 </yandex>