From e2bf5814de6b12046eff0f23a6ee9bef46f4329e Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Mon, 7 Apr 2025 14:27:24 -0500 Subject: [PATCH 1/2] Draft Kafka log how to Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- app/_how-tos/publish-logs-to-kafka.md | 130 ++++++++++++++++++++ tools/track-docs-changes/config/sources.yml | 2 + 2 files changed, 132 insertions(+) create mode 100644 app/_how-tos/publish-logs-to-kafka.md diff --git a/app/_how-tos/publish-logs-to-kafka.md b/app/_how-tos/publish-logs-to-kafka.md new file mode 100644 index 000000000..f062879c8 --- /dev/null +++ b/app/_how-tos/publish-logs-to-kafka.md @@ -0,0 +1,130 @@ +--- +title: Publish request and response logs to an Apache Kafka topic with the Kafka Log plugin +content_type: how_to +related_resources: + - text: Authentication + url: /gateway/authentication/ + +products: + - gateway + +plugins: + - kafka-log + +works_on: + - on-prem + - konnect + +min_version: + gateway: '3.4' + +entities: + - plugin + - service + - route + - consumer + +tags: + - authentication + +tldr: + q: How do I authenticate Consumers with basic authentication? + a: Create a Consumer with a username and password in the `basicauth_credentials` configuration. Enable the Basic Authentication plugin globally, and authenticate with the base64-encoded Consumer credentials. + +tools: + - deck + +prereqs: + entities: + services: + - example-service + routes: + - example-route + inline: + - title: Apache Kafka + content: | + In this tutorial, we'll be using Apache Kafka: + 1. Install Apache Kafka. + 2. Get the Kafka Docker image: + ```sh + docker pull apache/kafka-native:4.0.0 + docker run -d --name kafka --network kong-quickstart-net -p 9092:9092 apache/kafka-native:4.0.0 + ``` + +cleanup: + inline: + - title: Clean up Konnect environment + include_content: cleanup/platform/konnect + icon_url: /assets/icons/gateway.svg + - title: Destroy the {{site.base_gateway}} container + include_content: cleanup/products/gateway + icon_url: /assets/icons/gateway.svg +--- + +## Create Kafka topic + +Save the path to `/bin/kafka-topics` as an environment variable: + +```sh +export KAFKA_HOME='' +``` + +Create the `kong-log` Kafka topic: +```bash +$KAFKA_HOME/bin/kafka-topics --create \ + --bootstrap-server localhost:9092 \ + --replication-factor 1 \ + --partitions 10 \ + --topic kong-log +``` + +Since the topic can vary in production instances, set the Kafka topic as an environment variable: +```sh +export DECK_KAFKA_TOPIC=kong-log +``` + +## 2. Enable the Kafka Log plugin + +Set the `host.docker.internal` as your host. We're using this as our host in this tutorial because {{site.base_gateway}} is using `localhost` and both {{site.base_gateway}} and Kafka are in Docker containers. + +```sh +export DECK_HOST=host.docker.internal +``` + +Enable the plugin on the [`example-route` we created earlier](/how-to/publish-logs-to-kafka/#prerequisites): + +{% entity_examples %} +entities: + plugins: + - name: kafka-log + route: example-route + config: + bootstrap_servers: + - host: ${kafka_host} + port: 9092 + topic: ${kafka_topic} +variables: + kafka_host: + value: $HOST + kafka_topic: + value: $KAFKA_TOPIC +{% endentity_examples %} + +## Make sample requests + +```bash +for i in {1..50} ; do curl http://localhost:8000/anything/$i ; done +``` + +## 3. Validate + +Verify the contents of the Kafka `kong-log` topic: + +```sh +$KAFKA_HOME/bin/kafka-console-consumer \ + --bootstrap-server localhost:9092 \ + --topic kong-log \ + --partition 0 \ + --from-beginning \ + --timeout-ms 1000 +``` \ No newline at end of file diff --git a/tools/track-docs-changes/config/sources.yml b/tools/track-docs-changes/config/sources.yml index d98578482..60cf84571 100644 --- a/tools/track-docs-changes/config/sources.yml +++ b/tools/track-docs-changes/config/sources.yml @@ -208,6 +208,8 @@ app/_how-tos/adjust-header-names-in-request.md: - app/_hub/kong-inc/post-function/how-to/_index.md app/_how-tos/configure-oidc-with-client-credentials-grant.md: - /app/_hub/kong-inc/openid-connect/how-to/authentication/_client-credentials.md +app/_how-tos/publish-logs-to-kafka.md: + - app/_hub/kong-inc/kafka-log/overview/_index.md ## KIC app/_how-tos/kic-install.md: From 544145507f7dd4cb625c06a559914dedc548f3da Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Thu, 10 Apr 2025 11:20:29 -0500 Subject: [PATCH 2/2] Apply Rick's feedback Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- app/_how-tos/publish-logs-to-kafka.md | 60 +++++++++++++-------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/app/_how-tos/publish-logs-to-kafka.md b/app/_how-tos/publish-logs-to-kafka.md index f062879c8..49d761ae8 100644 --- a/app/_how-tos/publish-logs-to-kafka.md +++ b/app/_how-tos/publish-logs-to-kafka.md @@ -2,8 +2,8 @@ title: Publish request and response logs to an Apache Kafka topic with the Kafka Log plugin content_type: how_to related_resources: - - text: Authentication - url: /gateway/authentication/ + - text: Apache Kafka documentation + url: https://kafka.apache.org/documentation/ products: - gateway @@ -28,8 +28,8 @@ tags: - authentication tldr: - q: How do I authenticate Consumers with basic authentication? - a: Create a Consumer with a username and password in the `basicauth_credentials` configuration. Enable the Basic Authentication plugin globally, and authenticate with the base64-encoded Consumer credentials. + q: How do I publish request and response logs to an Apache Kafka topic with the Kafka Log plugin? + a: Create a Kafka topic, and set your Kafka host as an environment variable. Enable the Kafka Log plugin and specify the Kafka host, port, and topic when you configure it. tools: - deck @@ -40,16 +40,6 @@ prereqs: - example-service routes: - example-route - inline: - - title: Apache Kafka - content: | - In this tutorial, we'll be using Apache Kafka: - 1. Install Apache Kafka. - 2. Get the Kafka Docker image: - ```sh - docker pull apache/kafka-native:4.0.0 - docker run -d --name kafka --network kong-quickstart-net -p 9092:9092 apache/kafka-native:4.0.0 - ``` cleanup: inline: @@ -61,21 +51,30 @@ cleanup: icon_url: /assets/icons/gateway.svg --- -## Create Kafka topic +## 1. Run the Kafka broker in Docker -Save the path to `/bin/kafka-topics` as an environment variable: +To run the broker, this will run Kafka 4.0 in [KRaft mode](https://kafka.apache.org/40/documentation/zk2kraft.html). This allows for connections from within the Docker network, from the host machine, and from the Docker bridge network: ```sh -export KAFKA_HOME='' +docker run -d --rm --name=kafka -h kafka -p 9092:9092 -p 39092:39092 \ + -e KAFKA_NODE_ID=1 \ + -e KAFKA_LISTENERS='DOCKER://kafka:29092,CONTROLLER://kafka:29093,HOST://0.0.0.0:9092,BRIDGE://0.0.0.0:39092' \ + -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP='CONTROLLER:PLAINTEXT,DOCKER:PLAINTEXT,HOST:PLAINTEXT,BRIDGE:PLAINTEXT' \ + -e KAFKA_ADVERTISED_LISTENERS='DOCKER://kafka:29092,HOST://localhost:9092,BRIDGE://host.docker.internal:39092' \ + -e KAFKA_PROCESS_ROLES='broker,controller' \ + -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ + -e KAFKA_CONTROLLER_QUORUM_VOTERS='1@kafka:29093' \ + -e KAFKA_INTER_BROKER_LISTENER_NAME='DOCKER' \ + -e KAFKA_CONTROLLER_LISTENER_NAMES='CONTROLLER' \ + -e CLUSTER_ID='MkU3OEVBNTcwNTJENDM2Qk' \ + apache/kafka:4.0.0 ``` -Create the `kong-log` Kafka topic: -```bash -$KAFKA_HOME/bin/kafka-topics --create \ - --bootstrap-server localhost:9092 \ - --replication-factor 1 \ - --partitions 10 \ - --topic kong-log +## 2. Create Kafka topic + +Now that the broker is running, we can create the `kong-log` Kafka topic: +```sh +docker exec -it kafka /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 --create --topic kong-log ``` Since the topic can vary in production instances, set the Kafka topic as an environment variable: @@ -88,7 +87,7 @@ export DECK_KAFKA_TOPIC=kong-log Set the `host.docker.internal` as your host. We're using this as our host in this tutorial because {{site.base_gateway}} is using `localhost` and both {{site.base_gateway}} and Kafka are in Docker containers. ```sh -export DECK_HOST=host.docker.internal +export DECK_KAFKA_HOST=host.docker.internal ``` Enable the plugin on the [`example-route` we created earlier](/how-to/publish-logs-to-kafka/#prerequisites): @@ -101,7 +100,7 @@ entities: config: bootstrap_servers: - host: ${kafka_host} - port: 9092 + port: 39092 topic: ${kafka_topic} variables: kafka_host: @@ -112,6 +111,8 @@ variables: ## Make sample requests +Send the following request to generate logs: + ```bash for i in {1..50} ; do curl http://localhost:8000/anything/$i ; done ``` @@ -121,10 +122,5 @@ for i in {1..50} ; do curl http://localhost:8000/anything/$i ; done Verify the contents of the Kafka `kong-log` topic: ```sh -$KAFKA_HOME/bin/kafka-console-consumer \ - --bootstrap-server localhost:9092 \ - --topic kong-log \ - --partition 0 \ - --from-beginning \ - --timeout-ms 1000 +docker exec -it kafka /opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server kafka:29092 --topic kong-log --from-beginning ``` \ No newline at end of file