feat: Add kafka on eks datastack with example of streaming pipeline with monitoring

data-stacks/airflow-on-eks/examples/dag-bundles/spark-pi.py

@@ -6,7 +6,7 @@
 from airflow.providers.cncf.kubernetes.operators.spark_kubernetes import SparkKubernetesOperator
 from airflow.providers.cncf.kubernetes.sensors.spark_kubernetes import SparkKubernetesSensor
 
-with DAG(  
+with DAG(
     dag_id="spark_pi",
     start_date=datetime(2025, 10, 24),
     description="submit spark-pi as sparkApplication on kubernetes",
@@ -15,7 +15,7 @@
     schedule=timedelta(days=1),
     tags=["example", "data-on-eks"],
 ) as dag:
-    t1 = SparkKubernetesOperator(  
+    t1 = SparkKubernetesOperator(
         task_id="pyspark_pi_submit",
         namespace="spark-team-a",
         application_file="spark-pi.yaml",

data-stacks/airflow-on-eks/examples/dag-bundles/spark-pi.yaml

@@ -51,4 +51,4 @@ spec:
       runAsNonRoot: true
       allowPrivilegeEscalation: false
       seccompProfile:
-        type: RuntimeDefault
+        type: RuntimeDefault

data-stacks/airflow-on-eks/terraform/data-stack.tfvars

@@ -1,5 +1,5 @@
-name = "airflow-on-eks"
-region = "us-west-2"
+name                 = "airflow-on-eks"
+region               = "us-west-2"
 enable_ingress_nginx = true
-deployment_id = "abcdefg"
-enable_airflow = true
+deployment_id        = "abcdefg"
+enable_airflow       = true

data-stacks/kafka-on-eks/cleanup.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd terraform/_local
+source ./cleanup.sh

data-stacks/kafka-on-eks/deploy.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+
+# --- Configuration ---
+STACKS="kafka-on-eks"
+TERRAFORM_DIR="terraform"
+AWS_REGION="${AWS_REGION:-us-west-2}"
+KUBECONFIG_FILE="kubeconfig.yaml"
+
+
+# --- Get Repo Root ---
+REPO_PATH=$(git rev-parse --show-toplevel)
+
+# --- Source and Execute the Main Deployment Engine ---
+# The centralized install.sh handles all the heavy lifting.
+source "$REPO_PATH/infra/terraform/install.sh"
+
+# --- Post-Deployment Steps ---
+# Steps specific to this stack can be added here.
+print_status "Running stack-specific post-deployment steps..."
+
+# Backup the state file from the _local directory
+cp "$TERRAFORM_DIR/_local/terraform.tfstate" "$TERRAFORM_DIR/terraform.tfstate.bak"
+print_status "Backed up terraform.tfstate."
+
+# Setup kubeconfig
+setup_kubeconfig
+
+# Get ArgoCD admin password
+export KUBECONFIG=$KUBECONFIG_FILE
+ARGOCD_PASSWORD=$(kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d)
+
+print_next_steps

data-stacks/kafka-on-eks/examples/kafka-producers-consumers.yaml

@@ -0,0 +1,110 @@
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: java-kafka-producer
+  name: java-kafka-producer
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java-kafka-producer
+  template:
+    metadata:
+      labels:
+        app: java-kafka-producer
+    spec:
+      containers:
+        - name: java-kafka-producer
+          image: quay.io/strimzi-examples/java-kafka-producer:latest
+          env:
+            - name: STRIMZI_TOPIC
+              value: my-topic
+            - name: STRIMZI_DELAY_MS
+              value: "1000"
+            - name: STRIMZI_LOG_LEVEL
+              value: "INFO"
+            - name: STRIMZI_MESSAGE_COUNT
+              value: "1000000"
+            - name: KAFKA_BOOTSTRAP_SERVERS
+              value: data-on-eks-kafka-bootstrap:9092
+            - name: KAFKA_KEY_SERIALIZER
+              value: "org.apache.kafka.common.serialization.StringSerializer"
+            - name: KAFKA_VALUE_SERIALIZER
+              value: "org.apache.kafka.common.serialization.StringSerializer"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: java-kafka-streams
+  name: java-kafka-streams
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java-kafka-streams
+  template:
+    metadata:
+      labels:
+        app: java-kafka-streams
+    spec:
+      containers:
+        - name: java-kafka-streams
+          image: quay.io/strimzi-examples/java-kafka-streams:latest
+          env:
+            - name: STRIMZI_SOURCE_TOPIC
+              value: my-topic
+            - name: STRIMZI_TARGET_TOPIC
+              value: my-topic-reversed
+            - name: STRIMZI_LOG_LEVEL
+              value: "INFO"
+            - name: KAFKA_BOOTSTRAP_SERVERS
+              value: data-on-eks-kafka-bootstrap:9092
+            - name: KAFKA_APPLICATION_ID
+              value: java-kafka-streams
+            - name: KAFKA_DEFAULT_COMMIT_INTERVAL_MS
+              value: "5000"
+            - name: KAFKA_DEFAULT_KEY_SERDE
+              value: "org.apache.kafka.common.serialization.Serdes$StringSerde"
+            - name: KAFKA_DEFAULT_VALUE_SERDE
+              value: "org.apache.kafka.common.serialization.Serdes$StringSerde"
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: java-kafka-consumer
+  name: java-kafka-consumer
+  namespace: kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: java-kafka-consumer
+  template:
+    metadata:
+      labels:
+        app: java-kafka-consumer
+    spec:
+      containers:
+        - name: java-kafka-consumer
+          image: quay.io/strimzi-examples/java-kafka-consumer:latest
+          env:
+            - name: STRIMZI_TOPIC
+              value: my-topic-reversed
+            - name: STRIMZI_LOG_LEVEL
+              value: "INFO"
+            - name: STRIMZI_MESSAGE_COUNT
+              value: "1000000"
+            - name: KAFKA_BOOTSTRAP_SERVERS
+              value: data-on-eks-kafka-bootstrap:9092
+            - name: KAFKA_GROUP_ID
+              value: java-kafka-consumer
+            - name: KAFKA_KEY_DESERIALIZER
+              value: "org.apache.kafka.common.serialization.StringDeserializer"
+            - name: KAFKA_VALUE_DESERIALIZER
+              value: "org.apache.kafka.common.serialization.StringDeserializer"

data-stacks/kafka-on-eks/examples/kafka-topics.yaml

@@ -0,0 +1,21 @@
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: my-topic
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: data-on-eks
+spec:
+  replicas: 3
+  partitions: 12
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+  name: my-topic-reversed
+  namespace: kafka
+  labels:
+    strimzi.io/cluster: data-on-eks
+spec:
+  replicas: 3
+  partitions: 12

data-stacks/kafka-on-eks/examples/load-test.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+# Kafka Load Testing Script
+# This script contains commands specifically for load testing and performance validation
+
+case "$1" in
+  create-perf-test-topic)
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-topics.sh \
+    --create \
+    --topic test-topic-perf \
+    --partitions 3 \
+    --replication-factor 3 \
+    --bootstrap-server data-on-eks-kafka-bootstrap:9092
+    ;;
+  run-producer-perf-test)
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-producer-perf-test.sh \
+      --topic test-topic-perf \
+      --num-records 100000000 \
+      --throughput -1 \
+      --producer-props bootstrap.servers=data-on-eks-kafka-bootstrap:9092 \
+          acks=all \
+      --record-size 100 \
+      --print-metrics
+    ;;
+  run-consumer-perf-test)
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-consumer-perf-test.sh \
+      --topic test-topic-perf \
+      --messages 100000000 \
+      --broker-list data-on-eks-kafka-bootstrap:9092 | \
+      jq -R .|jq -sr 'map(./",")|transpose|map(join(": "))[]'
+    ;;
+  create-high-throughput-topic)
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-topics.sh \
+    --create \
+    --topic high-throughput-test \
+    --partitions 12 \
+    --replication-factor 3 \
+    --config min.insync.replicas=2 \
+    --bootstrap-server data-on-eks-kafka-bootstrap:9092
+    ;;
+  run-sustained-load-test)
+    echo "Starting sustained load test for 10 minutes..."
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-producer-perf-test.sh \
+      --topic test-topic-perf \
+      --num-records 60000000 \
+      --throughput 100000 \
+      --producer-props bootstrap.servers=data-on-eks-kafka-bootstrap:9092 \
+          acks=all \
+          batch.size=16384 \
+          linger.ms=5 \
+      --record-size 1024 \
+      --print-metrics
+    ;;
+  run-burst-load-test)
+    echo "Starting burst load test..."
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-producer-perf-test.sh \
+      --topic test-topic-perf \
+      --num-records 10000000 \
+      --throughput -1 \
+      --producer-props bootstrap.servers=data-on-eks-kafka-bootstrap:9092 \
+          acks=1 \
+          batch.size=32768 \
+          linger.ms=0 \
+      --record-size 512 \
+      --print-metrics
+    ;;
+  cleanup-perf-topics)
+    echo "Cleaning up performance test topics..."
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-topics.sh \
+      --delete \
+      --topic test-topic-perf \
+      --bootstrap-server data-on-eks-kafka-bootstrap:9092
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-topics.sh \
+      --delete \
+      --topic high-throughput-test \
+      --bootstrap-server data-on-eks-kafka-bootstrap:9092
+    ;;
+  list-perf-topics)
+    kubectl exec -it kafka-cli -n kafka -- bin/kafka-topics.sh \
+      --list \
+      --bootstrap-server data-on-eks-kafka-bootstrap:9092 | grep -E "(perf|throughput)"
+    ;;
+  *)
+    echo "Usage: $0 {create-perf-test-topic|run-producer-perf-test|run-consumer-perf-test|create-high-throughput-topic|run-sustained-load-test|run-burst-load-test|cleanup-perf-topics|list-perf-topics}"
+    echo ""
+    echo "Load Testing Commands:"
+    echo "  create-perf-test-topic      - Create a topic optimized for performance testing"
+    echo "  run-producer-perf-test      - Run producer performance test"
+    echo "  run-consumer-perf-test      - Run consumer performance test"
+    echo "  create-high-throughput-topic - Create topic with more partitions for high throughput"
+    echo "  run-sustained-load-test     - Run sustained load test for 10 minutes"
+    echo "  run-burst-load-test         - Run burst load test with maximum throughput"
+    echo "  cleanup-perf-topics         - Delete all performance test topics"
+    echo "  list-perf-topics            - List all performance test topics"
+    exit 1
+esac

data-stacks/kafka-on-eks/examples/pipeline/cleanup.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+echo "=================================================="
+echo "  Clickstream Analytics Pipeline Cleanup"
+echo "=================================================="
+echo ""
+
+# Delete deployments
+echo "🗑️  Deleting deployments..."
+kubectl delete -f deploy/consumer-deployment.yaml --ignore-not-found=true
+kubectl delete -f deploy/streams-deployment-python.yaml --ignore-not-found=true
+kubectl delete -f deploy/producer-deployment.yaml --ignore-not-found=true
+echo "✓ Deployments deleted"
+echo ""
+
+# Delete topics
+echo "🗑️  Deleting Kafka topics..."
+kubectl delete -f kafka-topics.yaml --ignore-not-found=true
+echo "✓ Topics deleted"
+echo ""
+
+echo "=================================================="
+echo "  ✅ Cleanup Complete!"
+echo "=================================================="
+echo ""