feat: automate the pipleline initializatgions

Author: anqorithm

docker-compose.yaml

@@ -28,41 +28,51 @@ services:
     networks:
       stock-net:
         ipv4_address: 172.28.1.2
+    volumes:
+      - ./scripts/init-kafka.sh:/init-kafka.sh
+    # entrypoint: ["/bin/bash", "init-kafka.sh"]
+    restart: always
 
   cassandra:
     image: cassandra:latest
     ports:
       - "9042:9042"
     volumes:
       - ./init-cassandra:/init-cassandra
+      - ./scripts/init-cassandra-schema.sh:/init-cassandra-schema.sh
     environment:
       - CASSANDRA_START_RPC=true
     networks:
       stock-net:
         ipv4_address: 172.28.1.3
+    # entrypoint: ["/bin/bash", "init-cassandra-schema.sh"]
+    restart: always
 
   spark:
     image: bitnami/spark:latest
     volumes:
       - ./spark:/opt/bitnami/spark/jobs
+      - ./scripts/submit-spark-job.sh:/opt/bitnami/spark/submit-spark-job.sh
     ports:
       - "8080:8080"
     depends_on:
       - kafka
     networks:
       stock-net:
         ipv4_address: 172.28.1.4
-    
+    entrypoint: ["sh", "-c", "./submit-spark-job.sh"]
+    restart: always
+
   kafka_producer:
     build:
-      context: ./kafka-producer 
-      dockerfile: kafka_producer.dockerfile 
+      context: ./kafka-producer
+      dockerfile: kafka_producer.dockerfile
     depends_on:
       - kafka
     networks:
       stock-net:
-        ipv4_address: 172.28.1.5
-
+        ipv4_address: 172.28.1.8
+    restart: always
 networks:
   stock-net:
     driver: bridge

scripts/init-cassandra-schema.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+# Function to check if Cassandra is up and running
+function check_cassandra {
+    while ! cqlsh -e "describe keyspaces" &>/dev/null; do
+        echo "Waiting for Cassandra to be up..."
+        sleep 10
+    done
+}
+
+echo "Setting up Cassandra schema..."
+
+# Wait for Cassandra to be ready
+check_cassandra
+
+# Execute schema setup commands
+cqlsh -f /init-cassandra/init.cql -u cassandra -p cassandra
+
+echo "Cassandra schema setup complete."

scripts/init-kafka.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+/opt/bitnami/scripts/kafka/entrypoint.sh /opt/bitnami/scripts/kafka/run.sh &
+
+echo "Waiting for Kafka to be up..."
+sleep 10
+
+kafka-topics.sh --create --topic stocks --bootstrap-server localhost:9092 --partitions 1 --replication-factor 1
+
+wait

scripts/submit-spark-job.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+spark-submit \
+  --conf "spark.jars.ivy=/opt/bitnami/spark/.ivy2" \
+  --conf "spark.executor.extraJavaOptions=-Djava.security.krb5.conf=/path/to/krb5.conf" \
+  --conf "spark.driver.extraJavaOptions=-Djava.security.krb5.conf=/path/to/krb5.conf" \
+  --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.3.1,com.datastax.spark:spark-cassandra-connector_2.12:3.0.0 \
+  /opt/bitnami/spark/jobs/spark_job.py stocks