Real-Time Stock Insight & Prediction Platform-FinanceLake

.dockerignore

@@ -0,0 +1,26 @@
+# Git
+.git
+.gitignore
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.venv/
+venv/
+env/
+
+# IDE / OS specific
+.idea/
+.vscode/
+*.DS_Store
+
+# Docker
+Dockerfile
+
+# Local data and logs (these are mounted as volumes anyway)
+logs/
+data/
+checkpoints/
+models/

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.gitignore

@@ -18,4 +18,27 @@ __pycache__/
 # Dataset
 data-ingestion-kafka/data/creditcard.csv
 
-package-lock.json 
+package-lock.json 
+venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+logs/
+.env
+checkpoints/
+jars/
+data/
+

Dockerfile.dashboard

@@ -0,0 +1,24 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# Install system deps & clean up
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Cache-mount pip install
+COPY requirements-dashboard.txt .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r requirements-dashboard.txt
+
+# Copy source
+COPY src /app/src
+
+ENV PYTHONPATH=/app/src
+EXPOSE 8501
+
+HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health || exit 1
+
+CMD ["streamlit", "run", "/app/src/dashboard/main.py", "--server.port=8501", "--server.address=0.0.0.0"]

Dockerfile.scraper

@@ -0,0 +1,13 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+COPY requirements-scraper.txt .
+RUN pip install --no-cache-dir -r requirements-scraper.txt
+
+COPY src /app/src
+
+RUN mkdir -p /app/logs && chmod 777 /app/logs
+ENV PYTHONPATH=/app/src:$PYTHONPATH
+
+CMD ["python", "-u", "/app/src/ingestion/connectors/yahoo_connector.py"]

Dockerfile.spark

@@ -0,0 +1,46 @@
+# Use the official image (already contains PySpark 3.4.0)
+FROM apache/spark-py:v3.4.0
+
+# Switch to root to install system deps
+USER root
+
+WORKDIR /app
+
+# 1. System Deps: Install & Clean in one layer to keep image small
+RUN apt-get update && apt-get install -y \
+    curl \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+# 2. Copy JARs (These change rarely)
+COPY jars /opt/spark/jars/
+
+# 3. SENIOR OPTIMIZATION: BuildKit Cache & CPU-Only Torch
+# This mounts a local cache for pip. If you rebuild, it uses the cache.
+# --index-url points to the CPU-only repo (Small download).
+# RUN --mount=type=cache,target=/root/.cache/pip \
+#     pip install --no-cache-dir \
+#     torch torchvision torchaudio \
+#     --index-url https://download.pytorch.org/whl/cpu
+
+# 4. Install other Python deps
+# Ensure 'pyspark' and 'torch' are REMOVED from this file!
+COPY requirements-spark.txt .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install --no-cache-dir -r requirements-spark.txt
+
+# 5. Copy Source Code (Changes most frequently, so it goes last)
+COPY src /app/src
+
+# 6. Runtime Directories
+RUN mkdir -p /app/logs /app/data /app/checkpoints /app/mlruns
+
+# 7. Environment
+ENV PYTHONPATH=/app:/app/src:$PYTHONPATH
+
+# Command
+# CMD ["spark-submit", \
+#     "--driver-memory", "2g", \
+#     "--executor-memory", "2g", \
+#     "--conf", "spark.driver.extraJavaOptions=-Dlog4j.configuration=file:/opt/spark/conf/log4j.properties", \
+#     "/app/src/processing/spark_streaming_main.py"]

docker-compose.yml

@@ -1,68 +1,143 @@
-version: "3.8"
-
 services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper:7.5.0
-    environment:
-      ZOOKEEPER_CLIENT_PORT: ${ZOOKEEPER_CLIENT_PORT}
+  # --- INFRASTRUCTURE ---
+  redpanda:
+    image: docker.redpanda.com/redpandadata/redpanda:v23.2.14
+    container_name: redpanda
     ports:
-      - "${ZOOKEEPER_CLIENT_PORT}:${ZOOKEEPER_CLIENT_PORT}"
+      - "9094:9094"
+      - "19094:19094"
+    command:
+      - redpanda start
+      - --smp 1
+      - --overprovisioned
+      - --kafka-addr internal://0.0.0.0:9094,external://0.0.0.0:19094
+      - --advertise-kafka-addr internal://redpanda:9094,external://localhost:19094
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
+      - --advertise-pandaproxy-addr internal://redpanda:8082,external://localhost:18082
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
+      - --rpc-addr redpanda:33145
+      # Removed auth flag
+    volumes:
+      - redpanda_data:/var/lib/redpanda/data
+    healthcheck:
+      test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy|Unhealthy'"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
-  kafka:
-    image: confluentinc/cp-kafka:7.5.0
-    depends_on:
-      - zookeeper
+  minio:
+    image: minio/minio
+    container_name: minio
     ports:
-      - "${KAFKA_PORT}:${KAFKA_PORT}"
+      - "9000:9000"
+      - "9001:9001"
     environment:
-      KAFKA_BROKER_ID: ${KAFKA_BROKER_ID}
-      KAFKA_ZOOKEEPER_CONNECT: ${KAFKA_ZOOKEEPER_CONNECT}
-      KAFKA_ADVERTISED_LISTENERS: ${KAFKA_ADVERTISED_LISTENER}
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      MINIO_ROOT_USER: minioadmin
+      MINIO_ROOT_PASSWORD: minioadmin
+    command: server /data --console-address ":9001"
+    volumes:
+      - minio_data:/data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
 
-  spark:
-    image: bitnami/spark:latest
-    environment:
-      - SPARK_MODE=master
-    ports:
-      - "${SPARK_MASTER_PORT}:${SPARK_MASTER_PORT}"
-      - "${SPARK_WEB_UI_PORT}:${SPARK_WEB_UI_PORT}"
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      minio:
+        condition: service_healthy
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set myminio http://minio:9000 minioadmin minioadmin;
+      /usr/bin/mc mb --ignore-existing myminio/finance-lake;
+      /usr/bin/mc mb --ignore-existing myminio/mlflow;
+      exit 0;
+      "
 
-  superset:
-    image: apache/superset
-    ports:
-      - ${SUPERSET_PORT}:${SUPERSET_PORT}
-    environment:
-      SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY}
+  # --- SPARK (SINGLE NODE MODE - The Fix) ---
+  spark-streaming:
+    build:
+      context: .
+      dockerfile: Dockerfile.spark
+    container_name: spark-streaming
     depends_on:
-      - spark
-    command: >
-      /bin/bash -c "
-        superset db upgrade &&
-        superset fab create-admin --username ${SUPERSET_ADMIN_USERNAME} --firstname Admin --lastname User --email ${SUPERSET_ADMIN_EMAIL} --password ${SUPERSET_ADMIN_PASSWORD} &&
-        superset init &&
-        superset run -h 0.0.0.0 -p 8088"
+      redpanda:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+    environment:
+      - AWS_ACCESS_KEY_ID=minioadmin
+      - AWS_SECRET_ACCESS_KEY=minioadmin
+      - AWS_REGION=us-east-1
+      - KAFKA_BROKER=redpanda:9094
+    volumes:
+      - ./src:/app/src
+      - ./logs:/app/logs
+      - ./data/mlruns:/mlruns
+    command: tail -f /dev/null
+    # command: >
+    #   spark-submit 
+    #   --master "local[*]" 
+    #   --driver-memory 2g 
+    #   --conf spark.driver.maxResultSize=1g
+    #   --conf spark.driver.extraJavaOptions=-Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties
+    #   /app/src/processing/spark_streaming_main.py
 
-  postgres:
-    image: postgres:15
+  # --- APPS ---
+  scraper:
+    build:
+      context: .
+      dockerfile: Dockerfile.scraper
+    container_name: scraper
+    depends_on:
+      redpanda:
+        condition: service_healthy
     environment:
-      POSTGRES_DB: ${POSTGRES_DB}
-      POSTGRES_USER: ${POSTGRES_USER}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      - KAFKA_BROKER=redpanda:9094
+      - PYTHONPATH=/app/src
     volumes:
-      - postgres_data:/var/lib/postgresql/data
-    ports:
-      - "5432:5432"
-
-  nifi:
-    image: apache/nifi:2.3.0
+      - ./src:/app/src
+      - ./logs:/app/logs
+
+  dashboard:
+    build:
+      context: .
+      dockerfile: Dockerfile.dashboard
+    container_name: dashboard
     ports:
-      - "${NIFI_HTTPS_PORT}:${NIFI_HTTPS_PORT}"
+      - "8501:8501"
     environment:
-      NIFI_WEB_HTTPS_PORT: ${NIFI_HTTPS_PORT}
+      - PYTHONPATH=/app/src
+      - AWS_ACCESS_KEY_ID=minioadmin
+      - AWS_SECRET_ACCESS_KEY=minioadmin
+      - S3_ENDPOINT=http://minio:9000
+    depends_on:
+      minio:
+        condition: service_healthy
     volumes:
-      - nifi_data:/opt/nifi/data
+      - ./src:/app/src
+
+  # mlflow:
+  #   image: ghcr.io/mlflow/mlflow:v2.10.0
+  #   container_name: mlflow
+  #   ports:
+  #     - "5000:5000"
+  #   volumes:
+  #     # Map host data/mlruns to container /mlruns (Root level is safer)
+  #     - ./data/mlruns:/mlruns
+  #   # Point backend store to /mlruns
+  #   command: mlflow ui --host 0.0.0.0 --port 5000 --backend-store-uri file:///mlruns
 
 volumes:
-  postgres_data:
-  nifi_data:
+  minio_data:
+    driver: local
+  redpanda_data:
+    driver: local
+  spark_data:
+    driver: local
+
+networks:
+  default:
+    name: financelake-net

download_jars.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+JARS_DIR="jars"
+mkdir -p "$JARS_DIR"
+
+# List of JAR URLs (Added AWS/Hadoop dependencies)
+declare -a JARS=(
+  # Kafka & Delta (Keep these)
+  "https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.4.0/spark-sql-kafka-0-10_2.12-3.4.0.jar"
+  "https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/3.4.0/kafka-clients-3.4.0.jar"
+  "https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.11.1/commons-pool2-2.11.1.jar"
+  "https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.4.0/spark-token-provider-kafka-0-10_2.12-3.4.0.jar"
+  "https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.4.0/delta-core_2.12-2.4.0.jar"
+  "https://repo1.maven.org/maven2/io/delta/delta-storage/2.4.0/delta-storage-2.4.0.jar"
+  "https://repo1.maven.org/maven2/org/antlr/antlr4-runtime/4.9.3/antlr4-runtime-4.9.3.jar"
+
+  # --- NEW: S3/MinIO Support (Hadoop 3.3.4 for Spark 3.4.0) ---
+  "https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar"
+  "https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar"
+)
+
+for jar_url in "${JARS[@]}"; do
+  filename=$(basename "$jar_url")
+  if [ -f "$JARS_DIR/$filename" ]; then
+    echo "$filename exists. Skipping."
+  else
+    echo "Downloading $filename..."
+    wget -P "$JARS_DIR" "$jar_url"
+  fi
+done

requirements-dashboard.txt

@@ -0,0 +1,5 @@
+streamlit==1.31.0
+plotly==5.18.0
+duckdb==0.9.2
+pandas==2.1.4
+watchdog==3.0.0

requirements-scraper.txt

@@ -0,0 +1,6 @@
+requests
+kafka-python
+yfinance
+dotenv
+# defeatbeta-api
+# stockdex