updates to backend, kafka and frontend

.gitignore

@@ -120,7 +120,7 @@ celerybeat.pid
 
 # SageMath parsed files
 *.sage.py
-
+mlruns/
 # Environments
 .env
 .venv

0.5.1

@@ -0,0 +1,35 @@
+Collecting nixtla
+  Downloading nixtla-0.5.1-py3-none-any.whl.metadata (4.8 kB)
+Requirement already satisfied: httpx in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from nixtla) (0.27.0)
+Requirement already satisfied: pandas in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from nixtla) (2.2.2)
+Requirement already satisfied: pydantic in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from nixtla) (2.7.4)
+Requirement already satisfied: requests in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from nixtla) (2.32.3)
+Collecting tenacity (from nixtla)
+  Downloading tenacity-8.4.1-py3-none-any.whl.metadata (1.2 kB)
+Collecting utilsforecast>=0.1.7 (from nixtla)
+  Downloading utilsforecast-0.1.11-py3-none-any.whl.metadata (7.4 kB)
+Requirement already satisfied: numpy in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from utilsforecast>=0.1.7->nixtla) (1.26.4)
+Requirement already satisfied: packaging in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from utilsforecast>=0.1.7->nixtla) (24.1)
+Requirement already satisfied: python-dateutil>=2.8.2 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from pandas->nixtla) (2.9.0.post0)
+Requirement already satisfied: pytz>=2020.1 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from pandas->nixtla) (2024.1)
+Requirement already satisfied: tzdata>=2022.7 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from pandas->nixtla) (2024.1)
+Requirement already satisfied: anyio in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from httpx->nixtla) (4.4.0)
+Requirement already satisfied: certifi in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from httpx->nixtla) (2024.6.2)
+Requirement already satisfied: httpcore==1.* in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from httpx->nixtla) (1.0.5)
+Requirement already satisfied: idna in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from httpx->nixtla) (3.7)
+Requirement already satisfied: sniffio in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from httpx->nixtla) (1.3.1)
+Requirement already satisfied: h11<0.15,>=0.13 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from httpcore==1.*->httpx->nixtla) (0.14.0)
+Requirement already satisfied: annotated-types>=0.4.0 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from pydantic->nixtla) (0.7.0)
+Requirement already satisfied: pydantic-core==2.18.4 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from pydantic->nixtla) (2.18.4)
+Requirement already satisfied: typing-extensions>=4.6.1 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from pydantic->nixtla) (4.12.2)
+Requirement already satisfied: charset-normalizer<4,>=2 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from requests->nixtla) (3.3.2)
+Requirement already satisfied: urllib3<3,>=1.21.1 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from requests->nixtla) (2.2.2)
+Requirement already satisfied: six>=1.5 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from python-dateutil>=2.8.2->pandas->nixtla) (1.16.0)
+Requirement already satisfied: exceptiongroup>=1.0.2 in c:\users\user\downloads\ten_academy\week9\scalable_backtesting_infrastructure_for_crypto_trading\backtest\lib\site-packages (from anyio->httpx->nixtla) (1.2.1)
+Downloading nixtla-0.5.1-py3-none-any.whl (71 kB)
+   ---------------------------------------- 71.4/71.4 kB 561.5 kB/s eta 0:00:00
+Downloading utilsforecast-0.1.11-py3-none-any.whl (40 kB)
+   ---------------------------------------- 40.6/40.6 kB 978.0 kB/s eta 0:00:00
+Downloading tenacity-8.4.1-py3-none-any.whl (27 kB)
+Installing collected packages: tenacity, utilsforecast, nixtla
+Successfully installed nixtla-0.5.1 tenacity-8.4.1 utilsforecast-0.1.11

Dockerfile.backend

@@ -0,0 +1,33 @@
+# Use the official Python base image
+FROM python:3.9-slim
+
+# Set the working directory in the container
+WORKDIR /app
+
+# Copy the requirements file into the container
+COPY requirements.txt .
+
+# Install the required Python packages
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the entire backend project into the container
+COPY ./backend .
+
+# Expose the port that FastAPI runs on
+EXPOSE 8000
+
+# Set environment variables for the FastAPI app
+ENV PYTHONUNBUFFERED=1
+
+ENV BINANCE_API_KEY="FN6iy8IhLMbDR3kVEYp1ZqsN6lj0fZXFRQaRZeJsWuLz6Is7DkVvyb70fwPGDY3B"
+ENV DATABASE_URL="postgresql://trading_db_av2v_user:210M6MA9QKEEgVdiasnUdMQDBNN417oy@dpg-cpqojbqj1k6c73bkqq3g-a.oregon-postgres.render.com/trading_db_av2v"
+ENV PYCOPG_DATABASE_URL = "postgresql+psycopg2://trading_db_av2v_user:210M6MA9QKEEgVdiasnUdMQDBNN417oy@dpg-cpqojbqj1k6c73bkqq3g-a.oregon-postgres.render.com/trading_db_av2v"
+ENV DB_USERNAME='group3'
+ENV DB_PASSWORD='group3@week9'
+ENV DB_HOST='g3.10academy.org'
+ENV DB_PORT=5432
+ENV DB_DATABASE='backtest'
+ENV AWS_DATABASE_URL="postgresql+psycopg2://group3:group3%[email protected]/backtest"
+
+# Run the FastAPI app with Uvicorn
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

Dockerfile.frontend

@@ -0,0 +1,29 @@
+# Use the official Node.js image as the base image
+FROM node:16-alpine
+
+# Set the working directory inside the container
+WORKDIR /app
+
+# Copy package.json and package-lock.json to the working directory
+COPY ./frontend/package*.json ./
+
+# Install the dependencies
+RUN npm install
+
+# Copy the rest of the application code to the working directory
+COPY ./frontend .
+
+# Build the React application
+RUN npm run build
+
+# Use the official Nginx image to serve the React application
+FROM nginx:alpine
+
+# Copy the built React application from the previous stage
+COPY --from=0 /app/build /usr/share/nginx/html
+
+# Expose port 80
+EXPOSE 80
+
+# Start Nginx when the container starts
+CMD ["nginx", "-g", "daemon off;"]

README.md

@@ -1,4 +1,14 @@
-# Scalable Crypto Backtesting Infrastructure
+# Scalable Stock and Crypto Backtesting Infrastructure
+
+## Table of Contents
+
+- [Installation](#installation)
+- [Usage](#usage)
+- [Project Structure](#project-structure)
+- [Endpoints](#endpoints)
+- [Kafka Integration](#kafka-integration)
+- [LSTM Integration](#lstm-integration)
+- [License](#license)
 
 ## Overview
 
@@ -56,7 +66,20 @@ pip install -r requirements.txt
 ```
 ## Project Structure
 ```bash
-├── data/
+├── backend/
+│   ├── main.py
+│   ├── models.py
+│   ├── schemas.py
+│   ├── database.py   
+│   ├── auth.py
+│   ├── requirements.txt
+│   ├── README.md
+│   └── scripts
+│       ├── backtesting
+│       ├── init_data.py
+│       ├── kafka_config.py
+│       │   ├── main.py
+│       └── init_db.py
 ├── scripts/
 │   └── download_data.py
 ├── tests/
@@ -66,13 +89,108 @@ pip install -r requirements.txt
 ├── requirements.txt
 └── README.md
 ```
+
+### Set Up Kafka
+
+If you do not have Kafka installed, you can run it using Docker:
+
+docker run -d --name zookeeper -p 2181:2181 zookeeper:3.4.9
+docker run -d --name kafka -p 9092:9092 --link zookeeper wurstmeister/kafka:latest
+
+Alternatively, follow the [Kafka Quickstart](https://kafka.apache.org/quickstart) guide to set up Kafka.
+
+### Configure Environment Variables
+
+Create a `.env` file in the root directory and add the following configurations:
+
+- KAFKA_BROKER_URL=localhost:9092
+- SCENE_TOPIC=scene_parameters
+- RESULT_TOPIC=backtest_results
+
+### Initialize the Database
+
+python -m scripts.init_db
+
+## Backend Usage
+
+### Run the FastAPI Application
+
+uvicorn main:app --reload
+
+### Sending Requests
+
+Use a tool like Postman, Thunder Client, or Curl to interact with the API.
+
+### Endpoints
+
+### Health Check
+
+**GET /health**
+
+### Create Indicator
+
+**POST /indicators/**
+
+```sh
+Body
+{
+  "name": "Simple Moving Average",
+  "symbol": "SMA",
+  "description": "A simple moving average indicator"
+}
+```
+
+### Read Indicators
+
+**GET /indicators/**
+
+### Create Stock
+
+**POST /stocks/**
+
+```sh
+Body
+{
+  "name": "Apple Inc.",
+  "symbol": "AAPL",
+  "description": "Apple Inc. stock"
+}
+```
+
+### Read Stocks
+
+**GET /stocks/**
+
+### Create Scene
+
+**POST /scenes/**
+
+```sh
+Body
+{
+  "period": 20,
+  "indicator_id": 1,
+  "stock_id": 1,
+  "start_date": "2023-01-01",
+  "end_date": "2023-12-31"
+}
+```
+
+### Read Scenes
+
+**GET /scenes/**
+
+### Perform Backtest
+
+**POST /backtests/{scene_id}**
+
+### Read Backtest Results
+
+**GET /backtest_results/**
+
 ## Contributors
 - Abubeker Shamil
-- Addisu Alemu
 - Michael George
-- Sheila Murugi
-
-
 
 ## License
 This project is licensed under the MIT License - see the LICENSE file for details.
@@ -85,4 +203,4 @@ This project is licensed under the MIT License - see the LICENSE file for detail
 - Rehmet
 
 **References:** 
-backtrader, Freqtrade, Vectorbt, Kafka, Airflow, MLflow, CML
+backtrader, Freqtrade, Vectorbt, Kafka, Airflow, MLflow, CML

airflow/Dockerfile

@@ -0,0 +1,26 @@
+FROM apache/airflow:2.9.1
+
+# Switch to root to install system dependencies
+USER root
+
+# Install system dependencies
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+       pkg-config \
+       libhdf5-dev \
+       gcc \
+       python3-dev \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+# Switch to airflow user to install Python packages
+USER airflow
+
+# Copy requirements and install Python packages
+COPY requirements.txt .
+
+COPY ../backend ./backend
+
+COPY ../scripts/backtesting ./scripts/backtesting
+
+RUN pip install --no-cache-dir -r requirements.txt