Initial commit

Author: ravikik

.github/CODEOWNERS

@@ -0,0 +1,10 @@
+# This is a comment.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+# @josephmachado will be requested for
+# review when someone opens a pull request.
+
+# REPLACE WITH YOUR GITHUB USER ID !!!!
+*       @josephmachado 

.github/workflows/cd.yml

@@ -0,0 +1,18 @@
+# Note: Uncomment to start CD
+# name: CD
+# on:
+#   push:
+#     branches:
+#       - main
+# jobs:
+#   deploy-to-ec2:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Deploy to server
+#         uses: easingthemes/ssh-deploy@main
+#         env:
+#           SSH_PRIVATE_KEY: ${{ secrets.SERVER_SSH_KEY }}
+#           REMOTE_HOST: ${{ secrets.REMOTE_HOST }}
+#           REMOTE_USER: ${{ secrets.REMOTE_USER }}
+#           SOURCE: "./"
+#           TARGET: "/home/ubuntu/data_engineering_project_template"

.github/workflows/ci.yml

@@ -0,0 +1,13 @@
+# Note: Uncomment to start CI
+# name: CI
+# on: [pull_request]
+# jobs:
+#   run-ci-tests:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: checkout repo
+#         uses: actions/checkout@v2
+#       - name: Spin up containers
+#         run: make up
+#       - name: Run CI test
+#         run: make ci

.gitignore

@@ -0,0 +1,81 @@
+### Vim ###
+[._]*.s[a-w][a-z]
+[._]s[a-w][a-z]
+*.un~
+Session.vim
+.netrwhist
+*~
+
+### SublimeText ###
+# cache files for sublime text
+*.tmlanguage.cache
+*.tmPreferences.cache
+*.stTheme.cache
+
+# workspace files are user-specific
+*.sublime-workspace
+
+# project files should be checked into the repository, unless a significant
+# proportion of contributors will probably not be using SublimeText
+# *.sublime-project
+
+# sftp configuration file
+sftp-config.json
+
+# Python
+__pycache__
+
+# policy
+trust-policy.json
+
+# logs
+logs/*
+*.log
+
+temp/*
+
+__MACOSX
+
+.dockerignore
+
+*.pem
+
+###### TERRAFORM IGNORE
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Crash log files
+crash.log
+crash.*.log
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version 
+# control as they are data points which are potentially sensitive and subject 
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Include override files you do wish to add to version control using negated pattern
+# !example_override.tf
+
+# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
+# example: *tfplan*
+
+# Ignore CLI configuration files
+.terraformrc
+terraform.rc
+Footer
+
+dashboard_files/
+

.tool-versions

@@ -0,0 +1 @@
+python 3.11.1

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Start Data Engineering
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,68 @@
+####################################################################################################################
+# Setup containers to run Airflow
+
+docker-spin-up:
+	docker compose up airflow-init && docker compose up --build -d
+
+perms:
+	sudo mkdir -p logs plugins temp dags tests migrations data visualization && sudo chmod -R u=rwx,g=rwx,o=rwx logs plugins temp dags tests migrations data visualization
+
+up: perms docker-spin-up 
+
+down:
+	docker compose down --volumes --rmi all
+
+restart: down up
+
+sh:
+	docker exec -ti webserver bash
+
+####################################################################################################################
+# Testing, auto formatting, type checks, & Lint checks
+
+pytest:
+	docker exec webserver pytest -p no:warnings -v /opt/airflow/tests
+
+format:
+	docker exec webserver python -m black -S --line-length 79 .
+
+isort:
+	docker exec webserver isort .
+
+type:
+	docker exec webserver mypy --ignore-missing-imports /opt/airflow
+
+lint: 
+	docker exec webserver flake8 /opt/airflow/dags
+
+ci: isort format type lint pytest
+
+####################################################################################################################
+# Set up cloud infrastructure
+
+tf-init:
+	terraform -chdir=./terraform init
+
+infra-up:
+	terraform -chdir=./terraform apply
+
+infra-down:
+	terraform -chdir=./terraform destroy
+
+infra-config:
+	terraform -chdir=./terraform output
+
+####################################################################################################################
+# Port forwarding to local machine
+
+cloud-metabase:
+	terraform -chdir=./terraform output -raw private_key > private_key.pem && chmod 600 private_key.pem && ssh -o "IdentitiesOnly yes" -i private_key.pem ubuntu@$$(terraform -chdir=./terraform output -raw ec2_public_dns) -N -f -L 3001:$$(terraform -chdir=./terraform output -raw ec2_public_dns):3000 && open http://localhost:3001 && rm private_key.pem
+
+cloud-airflow:
+	terraform -chdir=./terraform output -raw private_key > private_key.pem && chmod 600 private_key.pem && ssh -o "IdentitiesOnly yes" -i private_key.pem ubuntu@$$(terraform -chdir=./terraform output -raw ec2_public_dns) -N -f -L 8081:$$(terraform -chdir=./terraform output -raw ec2_public_dns):8080 && open http://localhost:8081 && rm private_key.pem
+
+####################################################################################################################
+# Helpers
+
+ssh-ec2:
+	terraform -chdir=./terraform output -raw private_key > private_key.pem && chmod 600 private_key.pem && ssh -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -i private_key.pem ubuntu@$$(terraform -chdir=./terraform output -raw ec2_public_dns) && rm private_key.pem

README.md

@@ -0,0 +1,139 @@
+
+* [Data engineering project template](#data-engineering-project-template)
+    * [Run Data Pipeline](#run-data-pipeline)
+        * [Run on codespaces](#run-on-codespaces)
+        * [Run locally](#run-locally)
+    * [Architecture and services in this template](#architecture-and-services-in-this-template)
+    * [Using template](#using-template)
+    * [Writing pipelines](#writing-pipelines)
+    * [(Optional) Advanced cloud setup](#optional-advanced-cloud-setup)
+        * [Prerequisites:](#prerequisites)
+        * [Tear down infra](#tear-down-infra)
+
+
+# Data engineering project template
+
+Detailed explanation can be found **[`in this post`](https://www.startdataengineering.com/post/data-engineering-projects-with-free-template/)**
+
+## Run Data Pipeline
+
+Code available at **[data_engineering_project_template](https://github.com/josephmachado/data_engineering_project_template/tree/main?tab=readme-ov-file#data-engineering-project-template)** repository.
+
+### Run on codespaces
+
+You can run this data pipeline using GitHub codespaces. Follow the instructions below.
+
+1. Create codespaces by going to the **[data_engineering_project_template](https://github.com/josephmachado/data_engineering_project_template/tree/main?tab=readme-ov-file#data-engineering-project-template)** repository, cloning it(or click `Use this template` button) and then clicking on `Create codespaces on main` button.
+2. Wait for codespaces to start, then in the terminal type `make up`.
+3. Wait for `make up` to complete, and then wait for 30s (for Airflow to start).
+4. After 30s go to the `ports` tab and click on the link exposing port `8080` to access Airflow UI (username and password is `airflow`).
+
+![codespaces start](./assets/images/cs1.png)
+![codespaces make up](./assets/images/cs2.png)
+![codespaces open url](./assets/images/cs3.png)
+
+### Run locally
+
+To run locally, you need:
+
+1. [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+2. [Github account](https://github.com/)
+3. [Docker](https://docs.docker.com/engine/install/) with at least 4GB of RAM and [Docker Compose](https://docs.docker.com/compose/install/) v1.27.0 or later
+
+Clone the repo and run the following commands to start the data pipeline:
+
+```bash
+git clone https://github.com/josephmachado/data_engineering_project_template.git
+cd data_engineering_project_template
+make up
+sleep 30 # wait for Airflow to start
+make ci # run checks and tests
+```
+Go to [http:localhost:8080](http:localhost:8080) to see the Airflow UI. Username and password are both `airflow`.
+
+## Architecture and services in this template
+
+This data engineering project template, includes the following:
+
+1. **`Airflow`**: To schedule and orchestrate DAGs.
+2. **`Postgres`**: To store Airflow's details (which you can see via Airflow UI) and also has a schema to represent upstream databases.
+3. **`DuckDB`**: To act as our warehouse
+4. **`Quarto with Plotly`**: To convert code in `markdown` format to html files that can be embedded in your app or servered as is.
+5. **`cuallee`**: To run data quality checks on the data we extracted from CoinCap API.
+6. **`minio`**: To provide an S3 compatible open source storage system.
+
+For simplicity services 1-5 of the above are installed and run in one container defined [here](./containers/airflow/Dockerfile).
+
+![DET](./assets/images/det2.png)
+
+The `coincap_elt` DAG in the [Airflow UI](http://localhost:8080) will look like the below image:
+
+![DAG](./assets/images/dag.png)
+
+You can see the rendered html at [./visualizations/dashboard.html](https://github.com/josephmachado/data_engineering_project_template/blob/main/visualization/dashboard.html).
+
+The file structure of our repo is as shown below:
+
+![File strucutre](./assets/images/fs.png)
+
+## Using template
+
+You can use this repo as a template and create your own, click on the `Use this template` button.
+
+![Template](./assets/images/template.png)
+
+## Writing pipelines
+
+We have a sample pipeline at [coincap_elt.py](./dags/coincap_elt.py) that you can use as a starter to create your own DAGs. The tests are available at [./tests](./tests) folder.
+
+Once the `coincap_elt` DAG runs, we can see the dashboard html at [./visualization/dashboard.html](./visualization/dashboard.html) and will look like ![Dashboard](./assets/images/dash.png).
+
+## (Optional) Advanced cloud setup
+
+If you want to run your code on an EC2 instance, with terraform, follow the steps below.
+
+### Prerequisites:
+
+1. [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli) 
+2. [AWS account](https://aws.amazon.com/) 
+3. [AWS CLI installed](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) and [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
+
+You can create your GitHub repository based on this template by clicking on the `Use this template button in the **[data_engineering_project_template](https://github.com/josephmachado/data_engineering_project_template)** repository. Clone your repository and replace content in the following files
+
+1. **[CODEOWNERS](https://github.com/josephmachado/data_engineering_project_template/blob/main/.github/CODEOWNERS)**: In this file change the user id from `@josephmachado` to your Github user id.
+2. **[cd.yml](https://github.com/josephmachado/data_engineering_project_template/blob/main/.github/workflows/cd.yml)**: In this file change the `data_engineering_project_template` part of the `TARGET` parameter to your repository name.
+3. **[variable.tf](https://github.com/josephmachado/data_engineering_project_template/blob/main/terraform/variable.tf)**: In this file change the default values for `alert_email_id` and `repo_url` variables with your email and [github repository url](https://www.theserverside.com/blog/Coffee-Talk-Java-News-Stories-and-Opinions/GitHub-URL-find-use-example) respectively.
+
+Run the following commands in your project directory.
+
+```shell
+# Create AWS services with Terraform
+make tf-init # Only needed on your first terraform run (or if you add new providers)
+make infra-up # type in yes after verifying the changes TF will make
+
+# Wait until the EC2 instance is initialized, you can check this via your AWS UI
+# See "Status Check" on the EC2 console, it should be "2/2 checks passed" before proceeding
+# Wait another 5 mins, Airflow takes a while to start up
+
+make cloud-airflow # this command will forward Airflow port from EC2 to your machine and opens it in the browser
+# the user name and password are both airflow
+
+make cloud-metabase # this command will forward Metabase port from EC2 to your machine and opens it in the browser
+# use https://github.com/josephmachado/data_engineering_project_template/blob/main/env file to connect to the warehouse from metabase
+```
+
+For the [continuous delivery](https://github.com/josephmachado/data_engineering_project_template/blob/main/.github/workflows/cd.yml) to work, set up the infrastructure with terraform, & defined the following repository secrets. You can set up the repository secrets by going to `Settings > Secrets > Actions > New repository secret`.
+
+1. **`SERVER_SSH_KEY`**: We can get this by running `terraform -chdir=./terraform output -raw private_key` in the project directory and paste the entire content in a new Action secret called SERVER_SSH_KEY.
+2. **`REMOTE_HOST`**: Get this by running `terraform -chdir=./terraform output -raw ec2_public_dns` in the project directory.
+3. **`REMOTE_USER`**: The value for this is **ubuntu**.
+
+### Tear down infra
+
+After you are done, make sure to destroy your cloud infrastructure.
+
+```shell
+make down # Stop docker containers on your computer
+make infra-down # type in yes after verifying the changes TF will make
+```
+