Initial project structure with ETL pipeline, Glue jobs, database scri… #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy to Staging | |
| on: | |
| push: | |
| branches: [ main ] | |
| workflow_dispatch: | |
| env: | |
| AWS_REGION: us-east-1 | |
| ENVIRONMENT: staging | |
| jobs: | |
| validate: | |
| name: Validate Code | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Install dependencies | |
| run: | | |
| pip install -r requirements.txt | |
| pip install flake8 pytest | |
| - name: Lint Python code | |
| run: flake8 glue_jobs/ --max-line-length=120 | |
| - name: Test Glue jobs | |
| run: pytest tests/ | |
| security-scan: | |
| name: Security Scanning | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run Checkov | |
| uses: bridgecrewio/checkov-action@master | |
| with: | |
| directory: terraform/ | |
| framework: terraform | |
| terraform-plan: | |
| name: Terraform Plan | |
| runs-on: ubuntu-latest | |
| needs: [validate, security-scan] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| with: | |
| terraform_version: 1.5.0 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Terraform Init | |
| working-directory: terraform/environments/staging | |
| run: terraform init | |
| - name: Terraform Plan | |
| working-directory: terraform/environments/staging | |
| run: terraform plan -var-file="../../../config/staging.tfvars" | |
| deploy: | |
| name: Deploy Infrastructure | |
| runs-on: ubuntu-latest | |
| needs: terraform-plan | |
| environment: staging | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| with: | |
| terraform_version: 1.5.0 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Terraform Init | |
| working-directory: terraform/environments/staging | |
| run: terraform init | |
| - name: Terraform Apply | |
| working-directory: terraform/environments/staging | |
| run: terraform apply -auto-approve -var-file="../../../config/staging.tfvars" | |
| - name: Deploy Glue Jobs | |
| run: | | |
| python scripts/deploy_glue_jobs.py --environment staging | |
| - name: Run Database Migrations | |
| run: | | |
| python scripts/run_migrations.py --environment staging | |
| smoke-tests: | |
| name: Smoke Tests | |
| runs-on: ubuntu-latest | |
| needs: deploy | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Run smoke tests | |
| run: | | |
| python -m pytest tests/smoke/ -v |