Skip to content
CI/CD Pipeline

Dspy refactor and enhance #1

Sign in to view logs

Dspy refactor and enhance

Dspy refactor and enhance #1

Workflow file for this run

.github/workflows/ci.yml at 303e58c
name: CI/CD Pipeline
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main, develop ]
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.9, 3.10, 3.11, 3.12]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip dependencies
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e macro_agents[dev]
pip install pytest-cov pytest-xdist
- name: Lint with ruff
run: |
ruff check macro_agents/
ruff format --check macro_agents/
- name: Type check with mypy
run: |
pip install mypy
mypy macro_agents/ --ignore-missing-imports
- name: Test with pytest
run: |
pytest macro_agents/tests/ -v --cov=macro_agents --cov-report=xml --cov-report=html --junitxml=pytest-results.xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
file: ./coverage.xml
flags: unittests
name: codecov-umbrella
fail_ci_if_error: false
- name: Upload test results
uses: actions/upload-artifact@v3
if: always()
with:
name: pytest-results-${{ matrix.python-version }}
path: pytest-results.xml
integration-test:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e macro_agents[dev]
pip install pytest-cov pytest-xdist
- name: Test Dagster definitions
run: |
cd macro_agents
dg check defs
- name: Test Dagster schedules
run: |
cd macro_agents
dg schedule list
dg sensor list
- name: Test Dagster asset materialization (dry run)
run: |
cd macro_agents
dg materialize --dry-run --select "*"
- name: Test scheduled job execution (dry run)
run: |
cd macro_agents
dg job execute --dry-run monthly_sector_analysis_job
dg job execute --dry-run weekly_cycle_analysis_job
dg job execute --dry-run weekly_allocation_job
dg job execute --dry-run daily_monitoring_job
- name: Test DBT models
run: |
cd dbt_project
dbt deps
dbt compile
dbt parse
security-scan:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e macro_agents[dev]
pip install bandit safety
- name: Security scan with bandit
run: |
bandit -r macro_agents/ -f json -o bandit-report.json || true
- name: Security scan with safety
run: |
safety check --json --output safety-report.json || true
- name: Upload security reports
uses: actions/upload-artifact@v3
if: always()
with:
name: security-reports
path: |
bandit-report.json
safety-report.json
build:
runs-on: ubuntu-latest
needs: [test, integration-test, security-scan]
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: 3.11
- name: Install build dependencies
run: |
python -m pip install --upgrade pip
pip install build twine
- name: Build package
run: |
cd macro_agents
python -m build
- name: Check package
run: |
cd macro_agents
twine check dist/*
- name: Upload build artifacts
uses: actions/upload-artifact@v3
with:
name: dist
path: macro_agents/dist/
deploy:
runs-on: ubuntu-latest
needs: build
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
environment: production
steps:
- uses: actions/checkout@v4
- name: Download build artifacts
uses: actions/download-artifact@v3
with:
name: dist
path: macro_agents/dist/
- name: Deploy to production
run: |
echo "Deploying to production..."
# Add your deployment commands here