Add cicd infrastructure #73
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: AI Config Validation | |
| # Updated: 2025-10-24 08:20 - Testing with fixed search embeddings, variation tracking, and 0.40 error threshold | |
| # Security Notes: | |
| # - pull_request_target is used to prevent external PRs from accessing secrets without approval | |
| # - External PRs require manual environment approval before secrets are exposed | |
| # - .env file is cleaned up after each run to prevent secret leakage | |
| # - Artifacts are sanitized before upload | |
| on: | |
| # Temporarily using pull_request for testing (will change back to pull_request_target) | |
| pull_request: | |
| branches: [main] | |
| types: [opened, synchronize, reopened, labeled] | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| description: 'LaunchDarkly environment to validate against' | |
| required: true | |
| default: 'production' | |
| type: choice | |
| options: | |
| - production | |
| - staging | |
| - development | |
| jobs: | |
| validate-configs: | |
| name: Validate AI Configs | |
| runs-on: ubuntu-latest | |
| # Require manual approval for external PRs to prevent secret exposure | |
| environment: | |
| name: ci | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "latest" | |
| enable-cache: true | |
| cache-dependency-glob: "**/pyproject.toml" | |
| - name: Set up Python | |
| run: uv python install | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| # Install dependencies without installing the package itself (to avoid dev dependency issues) | |
| uv pip install langchain langgraph langchain-anthropic fastapi "uvicorn[standard]" pydantic launchdarkly-server-sdk launchdarkly-server-sdk-ai numpy openai faiss-cpu PyMuPDF tiktoken streamlit requests python-dotenv PyYAML langchain-openai langchain-mcp-adapters beautifulsoup4 mcp semanticscholar rank-bm25 langchain-mistralai httpx boto3 langchain-aws | |
| uv pip install git+https://x-access-token:${{ secrets.GH_PAT }}@github.com/launchdarkly-labs/scarlett_ai_configs_ci_cd-.git@feature/initial-implementation | |
| - name: Validate required secrets | |
| run: | | |
| if [ -z "${{ secrets.LD_SDK_KEY }}" ]; then | |
| echo "::error::Missing required secret: LD_SDK_KEY" | |
| exit 1 | |
| fi | |
| if [ -z "${{ secrets.LD_API_KEY }}" ]; then | |
| echo "::error::Missing required secret: LD_API_KEY" | |
| exit 1 | |
| fi | |
| echo "✅ Required secrets are configured" | |
| - name: Run AI Config validation | |
| env: | |
| LD_SDK_KEY: ${{ secrets.LD_SDK_KEY }} | |
| LD_API_KEY: ${{ secrets.LD_API_KEY }} | |
| LD_PROJECT_KEY: ${{ secrets.LD_PROJECT_KEY }} | |
| run: | | |
| # Use ld-aic-cicd framework to validate our AI configs | |
| .venv/bin/ld-aic validate \ | |
| --environment ${{ github.event.inputs.environment || 'production' }} \ | |
| --config-keys "supervisor-agent,support-agent,security-agent" \ | |
| --report validation-report.json \ | |
| --fail-on-error | |
| - name: Upload validation report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: validation-report | |
| path: validation-report.json | |
| - name: Comment PR with results | |
| if: github.event_name == 'pull_request' && always() | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const report = JSON.parse(fs.readFileSync('validation-report.json', 'utf8')); | |
| let comment = '## 🔍 AI Config Validation Results\n\n'; | |
| comment += `**Environment:** ${report.environment}\n`; | |
| comment += `**Total Configs:** ${report.total_configs}\n\n`; | |
| // Count statuses | |
| const configs = Object.values(report.configs); | |
| const valid = configs.filter(c => c.valid).length; | |
| const errors = configs.filter(c => c.errors.length > 0).length; | |
| const warnings = configs.filter(c => c.warnings.length > 0).length; | |
| // Summary | |
| comment += '### Summary\n'; | |
| comment += `✅ Valid: ${valid}\n`; | |
| comment += `❌ Errors: ${errors}\n`; | |
| comment += `⚠️ Warnings: ${warnings}\n\n`; | |
| // Details for problematic configs | |
| if (errors > 0 || warnings > 0) { | |
| comment += '### Issues Found\n'; | |
| for (const [key, config] of Object.entries(report.configs)) { | |
| if (config.errors.length > 0 || config.warnings.length > 0) { | |
| comment += `\n**${key}**\n`; | |
| config.errors.forEach(e => comment += `- ❌ ${e}\n`); | |
| config.warnings.forEach(w => comment += `- ⚠️ ${w}\n`); | |
| } | |
| } | |
| } | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); | |
| evaluate-configs: | |
| name: Evaluate AI Configs with Judge | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' | |
| # Require manual approval for external PRs to prevent secret exposure | |
| environment: | |
| name: ci | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha || github.ref }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "latest" | |
| enable-cache: true | |
| cache-dependency-glob: "**/pyproject.toml" | |
| - name: Set up Python | |
| run: uv python install | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| # Install dependencies without installing the package itself (to avoid dev dependency issues) | |
| uv pip install langchain langgraph langchain-anthropic fastapi "uvicorn[standard]" pydantic launchdarkly-server-sdk launchdarkly-server-sdk-ai numpy openai faiss-cpu PyMuPDF tiktoken streamlit requests python-dotenv PyYAML langchain-openai langchain-mcp-adapters beautifulsoup4 mcp semanticscholar rank-bm25 langchain-mistralai httpx boto3 langchain-aws | |
| uv pip install git+https://x-access-token:${{ secrets.GH_PAT }}@github.com/launchdarkly-labs/scarlett_ai_configs_ci_cd-.git@feature/initial-implementation | |
| - name: Validate required secrets | |
| run: | | |
| # Check LaunchDarkly secrets | |
| if [ -z "${{ secrets.LD_SDK_KEY }}" ]; then | |
| echo "::error::Missing required secret: LD_SDK_KEY" | |
| exit 1 | |
| fi | |
| # Check at least one AI provider API key is set | |
| if [ -z "${{ secrets.OPENAI_API_KEY }}" ] && [ -z "${{ secrets.ANTHROPIC_API_KEY }}" ]; then | |
| echo "::error::At least one AI provider API key required: OPENAI_API_KEY or ANTHROPIC_API_KEY" | |
| exit 1 | |
| fi | |
| echo "✅ Required secrets are configured" | |
| - name: Initialize vector embeddings | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| run: | | |
| echo "📚 Initializing vector embeddings for search tools..." | |
| .venv/bin/python initialize_embeddings.py | |
| - name: Install MCP servers for research-enhanced variation | |
| run: | | |
| echo "📦 Installing MCP servers (ArXiv and Semantic Scholar)..." | |
| # Install ArXiv MCP server | |
| uv tool install arxiv-mcp-server | |
| # Clone and set up Semantic Scholar MCP server | |
| git clone https://github.com/JackKuo666/semanticscholar-MCP-Server.git | |
| echo "✅ MCP servers installed (available for research-enhanced variation)" | |
| - name: Run tests with API | |
| id: run-tests | |
| env: | |
| LD_SDK_KEY: ${{ secrets.LD_SDK_KEY }} | |
| LD_API_KEY: ${{ secrets.LD_API_KEY }} | |
| LD_PROJECT_KEY: ${{ secrets.LD_PROJECT_KEY }} | |
| LD_CICD_SDK_KEY: ${{ secrets.LD_CICD_SDK_KEY }} | |
| LD_CICD_PROJECT_KEY: ${{ secrets.LD_CICD_PROJECT_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| PYTHONPATH: ${{ github.workspace }} | |
| CI_SAFE_MODE: true | |
| run: | | |
| # Create .env file BEFORE starting API (API needs these for search tool embedding) | |
| echo "📝 Creating .env file for API and ld-aic framework..." | |
| echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > .env | |
| echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" >> .env | |
| echo "MISTRAL_API_KEY=${MISTRAL_API_KEY}" >> .env | |
| echo "LD_SDK_KEY=${LD_SDK_KEY}" >> .env | |
| echo "LD_API_KEY=${LD_API_KEY}" >> .env | |
| echo "LD_PROJECT_KEY=${LD_PROJECT_KEY}" >> .env | |
| echo "LD_CICD_SDK_KEY=${LD_CICD_SDK_KEY}" >> .env | |
| echo "LD_CICD_PROJECT_KEY=${LD_CICD_PROJECT_KEY}" >> .env | |
| # NOTE: The evaluator (evaluators/local_evaluator.py) expects the API on port 8000 | |
| # If you change the port here, update the port in the evaluator as well | |
| echo "🚀 Starting agents-demo API on port 8000..." | |
| .venv/bin/python -m uvicorn api.main:app --host 0.0.0.0 --port 8000 > /tmp/agents-demo-api.log 2>&1 & | |
| API_PID=$! | |
| # Wait for API to be ready (max 30 seconds) | |
| echo "Waiting for API to start..." | |
| for i in {1..30}; do | |
| if curl -s http://localhost:8000/health >/dev/null 2>&1; then | |
| echo "✅ API is ready!" | |
| break | |
| fi | |
| if ! kill -0 $API_PID 2>/dev/null; then | |
| echo "❌ API process died. Logs:" | |
| cat /tmp/agents-demo-api.log | |
| exit 1 | |
| fi | |
| sleep 1 | |
| done | |
| # Test the /chat endpoint with curl | |
| echo "🔍 Testing /chat endpoint with curl..." | |
| curl -X POST http://localhost:8000/chat \ | |
| -H "Content-Type: application/json" \ | |
| -d '{"message":"test","user_id":"test-user","user_context":{}}' \ | |
| || echo "Curl test failed" | |
| # Test with httpx (Python) | |
| echo "🐍 Testing /chat endpoint with httpx..." | |
| .venv/bin/python << 'PYTHON_SCRIPT' | |
| import httpx | |
| import asyncio | |
| async def test(): | |
| client = httpx.AsyncClient(timeout=10.0) | |
| try: | |
| response = await client.post( | |
| 'http://127.0.0.1:8000/chat', | |
| json={'message': 'test', 'user_id': 'test-user', 'user_context': {}} | |
| ) | |
| print(f'HTTPx test - Status: {response.status_code}') | |
| print(f'HTTPx test - Response length: {len(response.text)}') | |
| data = response.json() | |
| print(f'HTTPx test - Has response field: {"response" in data}') | |
| except Exception as e: | |
| print(f'HTTPx test FAILED: {type(e).__name__}: {str(e)}') | |
| import traceback | |
| traceback.print_exc() | |
| finally: | |
| await client.aclose() | |
| asyncio.run(test()) | |
| PYTHON_SCRIPT | |
| echo "🧪 Running AI Config test suite with judge evaluations..." | |
| echo "Working directory: $(pwd)" | |
| echo "Test data file exists: $(test -f test_data/ai_config_evaluation.yaml && echo 'YES' || echo 'NO')" | |
| # Export all env vars so they're inherited by subprocesses | |
| export LD_SDK_KEY="${LD_SDK_KEY}" | |
| export LD_API_KEY="${LD_API_KEY}" | |
| export LD_PROJECT_KEY="${LD_PROJECT_KEY}" | |
| export ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY}" | |
| export OPENAI_API_KEY="${OPENAI_API_KEY}" | |
| export MISTRAL_API_KEY="${MISTRAL_API_KEY}" | |
| export CI_SAFE_MODE="${CI_SAFE_MODE}" | |
| # Debug: verify env vars are set | |
| echo "🔍 Env vars check:" | |
| echo " LD_SDK_KEY: ${LD_SDK_KEY:0:10}... (${#LD_SDK_KEY} chars)" | |
| echo " ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:+SET}" | |
| echo " OPENAI_API_KEY: ${OPENAI_API_KEY:+SET}" | |
| echo " MISTRAL_API_KEY: ${MISTRAL_API_KEY:+SET}" | |
| # Test: verify evaluator can be imported and called | |
| echo "🧪 Testing evaluator import and basic call..." | |
| .venv/bin/python test_evaluator.py | |
| .venv/bin/ld-aic test \ | |
| --config-keys "supervisor-agent,support-agent,security-agent" \ | |
| --environment production \ | |
| --evaluation-dataset test_data/ai_config_evaluation.yaml \ | |
| --evaluator evaluators.local_evaluator:AgentsDemoEvaluator \ | |
| --report test-report.json | |
| TEST_EXIT_CODE=$? | |
| echo "🛑 Stopping API..." | |
| kill $API_PID 2>/dev/null || true | |
| # If tests failed, dump API logs for debugging | |
| if [ $TEST_EXIT_CODE -ne 0 ]; then | |
| echo "===== /tmp/agents-demo-api.log =====" | |
| if [ -f /tmp/agents-demo-api.log ]; then | |
| tail -n 500 /tmp/agents-demo-api.log || true | |
| else | |
| echo "No API log found at /tmp/agents-demo-api.log" | |
| fi | |
| fi | |
| exit $TEST_EXIT_CODE | |
| - name: Summarize test failures | |
| if: failure() | |
| run: | | |
| echo "📊 Generating human-readable failure summary..." | |
| .venv/bin/python tools/summarize_test_failures.py || true | |
| - name: Cleanup secrets | |
| if: always() | |
| run: | | |
| # Remove .env file containing secrets | |
| rm -f .env | |
| echo "🧹 Cleaned up .env file" | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-report | |
| path: test-report.json | |
| - name: Upload judge evaluation logs | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: judge-evaluation-logs | |
| path: logs/judge_evaluations/** | |
| - name: Upload API server log | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: api-server-log | |
| path: /tmp/agents-demo-api.log | |
| sync-production: | |
| name: Sync Production Configs | |
| runs-on: ubuntu-latest | |
| # CONFIGURATION: Change 'main' to your production branch name (e.g., 'production', 'master', 'release') | |
| # This job creates drift detection PRs when LaunchDarkly production configs change | |
| # Only runs on pushes to your canonical production branch | |
| if: github.ref == 'refs/heads/main' && github.event_name == 'push' | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v3 | |
| with: | |
| version: "latest" | |
| enable-cache: true | |
| cache-dependency-glob: "**/pyproject.toml" | |
| - name: Set up Python | |
| run: uv python install | |
| - name: Install dependencies | |
| run: | | |
| uv venv | |
| # Install dependencies without installing the package itself (to avoid dev dependency issues) | |
| uv pip install langchain langgraph langchain-anthropic fastapi "uvicorn[standard]" pydantic launchdarkly-server-sdk launchdarkly-server-sdk-ai numpy openai faiss-cpu PyMuPDF tiktoken streamlit requests python-dotenv PyYAML langchain-openai langchain-mcp-adapters beautifulsoup4 mcp semanticscholar rank-bm25 langchain-mistralai httpx boto3 langchain-aws | |
| uv pip install git+https://x-access-token:${{ secrets.GH_PAT }}@github.com/launchdarkly-labs/scarlett_ai_configs_ci_cd-.git@feature/initial-implementation | |
| - name: Validate required secrets | |
| run: | | |
| if [ -z "${{ secrets.LD_API_KEY }}" ]; then | |
| echo "::error::Missing required secret: LD_API_KEY" | |
| exit 1 | |
| fi | |
| if [ -z "${{ secrets.LD_PROJECT_KEY }}" ]; then | |
| echo "::error::Missing required secret: LD_PROJECT_KEY" | |
| exit 1 | |
| fi | |
| echo "✅ Required secrets are configured" | |
| - name: Sync production configs | |
| env: | |
| LD_API_KEY: ${{ secrets.LD_API_KEY }} | |
| LD_PROJECT_KEY: ${{ secrets.LD_PROJECT_KEY }} | |
| run: | | |
| mkdir -p configs | |
| .venv/bin/ld-aic sync \ | |
| --environment production \ | |
| --output-dir configs \ | |
| --format json \ | |
| --generate-module | |
| - name: Check for drift | |
| run: | | |
| # Check if there are changes to commit | |
| git diff --exit-code configs/ || echo "DRIFT_DETECTED=true" >> $GITHUB_ENV | |
| - name: Create PR for config updates | |
| if: env.DRIFT_DETECTED == 'true' | |
| uses: peter-evans/create-pull-request@v5 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| commit-message: 'Sync AI configs from production' | |
| title: 'Sync AI Configs from Production' | |
| body: | | |
| ## 🔄 Production Config Sync | |
| This PR updates the local AI config defaults to match production. | |
| ### AI Configs Synced | |
| - `supervisor-agent` - Multi-agent workflow orchestration | |
| - `support-agent` - RAG + MCP research capabilities | |
| - `security-agent` - PII detection and compliance | |
| ### Changes | |
| - Updated config snapshots in `configs/` | |
| - Regenerated `configs/production_defaults.py` | |
| Please review the changes to ensure they are expected. | |
| branch: sync/production-configs | |
| delete-branch: true |