Merge pull request #12 from langchain-ai/marco/action-cli #71
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Comprehensive Tests | |
| on: | |
| push: | |
| branches: [ main, develop ] | |
| pull_request: | |
| branches: [ main, develop ] | |
| types: [opened, synchronize, reopened] | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v1 | |
| with: | |
| version: "latest" | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .uv/cache | |
| .venv | |
| key: ${{ runner.os }}-uv-comprehensive-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-comprehensive- | |
| - name: Install dependencies | |
| run: uv sync | |
| quality-checks: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Needed for pre-commit hooks | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v1 | |
| with: | |
| version: "latest" | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .uv/cache | |
| .venv | |
| key: ${{ runner.os }}-uv-comprehensive-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-comprehensive- | |
| - name: Run Linting and Formatting Checks | |
| run: make lint | |
| - name: Run Pre-Commit Hooks | |
| run: make pre-commit | |
| test-coverage: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| LANGSMITH_TRACING: ${{ secrets.LANGSMITH_TRACING }} | |
| LANGSMITH_ENDPOINT: ${{ secrets.LANGSMITH_ENDPOINT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v1 | |
| with: | |
| version: "latest" | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .uv/cache | |
| .venv | |
| key: ${{ runner.os }}-uv-comprehensive-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-comprehensive- | |
| - name: Run Tests with Coverage | |
| run: make test | |
| unit-tests: | |
| needs: [setup, quality-checks, test-coverage] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| LANGSMITH_TRACING: ${{ secrets.LANGSMITH_TRACING }} | |
| LANGSMITH_ENDPOINT: ${{ secrets.LANGSMITH_ENDPOINT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v1 | |
| with: | |
| version: "latest" | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .uv/cache | |
| .venv | |
| key: ${{ runner.os }}-uv-comprehensive-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-comprehensive- | |
| - name: Run unit tests | |
| run: uv run pytest tests/unit/ --junitxml=unit-results.xml | |
| continue-on-error: false | |
| - name: Upload unit test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: unit-test-results | |
| path: unit-results.xml | |
| integration-tests: | |
| needs: [setup, quality-checks, test-coverage] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| LANGSMITH_TRACING: ${{ secrets.LANGSMITH_TRACING }} | |
| LANGSMITH_ENDPOINT: ${{ secrets.LANGSMITH_ENDPOINT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v1 | |
| with: | |
| version: "latest" | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .uv/cache | |
| .venv | |
| key: ${{ runner.os }}-uv-comprehensive-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-comprehensive- | |
| - name: Run integration tests | |
| run: uv run pytest tests/integrations/ --junitxml=integration-results.xml | |
| continue-on-error: false | |
| - name: Upload integration test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: integration-test-results | |
| path: integration-results.xml | |
| evaluation-tests: | |
| needs: [setup, quality-checks, test-coverage] | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| LANGSMITH_TRACING: ${{ secrets.LANGSMITH_TRACING }} | |
| LANGSMITH_ENDPOINT: ${{ secrets.LANGSMITH_ENDPOINT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v1 | |
| with: | |
| version: "latest" | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .uv/cache | |
| .venv | |
| key: ${{ runner.os }}-uv-comprehensive-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-comprehensive- | |
| - name: Run evaluation tests | |
| run: uv run pytest -m evaluator --junitxml=evaluator-results.xml | |
| continue-on-error: false | |
| - name: Upload evaluation test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: evaluation-test-results | |
| path: evaluator-results.xml | |
| # Upload evaluation config JSON (LangSmith experiment name + criteria) | |
| - name: Upload LangSmith evaluation config | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: langsmith-eval-configs | |
| path: evaluation_config__*.json | |
| # Post LangSmith evaluation feedback as a PR comment | |
| evaluation-report: | |
| needs: evaluation-tests | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'pull_request' | |
| env: | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| LANGSMITH_TRACING: ${{ secrets.LANGSMITH_TRACING }} | |
| LANGSMITH_ENDPOINT: ${{ secrets.LANGSMITH_ENDPOINT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Download all LangSmith configs | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: langsmith-eval-configs | |
| - name: Install LangSmith SDK | |
| run: pip install langsmith | |
| - name: Run LangSmith evaluation report | |
| run: | | |
| python .github/scripts/report_eval.py --verbose evaluation_config__*.json | |
| - name: Comment PR with evaluation summary | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const comment = fs.readFileSync('eval_comment.md', 'utf8'); | |
| await github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: comment | |
| }); | |
| e2e-tests: | |
| needs: [setup, quality-checks, test-coverage] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| LANGSMITH_TRACING: ${{ secrets.LANGSMITH_TRACING }} | |
| LANGSMITH_ENDPOINT: ${{ secrets.LANGSMITH_ENDPOINT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v1 | |
| with: | |
| version: "latest" | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .uv/cache | |
| .venv | |
| key: ${{ runner.os }}-uv-comprehensive-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-comprehensive- | |
| - name: Run e2e tests | |
| run: uv run pytest tests/e2e/ --junitxml=e2e-results.xml | |
| continue-on-error: false | |
| - name: Upload e2e test results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-test-results | |
| path: e2e-results.xml | |
| langgraph-dev-test: | |
| needs: [unit-tests, integration-tests, e2e-tests, evaluation-tests] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: ["3.11"] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| LANGSMITH_TRACING: ${{ secrets.LANGSMITH_TRACING }} | |
| LANGSMITH_ENDPOINT: ${{ secrets.LANGSMITH_ENDPOINT }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v1 | |
| with: | |
| version: "latest" | |
| - name: Cache uv dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: | | |
| .uv/cache | |
| .venv | |
| key: ${{ runner.os }}-uv-comprehensive-${{ hashFiles('**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-uv-comprehensive- | |
| - name: Install dependencies | |
| run: uv sync | |
| - name: Start LangGraph dev server | |
| run: | | |
| echo "🚀 Starting LangGraph dev server..." | |
| uv run langgraph dev --host 0.0.0.0 --port 2024 & | |
| SERVER_PID=$! | |
| echo "SERVER_PID=$SERVER_PID" >> $GITHUB_ENV | |
| # Wait for server to start | |
| echo "⏳ Waiting for server to start..." | |
| sleep 10 | |
| # Check if server is running | |
| if ps -p $SERVER_PID > /dev/null; then | |
| echo "✅ LangGraph dev server started successfully (PID: $SERVER_PID)" | |
| else | |
| echo "❌ Failed to start LangGraph dev server" | |
| exit 1 | |
| fi | |
| - name: Test server health endpoint | |
| run: | | |
| echo "🔍 Testing server health endpoint..." | |
| # Test the /ok endpoint | |
| for i in {1..30}; do | |
| echo "Attempt $i/30: Testing /ok endpoint..." | |
| if curl -f -s http://localhost:2024/ok > /dev/null; then | |
| echo "✅ Server health check passed!" | |
| break | |
| else | |
| echo "⏳ Server not ready yet, waiting..." | |
| sleep 2 | |
| fi | |
| done | |
| # Final health check with detailed response | |
| echo "🔍 Final health check..." | |
| HEALTH_RESPONSE=$(curl -s http://localhost:2024/ok || echo '{"ok":false}') | |
| echo "Health response: $HEALTH_RESPONSE" | |
| # Parse JSON response | |
| if echo "$HEALTH_RESPONSE" | jq -e '.ok == true' > /dev/null; then | |
| echo "✅ LangGraph dev server is healthy and responding correctly!" | |
| else | |
| echo "❌ LangGraph dev server health check failed" | |
| echo "Response: $HEALTH_RESPONSE" | |
| exit 1 | |
| fi | |
| - name: Test server endpoints | |
| run: | | |
| echo "🔍 Testing additional server endpoints..." | |
| # Test root endpoint | |
| echo "Testing root endpoint..." | |
| ROOT_RESPONSE=$(curl -s http://localhost:2024/ || echo "Failed") | |
| echo "Root response: $ROOT_RESPONSE" | |
| # Test if server is serving the LangGraph Studio interface | |
| if curl -f -s http://localhost:2024/ | grep -q "LangGraph" || curl -f -s http://localhost:2024/ | grep -q "langgraph"; then | |
| echo "✅ LangGraph Studio interface is accessible" | |
| else | |
| echo "⚠️ LangGraph Studio interface may not be fully loaded" | |
| fi | |
| - name: Stop LangGraph dev server | |
| if: always() | |
| run: | | |
| echo "🛑 Stopping LangGraph dev server..." | |
| if [ -n "$SERVER_PID" ]; then | |
| kill $SERVER_PID 2>/dev/null || true | |
| echo "✅ LangGraph dev server stopped" | |
| fi | |
| - name: LangGraph dev server test results | |
| if: always() | |
| run: | | |
| echo "📊 LangGraph dev server test completed" | |
| echo "✅ Server was able to start and respond to health checks" |