Harden api with validation and rate limits #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Optimized CI/CD Pipeline with caching, parallelization, and smart path detection | |
| name: ChaosLabs CI/CD | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main, develop] | |
| workflow_dispatch: | |
| inputs: | |
| skip_tests: | |
| description: 'Skip test execution' | |
| required: false | |
| default: 'false' | |
| deploy_environment: | |
| description: 'Deploy to environment' | |
| required: false | |
| default: 'none' | |
| type: choice | |
| options: | |
| - none | |
| - staging | |
| - production | |
| # Optimize workflow concurrency | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: chaoslabs | |
| GO_VERSION: '1.21' | |
| NODE_VERSION: '18' | |
| DOCKER_BUILDKIT: 1 | |
| COMPOSE_DOCKER_CLI_BUILD: 1 | |
| jobs: | |
| # Smart change detection to skip unnecessary work | |
| detect-changes: | |
| name: Detect Changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| go-changed: ${{ steps.changes.outputs.go }} | |
| frontend-changed: ${{ steps.changes.outputs.frontend }} | |
| docs-changed: ${{ steps.changes.outputs.docs }} | |
| infra-changed: ${{ steps.changes.outputs.infra }} | |
| tests-changed: ${{ steps.changes.outputs.tests }} | |
| should-deploy: ${{ steps.deploy-check.outputs.should-deploy }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect file changes | |
| uses: dorny/paths-filter@v2 | |
| id: changes | |
| with: | |
| filters: | | |
| go: | |
| - 'controller/**/*.go' | |
| - 'agent/**/*.go' | |
| - 'cli/**/*.go' | |
| - 'go.mod' | |
| - 'go.sum' | |
| - '**/*_test.go' | |
| frontend: | |
| - 'dashboard-v2/**' | |
| - 'Dashboard/**' | |
| docs: | |
| - 'docs/**' | |
| - '*.md' | |
| - '.github/**/*.md' | |
| infra: | |
| - 'infrastructure/**' | |
| - 'docker-compose*.yml' | |
| - '.github/workflows/**' | |
| - 'Dockerfile*' | |
| tests: | |
| - 'tests/**' | |
| - '**/*_test.go' | |
| - 'test/**' | |
| - name: Check if deployment needed | |
| id: deploy-check | |
| run: | | |
| if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then | |
| echo "should-deploy=true" >> $GITHUB_OUTPUT | |
| elif [[ "${{ github.event.inputs.deploy_environment }}" != "none" ]]; then | |
| echo "should-deploy=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "should-deploy=false" >> $GITHUB_OUTPUT | |
| fi | |
| # Fast documentation-only path | |
| docs-only: | |
| name: Documentation Only | |
| runs-on: ubuntu-latest | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.docs-changed == 'true' && needs.detect-changes.outputs.go-changed == 'false' && needs.detect-changes.outputs.frontend-changed == 'false' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'npm' | |
| cache-dependency-path: 'docs/package-lock.json' | |
| - name: Build documentation | |
| run: | | |
| cd docs | |
| npm ci | |
| npm run build | |
| - name: Deploy docs to GitHub Pages | |
| if: github.ref == 'refs/heads/main' | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./docs/dist | |
| # Parallel linting stage | |
| lint: | |
| name: Lint & Format Check | |
| runs-on: ubuntu-latest | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.go-changed == 'true' || needs.detect-changes.outputs.frontend-changed == 'true' | |
| strategy: | |
| matrix: | |
| component: [go, frontend] | |
| exclude: | |
| - component: go | |
| # Exclude if only frontend changed | |
| condition: ${{ needs.detect-changes.outputs.go-changed == 'false' }} | |
| - component: frontend | |
| # Exclude if only go changed | |
| condition: ${{ needs.detect-changes.outputs.frontend-changed == 'false' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| # Go linting | |
| - name: Setup Go | |
| if: matrix.component == 'go' | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Go lint | |
| if: matrix.component == 'go' | |
| uses: golangci/golangci-lint-action@v3 | |
| with: | |
| version: latest | |
| args: --timeout=5m --config=.golangci.yml | |
| skip-cache: false | |
| skip-save-cache: false | |
| # Frontend linting | |
| - name: Setup Node.js | |
| if: matrix.component == 'frontend' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'npm' | |
| cache-dependency-path: 'dashboard-v2/package-lock.json' | |
| - name: Install frontend dependencies | |
| if: matrix.component == 'frontend' | |
| run: | | |
| cd dashboard-v2 | |
| npm ci --prefer-offline --no-audit | |
| - name: Frontend lint | |
| if: matrix.component == 'frontend' | |
| run: | | |
| cd dashboard-v2 | |
| npm run lint | |
| npm run type-check | |
| # Unit tests with matrix strategy | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ubuntu-latest | |
| needs: [detect-changes, lint] | |
| if: needs.detect-changes.outputs.go-changed == 'true' && github.event.inputs.skip_tests != 'true' | |
| strategy: | |
| matrix: | |
| component: [controller, agent, cli] | |
| go-version: ['1.21'] # Could add ['1.20', '1.21'] for multiple versions | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ matrix.go-version }} | |
| cache: true | |
| - name: Download dependencies | |
| run: go mod download | |
| - name: Run unit tests | |
| run: | | |
| cd ${{ matrix.component }} | |
| go test -race -coverprofile=coverage.out -covermode=atomic ./... | |
| - name: Generate coverage report | |
| run: | | |
| cd ${{ matrix.component }} | |
| go tool cover -html=coverage.out -o coverage.html | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./${{ matrix.component }}/coverage.out | |
| flags: ${{ matrix.component }} | |
| name: ${{ matrix.component }}-coverage | |
| - name: Upload test artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: ${{ matrix.component }}-test-results | |
| path: | | |
| ${{ matrix.component }}/coverage.out | |
| ${{ matrix.component }}/coverage.html | |
| # Frontend tests | |
| frontend-tests: | |
| name: Frontend Tests | |
| runs-on: ubuntu-latest | |
| needs: [detect-changes, lint] | |
| if: needs.detect-changes.outputs.frontend-changed == 'true' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'npm' | |
| cache-dependency-path: 'dashboard-v2/package-lock.json' | |
| - name: Install dependencies | |
| run: | | |
| cd dashboard-v2 | |
| npm ci --prefer-offline --no-audit | |
| - name: Run tests | |
| run: | | |
| cd dashboard-v2 | |
| npm run test:coverage | |
| - name: Upload frontend coverage | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./dashboard-v2/coverage/lcov.info | |
| flags: frontend | |
| name: frontend-coverage | |
| # Integration tests with services | |
| integration-tests: | |
| name: Integration Tests | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, detect-changes] | |
| if: needs.detect-changes.outputs.go-changed == 'true' || needs.detect-changes.outputs.infra-changed == 'true' | |
| services: | |
| redis: | |
| image: redis:7-alpine | |
| ports: | |
| - 6379:6379 | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| nats: | |
| image: nats:2.10-alpine | |
| ports: | |
| - 4222:4222 | |
| options: >- | |
| --health-cmd "wget --no-verbose --tries=1 --spider http://localhost:8222/healthz || exit 1" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Wait for services | |
| run: | | |
| timeout 30s bash -c 'until redis-cli -h localhost ping; do sleep 1; done' | |
| timeout 30s bash -c 'until curl -f http://localhost:8222/healthz; do sleep 1; done' | |
| - name: Run integration tests | |
| env: | |
| REDIS_URL: redis://localhost:6379 | |
| NATS_URL: nats://localhost:4222 | |
| run: | | |
| go test -tags=integration -v ./tests/integration/... | |
| # Security scanning | |
| security: | |
| name: Security Scan | |
| runs-on: ubuntu-latest | |
| needs: detect-changes | |
| if: needs.detect-changes.outputs.go-changed == 'true' || needs.detect-changes.outputs.infra-changed == 'true' | |
| permissions: | |
| security-events: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache: true | |
| - name: Run Gosec Security Scanner | |
| uses: securecodewarrior/github-action-gosec@master | |
| with: | |
| args: '-fmt sarif -out gosec.sarif ./...' | |
| - name: Upload SARIF file | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: gosec.sarif | |
| - name: Run govulncheck | |
| run: | | |
| go install golang.org/x/vuln/cmd/govulncheck@latest | |
| govulncheck ./... | |
| # Build and push Docker images | |
| build-images: | |
| name: Build Images | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, integration-tests, detect-changes] | |
| if: always() && !cancelled() && (needs.detect-changes.outputs.go-changed == 'true' || needs.detect-changes.outputs.infra-changed == 'true') | |
| strategy: | |
| matrix: | |
| component: [controller, agent, dashboard] | |
| outputs: | |
| controller-digest: ${{ steps.build.outputs.controller-digest }} | |
| agent-digest: ${{ steps.build.outputs.agent-digest }} | |
| dashboard-digest: ${{ steps.build.outputs.dashboard-digest }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: | | |
| network=host | |
| - name: Log in to Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.component }} | |
| tags: | | |
| type=ref,event=branch | |
| type=ref,event=pr | |
| type=sha,prefix={{branch}}- | |
| type=raw,value=latest,enable={{is_default_branch}} | |
| - name: Build and push | |
| id: build | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ./infrastructure/Dockerfile.${{ matrix.component }}.optimized | |
| target: production | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha,scope=${{ matrix.component }} | |
| cache-to: type=gha,mode=max,scope=${{ matrix.component }} | |
| provenance: true | |
| sbom: true | |
| - name: Output digest | |
| run: echo "${{ matrix.component }}-digest=${{ steps.build.outputs.digest }}" >> $GITHUB_OUTPUT | |
| # Performance tests (soak tests) | |
| performance-tests: | |
| name: Performance Tests | |
| runs-on: ubuntu-latest | |
| needs: [build-images, detect-changes] | |
| if: needs.detect-changes.outputs.should-deploy == 'true' || github.event.inputs.skip_tests != 'true' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup k6 | |
| run: | | |
| sudo gpg -k | |
| sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69 | |
| echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list | |
| sudo apt-get update | |
| sudo apt-get install k6 | |
| - name: Start test environment | |
| run: | | |
| docker-compose -f infrastructure/docker-compose.test.yml up -d | |
| sleep 30 | |
| - name: Run performance tests | |
| run: | | |
| k6 run tests/performance/load-test.js | |
| k6 run tests/performance/stress-test.js | |
| - name: Cleanup test environment | |
| if: always() | |
| run: | | |
| docker-compose -f infrastructure/docker-compose.test.yml down -v | |
| # Deployment to staging/production | |
| deploy: | |
| name: Deploy | |
| runs-on: ubuntu-latest | |
| needs: [build-images, performance-tests, detect-changes] | |
| if: needs.detect-changes.outputs.should-deploy == 'true' | |
| environment: | |
| name: ${{ github.event.inputs.deploy_environment || 'staging' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Deploy to EKS | |
| run: | | |
| aws eks update-kubeconfig --name chaoslabs-cluster | |
| envsubst < infrastructure/k8s/deployment.yaml | kubectl apply -f - | |
| env: | |
| ENVIRONMENT: ${{ github.event.inputs.deploy_environment || 'staging' }} | |
| CONTROLLER_IMAGE: ${{ env.REGISTRY }}/${{ github.repository }}/controller@${{ needs.build-images.outputs.controller-digest }} | |
| AGENT_IMAGE: ${{ env.REGISTRY }}/${{ github.repository }}/agent@${{ needs.build-images.outputs.agent-digest }} | |
| DASHBOARD_IMAGE: ${{ env.REGISTRY }}/${{ github.repository }}/dashboard@${{ needs.build-images.outputs.dashboard-digest }} | |
| # Generate reports and notifications | |
| report: | |
| name: Generate Report | |
| runs-on: ubuntu-latest | |
| needs: [unit-tests, integration-tests, performance-tests, security, build-images] | |
| if: always() | |
| steps: | |
| - name: Download test artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: artifacts | |
| - name: Generate CI report | |
| run: | | |
| echo "## ChaosLabs CI/CD Report" > ci-report.md | |
| echo "" >> ci-report.md | |
| echo "**Workflow:** ${{ github.workflow }}" >> ci-report.md | |
| echo "**Run:** #${{ github.run_number }}" >> ci-report.md | |
| echo "**Trigger:** ${{ github.event_name }}" >> ci-report.md | |
| echo "**Branch:** ${{ github.ref_name }}" >> ci-report.md | |
| echo "**Commit:** ${{ github.sha }}" >> ci-report.md | |
| echo "" >> ci-report.md | |
| echo "### Job Status" >> ci-report.md | |
| echo "- Unit Tests: ${{ needs.unit-tests.result }}" >> ci-report.md | |
| echo "- Integration Tests: ${{ needs.integration-tests.result }}" >> ci-report.md | |
| echo "- Security Scan: ${{ needs.security.result }}" >> ci-report.md | |
| echo "- Build Images: ${{ needs.build-images.result }}" >> ci-report.md | |
| echo "- Performance Tests: ${{ needs.performance-tests.result }}" >> ci-report.md | |
| echo "" >> ci-report.md | |
| echo "### Performance Metrics" >> ci-report.md | |
| echo "- Workflow Duration: ${{ github.event.head_commit.timestamp }}" >> ci-report.md | |
| if [ -d "artifacts" ]; then | |
| echo "### Artifacts" >> ci-report.md | |
| find artifacts -name "*.out" -o -name "*.html" | while read file; do | |
| echo "- [$(basename $file)]($file)" >> ci-report.md | |
| done | |
| fi | |
| - name: Comment PR | |
| if: github.event_name == 'pull_request' | |
| uses: thollander/actions-comment-pull-request@v2 | |
| with: | |
| filePath: ci-report.md | |
| - name: Slack notification | |
| if: always() && (github.ref == 'refs/heads/main' || failure()) | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: ${{ job.status }} | |
| channel: '#ci-cd' | |
| text: | | |
| ChaosLabs CI/CD ${{ job.status }} | |
| Branch: ${{ github.ref_name }} | |
| Commit: ${{ github.sha }} | |
| Workflow: ${{ github.workflow }} | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} |