optimize log query performance by reducing time window to 24 hours an… #232
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main, develop] | |
| env: | |
| NODE_VERSION: '20' | |
| PNPM_VERSION: '10' | |
| # Restrict permissions for all jobs (principle of least privilege) | |
| permissions: | |
| contents: read | |
| actions: read | |
| jobs: | |
| # ==================== | |
| # Backend Tests | |
| # ==================== | |
| backend-test: | |
| name: Backend Tests | |
| runs-on: ubuntu-latest | |
| services: | |
| postgres: | |
| image: timescale/timescaledb:latest-pg16 | |
| env: | |
| POSTGRES_DB: logtide_test | |
| POSTGRES_USER: logtide_test | |
| POSTGRES_PASSWORD: test_password | |
| ports: | |
| - 5433:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| redis: | |
| image: redis:7-alpine | |
| ports: | |
| - 6380:6379 | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| mailhog: | |
| image: mailhog/mailhog:latest | |
| ports: | |
| - 1025:1025 | |
| - 8025:8025 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Build shared package | |
| run: pnpm --filter '@logtide/shared' build | |
| - name: Build reservoir package | |
| run: pnpm --filter '@logtide/reservoir' build | |
| - name: Run backend tests with coverage | |
| working-directory: packages/backend | |
| env: | |
| NODE_ENV: test | |
| DATABASE_URL: postgresql://logtide_test:test_password@localhost:5433/logtide_test | |
| DATABASE_HOST: localhost | |
| DATABASE_PORT: 5433 | |
| DB_USER: logtide_test | |
| DB_PASSWORD: test_password | |
| DB_NAME: logtide_test | |
| REDIS_URL: redis://localhost:6380 | |
| API_KEY_SECRET: test_secret_key_32_chars_long!!! | |
| SMTP_HOST: localhost | |
| SMTP_PORT: 1025 | |
| SMTP_USER: '' | |
| SMTP_PASS: '' | |
| SMTP_FROM: test@logtide.dev | |
| RATE_LIMIT_MAX: 1000 | |
| RATE_LIMIT_WINDOW: 60000 | |
| run: pnpm test:ci:coverage | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| files: ./packages/backend/coverage/lcov.info,./packages/backend/coverage/coverage-final.json | |
| flags: backend | |
| name: backend-coverage | |
| fail_ci_if_error: false | |
| verbose: true | |
| - name: Check coverage threshold | |
| working-directory: packages/backend | |
| run: | | |
| COVERAGE=$(cat coverage/coverage-summary.json | jq '.total.lines.pct') | |
| echo "Line coverage: $COVERAGE%" | |
| if (( $(echo "$COVERAGE < 80" | bc -l) )); then | |
| echo "::error::Coverage $COVERAGE% is below 80% threshold" | |
| exit 1 | |
| fi | |
| echo "::notice::Coverage $COVERAGE% meets the 80% threshold" | |
| # ==================== | |
| # Reservoir Tests | |
| # ==================== | |
| reservoir-test: | |
| name: Reservoir Tests | |
| runs-on: ubuntu-latest | |
| services: | |
| clickhouse: | |
| image: clickhouse/clickhouse-server:24.1 | |
| env: | |
| CLICKHOUSE_DB: logtide_test | |
| CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 | |
| ports: | |
| - 18123:8123 | |
| options: >- | |
| --health-cmd "clickhouse-client --query 'SELECT 1'" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Build shared package | |
| run: pnpm --filter '@logtide/shared' build | |
| - name: Build reservoir package | |
| run: pnpm --filter '@logtide/reservoir' build | |
| - name: Run reservoir tests | |
| working-directory: packages/reservoir | |
| run: npx vitest run --reporter=verbose | |
| # ==================== | |
| # Typecheck | |
| # ==================== | |
| typecheck: | |
| name: TypeScript Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Build shared package | |
| run: pnpm --filter '@logtide/shared' build | |
| - name: Build reservoir package | |
| run: pnpm --filter '@logtide/reservoir' build | |
| - name: Typecheck backend | |
| run: pnpm --filter '@logtide/backend' typecheck | |
| - name: Typecheck frontend | |
| env: | |
| PUBLIC_API_URL: http://localhost:8080 | |
| run: pnpm --filter '@logtide/frontend' typecheck | |
| # ==================== | |
| # E2E Tests (Playwright) | |
| # ==================== | |
| e2e-test: | |
| name: E2E Tests | |
| runs-on: ubuntu-latest | |
| needs: [backend-test, typecheck] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: ${{ env.PNPM_VERSION }} | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: 'pnpm' | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Install Playwright browsers | |
| working-directory: packages/frontend | |
| run: npx playwright install --with-deps chromium | |
| - name: Start test infrastructure | |
| run: | | |
| docker compose -f docker-compose.test.yml up -d --build | |
| # Wait for services to be healthy | |
| echo "Waiting for services to be ready..." | |
| timeout 120 bash -c 'until curl -s http://localhost:3001/health > /dev/null; do sleep 2; done' | |
| echo "Backend is ready" | |
| timeout 120 bash -c 'until curl -s http://localhost:3002 > /dev/null; do sleep 2; done' | |
| echo "Frontend is ready" | |
| - name: Run E2E tests | |
| working-directory: packages/frontend | |
| env: | |
| E2E: 'true' | |
| TEST_API_URL: http://localhost:3001 | |
| TEST_FRONTEND_URL: http://localhost:3002 | |
| run: npx playwright test --reporter=list | |
| - name: Upload Playwright report | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: playwright-report | |
| path: packages/frontend/playwright-report/ | |
| retention-days: 7 | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v4 | |
| if: failure() | |
| with: | |
| name: test-results | |
| path: packages/frontend/test-results/ | |
| retention-days: 7 | |
| - name: Stop test infrastructure | |
| if: always() | |
| run: docker compose -f docker-compose.test.yml down -v | |
| # ==================== | |
| # Build Docker Images | |
| # ==================== | |
| build: | |
| name: Build Docker Images | |
| runs-on: ubuntu-latest | |
| needs: [backend-test, reservoir-test, typecheck, e2e-test] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build backend image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: packages/backend/Dockerfile | |
| push: false | |
| tags: logtide/backend:${{ github.sha }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Build frontend image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: packages/frontend/Dockerfile | |
| push: false | |
| tags: logtide/frontend:${{ github.sha }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max |