get testing to work kind of

Author: chitalian

.github/workflows/supabase-start.yml

@@ -0,0 +1,46 @@
+name: Supabase Start
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  supabase-start:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "yarn"
+          cache-dependency-path: "yarn.lock"
+
+      - name: Enable Corepack
+        run: corepack enable
+
+      - name: Setup Supabase CLI
+        uses: supabase/setup-cli@v1
+        with:
+          version: latest
+
+      - name: Start Clickhouse and Minio
+        run: cd docker && docker compose up minio minio-setup clickhouse -d
+
+      - name: Start Supabase
+        run: echo 'y' | supabase start -x realtime,storage-api,imgproxy,mailpit,edge-runtime,logflare,vector,supavisor
+
+      - name: start gateway workers
+        run: |
+          cd worker
+          yarn install
+          npx wrangler dev --var WORKER_TYPE:HELICONE_API --port 8788 &
+          npx wrangler dev --var WORKER_TYPE:AI_GATEWAY_API --port 8793 --test-scheduled &
+          sleep 10
+
+      - name: start jawn

e2e/.env.example

@@ -0,0 +1,17 @@
+# AI Gateway URL (default: http://localhost:8793)
+AI_GATEWAY_URL=http://localhost:8793
+
+# Worker API URL (default: http://localhost:8788)
+WORKER_API_URL=http://localhost:8788
+
+# Jawn API URL (default: http://localhost:8585)
+JAWN_URL=http://localhost:8585
+
+# ClickHouse URL (default: http://localhost:8123)
+CLICKHOUSE_URL=http://localhost:8123
+
+# PostgreSQL URL (default: postgresql://postgres:postgres@localhost:54322/postgres)
+POSTGRES_URL=postgresql://postgres:postgres@localhost:54322/postgres
+
+# Test timeout in milliseconds (default: 30000)
+TEST_TIMEOUT=30000

e2e/README.md

@@ -0,0 +1,188 @@
+# Helicone E2E Tests
+
+End-to-end integration tests for the Helicone AI Gateway.
+
+## Overview
+
+This test suite validates the Helicone AI Gateway functionality by making real HTTP requests to the gateway running on port 8793.
+
+## Prerequisites
+
+- Node.js 20+
+- Yarn
+- Helicone AI Gateway running on port 8793
+
+## Getting Started
+
+### 1. Install Dependencies
+
+```bash
+cd e2e
+yarn install
+```
+
+### 2. Start the Gateway
+
+From the root of the Helicone project:
+
+```bash
+cd worker
+npx wrangler dev --var WORKER_TYPE:AI_GATEWAY_API --port 8793 --test-scheduled
+```
+
+Or use the convenience script:
+
+```bash
+./worker/run_all_workers.sh
+```
+
+### 3. Run Tests
+
+```bash
+# Run all tests
+yarn test
+
+# Run gateway tests only
+yarn test:gateway
+
+# Run in watch mode
+yarn test:watch
+
+# Run with coverage
+yarn test:coverage
+```
+
+## Configuration
+
+Create a `.env` file in the `e2e` directory to customize settings:
+
+```bash
+GATEWAY_URL=http://localhost:8793
+```
+
+## Test Structure
+
+```
+e2e/
+├── lib/                    # Shared utilities
+│   ├── constants.ts        # Test constants and config
+│   ├── http-client.ts      # HTTP client wrapper
+│   └── test-helpers.ts     # Common test helpers
+├── tests/
+│   ├── setup.ts            # Jest setup file
+│   └── gateway/            # Gateway-specific tests
+│       ├── health.test.ts          # Health check tests
+│       └── chat-completion.test.ts # Chat completion tests
+├── jest.config.ts          # Jest configuration
+├── tsconfig.json           # TypeScript configuration
+└── package.json
+```
+
+## Writing Tests
+
+### Example Test
+
+```typescript
+import { gatewayClient } from "../../lib/http-client";
+import { GATEWAY_ENDPOINTS } from "../../lib/constants";
+import { createChatCompletionRequest } from "../../lib/test-helpers";
+
+describe("My Feature", () => {
+  it("should work correctly", async () => {
+    const requestBody = createChatCompletionRequest({
+      model: "gpt-4o-mini",
+      messages: [{ role: "user", content: "Hello" }],
+    });
+
+    const response = await gatewayClient.post(
+      GATEWAY_ENDPOINTS.CHAT_COMPLETIONS,
+      requestBody
+    );
+
+    expect(response.status).toBe(200);
+  });
+});
+```
+
+## Available Test Utilities
+
+### HTTP Client
+
+```typescript
+import { gatewayClient } from "../../lib/http-client";
+
+// Make requests
+await gatewayClient.post("/endpoint", data);
+await gatewayClient.get("/endpoint");
+
+// Custom headers
+gatewayClient.setHeaders({ "X-Custom": "value" });
+gatewayClient.resetHeaders();
+```
+
+### Test Helpers
+
+```typescript
+import {
+  createChatCompletionRequest,
+  validateChatCompletionResponse,
+  retry,
+  sleep,
+} from "../../lib/test-helpers";
+
+// Create request body
+const request = createChatCompletionRequest({
+  model: "gpt-4o-mini",
+  messages: [{ role: "user", content: "Hello" }],
+});
+
+// Validate response
+validateChatCompletionResponse(response);
+
+// Retry with backoff
+await retry(() => apiCall(), { maxAttempts: 3 });
+```
+
+## Troubleshooting
+
+### Gateway Not Running
+
+If tests fail with connection errors:
+
+1. Ensure the gateway is running: `npx wrangler dev --var WORKER_TYPE:AI_GATEWAY_API --port 8793`
+2. Check the port is correct: `lsof -i :8793`
+3. Verify the gateway URL in `.env`
+
+### Timeout Errors
+
+If tests timeout:
+
+1. Increase timeout in `jest.config.ts`: `testTimeout: 60000`
+2. Check gateway logs for errors
+3. Verify network connectivity
+
+### Authentication Errors
+
+Tests use mock authentication headers. If you see auth errors:
+
+1. Check `lib/constants.ts` for header configuration
+2. Verify the gateway is configured for local testing
+3. Update test headers as needed
+
+## CI/CD Integration
+
+Add to your CI pipeline:
+
+```yaml
+- name: Install E2E dependencies
+  run: cd e2e && yarn install
+
+- name: Start Gateway
+  run: |
+    cd worker
+    npx wrangler dev --var WORKER_TYPE:AI_GATEWAY_API --port 8793 &
+    sleep 10
+
+- name: Run E2E tests
+  run: cd e2e && yarn test
+```

e2e/jest.config.ts

@@ -0,0 +1,33 @@
+import type { Config } from "jest";
+
+const config: Config = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  roots: ["<rootDir>/tests"],
+  testMatch: ["**/__tests__/**/*.ts", "**/?(*.)+(spec|test).ts"],
+  moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"],
+  collectCoverageFrom: [
+    "tests/**/*.{ts,tsx}",
+    "lib/**/*.{ts,tsx}",
+    "!tests/**/*.d.ts",
+    "!lib/**/*.d.ts",
+    "!tests/**/*.spec.ts",
+    "!tests/**/*.test.ts",
+  ],
+  coverageDirectory: "coverage",
+  verbose: true,
+  testTimeout: 30000,
+  setupFilesAfterEnv: ["<rootDir>/tests/setup.ts"],
+  transform: {
+    "^.+\\.ts$": [
+      "ts-jest",
+      {
+        tsconfig: {
+          esModuleInterop: true,
+        },
+      },
+    ],
+  },
+};
+
+export default config;

e2e/lib/constants.ts

@@ -0,0 +1,47 @@
+/**
+ * Test constants and configuration
+ */
+
+// Service URLs
+export const AI_GATEWAY_URL =
+  process.env.AI_GATEWAY_URL || "http://localhost:8793";
+export const WORKER_API_URL =
+  process.env.WORKER_API_URL || "http://localhost:8788";
+export const JAWN_URL = process.env.JAWN_URL || "http://localhost:8585";
+export const POSTGRES_URL =
+  process.env.POSTGRES_URL ||
+  "postgresql://postgres:postgres@localhost:54322/postgres";
+export const CLICKHOUSE_URL =
+  process.env.CLICKHOUSE_URL || "http://localhost:18123";
+
+// Gateway endpoints
+export const GATEWAY_ENDPOINTS = {
+  CHAT_COMPLETIONS: "/v1/chat/completions",
+  HEALTHCHECK: "/healthcheck",
+} as const;
+
+// Jawn endpoints
+export const JAWN_ENDPOINTS = {
+  HEALTHCHECK: "/healthcheck",
+} as const;
+
+export const TEST_ORG_ID = "83635a30-5ba6-41a8-8cc6-fb7df941b24a";
+
+export const TEST_ORG_API_KEY = "sk-helicone-aizk36y-5yue2my-qmy5tza-n7x3aqa";
+
+export const TEST_HEADERS = {
+  "Content-Type": "application/json",
+  Authorization: `Bearer ${TEST_ORG_API_KEY}`,
+  "__helicone-mock-response": "true",
+} as const;
+
+export const DEFAULT_TIMEOUT = 30000; // 30 seconds
+
+export const TEST_MESSAGES = {
+  SIMPLE: [
+    {
+      role: "user" as const,
+      content: "Say hello",
+    },
+  ],
+} as const;

e2e/lib/http-client.ts

@@ -0,0 +1,60 @@
+/**
+ * HTTP client for making requests to the Helicone Gateway
+ */
+
+import axios, { AxiosInstance, AxiosRequestConfig, AxiosResponse } from "axios";
+import { AI_GATEWAY_URL, TEST_HEADERS, DEFAULT_TIMEOUT } from "./constants";
+
+export class GatewayClient {
+  private client: AxiosInstance;
+
+  constructor(baseURL: string = AI_GATEWAY_URL) {
+    this.client = axios.create({
+      baseURL,
+      timeout: DEFAULT_TIMEOUT,
+      headers: TEST_HEADERS,
+      validateStatus: () => true, // Don't throw on any status code
+    });
+  }
+
+  /**
+   * Make a POST request to the gateway
+   */
+  async post<T = any>(
+    endpoint: string,
+    data: any,
+    config?: AxiosRequestConfig
+  ): Promise<AxiosResponse<T>> {
+    return this.client.post<T>(endpoint, data, config);
+  }
+
+  /**
+   * Make a GET request to the gateway
+   */
+  async get<T = any>(
+    endpoint: string,
+    config?: AxiosRequestConfig
+  ): Promise<AxiosResponse<T>> {
+    return this.client.get<T>(endpoint, config);
+  }
+
+  /**
+   * Set custom headers for the next request
+   */
+  setHeaders(headers: Record<string, string>): void {
+    Object.assign(this.client.defaults.headers, headers);
+  }
+
+  /**
+   * Reset headers to default
+   */
+  resetHeaders(): void {
+    this.client.defaults.headers = {
+      ...this.client.defaults.headers,
+      ...TEST_HEADERS,
+    } as any;
+  }
+}
+
+// Export a singleton instance
+export const gatewayClient = new GatewayClient();