Skip to content

Commit ffa12ea

Browse files
chitalianchitalian
authored
Justin/get full testing suite (#5033)
1 parent 2feb1a6 commit ffa12ea

File tree

18 files changed

+3472
-6
lines changed

18 files changed

+3472
-6
lines changed

‎.github/workflows/e2e-test-suite.yml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
name: E2E Test Suite
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
- justin/get-full-testing-suite
8+
- ci/*
9+
10+
jobs:
11+
e2e-tests:
12+
runs-on: ubuntu-latest
13+
timeout-minutes: 20
14+
15+
steps:
16+
- name: Checkout code
17+
uses: actions/checkout@v4
18+
19+
- name: Setup Node.js
20+
uses: actions/setup-node@v4
21+
with:
22+
node-version: "20"
23+
cache: "yarn"
24+
cache-dependency-path: "yarn.lock"
25+
26+
- name: Enable Corepack
27+
run: corepack enable
28+
29+
- name: Install dependencies
30+
run: yarn install
31+
32+
- name: Setup Supabase CLI
33+
uses: supabase/setup-cli@v1
34+
with:
35+
version: latest
36+
37+
- name: Start Clickhouse and Minio
38+
run: cd docker && docker compose up minio minio-setup clickhouse -d
39+
40+
- name: Start Supabase
41+
run: |
42+
echo 'y' | npx supabase start -x realtime,storage-api,imgproxy,mailpit,edge-runtime,logflare,vector,supavisor
43+
44+
- name: Start Gateway Workers
45+
run: |
46+
cd worker
47+
yarn install
48+
mkdir -p logs
49+
echo "Starting HELICONE_API worker on port 8788..."
50+
npx wrangler dev --var WORKER_TYPE:HELICONE_API --port 8788 --inspector-port=9240 > logs/wrangler-helicone-api.log 2>&1 &
51+
echo $! > logs/wrangler-helicone-api.pid
52+
echo "Starting AI_GATEWAY_API worker on port 8793..."
53+
npx wrangler dev --var WORKER_TYPE:AI_GATEWAY_API --var HELICONE_ORG_ID:"a75d76e3-02e7-4d02-8a2b-c65ed27c69b2" --port 8793 --inspector-port=9241 > logs/wrangler-ai-gateway.log 2>&1 &
54+
echo $! > logs/wrangler-ai-gateway.pid
55+
echo "Waiting for workers to start..."
56+
sleep 10
57+
58+
- name: Health Check Workers
59+
run: |
60+
echo "Checking if workers are running..."
61+
MAX_RETRIES=30
62+
RETRY_DELAY=2
63+
64+
# Check HELICONE_API worker (port 8788)
65+
for i in $(seq 1 $MAX_RETRIES); do
66+
if curl -f http://localhost:8788/healthcheck 2>/dev/null; then
67+
echo "✓ HELICONE_API worker is running on port 8788"
68+
break
69+
fi
70+
if [ $i -eq $MAX_RETRIES ]; then
71+
echo "✗ HELICONE_API worker failed to start on port 8788"
72+
echo "Last 50 lines of wrangler-helicone-api.log:"
73+
tail -50 worker/logs/wrangler-helicone-api.log || true
74+
exit 1
75+
fi
76+
echo "Waiting for HELICONE_API worker... (attempt $i/$MAX_RETRIES)"
77+
sleep $RETRY_DELAY
78+
done
79+
80+
# Check AI_GATEWAY_API worker (port 8793)
81+
for i in $(seq 1 $MAX_RETRIES); do
82+
if curl -f http://localhost:8793/healthcheck 2>/dev/null; then
83+
echo "✓ AI_GATEWAY_API worker is running on port 8793"
84+
break
85+
fi
86+
if [ $i -eq $MAX_RETRIES ]; then
87+
echo "✗ AI_GATEWAY_API worker failed to start on port 8793"
88+
echo "Last 50 lines of wrangler-ai-gateway.log:"
89+
tail -50 worker/logs/wrangler-ai-gateway.log || true
90+
exit 1
91+
fi
92+
echo "Waiting for AI_GATEWAY_API worker... (attempt $i/$MAX_RETRIES)"
93+
sleep $RETRY_DELAY
94+
done
95+
96+
echo "All workers are running!"
97+
98+
- name: Start Jawn
99+
run: |
100+
cd valhalla/jawn
101+
cp .env.hosted.example .env
102+
yarn install
103+
yarn dev &
104+
sleep 10
105+
106+
- name: Run E2E Tests
107+
run: |
108+
cd e2e
109+
yarn install
110+
yarn test
111+
112+
- name: Display Logs on Failure
113+
if: failure()
114+
run: |
115+
echo "========================================="
116+
echo "HELICONE_API Worker Logs (last 100 lines)"
117+
echo "========================================="
118+
tail -100 worker/logs/wrangler-helicone-api.log || echo "No logs found for HELICONE_API worker"
119+
echo ""
120+
echo "========================================="
121+
echo "AI_GATEWAY_API Worker Logs (last 100 lines)"
122+
echo "========================================="
123+
tail -100 worker/logs/wrangler-ai-gateway.log || echo "No logs found for AI_GATEWAY_API worker"
124+
echo ""
125+
echo "========================================="
126+
echo "Worker Process Status"
127+
echo "========================================="
128+
ps aux | grep wrangler || echo "No wrangler processes found"
129+
130+
- name: Cleanup
131+
if: always()
132+
run: |
133+
pkill -f wrangler || true
134+
pkill -f "yarn dev" || true
135+
supabase stop --no-backup || true
136+
cd docker && docker compose down || true

‎e2e/README.md‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Helicone E2E Tests
2+
3+
End-to-end integration tests for the Helicone AI Gateway.
4+
5+
**Note:** This directory is intentionally **not** part of the root monorepo workspaces. E2E tests are standalone test infrastructure with their own dependencies, separate from production code.
6+
7+
## Getting Started (running locally)
8+
9+
### 1. Install Dependencies
10+
11+
```bash
12+
cd e2e
13+
yarn install
14+
```
15+
16+
### 2. Start the Gateway
17+
18+
From the root of the Helicone project:
19+
20+
```bash
21+
cd worker
22+
npx wrangler dev --var WORKER_TYPE:AI_GATEWAY_API --port 8793 --test-scheduled
23+
npx wrangler dev --var WORKER_TYPE:HELICONE_API --port 8788 --inspector-port=9240
24+
```
25+
26+
Or use the convenience script:
27+
28+
```bash
29+
./worker/run_all_workers.sh
30+
```
31+
32+
make sure to run jawn too.
33+
34+
### 3. Run Tests
35+
36+
```bash
37+
# Run all tests
38+
yarn test
39+
```
40+
41+
That's it!!
42+
43+
## Debugging CI
44+
45+
If you want to debug a test that is failing on github actions, do not use `act`. `act` will use a network bridge that will fuck shit up, so just debug stuff on bare metal or push a branch that starts with `ci/*` to experiment with. it sucks and is slow, but ideally you will not have to fuck with it too much.

‎e2e/jest.config.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
import type { Config } from "jest";
2+
3+
const config: Config = {
4+
preset: "ts-jest",
5+
testEnvironment: "node",
6+
roots: ["<rootDir>/tests"],
7+
testMatch: ["**/__tests__/**/*.ts", "**/?(*.)+(spec|test).ts"],
8+
moduleFileExtensions: ["ts", "tsx", "js", "jsx", "json", "node"],
9+
collectCoverageFrom: [
10+
"tests/**/*.{ts,tsx}",
11+
"lib/**/*.{ts,tsx}",
12+
"!tests/**/*.d.ts",
13+
"!lib/**/*.d.ts",
14+
"!tests/**/*.spec.ts",
15+
"!tests/**/*.test.ts",
16+
],
17+
coverageDirectory: "coverage",
18+
verbose: true,
19+
testTimeout: 30000,
20+
setupFilesAfterEnv: ["<rootDir>/tests/setup.ts"],
21+
transform: {
22+
"^.+\\.ts$": [
23+
"ts-jest",
24+
{
25+
tsconfig: {
26+
esModuleInterop: true,
27+
},
28+
},
29+
],
30+
},
31+
};
32+
33+
export default config;

‎e2e/lib/constants.ts‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/**
2+
* Test constants and configuration
3+
*/
4+
5+
// Service URLs
6+
export const AI_GATEWAY_URL =
7+
process.env.AI_GATEWAY_URL || "http://localhost:8793";
8+
export const WORKER_API_URL =
9+
process.env.WORKER_API_URL || "http://localhost:8788";
10+
export const JAWN_URL = process.env.JAWN_URL || "http://localhost:8585";
11+
export const POSTGRES_URL =
12+
process.env.POSTGRES_URL ||
13+
"postgresql://postgres:postgres@localhost:54322/postgres";
14+
export const CLICKHOUSE_URL =
15+
process.env.CLICKHOUSE_URL || "http://localhost:18123";
16+
17+
// Gateway endpoints
18+
export const GATEWAY_ENDPOINTS = {
19+
CHAT_COMPLETIONS: "/v1/chat/completions",
20+
HEALTHCHECK: "/healthcheck",
21+
} as const;
22+
23+
// Jawn endpoints
24+
export const JAWN_ENDPOINTS = {
25+
HEALTHCHECK: "/healthcheck",
26+
} as const;
27+
28+
// Test credentials from supabase/seed.sql (local development only)
29+
// See seed.sql line 13 for org ID and line 22 for API key
30+
export const TEST_ORG_ID = "83635a30-5ba6-41a8-8cc6-fb7df941b24a";
31+
32+
export const TEST_ORG_API_KEY = "sk-helicone-aizk36y-5yue2my-qmy5tza-n7x3aqa";
33+
34+
// Mock OpenAI response for testing
35+
export const MOCK_OPENAI_RESPONSE = {
36+
id: "chatcmpl-1234567890",
37+
object: "chat.completion",
38+
created: 1759861728,
39+
model: "gpt-5-2025-08-07",
40+
choices: [
41+
{
42+
index: 0,
43+
message: {
44+
role: "assistant",
45+
content:
46+
"This is a mock response from the OpenAI API for testing purposes. - Helicone ooga booga",
47+
refusal: null,
48+
annotations: [],
49+
},
50+
finish_reason: "stop",
51+
},
52+
],
53+
usage: {
54+
prompt_tokens: 10,
55+
completion_tokens: 1870,
56+
total_tokens: 1880,
57+
completion_tokens_details: {
58+
reasoning_tokens: 1792,
59+
audio_tokens: 0,
60+
accepted_prediction_tokens: 0,
61+
rejected_prediction_tokens: 0,
62+
},
63+
},
64+
service_tier: "default",
65+
system_fingerprint: null,
66+
};
67+
68+
export const TEST_HEADERS = {
69+
"Content-Type": "application/json",
70+
Authorization: `Bearer ${TEST_ORG_API_KEY}`,
71+
"__helicone-mock-response": JSON.stringify(MOCK_OPENAI_RESPONSE),
72+
} as const;
73+
74+
export const DEFAULT_TIMEOUT = 30000; // 30 seconds
75+
76+
export const TEST_MESSAGES = {
77+
SIMPLE: [
78+
{
79+
role: "user" as const,
80+
content: "Say hello",
81+
},
82+
],
83+
} as const;

‎e2e/lib/http-client.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/**
2+
* HTTP client for making requests to the Helicone Gateway
3+
*/
4+
5+
import axios, { AxiosInstance, AxiosRequestConfig, AxiosResponse } from "axios";
6+
import { AI_GATEWAY_URL, TEST_HEADERS, DEFAULT_TIMEOUT } from "./constants";
7+
8+
export class GatewayClient {
9+
private client: AxiosInstance;
10+
11+
constructor(baseURL: string = AI_GATEWAY_URL) {
12+
this.client = axios.create({
13+
baseURL,
14+
timeout: DEFAULT_TIMEOUT,
15+
headers: TEST_HEADERS,
16+
validateStatus: () => true, // Don't throw on any status code
17+
});
18+
}
19+
20+
/**
21+
* Make a POST request to the gateway
22+
*/
23+
async post<T = any>(
24+
endpoint: string,
25+
data: any,
26+
config?: AxiosRequestConfig
27+
): Promise<AxiosResponse<T>> {
28+
return this.client.post<T>(endpoint, data, config);
29+
}
30+
31+
/**
32+
* Make a GET request to the gateway
33+
*/
34+
async get<T = any>(
35+
endpoint: string,
36+
config?: AxiosRequestConfig
37+
): Promise<AxiosResponse<T>> {
38+
return this.client.get<T>(endpoint, config);
39+
}
40+
41+
/**
42+
* Set custom headers for the next request
43+
*/
44+
setHeaders(headers: Record<string, string>): void {
45+
Object.assign(this.client.defaults.headers, headers);
46+
}
47+
48+
/**
49+
* Reset headers to default
50+
*/
51+
resetHeaders(): void {
52+
this.client.defaults.headers = { ...TEST_HEADERS } as any;
53+
}
54+
}
55+
56+
// Export a singleton instance
57+
export const gatewayClient = new GatewayClient();

0 commit comments

Comments
 (0)