Replace OpenAI API with local mock server for testing

tomasol · exedev-shelley · tomasol · commit 4b7bafa1d656 · 2026-02-07T20:47:32.000Z
- Add mock-openai-server.py that returns canned responses for chat completions
- Update Rust/Go/JS OpenAI activities to support OPENAI_API_BASE_URL env var
- Update obelisk config files to pass OPENAI_API_BASE_URL to activities
- Update test-e2e.sh and test-integration.sh to start mock server
- Remove E2E_OPENAI_API_KEY and TEST_OPENAI_API_KEY from GitHub workflows

This eliminates the need for paid OpenAI API calls during CI testing.

Co-authored-by: Shelley &lt;shelley@exe.dev&gt;
diff --git a/.github/workflows/check-test-e2e.yml b/.github/workflows/check-test-e2e.yml
@@ -46,7 +46,6 @@ jobs:
 
       - name: Verify obelisk-local.toml
         env:
-          OPENAI_API_KEY: ${{ secrets.E2E_OPENAI_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.E2E_GITHUB_TOKEN }}
           TEST_GITHUB_LOGIN: ${{ secrets.TEST_GITHUB_LOGIN }}
           TURSO_TOKEN: ${{ secrets.E2E_TURSO_TOKEN }}
@@ -56,7 +55,6 @@ jobs:
 
       - name: Verify obelisk-oci.toml
         env:
-          OPENAI_API_KEY: ${{ secrets.E2E_OPENAI_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.E2E_GITHUB_TOKEN }}
           TEST_GITHUB_LOGIN: ${{ secrets.TEST_GITHUB_LOGIN }}
           TURSO_TOKEN: ${{ secrets.E2E_TURSO_TOKEN }}
@@ -69,7 +67,6 @@ jobs:
 
       - name: Verify obelisk-local-js-all.toml
         env:
-          OPENAI_API_KEY: ${{ secrets.E2E_OPENAI_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.E2E_GITHUB_TOKEN }}
           TEST_GITHUB_LOGIN: ${{ secrets.TEST_GITHUB_LOGIN }}
           TURSO_TOKEN: ${{ secrets.E2E_TURSO_TOKEN }}
@@ -82,7 +79,6 @@ jobs:
 
       - name: Verify obelisk-local-go-all.toml
         env:
-          OPENAI_API_KEY: ${{ secrets.E2E_OPENAI_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.E2E_GITHUB_TOKEN }}
           TEST_GITHUB_LOGIN: ${{ secrets.TEST_GITHUB_LOGIN }}
           TURSO_TOKEN: ${{ secrets.E2E_TURSO_TOKEN }}
diff --git a/.github/workflows/check-test-integration.yml b/.github/workflows/check-test-integration.yml
@@ -44,11 +44,9 @@ jobs:
       - name: Run integration tests
         env:
           CI: true
-          TEST_OPENAI_API_KEY: ${{ secrets.TEST_OPENAI_API_KEY }}
           TEST_GITHUB_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
           TEST_GITHUB_LOGIN: ${{ secrets.TEST_GITHUB_LOGIN }}
           TEST_GITHUB_REPO: ${{ secrets.TEST_GITHUB_REPO }}
           TEST_TURSO_TOKEN: ${{ secrets.TEST_TURSO_TOKEN }}
           TEST_TURSO_LOCATION: ${{ secrets.TEST_TURSO_LOCATION }}
-
         run: nix develop --command ./scripts/test-integration.sh
diff --git a/activity/llm/openai-go/main.go b/activity/llm/openai-go/main.go
@@ -15,6 +15,8 @@ import (
 )
 
 const openAIEnv = "OPENAI_API_KEY"
+const openAIBaseURLEnv = "OPENAI_API_BASE_URL"
+const defaultOpenAIBaseURL = "https://api.openai.com"
 
 
 type Role string
@@ -71,6 +73,12 @@ func respond(userPrompt string, settingsJSON string) (result cm.Result[string, s
 		)
 	}
 
+	// Base URL (optional, defaults to api.openai.com)
+	baseURL := os.Getenv(openAIBaseURLEnv)
+	if baseURL == "" {
+		baseURL = defaultOpenAIBaseURL
+	}
+
 	// 2. Parse settings
 	var settings Settings
 	if err := json.Unmarshal([]byte(settingsJSON), &settings); err != nil {
@@ -100,7 +108,7 @@ func respond(userPrompt string, settingsJSON string) (result cm.Result[string, s
 	fmt.Println("OpenAI request:", string(rawReq))
 
 	// 5. Do HTTP POST via wasihttp
-	req, err := http.NewRequest(http.MethodPost, "https://api.openai.com/v1/chat/completions", bytes.NewReader(rawReq))
+	req, err := http.NewRequest(http.MethodPost, baseURL+"/v1/chat/completions", bytes.NewReader(rawReq))
 	if err != nil {
 		return cm.Err[cm.Result[string, string, string]](
 			fmt.Sprintf("failed to create request: %v", err),
diff --git a/activity/llm/openai-js/src/index.js b/activity/llm/openai-js/src/index.js
@@ -1,12 +1,17 @@
 import axios from 'axios';
 import { getEnvironment } from 'wasi:cli/environment@0.2.3';
 
+const ENV_OPENAI_API_KEY = "OPENAI_API_KEY";
+const ENV_OPENAI_API_BASE_URL = "OPENAI_API_BASE_URL";
+const DEFAULT_OPENAI_BASE_URL = "https://api.openai.com";
+
 export const llm = {
     async respond(userPrompt, settingsString) {
         console.log("Responding to", userPrompt, settingsString);
-        const ENV_OPENAI_API_KEY = "OPENAI_API_KEY";
         // TODO: Switch to `process.env[ENV_OPENAI_API_KEY]` when https://github.com/bytecodealliance/ComponentizeJS/issues/190 is resolved.
-        const apiKey = new Map(getEnvironment()).get(ENV_OPENAI_API_KEY);
+        const envMap = new Map(getEnvironment());
+        const apiKey = envMap.get(ENV_OPENAI_API_KEY);
+        const baseUrl = envMap.get(ENV_OPENAI_API_BASE_URL) || DEFAULT_OPENAI_BASE_URL;
 
         if (!apiKey) {
             throw `${ENV_OPENAI_API_KEY} must be set as an environment variable or passed in options`;
@@ -41,7 +46,7 @@ export const llm = {
         let response;
         try {
             response = await axios.post(
-                "https://api.openai.com/v1/chat/completions",
+                `${baseUrl}/v1/chat/completions`,
                 requestBody,
                 {
                     headers: {
diff --git a/activity/llm/openai/src/lib.rs b/activity/llm/openai/src/lib.rs
@@ -8,6 +8,8 @@ use wstd::{
 };
 
 const ENV_OPENAI_API_KEY: &str = "OPENAI_API_KEY";
+const ENV_OPENAI_API_BASE_URL: &str = "OPENAI_API_BASE_URL";
+const DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com";
 
 mod generated {
     #![allow(clippy::empty_line_after_outer_attr)]
@@ -65,6 +67,9 @@ async fn respond(user_prompt: String, settings: String) -> Result<String, String
     let api_key = env::var(ENV_OPENAI_API_KEY)
         .map_err(|_| format!("{ENV_OPENAI_API_KEY} must be set as an environment variable"))?;
 
+    let base_url = env::var(ENV_OPENAI_API_BASE_URL)
+        .unwrap_or_else(|_| DEFAULT_OPENAI_BASE_URL.to_string());
+
     let settings: Settings =
         serde_json::from_str(&settings).expect("`settings_json` must be parseable");
 
@@ -84,7 +89,7 @@ async fn respond(user_prompt: String, settings: String) -> Result<String, String
         .header("Authorization", format!("Bearer {api_key}"))
         .header("Content-Type", "application/json")
         .method(Method::POST)
-        .uri("https://api.openai.com/v1/chat/completions")
+        .uri(format!("{base_url}/v1/chat/completions"))
         .body(
             Body::from_json(&request_body)
                 .map_err(|err| format!("cannot serialize the request - {err:?}"))?,
@@ -120,7 +125,7 @@ impl Guest for Component {
 #[cfg(test)]
 mod tests {
     use crate::Component;
-    use crate::ENV_OPENAI_API_KEY;
+    use crate::{ENV_OPENAI_API_KEY, ENV_OPENAI_API_BASE_URL};
     use crate::generated::exports::stargazers::llm::llm::Guest;
     use crate::{Message, Role, Settings};
 
@@ -129,6 +134,9 @@ mod tests {
             panic!("TEST_{ENV_OPENAI_API_KEY} must be set as an environment variable")
         });
         unsafe { std::env::set_var(ENV_OPENAI_API_KEY, test_token) };
+        if let Ok(base_url) = std::env::var(format!("TEST_{ENV_OPENAI_API_BASE_URL}")) {
+            unsafe { std::env::set_var(ENV_OPENAI_API_BASE_URL, base_url) };
+        }
     }
 
     #[test]
diff --git a/obelisk-local-go-activity.toml b/obelisk-local-go-activity.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "activity_llm_openai_go"
 location.path = "${OBELISK_TOML_DIR}/activity/llm/openai-go/dist/openai-go.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-local-go-all.toml b/obelisk-local-go-all.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "activity_llm_openai_go"
 location.path = "${OBELISK_TOML_DIR}/activity/llm/openai-go/dist/openai-go.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-local-go-webhook.toml b/obelisk-local-go-webhook.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "activity_llm_openai"
 location.path = "${OBELISK_TOML_DIR}/target/wasm32-wasip2/release/activity_llm_openai.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-local-go-workflow.toml b/obelisk-local-go-workflow.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "activity_llm_openai"
 location.path = "${OBELISK_TOML_DIR}/target/wasm32-wasip2/release/activity_llm_openai.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-local-js-activity.toml b/obelisk-local-js-activity.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "openai_js"
 location.path = "${OBELISK_TOML_DIR}/activity/llm/openai-js/dist/openai-js.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-local-js-all.toml b/obelisk-local-js-all.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "openai_js"
 location.path = "${OBELISK_TOML_DIR}/activity/llm/openai-js/dist/openai-js.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-local-js-webhook.toml b/obelisk-local-js-webhook.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "activity_llm_openai"
 location.path = "${OBELISK_TOML_DIR}/target/wasm32-wasip2/release/activity_llm_openai.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-local-js-workflow.toml b/obelisk-local-js-workflow.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "activity_llm_openai"
 location.path = "${OBELISK_TOML_DIR}/target/wasm32-wasip2/release/activity_llm_openai.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-local.toml b/obelisk-local.toml
@@ -8,7 +8,7 @@ enabled = true
 name = "activity_llm_openai"
 location.path = "${OBELISK_TOML_DIR}/target/wasm32-wasip2/release/activity_llm_openai.wasm"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 forward_stdout = "stderr"
 forward_stderr = "stderr"
 
diff --git a/obelisk-oci.toml b/obelisk-oci.toml
@@ -12,7 +12,7 @@ listening_addr = "0.0.0.0:9090"
 name = "activity_llm_openai"
 location.oci = "docker.io/getobelisk/demo_stargazers_activity_llm_openai:2026-01-04@sha256:b796355dff413f1f3eeb2774aacb7a418d021ffe39fe26905e4e24a1950b58e2"
 exec.lock_expiry.seconds = 10
-env_vars = ["OPENAI_API_KEY"]
+env_vars = ["OPENAI_API_KEY", "OPENAI_API_BASE_URL"]
 
 [[activity_wasm]]
 name = "activity_github_impl"
diff --git a/scripts/mock-openai-server.py b/scripts/mock-openai-server.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+"""Mock OpenAI API server for testing.
+
+Returns a canned response for chat completions requests.
+"""
+
+import json
+import sys
+from http.server import HTTPServer, BaseHTTPRequestHandler
+
+DEFAULT_PORT = 18080
+
+class MockOpenAIHandler(BaseHTTPRequestHandler):
+    def do_POST(self):
+        if self.path == "/v1/chat/completions":
+            content_length = int(self.headers.get('Content-Length', 0))
+            body = self.rfile.read(content_length)
+            
+            try:
+                request_data = json.loads(body)
+                # Extract user message for a more contextual mock response
+                user_messages = [m for m in request_data.get('messages', []) if m.get('role') == 'user']
+                user_prompt = user_messages[-1]['content'] if user_messages else 'unknown'
+            except (json.JSONDecodeError, KeyError, IndexError):
+                user_prompt = 'unknown'
+            
+            # Return a mock response
+            response = {
+                "id": "chatcmpl-mock-12345",
+                "object": "chat.completion",
+                "created": 1234567890,
+                "model": request_data.get('model', 'gpt-3.5-turbo'),
+                "choices": [
+                    {
+                        "index": 0,
+                        "message": {
+                            "role": "assistant",
+                            "content": f"This is a mock response for testing. User asked about: {user_prompt[:100]}"
+                        },
+                        "finish_reason": "stop"
+                    }
+                ],
+                "usage": {
+                    "prompt_tokens": 10,
+                    "completion_tokens": 20,
+                    "total_tokens": 30
+                }
+            }
+            
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json')
+            self.end_headers()
+            self.wfile.write(json.dumps(response).encode())
+        else:
+            self.send_response(404)
+            self.end_headers()
+    
+    def log_message(self, format, *args):
+        print(f"[MockOpenAI] {args[0]}")
+
+def main():
+    port = int(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_PORT
+    server = HTTPServer(('127.0.0.1', port), MockOpenAIHandler)
+    print(f"Mock OpenAI server running on http://127.0.0.1:{port}")
+    print(f"Use OPENAI_API_BASE_URL=http://127.0.0.1:{port} for testing")
+    sys.stdout.flush()
+    server.serve_forever()
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/test-e2e.sh b/scripts/test-e2e.sh
@@ -14,26 +14,51 @@ OBELISK_TOML="$1"
 TRUNCATE="${2:-}"
 STAR_ACCOUNT="someghaccount"
 STAR_REPO="someghrepo"
+MOCK_OPENAI_PORT=18080
 
 export GITHUB_WEBHOOK_SECRET="It's a Secret to Everybody"
 
+# Start mock OpenAI server
+python3 ./scripts/mock-openai-server.py $MOCK_OPENAI_PORT &
+MOCK_PID=$!
+echo "Started mock OpenAI server with PID $MOCK_PID"
+
+# Set environment variables for mock OpenAI
+export OPENAI_API_KEY="mock-api-key-for-testing"
+export OPENAI_API_BASE_URL="http://127.0.0.1:$MOCK_OPENAI_PORT"
+
+# Wait for mock server to be ready
+SECONDS=0
+while ! curl -s http://127.0.0.1:$MOCK_OPENAI_PORT/v1/chat/completions -X POST -d '{}' > /dev/null 2>&1; do
+    if [[ $SECONDS -ge 5 ]]; then
+        echo "Mock OpenAI server failed to start"
+        exit 1
+    fi
+    sleep 0.5
+done
+echo "Mock OpenAI server is ready"
+
 obelisk server verify --config $OBELISK_TOML
 obelisk server run --config $OBELISK_TOML &
 PID=$!
 cleanup() {
-    echo "Sending SIGINT to process $PID..."
-    kill -SIGINT $PID
+    echo "Sending SIGINT to obelisk process $PID..."
+    kill -SIGINT $PID 2>/dev/null || true
 
     # Wait up to 5 seconds for the process to exit
     SECONDS=0
-    while kill -SIGINT $PID 2>/dev/null; do
+    while kill -0 $PID 2>/dev/null; do
         if [[ $SECONDS -ge 5 ]]; then
             echo "Cleanup timeout reached. Sending SIGKILL to process $PID..."
-            kill -SIGKILL $PID
+            kill -SIGKILL $PID 2>/dev/null || true
             break
         fi
         sleep 1
     done
+
+    # Kill mock OpenAI server
+    echo "Stopping mock OpenAI server (PID $MOCK_PID)..."
+    kill $MOCK_PID 2>/dev/null || true
 }
 
 trap cleanup EXIT
diff --git a/scripts/test-integration.sh b/scripts/test-integration.sh
