bring back openai

Author: gr2m

package-lock.json

@@ -12,7 +12,8 @@
   "license": "ISC",
   "description": "",
   "dependencies": {
-    "@copilot-extensions/preview-sdk": "^3.0.0"
+    "@copilot-extensions/preview-sdk": "^3.0.0",
+    "openai": "^4.55.0"
   },
   "devDependencies": {
     "@types/express": "^4.17.21",

package.json

@@ -1,5 +1,4 @@
-import type { PromptFunction, InteropMessage } from "@copilot-extensions/preview-sdk";
-
+import OpenAI from "openai";
 import { ModelsAPI } from "./models-api.js";
 
 // defaultModel is the model used for internal calls - for tool calling,
@@ -9,26 +8,26 @@ export const defaultModel = "gpt-4o-mini";
 // RunnerResponse is the response from a function call.
 export interface RunnerResponse {
   model: string;
-  messages: InteropMessage[];
+  messages: OpenAI.ChatCompletionMessageParam[];
 }
 
 export abstract class Tool {
   modelsAPI: ModelsAPI;
-  static definition: PromptFunction["function"];
+  static definition: OpenAI.FunctionDefinition;
 
   constructor(modelsAPI: ModelsAPI) {
     this.modelsAPI = modelsAPI;
   }
 
-  static get tool(): PromptFunction {
+  static get tool(): OpenAI.Chat.Completions.ChatCompletionTool {
     return {
       type: "function",
       function: this.definition,
     };
   }
 
   abstract execute(
-    messages: InteropMessage[],
-    args: Record<string, unknown>
+    messages: OpenAI.ChatCompletionMessageParam[],
+    args: object
   ): Promise<RunnerResponse>;
 }

src/functions.ts

@@ -1,5 +1,4 @@
-import type { InteropMessage } from "@copilot-extensions/preview-sdk";
-
+import OpenAI from "openai";
 import { RunnerResponse, defaultModel, Tool } from "../functions.js";
 
 export class describeModel extends Tool {
@@ -20,7 +19,7 @@ export class describeModel extends Tool {
   };
 
   async execute(
-    messages: InteropMessage[],
+    messages: OpenAI.ChatCompletionMessageParam[],
     args: { model: string }
   ): Promise<RunnerResponse> {
     const [model, modelSchema] = await Promise.all([

src/functions/describe-model.ts

@@ -1,8 +1,7 @@
-import type { InteropMessage } from "@copilot-extensions/preview-sdk";
-
+import OpenAI from "openai";
 import { RunnerResponse, Tool } from "../functions.js";
 
-type MessageWithReferences = InteropMessage & {
+type MessageWithReferences = OpenAI.ChatCompletionMessageParam & {
   copilot_references: Reference[];
 };
 

src/functions/execute-model.ts

@@ -1,5 +1,4 @@
-import type { InteropMessage } from "@copilot-extensions/preview-sdk";
-
+import OpenAI from "openai";
 import { RunnerResponse, defaultModel, Tool } from "../functions.js";
 
 export class listModels extends Tool {
@@ -16,7 +15,7 @@ export class listModels extends Tool {
   };
 
   async execute(
-    messages: InteropMessage[]
+    messages: OpenAI.ChatCompletionMessageParam[]
   ): Promise<RunnerResponse> {
     const models = await this.modelsAPI.listModels();
 

src/functions/list-models.ts

@@ -1,5 +1,4 @@
-import type { InteropMessage } from "@copilot-extensions/preview-sdk";
-
+import OpenAI from "openai";
 import { RunnerResponse, defaultModel, Tool } from "../functions.js";
 
 export class recommendModel extends Tool {
@@ -16,7 +15,7 @@ export class recommendModel extends Tool {
   };
 
   async execute(
-    messages: InteropMessage[]
+    messages: OpenAI.ChatCompletionMessageParam[]
   ): Promise<RunnerResponse> {
     const models = await this.modelsAPI.listModels();
 

src/functions/recommend-model.ts

@@ -1,6 +1,7 @@
-import { createServer } from "node:http";
+import { createServer, type IncomingMessage } from "node:http";
 
-import { prompt, getFunctionCalls, createDoneEvent, verifyAndParseRequest } from "@copilot-extensions/preview-sdk";
+import { getFunctionCalls, createDoneEvent, verifyAndParseRequest } from "@copilot-extensions/preview-sdk";
+import OpenAI from "openai";
 
 import { describeModel } from "./functions/describe-model.js";
 import { executeModel } from "./functions/execute-model.js";
@@ -52,9 +53,15 @@ const server = createServer(async (request, response) => {
   }
 
   // List of functions that are available to be called
-  const modelsAPI = new ModelsAPI();
+  const modelsAPI = new ModelsAPI(apiKey);
   const functions = [listModels, describeModel, executeModel, recommendModel];
 
+  // Use the Copilot API to determine which function to execute
+  const capiClient = new OpenAI({
+    baseURL: "https://api.githubcopilot.com",
+    apiKey,
+  });
+
   // Prepend a system message that includes the list of models, so that
   // tool calls can better select the right model to use.
   const models = await modelsAPI.listModels();
@@ -82,27 +89,29 @@ const server = createServer(async (request, response) => {
   ].concat(payload.messages);
 
   console.time("tool-call");
-  const promptResult = await prompt({
+  const toolCaller = await capiClient.chat.completions.create({
     messages: toolCallMessages,
-    token: apiKey,
+    stream: false,
+    model: "gpt-4",
     tools: functions.map((f) => f.tool),
   })
   console.timeEnd("tool-call");
 
-  const [functionToCall] = getFunctionCalls(promptResult)
+  const [functionToCall] = getFunctionCalls(toolCaller)
 
   if (
     !functionToCall
   ) {
     console.log("No tool call found");
-
-    const { stream } = await prompt.stream({
-      messages: payload.messages,
-      token: apiKey,
-    })
+    // No tool to call, so just call the model with the original messages
+    const stream = await capiClient.chat.completions.create({
+      stream: true,
+      model: "gpt-4",
+    });
 
     for await (const chunk of stream) {
-      response.write(new TextDecoder().decode(chunk));
+      const chunkStr = "data: " + JSON.stringify(chunk) + "\n\n";
+      response.write(chunkStr);
     }
 
     response.end(createDoneEvent().toString());
@@ -134,16 +143,19 @@ const server = createServer(async (request, response) => {
   console.timeEnd("function-exec");
 
   try {
-    console.time("streaming");
-    const { stream } = await prompt.stream({
-      endpoint: 'https://models.inference.ai.azure.com/chat/completions',
+    const stream = await modelsAPI.inference.chat.completions.create({
       model: functionCallRes.model,
       messages: functionCallRes.messages,
-      token: apiKey,
-    })
+      stream: true,
+      stream_options: {
+        include_usage: false,
+      },
+    });
 
+    console.time("streaming");
     for await (const chunk of stream) {
-      response.write(new TextDecoder().decode(chunk));
+      const chunkStr = "data: " + JSON.stringify(chunk) + "\n\n";
+      response.write(chunkStr);
     }
 
     response.end(createDoneEvent().toString());
@@ -159,9 +171,9 @@ const port = process.env.PORT || "3000"
 server.listen(port);
 console.log(`Server running at http://localhost:${port}`);
 
-function getBody(request: any): Promise<string> {
+function getBody(request: IncomingMessage): Promise<string> {
   return new Promise((resolve) => {
-    const bodyParts: any[] = [];
+    const bodyParts: Buffer[] = [];
     let body;
     request
       .on("data", (chunk: Buffer) => {

src/index.ts

@@ -1,3 +1,5 @@
+import OpenAI from "openai";
+
 // Model is the structure of a model in the model catalog.
 export interface Model {
   id: string;
@@ -31,8 +33,16 @@ export type ModelSchemaParameter = {
 };
 
 export class ModelsAPI {
+  inference: OpenAI;
   private _models: Model[] | null = null;
 
+  constructor(apiKey: string) {
+    this.inference = new OpenAI({
+      baseURL: "https://models.inference.ai.azure.com",
+      apiKey,
+    });
+  }
+
   async getModel(modelName: string): Promise<Model> {
     const modelRes = await fetch(
       "https://modelcatalog.azure-api.net/v1/model/" + modelName