From bc01e0d149d74a8c485c408005d766574636b119 Mon Sep 17 00:00:00 2001 From: Mikhail Simin Date: Mon, 10 Feb 2025 14:28:35 -0800 Subject: [PATCH 1/8] Remove Matrix and 400 error ignoring from chatCompletions --- sdk/openai/openai/.gitignore | 2 + .../test/public/chatCompletions.spec.ts | 613 +++++++++--------- sdk/openai/openai/test/utils/utils.ts | 27 +- 3 files changed, 323 insertions(+), 319 deletions(-) create mode 100644 sdk/openai/openai/.gitignore diff --git a/sdk/openai/openai/.gitignore b/sdk/openai/openai/.gitignore new file mode 100644 index 000000000000..fca29659d191 --- /dev/null +++ b/sdk/openai/openai/.gitignore @@ -0,0 +1,2 @@ +# generated by tests +deployments.json \ No newline at end of file diff --git a/sdk/openai/openai/test/public/chatCompletions.spec.ts b/sdk/openai/openai/test/public/chatCompletions.spec.ts index 64ca938a5353..9d3a999d5bb1 100644 --- a/sdk/openai/openai/test/public/chatCompletions.spec.ts +++ b/sdk/openai/openai/test/public/chatCompletions.spec.ts @@ -1,361 +1,358 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { matrix } from "@azure-tools/test-utils-vitest"; -import { assert, describe, beforeEach, it } from "vitest"; -import { createClientsAndDeployments } from "../utils/createClients.js"; -import type { APIVersion } from "../utils/utils.js"; -import { assertChatCompletions, assertChatCompletionsList } from "../utils/asserts.js"; +import {assert, describe, beforeEach, it} from "vitest"; +import {createClientsAndDeployments} from "../utils/createClients.js"; +import type {APIVersion} from "../utils/utils.js"; +import {assertChatCompletions, assertChatCompletionsList} from "../utils/asserts.js"; import { APIMatrix, bufferAsyncIterable, createAzureSearchExtension, withDeployments, } from "../utils/utils.js"; -import { type ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; -import { functionCallModelsToSkip, jsonResponseModelsToSkip } from "../utils/models.js"; +import {type ChatCompletionMessageParam} from "openai/resources/chat/completions.mjs"; +import {functionCallModelsToSkip, jsonResponseModelsToSkip} from "../utils/models.js"; import "../../src/types/index.js"; -import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import type {ClientsAndDeploymentsInfo} from "../utils/types.js"; -describe("Chat Completions", function () { - matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { - describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; +describe.each(APIMatrix)("Chat Completions", function (apiVersion: APIVersion) { + describe(`[${apiVersion}] Client`, () => { + let clientsAndDeployments: ClientsAndDeploymentsInfo; - beforeEach(async function () { - clientsAndDeployments = createClientsAndDeployments( - apiVersion, - { chatCompletion: "true" }, - { - deploymentsToSkip: ["o1" /** It gets stuck and never returns */], - modelsToSkip: [{ name: "gpt-4o-audio-preview" }], + beforeEach(async function () { + clientsAndDeployments = createClientsAndDeployments( + apiVersion, + {chatCompletion: "true"}, + { + deploymentsToSkip: ["o1" /** It gets stuck and never returns */], + modelsToSkip: [{name: "gpt-4o-audio-preview"}], + }, + ); + }); + + describe("chat.completions.create", function () { + const pirateMessages = [ + { + role: "system", + content: "You are a helpful assistant. You will talk like a pirate.", + } as const, + {role: "user", content: "Can you help me?"} as const, + { + role: "assistant", + content: "Arrrr! Of course, me hearty! What can I do for ye?", + } as const, + {role: "user", content: "What's the best way to train a parrot?"} as const, + ]; + const byodMessages = [ + { + role: "user", + content: + "What's the most common feedback we received from our customers about the product?", + } as const, + ]; + const getCurrentWeather = { + name: "get_current_weather", + description: "Get the current weather in a given location", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state, e.g. San Francisco, CA", + }, + unit: { + type: "string", + enum: ["celsius", "fahrenheit"], + }, + }, + required: ["location"], + }, + }; + + it("returns completions across all models", async function () { + await withDeployments( + clientsAndDeployments, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: pirateMessages, + }), + assertChatCompletions, + ); + }); + + it("calls functions", async function () { + await withDeployments( + clientsAndDeployments, + async (client, deploymentName) => { + const weatherMessages: ChatCompletionMessageParam[] = [ + {role: "user", content: "What's the weather like in Boston?"}, + ]; + const result = await client.chat.completions.create({ + model: deploymentName, + messages: weatherMessages, + functions: [getCurrentWeather], + }); + assertChatCompletions(result, {functions: true}); + const responseMessage = result.choices[0].message; + if (!responseMessage?.function_call) { + assert.fail("Undefined function call"); + } + const functionArgs = JSON.parse(responseMessage.function_call.arguments); + weatherMessages.push(responseMessage); + weatherMessages.push({ + role: "function", + name: responseMessage.function_call.name, + content: JSON.stringify({ + location: functionArgs.location, + temperature: "72", + unit: functionArgs.unit, + forecast: ["sunny", "windy"], + }), + }); + return client.chat.completions.create({ + model: deploymentName, + messages: weatherMessages, + }); + }, + (result) => assertChatCompletions(result, {functions: true}), + functionCallModelsToSkip, + ); + }); + + it("doesn't call tools if toolChoice is set to none", async function () { + await withDeployments( + clientsAndDeployments, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: [{role: "user", content: "What's the weather like in Boston?"}], + tool_choice: "none", + tools: [{type: "function", function: getCurrentWeather}], + }), + (res) => { + assertChatCompletions(res, {functions: false}); + assert.isUndefined(res.choices[0].message?.tool_calls); }, ); }); - describe("chat.completions.create", function () { - const pirateMessages = [ - { - role: "system", - content: "You are a helpful assistant. You will talk like a pirate.", - } as const, - { role: "user", content: "Can you help me?" } as const, - { - role: "assistant", - content: "Arrrr! Of course, me hearty! What can I do for ye?", - } as const, - { role: "user", content: "What's the best way to train a parrot?" } as const, - ]; - const byodMessages = [ - { - role: "user", - content: - "What's the most common feedback we received from our customers about the product?", - } as const, - ]; - const getCurrentWeather = { - name: "get_current_weather", - description: "Get the current weather in a given location", + it("calls a specific tool if its name is specified", async function () { + await withDeployments( + clientsAndDeployments, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: [{role: "user", content: "What's the weather like in Boston?"}], + + tool_choice: { + type: "function", + function: {name: getCurrentWeather.name}, + }, + tools: [ + {type: "function", function: getCurrentWeather}, + { + type: "function", + function: { + name: "get_current_weather2", + description: "Get the current weather in a given location in the US", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state, e.g. San Francisco, CA", + }, + unit: { + type: "string", + enum: ["celsius", "fahrenheit"], + }, + }, + required: ["location"], + }, + }, + }, + ], + }), + (res) => { + assertChatCompletions(res, {functions: true}); + const toolCalls = res.choices[0].message?.tool_calls; + if (!toolCalls) { + throw new Error("toolCalls should be defined here"); + } + assert.equal(toolCalls[0].function.name, getCurrentWeather.name); + assert.isUndefined(res.choices[0].message?.function_call); + }, + ); + }); + + it("ensures schema name is not transformed with snake case", async function () { + const getAssetInfo = { + name: "getAssetInfo", + description: "Returns information about an asset", parameters: { type: "object", properties: { - location: { + assetName: { type: "string", - description: "The city and state, e.g. San Francisco, CA", - }, - unit: { - type: "string", - enum: ["celsius", "fahrenheit"], + description: "The asset name. This is a required parameter.", }, }, - required: ["location"], + required: ["assetName"], }, }; + await withDeployments( + clientsAndDeployments, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: [{role: "user", content: "Give me information about Asset No1"}], + tools: [{type: "function", function: getAssetInfo}], + }), + (res) => { + assertChatCompletions(res, {functions: true}); + const toolCalls = res.choices[0].message?.tool_calls; + if (!toolCalls) { + throw new Error("toolCalls should be defined here"); + } + const argument = toolCalls[0].function.arguments; + assert.isTrue(argument?.includes("assetName")); + }, + functionCallModelsToSkip, + ); + }); - it("returns completions across all models", async function () { - await withDeployments( - clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: pirateMessages, - }), - assertChatCompletions, - ); - }); - - it("calls functions", async function () { - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => { - const weatherMessages: ChatCompletionMessageParam[] = [ - { role: "user", content: "What's the weather like in Boston?" }, - ]; - const result = await client.chat.completions.create({ - model: deploymentName, - messages: weatherMessages, - functions: [getCurrentWeather], - }); - assertChatCompletions(result, { functions: true }); - const responseMessage = result.choices[0].message; - if (!responseMessage?.function_call) { - assert.fail("Undefined function call"); - } - const functionArgs = JSON.parse(responseMessage.function_call.arguments); - weatherMessages.push(responseMessage); - weatherMessages.push({ - role: "function", - name: responseMessage.function_call.name, - content: JSON.stringify({ - location: functionArgs.location, - temperature: "72", - unit: functionArgs.unit, - forecast: ["sunny", "windy"], - }), - }); - return client.chat.completions.create({ - model: deploymentName, - messages: weatherMessages, - }); - }, - (result) => assertChatCompletions(result, { functions: true }), - functionCallModelsToSkip, - ); + it("respects json_object responseFormat", async function () { + clientsAndDeployments = createClientsAndDeployments(apiVersion, { + chatCompletion: "true", + jsonObjectResponse: "true", }); + await withDeployments( + clientsAndDeployments, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: [ + { + role: "user", + content: + "Answer the following question in JSON format: What are the capital cities in Africa?", + }, + ], + response_format: {type: "json_object"}, + }), + (res) => { + assertChatCompletions(res, {functions: false}); + const content = res.choices[0].message?.content; + if (!content) assert.fail("Undefined content"); + try { + JSON.parse(content); + } catch { + assert.fail(`Invalid JSON: ${content}`); + } + }, + jsonResponseModelsToSkip, + ); + }); - it("doesn't call tools if toolChoice is set to none", async function () { - await withDeployments( - clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: [{ role: "user", content: "What's the weather like in Boston?" }], - tool_choice: "none", - tools: [{ type: "function", function: getCurrentWeather }], - }), - (res) => { - assertChatCompletions(res, { functions: false }); - assert.isUndefined(res.choices[0].message?.tool_calls); - }, - ); - }); + it("works with custom data sources", async function () { + await withDeployments( + clientsAndDeployments, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: byodMessages, + data_sources: [createAzureSearchExtension()], + }), + assertChatCompletions, + ); + }); - it("calls a specific tool if its name is specified", async function () { + describe("return stream", function () { + it("returns completions across all models", async function () { await withDeployments( clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: [{ role: "user", content: "What's the weather like in Boston?" }], - - tool_choice: { - type: "function", - function: { name: getCurrentWeather.name }, - }, - tools: [ - { type: "function", function: getCurrentWeather }, - { - type: "function", - function: { - name: "get_current_weather2", - description: "Get the current weather in a given location in the US", - parameters: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state, e.g. San Francisco, CA", - }, - unit: { - type: "string", - enum: ["celsius", "fahrenheit"], - }, - }, - required: ["location"], - }, - }, - }, - ], + async (client, deploymentName) => + bufferAsyncIterable( + await client.chat.completions.create({ + model: deploymentName, + messages: pirateMessages, + stream: true, + }), + ), + (res) => + assertChatCompletionsList(res, { + // The API returns an empty choice in the first event for some + // reason. This should be fixed in the API. + allowEmptyChoices: true, + // The API returns an empty ID in the first event for some + // reason. This should be fixed in the API. + allowEmptyId: true, }), - (res) => { - assertChatCompletions(res, { functions: true }); - const toolCalls = res.choices[0].message?.tool_calls; - if (!toolCalls) { - throw new Error("toolCalls should be defined here"); - } - assert.equal(toolCalls[0].function.name, getCurrentWeather.name); - assert.isUndefined(res.choices[0].message?.function_call); - }, ); }); - it("ensure schema name is not transformed with snake case", async function () { - const getAssetInfo = { - name: "getAssetInfo", - description: "Returns information about an asset", - parameters: { - type: "object", - properties: { - assetName: { - type: "string", - description: "The asset name. This is a required parameter.", - }, - }, - required: ["assetName"], - }, - }; + it("calls functions", async function () { await withDeployments( clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: [{ role: "user", content: "Give me information about Asset No1" }], - tools: [{ type: "function", function: getAssetInfo }], + async (client, deploymentName) => + bufferAsyncIterable( + await client.chat.completions.create({ + model: deploymentName, + messages: [{role: "user", content: "What's the weather like in Boston?"}], + stream: true, + functions: [getCurrentWeather], + }), + ), + (res) => + assertChatCompletionsList(res, { + functions: true, + // The API returns an empty choice in the first event for some + // reason. This should be fixed in the API. + allowEmptyChoices: true, }), - (res) => { - assertChatCompletions(res, { functions: true }); - const toolCalls = res.choices[0].message?.tool_calls; - if (!toolCalls) { - throw new Error("toolCalls should be defined here"); - } - const argument = toolCalls[0].function.arguments; - assert.isTrue(argument?.includes("assetName")); - }, - functionCallModelsToSkip, ); }); - it("respects json_object responseFormat", async function () { - clientsAndDeployments = createClientsAndDeployments(apiVersion, { - chatCompletion: "true", - jsonObjectResponse: "true", - }); + it("calls toolCalls", async function () { await withDeployments( clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: [ - { - role: "user", - content: - "Answer the following question in JSON format: What are the capital cities in Africa?", - }, - ], - response_format: { type: "json_object" }, + async (client, deploymentName) => + bufferAsyncIterable( + await client.chat.completions.create({ + model: deploymentName, + messages: [{role: "user", content: "What's the weather like in Boston?"}], + stream: true, + tools: [{type: "function", function: getCurrentWeather}], + }), + ), + (res) => + assertChatCompletionsList(res, { + functions: true, + // The API returns an empty choice in the first event for some + // reason. This should be fixed in the API. + allowEmptyChoices: true, }), - (res) => { - assertChatCompletions(res, { functions: false }); - const content = res.choices[0].message?.content; - if (!content) assert.fail("Undefined content"); - try { - JSON.parse(content); - } catch { - assert.fail(`Invalid JSON: ${content}`); - } - }, - jsonResponseModelsToSkip, ); }); it("bring your data", async function () { + const dataSources = {data_sources: [createAzureSearchExtension()]}; await withDeployments( clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: byodMessages, - data_sources: [createAzureSearchExtension()], - }), - assertChatCompletions, - ); - }); - - describe("return stream", function () { - it("returns completions across all models", async function () { - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => - bufferAsyncIterable( - await client.chat.completions.create({ - model: deploymentName, - messages: pirateMessages, - stream: true, - }), - ), - (res) => - assertChatCompletionsList(res, { - // The API returns an empty choice in the first event for some - // reason. This should be fixed in the API. - allowEmptyChoices: true, - // The API returns an empty ID in the first event for some - // reason. This should be fixed in the API. - allowEmptyId: true, - }), - ); - }); - - it("calls functions", async function () { - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => - bufferAsyncIterable( - await client.chat.completions.create({ - model: deploymentName, - messages: [{ role: "user", content: "What's the weather like in Boston?" }], - stream: true, - functions: [getCurrentWeather], - }), - ), - (res) => - assertChatCompletionsList(res, { - functions: true, - // The API returns an empty choice in the first event for some - // reason. This should be fixed in the API. - allowEmptyChoices: true, - }), - ); - }); - - it("calls toolCalls", async function () { - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => - bufferAsyncIterable( - await client.chat.completions.create({ - model: deploymentName, - messages: [{ role: "user", content: "What's the weather like in Boston?" }], - stream: true, - tools: [{ type: "function", function: getCurrentWeather }], - }), - ), - (res) => - assertChatCompletionsList(res, { - functions: true, - // The API returns an empty choice in the first event for some - // reason. This should be fixed in the API. - allowEmptyChoices: true, + async (client, deploymentName) => + bufferAsyncIterable( + await client.chat.completions.create({ + model: deploymentName, + messages: byodMessages, + stream: true, + ...dataSources, }), - ); - }); - - it("bring your data", async function () { - const dataSources = { data_sources: [createAzureSearchExtension()] }; - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => - bufferAsyncIterable( - await client.chat.completions.create({ - model: deploymentName, - messages: byodMessages, - stream: true, - ...dataSources, - }), - ), - assertChatCompletionsList, - [{ name: "gpt-4", version: "vision-preview" }], - ); - }); + ), + assertChatCompletionsList, + [{name: "gpt-4", version: "vision-preview"}], + ); }); }); }); diff --git a/sdk/openai/openai/test/utils/utils.ts b/sdk/openai/openai/test/utils/utils.ts index 870cac64995e..2090377f95c4 100644 --- a/sdk/openai/openai/test/utils/utils.ts +++ b/sdk/openai/openai/test/utils/utils.ts @@ -49,16 +49,16 @@ export async function withDeployments( let i = 0; for (const { client, deployments } of clientsAndDeployments) { for (const deployment of deployments) { - try { - logger.info( + logger.info( `[${++i}/${count}] testing with deployment: ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, - ); - if (modelsListToSkip && isModelInList(deployment.model, modelsListToSkip)) { - logger.info( + ); + if (modelsListToSkip && isModelInList(deployment.model, modelsListToSkip)) { + logger.info( `Skipping deployment ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, - ); - continue; - } + ); + continue; + } + try { const res = await run(client, deployment.deploymentName); validate?.(res); succeeded.push(deployment); @@ -74,18 +74,23 @@ export async function withDeployments( "ModelDeprecated", "429", "UserError", - 400, ].includes(error.code) || error.type === "invalid_request_error" || error.name === "AbortError" || errorStr.includes("Connection error") || errorStr.includes("toolCalls") || + ["ManagedIdentityIsNotEnabled", + "Rate limit is exceeded", + "Invalid AzureCognitiveSearch configuration detected", + "Unsupported Model", + "does not support 'system' with this model", + ].some(match => error.message.includes(match)) || error.status === 404 ) { - logger.info("Handled error: ", error); + logger.warning("Handled error: ", error); continue; } - logger.info(`Error in deployment ${deployment.deploymentName}: `, error); + logger.error(`Error in deployment ${deployment.deploymentName}: `, error); errors.push(e); } } From ddd85addaacd1d99b9a2a9b32e35536912396f72 Mon Sep 17 00:00:00 2001 From: Mikhail Simin Date: Tue, 11 Feb 2025 14:38:21 -0800 Subject: [PATCH 2/8] Remove withDeployments from chatCompletions.spec --- .../test/public/chatCompletions.spec.ts | 671 ++++++++++-------- sdk/openai/openai/test/utils/utils.ts | 12 +- 2 files changed, 368 insertions(+), 315 deletions(-) diff --git a/sdk/openai/openai/test/public/chatCompletions.spec.ts b/sdk/openai/openai/test/public/chatCompletions.spec.ts index 9d3a999d5bb1..e2acf4bbc260 100644 --- a/sdk/openai/openai/test/public/chatCompletions.spec.ts +++ b/sdk/openai/openai/test/public/chatCompletions.spec.ts @@ -1,359 +1,410 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import {assert, describe, beforeEach, it} from "vitest"; -import {createClientsAndDeployments} from "../utils/createClients.js"; -import type {APIVersion} from "../utils/utils.js"; -import {assertChatCompletions, assertChatCompletionsList} from "../utils/asserts.js"; +import { assert, describe, it } from "vitest"; +import { createClientsAndDeployments } from "../utils/createClients.js"; +import type { APIVersion } from "../utils/utils.js"; import { APIMatrix, bufferAsyncIterable, createAzureSearchExtension, withDeployments, } from "../utils/utils.js"; -import {type ChatCompletionMessageParam} from "openai/resources/chat/completions.mjs"; -import {functionCallModelsToSkip, jsonResponseModelsToSkip} from "../utils/models.js"; +import { assertChatCompletions, assertChatCompletionsList } from "../utils/asserts.js"; +import { type ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; +import { functionCallModelsToSkip, jsonResponseModelsToSkip } from "../utils/models.js"; import "../../src/types/index.js"; -import type {ClientsAndDeploymentsInfo} from "../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import { logger } from "../utils/logger.js"; -describe.each(APIMatrix)("Chat Completions", function (apiVersion: APIVersion) { - describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; +describe.each(APIMatrix)("Chat Completions [%s]", function (apiVersion: APIVersion) { + let clientsAndDeploymentsInfo: ClientsAndDeploymentsInfo; - beforeEach(async function () { - clientsAndDeployments = createClientsAndDeployments( - apiVersion, - {chatCompletion: "true"}, - { - deploymentsToSkip: ["o1" /** It gets stuck and never returns */], - modelsToSkip: [{name: "gpt-4o-audio-preview"}], + clientsAndDeploymentsInfo = createClientsAndDeployments( + apiVersion, + { chatCompletion: "true" }, + { + deploymentsToSkip: ["o1" /** It gets stuck and never returns */], + modelsToSkip: [{ name: "gpt-4o-audio-preview" }], + }, + ); + + describe("chat.completions.create", function () { + const pirateMessages = [ + { + role: "system", + content: "You are a helpful assistant. You will talk like a pirate.", + } as const, + { role: "user", content: "Can you help me?" } as const, + { + role: "assistant", + content: "Arrrr! Of course, me hearty! What can I do for ye?", + } as const, + { role: "user", content: "What's the best way to train a parrot?" } as const, + ]; + const byodMessages = [ + { + role: "user", + content: + "What's the most common feedback we received from our customers about the product?", + } as const, + ]; + const getCurrentWeather = { + name: "get_current_weather", + description: "Get the current weather in a given location", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state, e.g. San Francisco, CA", + }, + unit: { + type: "string", + enum: ["celsius", "fahrenheit"], + }, + }, + required: ["location"], + }, + }; + + it("returns completions across all models", async function () { + await withDeployments( + clientsAndDeploymentsInfo, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: pirateMessages, + }), + assertChatCompletions, + ); + }); + + it("calls functions", async function () { + await withDeployments( + clientsAndDeploymentsInfo, + async (client, deploymentName) => { + const weatherMessages: ChatCompletionMessageParam[] = [ + { role: "user", content: "What's the weather like in Boston?" }, + ]; + const result = await client.chat.completions.create({ + model: deploymentName, + messages: weatherMessages, + functions: [getCurrentWeather], + }); + assertChatCompletions(result, { functions: true }); + const responseMessage = result.choices[0].message; + if (!responseMessage?.function_call) { + assert.fail("Undefined function call"); + } + const functionArgs = JSON.parse(responseMessage.function_call.arguments); + weatherMessages.push(responseMessage); + weatherMessages.push({ + role: "function", + name: responseMessage.function_call.name, + content: JSON.stringify({ + location: functionArgs.location, + temperature: "72", + unit: functionArgs.unit, + forecast: ["sunny", "windy"], + }), + }); + return client.chat.completions.create({ + model: deploymentName, + messages: weatherMessages, + }); + }, + (result) => assertChatCompletions(result, { functions: true }), + functionCallModelsToSkip, + ); + }); + + it("doesn't call tools if toolChoice is set to none", async function () { + await withDeployments( + clientsAndDeploymentsInfo, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: [{ role: "user", content: "What's the weather like in Boston?" }], + tool_choice: "none", + tools: [{ type: "function", function: getCurrentWeather }], + }), + (res) => { + assertChatCompletions(res, { functions: false }); + assert.isUndefined(res.choices[0].message?.tool_calls); + }, + ); + }); + + it("calls a specific tool if its name is specified", async function () { + await withDeployments( + clientsAndDeploymentsInfo, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: [{ role: "user", content: "What's the weather like in Boston?" }], + + tool_choice: { + type: "function", + function: { name: getCurrentWeather.name }, + }, + tools: [ + { type: "function", function: getCurrentWeather }, + { + type: "function", + function: { + name: "get_current_weather2", + description: "Get the current weather in a given location in the US", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "The city and state, e.g. San Francisco, CA", + }, + unit: { + type: "string", + enum: ["celsius", "fahrenheit"], + }, + }, + required: ["location"], + }, + }, + }, + ], + }), + (res) => { + assertChatCompletions(res, { functions: true }); + const toolCalls = res.choices[0].message?.tool_calls; + if (!toolCalls) { + throw new Error("toolCalls should be defined here"); + } + assert.equal(toolCalls[0].function.name, getCurrentWeather.name); + assert.isUndefined(res.choices[0].message?.function_call); }, ); }); - describe("chat.completions.create", function () { - const pirateMessages = [ - { - role: "system", - content: "You are a helpful assistant. You will talk like a pirate.", - } as const, - {role: "user", content: "Can you help me?"} as const, - { - role: "assistant", - content: "Arrrr! Of course, me hearty! What can I do for ye?", - } as const, - {role: "user", content: "What's the best way to train a parrot?"} as const, - ]; - const byodMessages = [ - { - role: "user", - content: - "What's the most common feedback we received from our customers about the product?", - } as const, - ]; - const getCurrentWeather = { - name: "get_current_weather", - description: "Get the current weather in a given location", + it("ensures schema name is not transformed with snake case", async function () { + const getAssetInfo = { + name: "getAssetInfo", + description: "Returns information about an asset", parameters: { type: "object", properties: { - location: { - type: "string", - description: "The city and state, e.g. San Francisco, CA", - }, - unit: { + assetName: { type: "string", - enum: ["celsius", "fahrenheit"], + description: "The asset name. This is a required parameter.", }, }, - required: ["location"], + required: ["assetName"], }, }; + await withDeployments( + clientsAndDeploymentsInfo, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: [{ role: "user", content: "Give me information about Asset No1" }], + tools: [{ type: "function", function: getAssetInfo }], + }), + (res) => { + assertChatCompletions(res, { functions: true }); + const toolCalls = res.choices[0].message?.tool_calls; + if (!toolCalls) { + throw new Error("toolCalls should be defined here"); + } + const argument = toolCalls[0].function.arguments; + assert.isTrue(argument?.includes("assetName")); + }, + functionCallModelsToSkip, + ); + }); - it("returns completions across all models", async function () { - await withDeployments( - clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: pirateMessages, - }), - assertChatCompletions, - ); + it("respects json_object responseFormat", async function () { + clientsAndDeploymentsInfo = createClientsAndDeployments(apiVersion, { + chatCompletion: "true", + jsonObjectResponse: "true", }); + await withDeployments( + clientsAndDeploymentsInfo, + (client, deploymentName) => + client.chat.completions.create({ + model: deploymentName, + messages: [ + { + role: "user", + content: + "Answer the following question in JSON format: What are the capital cities in Africa?", + }, + ], + response_format: { type: "json_object" }, + }), + (res) => { + assertChatCompletions(res, { functions: false }); + const content = res.choices[0].message?.content; + if (!content) assert.fail("Undefined content"); + try { + JSON.parse(content); + } catch { + assert.fail(`Invalid JSON: ${content}`); + } + }, + jsonResponseModelsToSkip, + ); + }); - it("calls functions", async function () { - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => { - const weatherMessages: ChatCompletionMessageParam[] = [ - {role: "user", content: "What's the weather like in Boston?"}, - ]; - const result = await client.chat.completions.create({ - model: deploymentName, - messages: weatherMessages, - functions: [getCurrentWeather], - }); - assertChatCompletions(result, {functions: true}); - const responseMessage = result.choices[0].message; - if (!responseMessage?.function_call) { - assert.fail("Undefined function call"); - } - const functionArgs = JSON.parse(responseMessage.function_call.arguments); - weatherMessages.push(responseMessage); - weatherMessages.push({ - role: "function", - name: responseMessage.function_call.name, - content: JSON.stringify({ - location: functionArgs.location, - temperature: "72", - unit: functionArgs.unit, - forecast: ["sunny", "windy"], - }), - }); - return client.chat.completions.create({ - model: deploymentName, - messages: weatherMessages, - }); - }, - (result) => assertChatCompletions(result, {functions: true}), - functionCallModelsToSkip, - ); - }); + describe("works with custom data sources", async function () { + logger.info("Starting..."); + let { clientsAndDeployments, count } = clientsAndDeploymentsInfo; + assert.isNotEmpty(clientsAndDeployments, "No deployments found"); + let i = 0; + describe.each(clientsAndDeployments)( + "Chat Completions", + async function ({ client, deployments }) { + deployments.forEach((deployment) => { + it(deployment.deploymentName, async function (done) { + logger.info( + `[${++i}/${count}] testing with deployment: ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, + ); - it("doesn't call tools if toolChoice is set to none", async function () { - await withDeployments( - clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: [{role: "user", content: "What's the weather like in Boston?"}], - tool_choice: "none", - tools: [{type: "function", function: getCurrentWeather}], - }), - (res) => { - assertChatCompletions(res, {functions: false}); - assert.isUndefined(res.choices[0].message?.tool_calls); - }, - ); - }); + try { + assert.isNotNull(client); + logger.info("Client: ", client); + logger.info("Starting to run deployment name ", deployment.deploymentName); + const res = await client.chat.completions.create({ + model: deployment.deploymentName, + messages: byodMessages, + data_sources: [createAzureSearchExtension()], + }); + logger.info("Result: ", res); + assertChatCompletions?.(res); + logger.info("Chat completions passed"); + } catch (e) { + const error = e as any; + if (!e) return; + const errorStr = JSON.stringify(error); + if ( + [ + "OperationNotSupported", + "model_not_found", + "rate_limit_exceeded", + "ModelDeprecated", + "429", + "UserError", + ].includes(error.code) || + error.type === "invalid_request_error" || + error.name === "AbortError" || + errorStr.includes("Connection error") || + errorStr.includes("toolCalls") || + [ + "ManagedIdentityIsNotEnabled", + "Rate limit is exceeded", + "Invalid AzureCognitiveSearch configuration detected", + "Unsupported Model", + "does not support 'system' with this model", + "Cannot cancel run with status 'completed'", + ].some((match) => error.message.includes(match)) || + error.status === 404 + ) { + logger.warning("Handled error: ", error); + done.skip("Skipping test due to handled error"); + return; + } else { + logger.error(`Error in deployment ${deployment.deploymentName}: `, error); + throw e; + } + } + }); + }); + }, + ); + }); - it("calls a specific tool if its name is specified", async function () { + describe("return stream", function () { + it("returns completions across all models", async function () { await withDeployments( - clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: [{role: "user", content: "What's the weather like in Boston?"}], - - tool_choice: { - type: "function", - function: {name: getCurrentWeather.name}, - }, - tools: [ - {type: "function", function: getCurrentWeather}, - { - type: "function", - function: { - name: "get_current_weather2", - description: "Get the current weather in a given location in the US", - parameters: { - type: "object", - properties: { - location: { - type: "string", - description: "The city and state, e.g. San Francisco, CA", - }, - unit: { - type: "string", - enum: ["celsius", "fahrenheit"], - }, - }, - required: ["location"], - }, - }, - }, - ], + clientsAndDeploymentsInfo, + async (client, deploymentName) => + bufferAsyncIterable( + await client.chat.completions.create({ + model: deploymentName, + messages: pirateMessages, + stream: true, + }), + ), + (res) => + assertChatCompletionsList(res, { + // The API returns an empty choice in the first event for some + // reason. This should be fixed in the API. + allowEmptyChoices: true, + // The API returns an empty ID in the first event for some + // reason. This should be fixed in the API. + allowEmptyId: true, }), - (res) => { - assertChatCompletions(res, {functions: true}); - const toolCalls = res.choices[0].message?.tool_calls; - if (!toolCalls) { - throw new Error("toolCalls should be defined here"); - } - assert.equal(toolCalls[0].function.name, getCurrentWeather.name); - assert.isUndefined(res.choices[0].message?.function_call); - }, ); }); - it("ensures schema name is not transformed with snake case", async function () { - const getAssetInfo = { - name: "getAssetInfo", - description: "Returns information about an asset", - parameters: { - type: "object", - properties: { - assetName: { - type: "string", - description: "The asset name. This is a required parameter.", - }, - }, - required: ["assetName"], - }, - }; + it("calls functions", async function () { await withDeployments( - clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: [{role: "user", content: "Give me information about Asset No1"}], - tools: [{type: "function", function: getAssetInfo}], + clientsAndDeploymentsInfo, + async (client, deploymentName) => + bufferAsyncIterable( + await client.chat.completions.create({ + model: deploymentName, + messages: [{ role: "user", content: "What's the weather like in Boston?" }], + stream: true, + functions: [getCurrentWeather], + }), + ), + (res) => + assertChatCompletionsList(res, { + functions: true, + // The API returns an empty choice in the first event for some + // reason. This should be fixed in the API. + allowEmptyChoices: true, }), - (res) => { - assertChatCompletions(res, {functions: true}); - const toolCalls = res.choices[0].message?.tool_calls; - if (!toolCalls) { - throw new Error("toolCalls should be defined here"); - } - const argument = toolCalls[0].function.arguments; - assert.isTrue(argument?.includes("assetName")); - }, - functionCallModelsToSkip, ); }); - it("respects json_object responseFormat", async function () { - clientsAndDeployments = createClientsAndDeployments(apiVersion, { - chatCompletion: "true", - jsonObjectResponse: "true", - }); + it("calls toolCalls", async function () { await withDeployments( - clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: [ - { - role: "user", - content: - "Answer the following question in JSON format: What are the capital cities in Africa?", - }, - ], - response_format: {type: "json_object"}, + clientsAndDeploymentsInfo, + async (client, deploymentName) => + bufferAsyncIterable( + await client.chat.completions.create({ + model: deploymentName, + messages: [{ role: "user", content: "What's the weather like in Boston?" }], + stream: true, + tools: [{ type: "function", function: getCurrentWeather }], + }), + ), + (res) => + assertChatCompletionsList(res, { + functions: true, + // The API returns an empty choice in the first event for some + // reason. This should be fixed in the API. + allowEmptyChoices: true, }), - (res) => { - assertChatCompletions(res, {functions: false}); - const content = res.choices[0].message?.content; - if (!content) assert.fail("Undefined content"); - try { - JSON.parse(content); - } catch { - assert.fail(`Invalid JSON: ${content}`); - } - }, - jsonResponseModelsToSkip, ); }); - it("works with custom data sources", async function () { + it("bring your data", async function () { + const dataSources = { data_sources: [createAzureSearchExtension()] }; await withDeployments( - clientsAndDeployments, - (client, deploymentName) => - client.chat.completions.create({ - model: deploymentName, - messages: byodMessages, - data_sources: [createAzureSearchExtension()], - }), - assertChatCompletions, - ); - }); - - describe("return stream", function () { - it("returns completions across all models", async function () { - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => - bufferAsyncIterable( - await client.chat.completions.create({ - model: deploymentName, - messages: pirateMessages, - stream: true, - }), - ), - (res) => - assertChatCompletionsList(res, { - // The API returns an empty choice in the first event for some - // reason. This should be fixed in the API. - allowEmptyChoices: true, - // The API returns an empty ID in the first event for some - // reason. This should be fixed in the API. - allowEmptyId: true, - }), - ); - }); - - it("calls functions", async function () { - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => - bufferAsyncIterable( - await client.chat.completions.create({ - model: deploymentName, - messages: [{role: "user", content: "What's the weather like in Boston?"}], - stream: true, - functions: [getCurrentWeather], - }), - ), - (res) => - assertChatCompletionsList(res, { - functions: true, - // The API returns an empty choice in the first event for some - // reason. This should be fixed in the API. - allowEmptyChoices: true, + clientsAndDeploymentsInfo, + async (client, deploymentName) => + bufferAsyncIterable( + await client.chat.completions.create({ + model: deploymentName, + messages: byodMessages, + stream: true, + ...dataSources, }), - ); - }); - - it("calls toolCalls", async function () { - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => - bufferAsyncIterable( - await client.chat.completions.create({ - model: deploymentName, - messages: [{role: "user", content: "What's the weather like in Boston?"}], - stream: true, - tools: [{type: "function", function: getCurrentWeather}], - }), - ), - (res) => - assertChatCompletionsList(res, { - functions: true, - // The API returns an empty choice in the first event for some - // reason. This should be fixed in the API. - allowEmptyChoices: true, - }), - ); - }); - - it("bring your data", async function () { - const dataSources = {data_sources: [createAzureSearchExtension()]}; - await withDeployments( - clientsAndDeployments, - async (client, deploymentName) => - bufferAsyncIterable( - await client.chat.completions.create({ - model: deploymentName, - messages: byodMessages, - stream: true, - ...dataSources, - }), - ), - assertChatCompletionsList, - [{name: "gpt-4", version: "vision-preview"}], - ); - }); + ), + assertChatCompletionsList, + [{ name: "gpt-4", version: "vision-preview" }], + ); }); }); }); diff --git a/sdk/openai/openai/test/utils/utils.ts b/sdk/openai/openai/test/utils/utils.ts index 2090377f95c4..bdac8bf4cdcd 100644 --- a/sdk/openai/openai/test/utils/utils.ts +++ b/sdk/openai/openai/test/utils/utils.ts @@ -50,11 +50,11 @@ export async function withDeployments( for (const { client, deployments } of clientsAndDeployments) { for (const deployment of deployments) { logger.info( - `[${++i}/${count}] testing with deployment: ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, + `[${++i}/${count}] testing with deployment: ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, ); if (modelsListToSkip && isModelInList(deployment.model, modelsListToSkip)) { logger.info( - `Skipping deployment ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, + `Skipping deployment ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, ); continue; } @@ -79,15 +79,17 @@ export async function withDeployments( error.name === "AbortError" || errorStr.includes("Connection error") || errorStr.includes("toolCalls") || - ["ManagedIdentityIsNotEnabled", + [ + "ManagedIdentityIsNotEnabled", "Rate limit is exceeded", "Invalid AzureCognitiveSearch configuration detected", "Unsupported Model", "does not support 'system' with this model", - ].some(match => error.message.includes(match)) || + "Cannot cancel run with status 'completed'", + ].some((match) => error.message.includes(match)) || error.status === 404 ) { - logger.warning("Handled error: ", error); + logger.warning("WARNING: Handled error: ", error); continue; } logger.error(`Error in deployment ${deployment.deploymentName}: `, error); From 012a7a906d798ac8bf16e45b6d84cdf392e91afb Mon Sep 17 00:00:00 2001 From: Mikhail Simin Date: Thu, 13 Feb 2025 11:19:07 -0800 Subject: [PATCH 3/8] Create new testWithDeployments --- .../test/public/chatCompletions.spec.ts | 88 +++++-------------- sdk/openai/openai/test/utils/utils.ts | 62 ++++++++++--- 2 files changed, 75 insertions(+), 75 deletions(-) diff --git a/sdk/openai/openai/test/public/chatCompletions.spec.ts b/sdk/openai/openai/test/public/chatCompletions.spec.ts index e2acf4bbc260..e62e22db3041 100644 --- a/sdk/openai/openai/test/public/chatCompletions.spec.ts +++ b/sdk/openai/openai/test/public/chatCompletions.spec.ts @@ -3,11 +3,12 @@ import { assert, describe, it } from "vitest"; import { createClientsAndDeployments } from "../utils/createClients.js"; -import type { APIVersion } from "../utils/utils.js"; import { APIMatrix, + APIVersion, bufferAsyncIterable, createAzureSearchExtension, + testWithDeployments, withDeployments, } from "../utils/utils.js"; import { assertChatCompletions, assertChatCompletionsList } from "../utils/asserts.js"; @@ -15,7 +16,6 @@ import { type ChatCompletionMessageParam } from "openai/resources/chat/completio import { functionCallModelsToSkip, jsonResponseModelsToSkip } from "../utils/models.js"; import "../../src/types/index.js"; import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; -import { logger } from "../utils/logger.js"; describe.each(APIMatrix)("Chat Completions [%s]", function (apiVersion: APIVersion) { let clientsAndDeploymentsInfo: ClientsAndDeploymentsInfo; @@ -255,70 +255,28 @@ describe.each(APIMatrix)("Chat Completions [%s]", function (apiVersion: APIVersi }); describe("works with custom data sources", async function () { - logger.info("Starting..."); - let { clientsAndDeployments, count } = clientsAndDeploymentsInfo; - assert.isNotEmpty(clientsAndDeployments, "No deployments found"); - let i = 0; - describe.each(clientsAndDeployments)( - "Chat Completions", - async function ({ client, deployments }) { - deployments.forEach((deployment) => { - it(deployment.deploymentName, async function (done) { - logger.info( - `[${++i}/${count}] testing with deployment: ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, - ); - - try { - assert.isNotNull(client); - logger.info("Client: ", client); - logger.info("Starting to run deployment name ", deployment.deploymentName); - const res = await client.chat.completions.create({ - model: deployment.deploymentName, - messages: byodMessages, - data_sources: [createAzureSearchExtension()], - }); - logger.info("Result: ", res); - assertChatCompletions?.(res); - logger.info("Chat completions passed"); - } catch (e) { - const error = e as any; - if (!e) return; - const errorStr = JSON.stringify(error); - if ( - [ - "OperationNotSupported", - "model_not_found", - "rate_limit_exceeded", - "ModelDeprecated", - "429", - "UserError", - ].includes(error.code) || - error.type === "invalid_request_error" || - error.name === "AbortError" || - errorStr.includes("Connection error") || - errorStr.includes("toolCalls") || - [ - "ManagedIdentityIsNotEnabled", - "Rate limit is exceeded", - "Invalid AzureCognitiveSearch configuration detected", - "Unsupported Model", - "does not support 'system' with this model", - "Cannot cancel run with status 'completed'", - ].some((match) => error.message.includes(match)) || - error.status === 404 - ) { - logger.warning("Handled error: ", error); - done.skip("Skipping test due to handled error"); - return; - } else { - logger.error(`Error in deployment ${deployment.deploymentName}: `, error); - throw e; - } - } - }); - }); + assert.isNotEmpty(clientsAndDeploymentsInfo.clientsAndDeployments, "No deployments found"); + await testWithDeployments({ + clientsAndDeployments: clientsAndDeploymentsInfo, + run: (client, model) => + client.chat.completions.create({ + model: model, + messages: byodMessages, + data_sources: [createAzureSearchExtension()], + }), + validate: assertChatCompletions, + modelsListToSkip: [ + { name: "gpt-35-turbo-0613" }, // Unsupported model + { name: "gpt-4-32k" }, // Managed identity is not enabled + { name: "o1-preview" }, // o-series models are not supported with OYD. + ], + acceptableErrors: { + messageSubstring: [ + "Rate limit is exceeded", + "Invalid AzureCognitiveSearch configuration detected", // gpt-4-1106-preview and others + ], }, - ); + }); }); describe("return stream", function () { diff --git a/sdk/openai/openai/test/utils/utils.ts b/sdk/openai/openai/test/utils/utils.ts index bdac8bf4cdcd..48ba2880995a 100644 --- a/sdk/openai/openai/test/utils/utils.ts +++ b/sdk/openai/openai/test/utils/utils.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { assert } from "vitest"; +import { assert, test } from "vitest"; import { type PipelineRequest, type PipelineResponse, @@ -14,16 +14,15 @@ import { import type { Run } from "openai/resources/beta/threads/runs/runs.mjs"; import type { AzureChatExtensionConfiguration } from "../../src/types/index.js"; import { getAzureSearchEndpoint, getAzureSearchIndex } from "./injectables.js"; -import type { - ClientsAndDeploymentsInfo, - ModelCapabilities, - ModelInfo, - ResourceInfo, -} from "./types.js"; +import { ClientsAndDeploymentsInfo, ModelCapabilities, ModelInfo, ResourceInfo } from "./types.js"; import { logger } from "./logger.js"; import type { OpenAI } from "openai"; import type { Sku } from "@azure/arm-cognitiveservices"; +export type AcceptableErrors = { + messageSubstring: string[]; +}; + export const maxRetriesOption = { maxRetries: 0 }; export enum APIVersion { @@ -31,6 +30,7 @@ export enum APIVersion { Stable = "2024-10-21", OpenAI = "OpenAI", } + export const APIMatrix = [APIVersion.Preview, APIVersion.Stable]; function toString(error: any): string { @@ -86,8 +86,7 @@ export async function withDeployments( "Unsupported Model", "does not support 'system' with this model", "Cannot cancel run with status 'completed'", - ].some((match) => error.message.includes(match)) || - error.status === 404 + ].some((match) => error.message.includes(match)) ) { logger.warning("WARNING: Handled error: ", error); continue; @@ -105,6 +104,49 @@ export async function withDeployments( ); } +export type DeploymentTestingParameters = { + clientsAndDeployments: ClientsAndDeploymentsInfo; + run: (client: OpenAI, model: string) => Promise; + validate?: (result: T) => void; + modelsListToSkip?: Partial[]; + acceptableErrors?: AcceptableErrors; +}; + +export async function testWithDeployments({ + clientsAndDeployments, + run, + validate, + modelsListToSkip, + acceptableErrors, +}: DeploymentTestingParameters): Promise { + assert.isNotEmpty(clientsAndDeployments, "No deployments found"); + for (const { client, deployments } of clientsAndDeployments.clientsAndDeployments) { + for (const deployment of deployments) { + test(deployment.model.name, async (done) => { + if (modelsListToSkip && isModelInList(deployment.model, modelsListToSkip)) { + done.skip( + `Skipping ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, + ); + return; + } + let result; + try { + result = await run(client, deployment.deploymentName); + } catch (e) { + const error = e as any; + if (acceptableErrors?.messageSubstring.some((match) => error.message.includes(match))) { + done.skip(`WARNING: Handled error: ${error}`); + return; + } + throw e; + } + validate?.(result); + return; + }); + } + } +} + export function filterDeployments( resourcesInfo: ResourceInfo[], filters: { @@ -194,7 +236,7 @@ function isModelInList( for (const model of modelsList) { if ( expectedModel.name === model.name && - (!expectedModel.version || expectedModel.version === model.version) + (!expectedModel.version || !model.version || expectedModel.version === model.version) ) { return true; } From bd36600049dabe0f7f92eaf6c2f337fa9bcd656d Mon Sep 17 00:00:00 2001 From: Mikhail Simin Date: Thu, 13 Feb 2025 14:51:13 -0800 Subject: [PATCH 4/8] Concurrent and shuffled tests. Global rate limit handler --- .../test/public/chatCompletions.spec.ts | 3 +-- sdk/openai/openai/test/utils/utils.ts | 27 +++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/sdk/openai/openai/test/public/chatCompletions.spec.ts b/sdk/openai/openai/test/public/chatCompletions.spec.ts index e62e22db3041..fa9115ebfb2b 100644 --- a/sdk/openai/openai/test/public/chatCompletions.spec.ts +++ b/sdk/openai/openai/test/public/chatCompletions.spec.ts @@ -17,7 +17,7 @@ import { functionCallModelsToSkip, jsonResponseModelsToSkip } from "../utils/mod import "../../src/types/index.js"; import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; -describe.each(APIMatrix)("Chat Completions [%s]", function (apiVersion: APIVersion) { +describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (apiVersion: APIVersion) { let clientsAndDeploymentsInfo: ClientsAndDeploymentsInfo; clientsAndDeploymentsInfo = createClientsAndDeployments( @@ -272,7 +272,6 @@ describe.each(APIMatrix)("Chat Completions [%s]", function (apiVersion: APIVersi ], acceptableErrors: { messageSubstring: [ - "Rate limit is exceeded", "Invalid AzureCognitiveSearch configuration detected", // gpt-4-1106-preview and others ], }, diff --git a/sdk/openai/openai/test/utils/utils.ts b/sdk/openai/openai/test/utils/utils.ts index 48ba2880995a..8762268a8f45 100644 --- a/sdk/openai/openai/test/utils/utils.ts +++ b/sdk/openai/openai/test/utils/utils.ts @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { assert, test } from "vitest"; +import { assert, test, describe } from "vitest"; import { type PipelineRequest, type PipelineResponse, @@ -23,6 +23,10 @@ export type AcceptableErrors = { messageSubstring: string[]; }; +const GlobalAcceptedErrors: AcceptableErrors = { + messageSubstring: ["Rate limit is exceeded"], +} + export const maxRetriesOption = { maxRetries: 0 }; export enum APIVersion { @@ -88,7 +92,7 @@ export async function withDeployments( "Cannot cancel run with status 'completed'", ].some((match) => error.message.includes(match)) ) { - logger.warning("WARNING: Handled error: ", error); + logger.warning("WARNING: Handled error: ", error.message); continue; } logger.error(`Error in deployment ${deployment.deploymentName}: `, error); @@ -120,23 +124,24 @@ export async function testWithDeployments({ acceptableErrors, }: DeploymentTestingParameters): Promise { assert.isNotEmpty(clientsAndDeployments, "No deployments found"); - for (const { client, deployments } of clientsAndDeployments.clientsAndDeployments) { + describe.each(clientsAndDeployments.clientsAndDeployments)( + "$client.baseURL", async function({ client, deployments }) { for (const deployment of deployments) { - test(deployment.model.name, async (done) => { + test.concurrent(deployment.model.name, async (done) => { if (modelsListToSkip && isModelInList(deployment.model, modelsListToSkip)) { - done.skip( - `Skipping ${deployment.deploymentName} (${deployment.model.name}: ${deployment.model.version})`, - ); - return; + done.skip(`Skipping ${deployment.model.name} : ${deployment.model.version}`); } + let result; try { result = await run(client, deployment.deploymentName); } catch (e) { const error = e as any; if (acceptableErrors?.messageSubstring.some((match) => error.message.includes(match))) { - done.skip(`WARNING: Handled error: ${error}`); - return; + done.skip(`Skipping due to acceptable error: ${error}`); + } + if (GlobalAcceptedErrors.messageSubstring.some((match) => error.message.includes(match))) { + done.skip(`Skipping due to global acceptable error: ${error}`); } throw e; } @@ -144,7 +149,7 @@ export async function testWithDeployments({ return; }); } - } + }); } export function filterDeployments( From 25c67b26dfc5234ba070a9e2ace02eaf338de97a Mon Sep 17 00:00:00 2001 From: Mikhail Simin Date: Thu, 6 Mar 2025 13:14:32 -0800 Subject: [PATCH 5/8] Convert zod to new testWithDeployments function --- .../openai/test/public/abortsignal.spec.ts | 4 +- .../openai/test/public/assistants.spec.ts | 4 +- .../public/browser/webSocketRealtime.spec.ts | 4 +- .../test/public/chatCompletions.spec.ts | 201 +++++++++--------- .../openai/test/public/completions.spec.ts | 4 +- .../openai/test/public/embeddings.spec.ts | 4 +- sdk/openai/openai/test/public/images.spec.ts | 4 +- .../openai/test/public/node/batches.spec.ts | 4 +- .../openai/test/public/node/whisper.spec.ts | 4 +- .../test/public/node/wsRealtime.spec.ts | 4 +- sdk/openai/openai/test/public/tts.spec.ts | 4 +- sdk/openai/openai/test/public/vision.spec.ts | 4 +- sdk/openai/openai/test/utils/asserts.ts | 2 +- sdk/openai/openai/test/utils/createClients.ts | 4 +- sdk/openai/openai/test/utils/setup.ts | 4 +- sdk/openai/openai/test/utils/types.ts | 2 +- sdk/openai/openai/test/utils/utils.ts | 18 +- 17 files changed, 146 insertions(+), 129 deletions(-) diff --git a/sdk/openai/openai/test/public/abortsignal.spec.ts b/sdk/openai/openai/test/public/abortsignal.spec.ts index d1a05e034b50..e3313441798a 100644 --- a/sdk/openai/openai/test/public/abortsignal.spec.ts +++ b/sdk/openai/openai/test/public/abortsignal.spec.ts @@ -5,10 +5,10 @@ import { assert, describe, beforeEach, it } from "vitest"; import { matrix } from "@azure-tools/test-utils-vitest"; import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIVersion } from "../utils/utils.js"; -import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; describe("AbortSignal", () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; + let clientsAndDeployments: ClientsAndDeploymentsCountInfo; matrix([[APIVersion.Stable]] as const, async function (apiVersion: APIVersion) { beforeEach(async () => { diff --git a/sdk/openai/openai/test/public/assistants.spec.ts b/sdk/openai/openai/test/public/assistants.spec.ts index f1351df4f70c..c6eeca834ed5 100644 --- a/sdk/openai/openai/test/public/assistants.spec.ts +++ b/sdk/openai/openai/test/public/assistants.spec.ts @@ -6,13 +6,13 @@ import { assert, describe, beforeEach, it } from "vitest"; import { assertAssistantEquality } from "../utils/asserts.js"; import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIVersion, isRateLimitRun, withDeployments } from "../utils/utils.js"; -import type { ClientsAndDeploymentsInfo, Metadata } from "../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo, Metadata } from "../utils/types.js"; import type { AssistantCreateParams } from "openai/resources/beta/assistants.mjs"; describe("Assistants", () => { matrix([[APIVersion.Preview]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; + let clientsAndDeployments: ClientsAndDeploymentsCountInfo; function createCodeAssistant(deploymentName: string): AssistantCreateParams { return { diff --git a/sdk/openai/openai/test/public/browser/webSocketRealtime.spec.ts b/sdk/openai/openai/test/public/browser/webSocketRealtime.spec.ts index 08dc8c7e4fa2..4f20f1b176f6 100644 --- a/sdk/openai/openai/test/public/browser/webSocketRealtime.spec.ts +++ b/sdk/openai/openai/test/public/browser/webSocketRealtime.spec.ts @@ -11,13 +11,13 @@ import { assertResponseTextDoneEvent, assertSessionCreatedEvent, } from "../../utils/asserts.js"; -import type { ClientsAndDeploymentsInfo } from "../../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../../utils/types.js"; import type { AzureOpenAI } from "openai"; describe("Realtime", () => { matrix([[APIVersion["2024_10_01_preview"]]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientAndDeployments: ClientsAndDeploymentsInfo; + let clientAndDeployments: ClientsAndDeploymentsCountInfo; beforeEach(async () => { clientAndDeployments = createClientsAndDeployments(apiVersion, { realtime: "true" }); diff --git a/sdk/openai/openai/test/public/chatCompletions.spec.ts b/sdk/openai/openai/test/public/chatCompletions.spec.ts index 20930c77cc09..a2aaeab7f6c0 100644 --- a/sdk/openai/openai/test/public/chatCompletions.spec.ts +++ b/sdk/openai/openai/test/public/chatCompletions.spec.ts @@ -1,56 +1,53 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import { matrix } from "@azure-tools/test-utils-vitest"; -import { assert, describe, beforeEach, it } from "vitest"; -import { createClientsAndDeployments } from "../utils/createClients.js"; -import {APIVersion, testWithDeployments} from "../utils/utils.js"; -import { - assertChatCompletions, - assertChatCompletionsList, - assertParsedChatCompletion, -} from "../utils/asserts.js"; -import { z } from "zod"; -import { zodResponseFormat } from "openai/helpers/zod"; +import {assert, describe, it} from "vitest"; +import {createClientsAndDeployments} from "../utils/createClients.js"; +import type {APIVersion} from "../utils/utils.js"; import { APIMatrix, bufferAsyncIterable, createAzureSearchExtension, + testWithDeployments, withDeployments, } from "../utils/utils.js"; -import { type ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; -import { functionCallModelsToSkip, jsonResponseModelsToSkip } from "../utils/models.js"; +import { + assertChatCompletions, + assertChatCompletionsList, + assertParsedChatCompletion, +} from "../utils/asserts.js"; +import {z} from "zod"; +import {zodResponseFormat} from "openai/helpers/zod"; +import {type ChatCompletionMessageParam} from "openai/resources/chat/completions.mjs"; +import {functionCallModelsToSkip, jsonResponseModelsToSkip} from "../utils/models.js"; import "../../src/types/index.js"; -import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; -import { type MathResponse, assertMathResponseOutput } from "../utils/structuredOutputUtils.js"; +import type {ClientsAndDeploymentsCountInfo} from "../utils/types.js"; +import {assertMathResponseOutput, type MathResponse} from "../utils/structuredOutputUtils.js"; -describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (apiVersion: APIVersion) { - let clientsAndDeploymentsInfo: ClientsAndDeploymentsInfo; +describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersion) => { + let clientsAndDeploymentsInfo: ClientsAndDeploymentsCountInfo; - beforeEach(async () => { - clientsAndDeploymentsInfo = createClientsAndDeployments( - apiVersion, - { chatCompletion: "true" }, - { - deploymentsToSkip: ["o1" /** It gets stuck and never returns */], - modelsToSkip: [{ name: "gpt-4o-audio-preview" }, { name: "o3-mini" }], - }, - ); - }); + clientsAndDeploymentsInfo = createClientsAndDeployments( + apiVersion, + {chatCompletion: "true"}, + { + deploymentsToSkip: ["o1" /** It gets stuck and never returns */], + modelsToSkip: [{name: "gpt-4o-audio-preview"}, {name: "o3-mini"}], + }, + ); - - describe("chat.completions.create", function () { + describe("chat.completions.create", () => { const pirateMessages = [ { role: "system", content: "You are a helpful assistant. You will talk like a pirate.", } as const, - { role: "user", content: "Can you help me?" } as const, + {role: "user", content: "Can you help me?"} as const, { role: "assistant", content: "Arrrr! Of course, me hearty! What can I do for ye?", } as const, - { role: "user", content: "What's the best way to train a parrot?" } as const, + {role: "user", content: "What's the best way to train a parrot?"} as const, ]; const byodMessages = [ { @@ -78,7 +75,7 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a }, }; // TODO: Change to arrow functions - it("returns completions across all models", async function () { + it("returns completions across all models", async () => { await withDeployments( clientsAndDeploymentsInfo, (client, deploymentName) => @@ -90,19 +87,19 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a ); }); - it("calls functions", async function () { + it("calls functions", async () => { await withDeployments( clientsAndDeploymentsInfo, async (client, deploymentName) => { const weatherMessages: ChatCompletionMessageParam[] = [ - { role: "user", content: "What's the weather like in Boston?" }, + {role: "user", content: "What's the weather like in Boston?"}, ]; const result = await client.chat.completions.create({ model: deploymentName, messages: weatherMessages, functions: [getCurrentWeather], }); - assertChatCompletions(result, { functions: true }); + assertChatCompletions(result, {functions: true}); const responseMessage = result.choices[0].message; if (!responseMessage?.function_call) { assert.fail("Undefined function call"); @@ -124,42 +121,42 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a messages: weatherMessages, }); }, - (result) => assertChatCompletions(result, { functions: true }), + (result) => assertChatCompletions(result, {functions: true}), functionCallModelsToSkip, ); }); - it("doesn't call tools if toolChoice is set to none", async function () { + it("doesn't call tools if toolChoice is set to none", async () => { await withDeployments( clientsAndDeploymentsInfo, (client, deploymentName) => client.chat.completions.create({ model: deploymentName, - messages: [{ role: "user", content: "What's the weather like in Boston?" }], + messages: [{role: "user", content: "What's the weather like in Boston?"}], tool_choice: "none", - tools: [{ type: "function", function: getCurrentWeather }], + tools: [{type: "function", function: getCurrentWeather}], }), (res) => { - assertChatCompletions(res, { functions: false }); + assertChatCompletions(res, {functions: false}); assert.isUndefined(res.choices[0].message?.tool_calls); }, ); }); - it("calls a specific tool if its name is specified", async function () { + it("calls a specific tool if its name is specified", async () => { await withDeployments( clientsAndDeploymentsInfo, (client, deploymentName) => client.chat.completions.create({ model: deploymentName, - messages: [{ role: "user", content: "What's the weather like in Boston?" }], + messages: [{role: "user", content: "What's the weather like in Boston?"}], tool_choice: { type: "function", - function: { name: getCurrentWeather.name }, + function: {name: getCurrentWeather.name}, }, tools: [ - { type: "function", function: getCurrentWeather }, + {type: "function", function: getCurrentWeather}, { type: "function", function: { @@ -184,7 +181,7 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a ], }), (res) => { - assertChatCompletions(res, { functions: true }); + assertChatCompletions(res, {functions: true}); const toolCalls = res.choices[0].message?.tool_calls; if (!toolCalls) { throw new Error("toolCalls should be defined here"); @@ -195,7 +192,7 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a ); }); - it("ensures schema name is not transformed with snake case", async function () { + it("ensures schema name is not transformed with snake case", async () => { const getAssetInfo = { name: "getAssetInfo", description: "Returns information about an asset", @@ -215,11 +212,11 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a (client, deploymentName) => client.chat.completions.create({ model: deploymentName, - messages: [{ role: "user", content: "Give me information about Asset No1" }], - tools: [{ type: "function", function: getAssetInfo }], + messages: [{role: "user", content: "Give me information about Asset No1"}], + tools: [{type: "function", function: getAssetInfo}], }), (res) => { - assertChatCompletions(res, { functions: true }); + assertChatCompletions(res, {functions: true}); const toolCalls = res.choices[0].message?.tool_calls; if (!toolCalls) { throw new Error("toolCalls should be defined here"); @@ -231,7 +228,7 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a ); }); - it("respects json_object responseFormat", async function () { + it("respects json_object responseFormat", async () => { clientsAndDeploymentsInfo = createClientsAndDeployments(apiVersion, { chatCompletion: "true", jsonObjectResponse: "true", @@ -248,10 +245,10 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a "Answer the following question in JSON format: What are the capital cities in Africa?", }, ], - response_format: { type: "json_object" }, + response_format: {type: "json_object"}, }), (res) => { - assertChatCompletions(res, { functions: false }); + assertChatCompletions(res, {functions: false}); const content = res.choices[0].message?.content; if (!content) assert.fail("Undefined content"); try { @@ -264,7 +261,7 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a ); }); - describe("works with custom data sources", async function () { + describe("works with custom data sources", async () => { assert.isNotEmpty(clientsAndDeploymentsInfo.clientsAndDeployments, "No deployments found"); await testWithDeployments({ clientsAndDeployments: clientsAndDeploymentsInfo, @@ -276,9 +273,9 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a }), validate: assertChatCompletions, modelsListToSkip: [ - { name: "gpt-35-turbo-0613" }, // Unsupported model - { name: "gpt-4-32k" }, // Managed identity is not enabled - { name: "o1-preview" }, // o-series models are not supported with OYD. + {name: "gpt-35-turbo-0613"}, // Unsupported model + {name: "gpt-4-32k"}, // Managed identity is not enabled + {name: "o1-preview"}, // o-series models are not supported with OYD. ], acceptableErrors: { messageSubstring: [ @@ -288,8 +285,8 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a }); }); - describe("return stream", function () { - it("returns completions across all models", async function () { + describe("return stream", () => { + it("returns completions across all models", async () => { await withDeployments( clientsAndDeploymentsInfo, async (client, deploymentName) => @@ -312,14 +309,14 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a ); }); - it("calls functions", async function () { + it("calls functions", async () => { await withDeployments( clientsAndDeploymentsInfo, async (client, deploymentName) => bufferAsyncIterable( await client.chat.completions.create({ model: deploymentName, - messages: [{ role: "user", content: "What's the weather like in Boston?" }], + messages: [{role: "user", content: "What's the weather like in Boston?"}], stream: true, functions: [getCurrentWeather], }), @@ -334,16 +331,16 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a ); }); - it("calls toolCalls", async function () { + it("calls toolCalls", async () => { await withDeployments( clientsAndDeploymentsInfo, async (client, deploymentName) => bufferAsyncIterable( await client.chat.completions.create({ model: deploymentName, - messages: [{ role: "user", content: "What's the weather like in Boston?" }], + messages: [{role: "user", content: "What's the weather like in Boston?"}], stream: true, - tools: [{ type: "function", function: getCurrentWeather }], + tools: [{type: "function", function: getCurrentWeather}], }), ), (res) => @@ -356,8 +353,8 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a ); }); - it("bring your data", async function () { - const dataSources = { data_sources: [createAzureSearchExtension()] }; + it("bring your data", async () => { + const dataSources = {data_sources: [createAzureSearchExtension()]}; await withDeployments( clientsAndDeploymentsInfo, async (client, deploymentName) => @@ -370,42 +367,54 @@ describe.concurrent.shuffle.each(APIMatrix)("Chat Completions [%s]", function (a }), ), assertChatCompletionsList, - [{ name: "gpt-4", version: "vision-preview" }], + [{name: "gpt-4", version: "vision-preview"}], ); }); - describe("chat.completions.parse", function () { - it("structured output for chat completions", async () => { - await withDeployments( - clientsAndDeploymentsInfo, - async (client, deploymentName) => { - const step = z.object({ - explanation: z.string(), - output: z.string(), - }); + describe("chat.completions.parse", () => { + describe("structured output for chat completions", async () => { + await testWithDeployments({ + clientsAndDeployments: clientsAndDeploymentsInfo, + run: async (client, deploymentName) => { + const step = z.object({ + explanation: z.string(), + output: z.string(), + }); - const mathResponse = z.object({ - steps: z.array(step), - final_answer: z.string(), - }); + const mathResponse = z.object({ + steps: z.array(step), + final_answer: z.string(), + }); - return client.beta.chat.completions.parse({ - model: deploymentName, - messages: [ - { - role: "system", - content: - "You are a helpful math tutor. Only use the schema for math responses.", - }, - { role: "user", content: "solve 8x + 3 = 21" }, - ], - response_format: zodResponseFormat(mathResponse, "mathResponse"), - }); - }, - (result) => { - assertParsedChatCompletion(result, assertMathResponseOutput, { - allowEmptyChoices: true, - }); + return client.beta.chat.completions.parse({ + model: deploymentName, + messages: [ + { + role: "system", + content: + "You are a helpful math tutor. Only use the schema for math responses.", + }, + {role: "user", content: "solve 8x + 3 = 21"}, + ], + response_format: zodResponseFormat(mathResponse, "mathResponse"), + }); + } + , + validate: (result) => { + assertParsedChatCompletion(result, assertMathResponseOutput, { + allowEmptyChoices: true, + }); + }, + modelsListToSkip: [ + // structured output is not supported + {name: "gpt-35-turbo"}, + {name: "gpt-4"}, + {name: "gpt-4-32k"}, + {name: "gpt-35-turbo-16k"}, + {name: "o1-preview"}, + {name: "gpt-4-32k"}, + {name: "gpt-4o", version: "2024-05-13"}, + ], }, ); }); diff --git a/sdk/openai/openai/test/public/completions.spec.ts b/sdk/openai/openai/test/public/completions.spec.ts index 32a88562084c..79a2f055cfde 100644 --- a/sdk/openai/openai/test/public/completions.spec.ts +++ b/sdk/openai/openai/test/public/completions.spec.ts @@ -9,12 +9,12 @@ import { assertCompletions, assertCompletionsStream } from "../utils/asserts.js" import { APIMatrix, withDeployments } from "../utils/utils.js"; import { completionsModelsToSkip } from "../utils/models.js"; import "../../src/types/index.js"; -import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; describe("Legacy Completions", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; + let clientsAndDeployments: ClientsAndDeploymentsCountInfo; beforeEach(async () => { clientsAndDeployments = createClientsAndDeployments(apiVersion, { completion: "true" }); diff --git a/sdk/openai/openai/test/public/embeddings.spec.ts b/sdk/openai/openai/test/public/embeddings.spec.ts index 97f79d7b66e8..d3697adbd42c 100644 --- a/sdk/openai/openai/test/public/embeddings.spec.ts +++ b/sdk/openai/openai/test/public/embeddings.spec.ts @@ -6,12 +6,12 @@ import { describe, it, beforeAll } from "vitest"; import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIMatrix, type APIVersion, withDeployments } from "../utils/utils.js"; import { assertEmbeddings } from "../utils/asserts.js"; -import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; describe("Embeddings", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; + let clientsAndDeployments: ClientsAndDeploymentsCountInfo; beforeAll(async function () { clientsAndDeployments = createClientsAndDeployments(apiVersion, { embeddings: "true" }); diff --git a/sdk/openai/openai/test/public/images.spec.ts b/sdk/openai/openai/test/public/images.spec.ts index b2af35cf5ae7..b8df5150ae01 100644 --- a/sdk/openai/openai/test/public/images.spec.ts +++ b/sdk/openai/openai/test/public/images.spec.ts @@ -6,12 +6,12 @@ import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIMatrix, type APIVersion, withDeployments } from "../utils/utils.js"; import { assertImagesWithJSON, assertImagesWithURLs } from "../utils/asserts.js"; import { describe, it, beforeAll } from "vitest"; -import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; describe("Images", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; + let clientsAndDeployments: ClientsAndDeploymentsCountInfo; beforeAll(async function () { clientsAndDeployments = createClientsAndDeployments(apiVersion, { diff --git a/sdk/openai/openai/test/public/node/batches.spec.ts b/sdk/openai/openai/test/public/node/batches.spec.ts index a04e3026cb89..112edd42130d 100644 --- a/sdk/openai/openai/test/public/node/batches.spec.ts +++ b/sdk/openai/openai/test/public/node/batches.spec.ts @@ -9,12 +9,12 @@ import { APIVersion, withDeployments } from "../../utils/utils.js"; import { assertBatch, assertNonEmptyArray } from "../../utils/asserts.js"; import { delay } from "@azure/core-util"; import type { FileObject } from "openai/resources/index"; -import type { ClientsAndDeploymentsInfo } from "../../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../../utils/types.js"; describe("Batches", () => { matrix([[APIVersion.Preview]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientAndDeployments: ClientsAndDeploymentsInfo; + let clientAndDeployments: ClientsAndDeploymentsCountInfo; beforeEach(async () => { clientAndDeployments = createClientsAndDeployments( diff --git a/sdk/openai/openai/test/public/node/whisper.spec.ts b/sdk/openai/openai/test/public/node/whisper.spec.ts index fe9eb409352d..f6c820ef4ee8 100644 --- a/sdk/openai/openai/test/public/node/whisper.spec.ts +++ b/sdk/openai/openai/test/public/node/whisper.spec.ts @@ -13,12 +13,12 @@ import { } from "../../utils/utils.js"; import { assertAudioResult } from "../../utils/asserts.js"; import type { AudioResultFormat } from "../../utils/audioTypes.js"; -import type { ClientsAndDeploymentsInfo } from "../../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../../utils/types.js"; describe("Whisper", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; + let clientsAndDeployments: ClientsAndDeploymentsCountInfo; beforeEach(async () => { clientsAndDeployments = createClientsAndDeployments( diff --git a/sdk/openai/openai/test/public/node/wsRealtime.spec.ts b/sdk/openai/openai/test/public/node/wsRealtime.spec.ts index 68c7fb6f8a2b..c1568769bb1e 100644 --- a/sdk/openai/openai/test/public/node/wsRealtime.spec.ts +++ b/sdk/openai/openai/test/public/node/wsRealtime.spec.ts @@ -11,13 +11,13 @@ import { assertSessionCreatedEvent, } from "../../utils/asserts.js"; import { createClientsAndDeployments } from "../../utils/createClients.js"; -import type { ClientsAndDeploymentsInfo } from "../../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../../utils/types.js"; import type { AzureOpenAI } from "openai"; describe("Realtime", () => { matrix([[APIVersion["2024_10_01_preview"]]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientAndDeployments: ClientsAndDeploymentsInfo; + let clientAndDeployments: ClientsAndDeploymentsCountInfo; beforeEach(async () => { clientAndDeployments = createClientsAndDeployments(apiVersion, { realtime: "true" }); diff --git a/sdk/openai/openai/test/public/tts.spec.ts b/sdk/openai/openai/test/public/tts.spec.ts index d923d093287e..c13bc475628f 100644 --- a/sdk/openai/openai/test/public/tts.spec.ts +++ b/sdk/openai/openai/test/public/tts.spec.ts @@ -5,12 +5,12 @@ import { matrix } from "@azure-tools/test-utils-vitest"; import { describe, beforeEach, it, assert } from "vitest"; import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIVersion, withDeployments } from "../utils/utils.js"; -import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; describe("Text to speech", function () { matrix([[APIVersion.Preview]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; + let clientsAndDeployments: ClientsAndDeploymentsCountInfo; beforeEach(async () => { clientsAndDeployments = createClientsAndDeployments( diff --git a/sdk/openai/openai/test/public/vision.spec.ts b/sdk/openai/openai/test/public/vision.spec.ts index fc60f36b5817..b69a282a749e 100644 --- a/sdk/openai/openai/test/public/vision.spec.ts +++ b/sdk/openai/openai/test/public/vision.spec.ts @@ -7,13 +7,13 @@ import { createClientsAndDeployments } from "../utils/createClients.js"; import { assertChatCompletions } from "../utils/asserts.js"; import { APIMatrix, type APIVersion, withDeployments } from "../utils/utils.js"; import { RestError } from "@azure/core-rest-pipeline"; -import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; import { logger } from "../utils/logger.js"; describe("Vision", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsInfo; + let clientsAndDeployments: ClientsAndDeploymentsCountInfo; beforeEach(async () => { clientsAndDeployments = createClientsAndDeployments( diff --git a/sdk/openai/openai/test/utils/asserts.ts b/sdk/openai/openai/test/utils/asserts.ts index c2d8fce5b1af..da0dc61cac98 100644 --- a/sdk/openai/openai/test/utils/asserts.ts +++ b/sdk/openai/openai/test/utils/asserts.ts @@ -569,7 +569,7 @@ function assertParsedMessage( if (message.content && message.parsed) { assert.deepEqual(message.content, JSON.stringify(message.parsed)); } - for (const item of message.tool_calls) { + for (const item of message.tool_calls || []) { assertParsedFunctionToolCall(item); } } diff --git a/sdk/openai/openai/test/utils/createClients.ts b/sdk/openai/openai/test/utils/createClients.ts index 59764c53bb26..2b130e394c7e 100644 --- a/sdk/openai/openai/test/utils/createClients.ts +++ b/sdk/openai/openai/test/utils/createClients.ts @@ -6,7 +6,7 @@ import { getBearerTokenProvider } from "@azure/identity"; import { APIVersion, filterDeployments } from "./utils.js"; import { createLiveCredential } from "@azure-tools/test-credential"; import { getResourcesInfo } from "./injectables.js"; -import type { ClientsAndDeploymentsInfo, CreateClientOptions, ModelCapabilities } from "./types.js"; +import type { ClientsAndDeploymentsCountInfo, CreateClientOptions, ModelCapabilities } from "./types.js"; const scope = "https://cognitiveservices.azure.com/.default"; @@ -14,7 +14,7 @@ export function createClientsAndDeployments( apiVersion: APIVersion, capabilities: ModelCapabilities, options: CreateClientOptions = {}, -): ClientsAndDeploymentsInfo { +): ClientsAndDeploymentsCountInfo { const { clientOptions, sku, deploymentsToSkip, modelsToSkip } = options; const { resourcesInfo } = getResourcesInfo(); switch (apiVersion) { diff --git a/sdk/openai/openai/test/utils/setup.ts b/sdk/openai/openai/test/utils/setup.ts index ed9cd916050e..708fcd83de49 100644 --- a/sdk/openai/openai/test/utils/setup.ts +++ b/sdk/openai/openai/test/utils/setup.ts @@ -73,7 +73,7 @@ export async function getDeployments(): Promise { const filePath = "./deployments.json"; try { const content = await readFile(filePath, "utf-8"); - logger.info(`Reading deployments from file: ${filePath}: ${content}`); + logger.verbose(`Reading deployments from file: ${filePath}: ${content}`); return JSON.parse(content); } catch { const resourcesInfo: Omit[] = []; @@ -92,7 +92,7 @@ export async function getDeployments(): Promise { }); } } - logger.info(`Available deployments [${count}]: ${JSON.stringify(resourcesInfo, null, 2)}`); + logger.verbose(`Available deployments [${count}]: ${JSON.stringify(resourcesInfo, null, 2)}`); await writeFile(filePath, JSON.stringify({ resourcesInfo, count }, null, 2)); return { resourcesInfo, count }; } diff --git a/sdk/openai/openai/test/utils/types.ts b/sdk/openai/openai/test/utils/types.ts index 1d7235001b5e..3f56e90528ad 100644 --- a/sdk/openai/openai/test/utils/types.ts +++ b/sdk/openai/openai/test/utils/types.ts @@ -49,7 +49,7 @@ export interface ClientAndDeploymentsInfo { deployments: DeploymentInfo[]; } -export interface ClientsAndDeploymentsInfo { +export interface ClientsAndDeploymentsCountInfo { clientsAndDeployments: ClientAndDeploymentsInfo[]; count: number; } diff --git a/sdk/openai/openai/test/utils/utils.ts b/sdk/openai/openai/test/utils/utils.ts index 9f54a827916d..f771aa16902a 100644 --- a/sdk/openai/openai/test/utils/utils.ts +++ b/sdk/openai/openai/test/utils/utils.ts @@ -14,7 +14,7 @@ import { import type { Run } from "openai/resources/beta/threads/runs/runs.mjs"; import type { AzureChatExtensionConfiguration } from "../../src/types/index.js"; import { getAzureSearchEndpoint, getAzureSearchIndex } from "./injectables.js"; -import { ClientsAndDeploymentsInfo, ModelCapabilities, ModelInfo, ResourceInfo } from "./types.js"; +import { ClientsAndDeploymentsCountInfo, ModelCapabilities, ModelInfo, ResourceInfo } from "./types.js"; import { logger } from "./logger.js"; import type { OpenAI } from "openai"; import type { Sku } from "@azure/arm-cognitiveservices"; @@ -43,7 +43,7 @@ function toString(error: any): string { } export async function withDeployments( - { clientsAndDeployments, count }: ClientsAndDeploymentsInfo, + { clientsAndDeployments, count }: ClientsAndDeploymentsCountInfo, run: (client: OpenAI, model: string) => Promise, validate?: (result: T) => void, modelsListToSkip?: Partial[], @@ -110,13 +110,21 @@ export async function withDeployments( } export type DeploymentTestingParameters = { - clientsAndDeployments: ClientsAndDeploymentsInfo; + clientsAndDeployments: ClientsAndDeploymentsCountInfo; run: (client: OpenAI, model: string) => Promise; validate?: (result: T) => void; modelsListToSkip?: Partial[]; acceptableErrors?: AcceptableErrors; }; +/** + * Test with deployments invokes `test` call, so it should be inside of `describe` and not `it`. + * @param clientsAndDeployments + * @param run + * @param validate + * @param modelsListToSkip + * @param acceptableErrors + */ export async function testWithDeployments({ clientsAndDeployments, run, @@ -125,10 +133,10 @@ export async function testWithDeployments({ acceptableErrors, }: DeploymentTestingParameters): Promise { assert.isNotEmpty(clientsAndDeployments, "No deployments found"); - describe.each(clientsAndDeployments.clientsAndDeployments)( + describe.concurrent.each(clientsAndDeployments.clientsAndDeployments)( "$client.baseURL", async function({ client, deployments }) { for (const deployment of deployments) { - test.concurrent(deployment.model.name, async (done) => { + test.concurrent(`${deployment.model.name} (${deployment.model.version})`, async (done) => { if (modelsListToSkip && isModelInList(deployment.model, modelsListToSkip)) { done.skip(`Skipping ${deployment.model.name} : ${deployment.model.version}`); } From fb82c9842f10fc98ae8c6b22d7e8ff5281ddc1d2 Mon Sep 17 00:00:00 2001 From: Mikhail Simin Date: Thu, 6 Mar 2025 13:17:54 -0800 Subject: [PATCH 6/8] Undo type renaming --- sdk/openai/openai/test/public/abortsignal.spec.ts | 4 ++-- sdk/openai/openai/test/public/assistants.spec.ts | 4 ++-- .../openai/test/public/browser/webSocketRealtime.spec.ts | 4 ++-- sdk/openai/openai/test/public/chatCompletions.spec.ts | 4 ++-- sdk/openai/openai/test/public/completions.spec.ts | 4 ++-- sdk/openai/openai/test/public/embeddings.spec.ts | 4 ++-- sdk/openai/openai/test/public/images.spec.ts | 4 ++-- sdk/openai/openai/test/public/node/batches.spec.ts | 4 ++-- sdk/openai/openai/test/public/node/whisper.spec.ts | 4 ++-- sdk/openai/openai/test/public/node/wsRealtime.spec.ts | 4 ++-- sdk/openai/openai/test/public/tts.spec.ts | 4 ++-- sdk/openai/openai/test/public/vision.spec.ts | 4 ++-- sdk/openai/openai/test/utils/createClients.ts | 4 ++-- sdk/openai/openai/test/utils/types.ts | 2 +- sdk/openai/openai/test/utils/utils.ts | 6 +++--- 15 files changed, 30 insertions(+), 30 deletions(-) diff --git a/sdk/openai/openai/test/public/abortsignal.spec.ts b/sdk/openai/openai/test/public/abortsignal.spec.ts index e3313441798a..d1a05e034b50 100644 --- a/sdk/openai/openai/test/public/abortsignal.spec.ts +++ b/sdk/openai/openai/test/public/abortsignal.spec.ts @@ -5,10 +5,10 @@ import { assert, describe, beforeEach, it } from "vitest"; import { matrix } from "@azure-tools/test-utils-vitest"; import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIVersion } from "../utils/utils.js"; -import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; describe("AbortSignal", () => { - let clientsAndDeployments: ClientsAndDeploymentsCountInfo; + let clientsAndDeployments: ClientsAndDeploymentsInfo; matrix([[APIVersion.Stable]] as const, async function (apiVersion: APIVersion) { beforeEach(async () => { diff --git a/sdk/openai/openai/test/public/assistants.spec.ts b/sdk/openai/openai/test/public/assistants.spec.ts index c6eeca834ed5..f1351df4f70c 100644 --- a/sdk/openai/openai/test/public/assistants.spec.ts +++ b/sdk/openai/openai/test/public/assistants.spec.ts @@ -6,13 +6,13 @@ import { assert, describe, beforeEach, it } from "vitest"; import { assertAssistantEquality } from "../utils/asserts.js"; import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIVersion, isRateLimitRun, withDeployments } from "../utils/utils.js"; -import type { ClientsAndDeploymentsCountInfo, Metadata } from "../utils/types.js"; +import type { ClientsAndDeploymentsInfo, Metadata } from "../utils/types.js"; import type { AssistantCreateParams } from "openai/resources/beta/assistants.mjs"; describe("Assistants", () => { matrix([[APIVersion.Preview]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsCountInfo; + let clientsAndDeployments: ClientsAndDeploymentsInfo; function createCodeAssistant(deploymentName: string): AssistantCreateParams { return { diff --git a/sdk/openai/openai/test/public/browser/webSocketRealtime.spec.ts b/sdk/openai/openai/test/public/browser/webSocketRealtime.spec.ts index 4f20f1b176f6..08dc8c7e4fa2 100644 --- a/sdk/openai/openai/test/public/browser/webSocketRealtime.spec.ts +++ b/sdk/openai/openai/test/public/browser/webSocketRealtime.spec.ts @@ -11,13 +11,13 @@ import { assertResponseTextDoneEvent, assertSessionCreatedEvent, } from "../../utils/asserts.js"; -import type { ClientsAndDeploymentsCountInfo } from "../../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../../utils/types.js"; import type { AzureOpenAI } from "openai"; describe("Realtime", () => { matrix([[APIVersion["2024_10_01_preview"]]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientAndDeployments: ClientsAndDeploymentsCountInfo; + let clientAndDeployments: ClientsAndDeploymentsInfo; beforeEach(async () => { clientAndDeployments = createClientsAndDeployments(apiVersion, { realtime: "true" }); diff --git a/sdk/openai/openai/test/public/chatCompletions.spec.ts b/sdk/openai/openai/test/public/chatCompletions.spec.ts index a2aaeab7f6c0..24f297ae98bd 100644 --- a/sdk/openai/openai/test/public/chatCompletions.spec.ts +++ b/sdk/openai/openai/test/public/chatCompletions.spec.ts @@ -21,11 +21,11 @@ import {zodResponseFormat} from "openai/helpers/zod"; import {type ChatCompletionMessageParam} from "openai/resources/chat/completions.mjs"; import {functionCallModelsToSkip, jsonResponseModelsToSkip} from "../utils/models.js"; import "../../src/types/index.js"; -import type {ClientsAndDeploymentsCountInfo} from "../utils/types.js"; +import type {ClientsAndDeploymentsInfo} from "../utils/types.js"; import {assertMathResponseOutput, type MathResponse} from "../utils/structuredOutputUtils.js"; describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersion) => { - let clientsAndDeploymentsInfo: ClientsAndDeploymentsCountInfo; + let clientsAndDeploymentsInfo: ClientsAndDeploymentsInfo; clientsAndDeploymentsInfo = createClientsAndDeployments( apiVersion, diff --git a/sdk/openai/openai/test/public/completions.spec.ts b/sdk/openai/openai/test/public/completions.spec.ts index 79a2f055cfde..32a88562084c 100644 --- a/sdk/openai/openai/test/public/completions.spec.ts +++ b/sdk/openai/openai/test/public/completions.spec.ts @@ -9,12 +9,12 @@ import { assertCompletions, assertCompletionsStream } from "../utils/asserts.js" import { APIMatrix, withDeployments } from "../utils/utils.js"; import { completionsModelsToSkip } from "../utils/models.js"; import "../../src/types/index.js"; -import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; describe("Legacy Completions", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsCountInfo; + let clientsAndDeployments: ClientsAndDeploymentsInfo; beforeEach(async () => { clientsAndDeployments = createClientsAndDeployments(apiVersion, { completion: "true" }); diff --git a/sdk/openai/openai/test/public/embeddings.spec.ts b/sdk/openai/openai/test/public/embeddings.spec.ts index d3697adbd42c..97f79d7b66e8 100644 --- a/sdk/openai/openai/test/public/embeddings.spec.ts +++ b/sdk/openai/openai/test/public/embeddings.spec.ts @@ -6,12 +6,12 @@ import { describe, it, beforeAll } from "vitest"; import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIMatrix, type APIVersion, withDeployments } from "../utils/utils.js"; import { assertEmbeddings } from "../utils/asserts.js"; -import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; describe("Embeddings", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsCountInfo; + let clientsAndDeployments: ClientsAndDeploymentsInfo; beforeAll(async function () { clientsAndDeployments = createClientsAndDeployments(apiVersion, { embeddings: "true" }); diff --git a/sdk/openai/openai/test/public/images.spec.ts b/sdk/openai/openai/test/public/images.spec.ts index b8df5150ae01..b2af35cf5ae7 100644 --- a/sdk/openai/openai/test/public/images.spec.ts +++ b/sdk/openai/openai/test/public/images.spec.ts @@ -6,12 +6,12 @@ import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIMatrix, type APIVersion, withDeployments } from "../utils/utils.js"; import { assertImagesWithJSON, assertImagesWithURLs } from "../utils/asserts.js"; import { describe, it, beforeAll } from "vitest"; -import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; describe("Images", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsCountInfo; + let clientsAndDeployments: ClientsAndDeploymentsInfo; beforeAll(async function () { clientsAndDeployments = createClientsAndDeployments(apiVersion, { diff --git a/sdk/openai/openai/test/public/node/batches.spec.ts b/sdk/openai/openai/test/public/node/batches.spec.ts index 112edd42130d..a04e3026cb89 100644 --- a/sdk/openai/openai/test/public/node/batches.spec.ts +++ b/sdk/openai/openai/test/public/node/batches.spec.ts @@ -9,12 +9,12 @@ import { APIVersion, withDeployments } from "../../utils/utils.js"; import { assertBatch, assertNonEmptyArray } from "../../utils/asserts.js"; import { delay } from "@azure/core-util"; import type { FileObject } from "openai/resources/index"; -import type { ClientsAndDeploymentsCountInfo } from "../../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../../utils/types.js"; describe("Batches", () => { matrix([[APIVersion.Preview]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientAndDeployments: ClientsAndDeploymentsCountInfo; + let clientAndDeployments: ClientsAndDeploymentsInfo; beforeEach(async () => { clientAndDeployments = createClientsAndDeployments( diff --git a/sdk/openai/openai/test/public/node/whisper.spec.ts b/sdk/openai/openai/test/public/node/whisper.spec.ts index f6c820ef4ee8..fe9eb409352d 100644 --- a/sdk/openai/openai/test/public/node/whisper.spec.ts +++ b/sdk/openai/openai/test/public/node/whisper.spec.ts @@ -13,12 +13,12 @@ import { } from "../../utils/utils.js"; import { assertAudioResult } from "../../utils/asserts.js"; import type { AudioResultFormat } from "../../utils/audioTypes.js"; -import type { ClientsAndDeploymentsCountInfo } from "../../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../../utils/types.js"; describe("Whisper", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsCountInfo; + let clientsAndDeployments: ClientsAndDeploymentsInfo; beforeEach(async () => { clientsAndDeployments = createClientsAndDeployments( diff --git a/sdk/openai/openai/test/public/node/wsRealtime.spec.ts b/sdk/openai/openai/test/public/node/wsRealtime.spec.ts index c1568769bb1e..68c7fb6f8a2b 100644 --- a/sdk/openai/openai/test/public/node/wsRealtime.spec.ts +++ b/sdk/openai/openai/test/public/node/wsRealtime.spec.ts @@ -11,13 +11,13 @@ import { assertSessionCreatedEvent, } from "../../utils/asserts.js"; import { createClientsAndDeployments } from "../../utils/createClients.js"; -import type { ClientsAndDeploymentsCountInfo } from "../../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../../utils/types.js"; import type { AzureOpenAI } from "openai"; describe("Realtime", () => { matrix([[APIVersion["2024_10_01_preview"]]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientAndDeployments: ClientsAndDeploymentsCountInfo; + let clientAndDeployments: ClientsAndDeploymentsInfo; beforeEach(async () => { clientAndDeployments = createClientsAndDeployments(apiVersion, { realtime: "true" }); diff --git a/sdk/openai/openai/test/public/tts.spec.ts b/sdk/openai/openai/test/public/tts.spec.ts index c13bc475628f..d923d093287e 100644 --- a/sdk/openai/openai/test/public/tts.spec.ts +++ b/sdk/openai/openai/test/public/tts.spec.ts @@ -5,12 +5,12 @@ import { matrix } from "@azure-tools/test-utils-vitest"; import { describe, beforeEach, it, assert } from "vitest"; import { createClientsAndDeployments } from "../utils/createClients.js"; import { APIVersion, withDeployments } from "../utils/utils.js"; -import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; describe("Text to speech", function () { matrix([[APIVersion.Preview]] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsCountInfo; + let clientsAndDeployments: ClientsAndDeploymentsInfo; beforeEach(async () => { clientsAndDeployments = createClientsAndDeployments( diff --git a/sdk/openai/openai/test/public/vision.spec.ts b/sdk/openai/openai/test/public/vision.spec.ts index b69a282a749e..fc60f36b5817 100644 --- a/sdk/openai/openai/test/public/vision.spec.ts +++ b/sdk/openai/openai/test/public/vision.spec.ts @@ -7,13 +7,13 @@ import { createClientsAndDeployments } from "../utils/createClients.js"; import { assertChatCompletions } from "../utils/asserts.js"; import { APIMatrix, type APIVersion, withDeployments } from "../utils/utils.js"; import { RestError } from "@azure/core-rest-pipeline"; -import type { ClientsAndDeploymentsCountInfo } from "../utils/types.js"; +import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; import { logger } from "../utils/logger.js"; describe("Vision", function () { matrix([APIMatrix] as const, async function (apiVersion: APIVersion) { describe(`[${apiVersion}] Client`, () => { - let clientsAndDeployments: ClientsAndDeploymentsCountInfo; + let clientsAndDeployments: ClientsAndDeploymentsInfo; beforeEach(async () => { clientsAndDeployments = createClientsAndDeployments( diff --git a/sdk/openai/openai/test/utils/createClients.ts b/sdk/openai/openai/test/utils/createClients.ts index 2b130e394c7e..59764c53bb26 100644 --- a/sdk/openai/openai/test/utils/createClients.ts +++ b/sdk/openai/openai/test/utils/createClients.ts @@ -6,7 +6,7 @@ import { getBearerTokenProvider } from "@azure/identity"; import { APIVersion, filterDeployments } from "./utils.js"; import { createLiveCredential } from "@azure-tools/test-credential"; import { getResourcesInfo } from "./injectables.js"; -import type { ClientsAndDeploymentsCountInfo, CreateClientOptions, ModelCapabilities } from "./types.js"; +import type { ClientsAndDeploymentsInfo, CreateClientOptions, ModelCapabilities } from "./types.js"; const scope = "https://cognitiveservices.azure.com/.default"; @@ -14,7 +14,7 @@ export function createClientsAndDeployments( apiVersion: APIVersion, capabilities: ModelCapabilities, options: CreateClientOptions = {}, -): ClientsAndDeploymentsCountInfo { +): ClientsAndDeploymentsInfo { const { clientOptions, sku, deploymentsToSkip, modelsToSkip } = options; const { resourcesInfo } = getResourcesInfo(); switch (apiVersion) { diff --git a/sdk/openai/openai/test/utils/types.ts b/sdk/openai/openai/test/utils/types.ts index 3f56e90528ad..1d7235001b5e 100644 --- a/sdk/openai/openai/test/utils/types.ts +++ b/sdk/openai/openai/test/utils/types.ts @@ -49,7 +49,7 @@ export interface ClientAndDeploymentsInfo { deployments: DeploymentInfo[]; } -export interface ClientsAndDeploymentsCountInfo { +export interface ClientsAndDeploymentsInfo { clientsAndDeployments: ClientAndDeploymentsInfo[]; count: number; } diff --git a/sdk/openai/openai/test/utils/utils.ts b/sdk/openai/openai/test/utils/utils.ts index f771aa16902a..370254340fd7 100644 --- a/sdk/openai/openai/test/utils/utils.ts +++ b/sdk/openai/openai/test/utils/utils.ts @@ -14,7 +14,7 @@ import { import type { Run } from "openai/resources/beta/threads/runs/runs.mjs"; import type { AzureChatExtensionConfiguration } from "../../src/types/index.js"; import { getAzureSearchEndpoint, getAzureSearchIndex } from "./injectables.js"; -import { ClientsAndDeploymentsCountInfo, ModelCapabilities, ModelInfo, ResourceInfo } from "./types.js"; +import { ClientsAndDeploymentsInfo, ModelCapabilities, ModelInfo, ResourceInfo } from "./types.js"; import { logger } from "./logger.js"; import type { OpenAI } from "openai"; import type { Sku } from "@azure/arm-cognitiveservices"; @@ -43,7 +43,7 @@ function toString(error: any): string { } export async function withDeployments( - { clientsAndDeployments, count }: ClientsAndDeploymentsCountInfo, + { clientsAndDeployments, count }: ClientsAndDeploymentsInfo, run: (client: OpenAI, model: string) => Promise, validate?: (result: T) => void, modelsListToSkip?: Partial[], @@ -110,7 +110,7 @@ export async function withDeployments( } export type DeploymentTestingParameters = { - clientsAndDeployments: ClientsAndDeploymentsCountInfo; + clientsAndDeployments: ClientsAndDeploymentsInfo; run: (client: OpenAI, model: string) => Promise; validate?: (result: T) => void; modelsListToSkip?: Partial[]; From 77f7733a3323a131723f3d600be3a2b01eb4d3d8 Mon Sep 17 00:00:00 2001 From: Deyaaeldeen Almahallawi Date: Thu, 6 Mar 2025 21:25:31 -0800 Subject: [PATCH 7/8] fix lint and format --- .../test/public/chatCompletions.spec.ts | 152 +++++++++--------- sdk/openai/openai/test/utils/utils.ts | 69 ++++---- 2 files changed, 114 insertions(+), 107 deletions(-) diff --git a/sdk/openai/openai/test/public/chatCompletions.spec.ts b/sdk/openai/openai/test/public/chatCompletions.spec.ts index 24f297ae98bd..c0569903ab92 100644 --- a/sdk/openai/openai/test/public/chatCompletions.spec.ts +++ b/sdk/openai/openai/test/public/chatCompletions.spec.ts @@ -1,9 +1,9 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -import {assert, describe, it} from "vitest"; -import {createClientsAndDeployments} from "../utils/createClients.js"; -import type {APIVersion} from "../utils/utils.js"; +import { assert, describe, it } from "vitest"; +import { createClientsAndDeployments } from "../utils/createClients.js"; +import type { APIVersion } from "../utils/utils.js"; import { APIMatrix, bufferAsyncIterable, @@ -16,23 +16,23 @@ import { assertChatCompletionsList, assertParsedChatCompletion, } from "../utils/asserts.js"; -import {z} from "zod"; -import {zodResponseFormat} from "openai/helpers/zod"; -import {type ChatCompletionMessageParam} from "openai/resources/chat/completions.mjs"; -import {functionCallModelsToSkip, jsonResponseModelsToSkip} from "../utils/models.js"; +import { z } from "zod"; +import { zodResponseFormat } from "openai/helpers/zod"; +import { type ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs"; +import { functionCallModelsToSkip, jsonResponseModelsToSkip } from "../utils/models.js"; import "../../src/types/index.js"; -import type {ClientsAndDeploymentsInfo} from "../utils/types.js"; -import {assertMathResponseOutput, type MathResponse} from "../utils/structuredOutputUtils.js"; +import type { ClientsAndDeploymentsInfo } from "../utils/types.js"; +import { assertMathResponseOutput, type MathResponse } from "../utils/structuredOutputUtils.js"; describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersion) => { let clientsAndDeploymentsInfo: ClientsAndDeploymentsInfo; clientsAndDeploymentsInfo = createClientsAndDeployments( apiVersion, - {chatCompletion: "true"}, + { chatCompletion: "true" }, { deploymentsToSkip: ["o1" /** It gets stuck and never returns */], - modelsToSkip: [{name: "gpt-4o-audio-preview"}, {name: "o3-mini"}], + modelsToSkip: [{ name: "gpt-4o-audio-preview" }, { name: "o3-mini" }], }, ); @@ -42,12 +42,12 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio role: "system", content: "You are a helpful assistant. You will talk like a pirate.", } as const, - {role: "user", content: "Can you help me?"} as const, + { role: "user", content: "Can you help me?" } as const, { role: "assistant", content: "Arrrr! Of course, me hearty! What can I do for ye?", } as const, - {role: "user", content: "What's the best way to train a parrot?"} as const, + { role: "user", content: "What's the best way to train a parrot?" } as const, ]; const byodMessages = [ { @@ -74,7 +74,7 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio required: ["location"], }, }; - // TODO: Change to arrow functions + it("returns completions across all models", async () => { await withDeployments( clientsAndDeploymentsInfo, @@ -92,14 +92,14 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio clientsAndDeploymentsInfo, async (client, deploymentName) => { const weatherMessages: ChatCompletionMessageParam[] = [ - {role: "user", content: "What's the weather like in Boston?"}, + { role: "user", content: "What's the weather like in Boston?" }, ]; const result = await client.chat.completions.create({ model: deploymentName, messages: weatherMessages, functions: [getCurrentWeather], }); - assertChatCompletions(result, {functions: true}); + assertChatCompletions(result, { functions: true }); const responseMessage = result.choices[0].message; if (!responseMessage?.function_call) { assert.fail("Undefined function call"); @@ -121,7 +121,7 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio messages: weatherMessages, }); }, - (result) => assertChatCompletions(result, {functions: true}), + (result) => assertChatCompletions(result, { functions: true }), functionCallModelsToSkip, ); }); @@ -132,12 +132,12 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio (client, deploymentName) => client.chat.completions.create({ model: deploymentName, - messages: [{role: "user", content: "What's the weather like in Boston?"}], + messages: [{ role: "user", content: "What's the weather like in Boston?" }], tool_choice: "none", - tools: [{type: "function", function: getCurrentWeather}], + tools: [{ type: "function", function: getCurrentWeather }], }), (res) => { - assertChatCompletions(res, {functions: false}); + assertChatCompletions(res, { functions: false }); assert.isUndefined(res.choices[0].message?.tool_calls); }, ); @@ -149,14 +149,14 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio (client, deploymentName) => client.chat.completions.create({ model: deploymentName, - messages: [{role: "user", content: "What's the weather like in Boston?"}], + messages: [{ role: "user", content: "What's the weather like in Boston?" }], tool_choice: { type: "function", - function: {name: getCurrentWeather.name}, + function: { name: getCurrentWeather.name }, }, tools: [ - {type: "function", function: getCurrentWeather}, + { type: "function", function: getCurrentWeather }, { type: "function", function: { @@ -181,7 +181,7 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio ], }), (res) => { - assertChatCompletions(res, {functions: true}); + assertChatCompletions(res, { functions: true }); const toolCalls = res.choices[0].message?.tool_calls; if (!toolCalls) { throw new Error("toolCalls should be defined here"); @@ -212,11 +212,11 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio (client, deploymentName) => client.chat.completions.create({ model: deploymentName, - messages: [{role: "user", content: "Give me information about Asset No1"}], - tools: [{type: "function", function: getAssetInfo}], + messages: [{ role: "user", content: "Give me information about Asset No1" }], + tools: [{ type: "function", function: getAssetInfo }], }), (res) => { - assertChatCompletions(res, {functions: true}); + assertChatCompletions(res, { functions: true }); const toolCalls = res.choices[0].message?.tool_calls; if (!toolCalls) { throw new Error("toolCalls should be defined here"); @@ -245,10 +245,10 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio "Answer the following question in JSON format: What are the capital cities in Africa?", }, ], - response_format: {type: "json_object"}, + response_format: { type: "json_object" }, }), (res) => { - assertChatCompletions(res, {functions: false}); + assertChatCompletions(res, { functions: false }); const content = res.choices[0].message?.content; if (!content) assert.fail("Undefined content"); try { @@ -273,9 +273,9 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio }), validate: assertChatCompletions, modelsListToSkip: [ - {name: "gpt-35-turbo-0613"}, // Unsupported model - {name: "gpt-4-32k"}, // Managed identity is not enabled - {name: "o1-preview"}, // o-series models are not supported with OYD. + { name: "gpt-35-turbo-0613" }, // Unsupported model + { name: "gpt-4-32k" }, // Managed identity is not enabled + { name: "o1-preview" }, // o-series models are not supported with OYD. ], acceptableErrors: { messageSubstring: [ @@ -316,7 +316,7 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio bufferAsyncIterable( await client.chat.completions.create({ model: deploymentName, - messages: [{role: "user", content: "What's the weather like in Boston?"}], + messages: [{ role: "user", content: "What's the weather like in Boston?" }], stream: true, functions: [getCurrentWeather], }), @@ -338,9 +338,9 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio bufferAsyncIterable( await client.chat.completions.create({ model: deploymentName, - messages: [{role: "user", content: "What's the weather like in Boston?"}], + messages: [{ role: "user", content: "What's the weather like in Boston?" }], stream: true, - tools: [{type: "function", function: getCurrentWeather}], + tools: [{ type: "function", function: getCurrentWeather }], }), ), (res) => @@ -354,7 +354,7 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio }); it("bring your data", async () => { - const dataSources = {data_sources: [createAzureSearchExtension()]}; + const dataSources = { data_sources: [createAzureSearchExtension()] }; await withDeployments( clientsAndDeploymentsInfo, async (client, deploymentName) => @@ -367,56 +367,54 @@ describe.shuffle.each(APIMatrix)("Chat Completions [%s]", (apiVersion: APIVersio }), ), assertChatCompletionsList, - [{name: "gpt-4", version: "vision-preview"}], + [{ name: "gpt-4", version: "vision-preview" }], ); }); describe("chat.completions.parse", () => { describe("structured output for chat completions", async () => { await testWithDeployments({ - clientsAndDeployments: clientsAndDeploymentsInfo, - run: async (client, deploymentName) => { - const step = z.object({ - explanation: z.string(), - output: z.string(), - }); + clientsAndDeployments: clientsAndDeploymentsInfo, + run: async (client, deploymentName) => { + const step = z.object({ + explanation: z.string(), + output: z.string(), + }); - const mathResponse = z.object({ - steps: z.array(step), - final_answer: z.string(), - }); + const mathResponse = z.object({ + steps: z.array(step), + final_answer: z.string(), + }); - return client.beta.chat.completions.parse({ - model: deploymentName, - messages: [ - { - role: "system", - content: - "You are a helpful math tutor. Only use the schema for math responses.", - }, - {role: "user", content: "solve 8x + 3 = 21"}, - ], - response_format: zodResponseFormat(mathResponse, "mathResponse"), - }); - } - , - validate: (result) => { - assertParsedChatCompletion(result, assertMathResponseOutput, { - allowEmptyChoices: true, - }); - }, - modelsListToSkip: [ - // structured output is not supported - {name: "gpt-35-turbo"}, - {name: "gpt-4"}, - {name: "gpt-4-32k"}, - {name: "gpt-35-turbo-16k"}, - {name: "o1-preview"}, - {name: "gpt-4-32k"}, - {name: "gpt-4o", version: "2024-05-13"}, - ], + return client.beta.chat.completions.parse({ + model: deploymentName, + messages: [ + { + role: "system", + content: + "You are a helpful math tutor. Only use the schema for math responses.", + }, + { role: "user", content: "solve 8x + 3 = 21" }, + ], + response_format: zodResponseFormat(mathResponse, "mathResponse"), + }); + }, + validate: (result) => { + assertParsedChatCompletion(result, assertMathResponseOutput, { + allowEmptyChoices: true, + }); }, - ); + modelsListToSkip: [ + // structured output is not supported + { name: "gpt-35-turbo" }, + { name: "gpt-4" }, + { name: "gpt-4-32k" }, + { name: "gpt-35-turbo-16k" }, + { name: "o1-preview" }, + { name: "gpt-4-32k" }, + { name: "gpt-4o", version: "2024-05-13" }, + ], + }); }); }); }); diff --git a/sdk/openai/openai/test/utils/utils.ts b/sdk/openai/openai/test/utils/utils.ts index 370254340fd7..4b0fb802b458 100644 --- a/sdk/openai/openai/test/utils/utils.ts +++ b/sdk/openai/openai/test/utils/utils.ts @@ -14,7 +14,12 @@ import { import type { Run } from "openai/resources/beta/threads/runs/runs.mjs"; import type { AzureChatExtensionConfiguration } from "../../src/types/index.js"; import { getAzureSearchEndpoint, getAzureSearchIndex } from "./injectables.js"; -import { ClientsAndDeploymentsInfo, ModelCapabilities, ModelInfo, ResourceInfo } from "./types.js"; +import type { + ClientsAndDeploymentsInfo, + ModelCapabilities, + ModelInfo, + ResourceInfo, +} from "./types.js"; import { logger } from "./logger.js"; import type { OpenAI } from "openai"; import type { Sku } from "@azure/arm-cognitiveservices"; @@ -25,7 +30,7 @@ export type AcceptableErrors = { const GlobalAcceptedErrors: AcceptableErrors = { messageSubstring: ["Rate limit is exceeded"], -} +}; export const maxRetriesOption = { maxRetries: 0 }; @@ -119,11 +124,11 @@ export type DeploymentTestingParameters = { /** * Test with deployments invokes `test` call, so it should be inside of `describe` and not `it`. - * @param clientsAndDeployments - * @param run - * @param validate - * @param modelsListToSkip - * @param acceptableErrors + * @param clientsAndDeployments - + * @param run - + * @param validate - + * @param modelsListToSkip - + * @param acceptableErrors - */ export async function testWithDeployments({ clientsAndDeployments, @@ -134,31 +139,35 @@ export async function testWithDeployments({ }: DeploymentTestingParameters): Promise { assert.isNotEmpty(clientsAndDeployments, "No deployments found"); describe.concurrent.each(clientsAndDeployments.clientsAndDeployments)( - "$client.baseURL", async function({ client, deployments }) { - for (const deployment of deployments) { - test.concurrent(`${deployment.model.name} (${deployment.model.version})`, async (done) => { - if (modelsListToSkip && isModelInList(deployment.model, modelsListToSkip)) { - done.skip(`Skipping ${deployment.model.name} : ${deployment.model.version}`); - } - - let result; - try { - result = await run(client, deployment.deploymentName); - } catch (e) { - const error = e as any; - if (acceptableErrors?.messageSubstring.some((match) => error.message.includes(match))) { - done.skip(`Skipping due to acceptable error: ${error}`); + "$client.baseURL", + async function ({ client, deployments }) { + for (const deployment of deployments) { + test.concurrent(`${deployment.model.name} (${deployment.model.version})`, async (done) => { + if (modelsListToSkip && isModelInList(deployment.model, modelsListToSkip)) { + done.skip(`Skipping ${deployment.model.name} : ${deployment.model.version}`); } - if (GlobalAcceptedErrors.messageSubstring.some((match) => error.message.includes(match))) { - done.skip(`Skipping due to global acceptable error: ${error}`); + + let result; + try { + result = await run(client, deployment.deploymentName); + } catch (e) { + const error = e as any; + if (acceptableErrors?.messageSubstring.some((match) => error.message.includes(match))) { + done.skip(`Skipping due to acceptable error: ${error}`); + } + if ( + GlobalAcceptedErrors.messageSubstring.some((match) => error.message.includes(match)) + ) { + done.skip(`Skipping due to global acceptable error: ${error}`); + } + throw e; } - throw e; - } - validate?.(result); - return; - }); - } - }); + validate?.(result); + return; + }); + } + }, + ); } export function filterDeployments( From a3a3578481fea0a9d5dc145ab83a8ca3a354c4c9 Mon Sep 17 00:00:00 2001 From: Deyaaeldeen Almahallawi Date: Mon, 10 Mar 2025 12:08:40 -0700 Subject: [PATCH 8/8] fix live tests --- sdk/openai/openai/test/snippets.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/openai/openai/test/snippets.spec.ts b/sdk/openai/openai/test/snippets.spec.ts index d8e0bfcef329..f44cf06df96e 100644 --- a/sdk/openai/openai/test/snippets.spec.ts +++ b/sdk/openai/openai/test/snippets.spec.ts @@ -10,7 +10,7 @@ import "@azure/openai/types"; describe("snippets", () => { it("ReadmeSampleAnalyzeBusinessData", async () => { // Import OpenAI Type Helpers - import "@azure/openai/types"; + // FIX: import "@azure/openai/types"; // @ts-preserve-whitespace const credential = new DefaultAzureCredential(); const scope = "https://cognitiveservices.azure.com/.default"; @@ -57,7 +57,7 @@ describe("snippets", () => { it("ReadmeSampleContentFilteredChatCompletions", async () => { // Import OpenAI Type Helpers - import "@azure/openai/types"; + // FIX: import "@azure/openai/types"; // @ts-preserve-whitespace const credential = new DefaultAzureCredential(); const scope = "https://cognitiveservices.azure.com/.default";