diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 33a65d75c4..21621582fa 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.77.0" + ".": "1.78.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 0c8278866d..5f1bee851b 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 97 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-0ee6b36cf3cc278cef4199a6aec5f7d530a6c1f17a74830037e96d50ca1edc50.yml -openapi_spec_hash: e8ec5f46bc0655b34f292422d58a60f6 -config_hash: d9b6b6e6bc85744663e300eebc482067 +configured_endpoints: 101 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/openai%2Fopenai-794a6ed3c3d3d77887564755168056af8a426b17cf1ec721e3a300503dc22a41.yml +openapi_spec_hash: 25a81c220713cd5b0bafc221d1dfa79a +config_hash: 0b768ed1b56c6d82816f0fa40dc4aaf5 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9097cdc65a..8648497457 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## 1.78.0 (2025-05-08) + +Full Changelog: [v1.77.0...v1.78.0](https://github.com/openai/openai-python/compare/v1.77.0...v1.78.0) + +### Features + +* **api:** Add reinforcement fine-tuning api support ([bebe361](https://github.com/openai/openai-python/commit/bebe36104bd3062d09ab9bbfb4bacfc99e737cb2)) + + +### Bug Fixes + +* ignore errors in isinstance() calls on LazyProxy subclasses ([#2343](https://github.com/openai/openai-python/issues/2343)) ([52cbbdf](https://github.com/openai/openai-python/commit/52cbbdf2207567741f16d18f1ea1b0d13d667375)), closes [#2056](https://github.com/openai/openai-python/issues/2056) + + +### Chores + +* **internal:** update proxy tests ([b8e848d](https://github.com/openai/openai-python/commit/b8e848d5fb58472cbfa27fb3ed01efc25a05d944)) +* use lazy imports for module level client ([4d0f409](https://github.com/openai/openai-python/commit/4d0f409e79a18cce9855fe076f5a50e52b8bafd8)) +* use lazy imports for resources ([834813c](https://github.com/openai/openai-python/commit/834813c5cb1a84effc34e5eabed760393e1de806)) + ## 1.77.0 (2025-05-02) Full Changelog: [v1.76.2...v1.77.0](https://github.com/openai/openai-python/compare/v1.76.2...v1.77.0) diff --git a/api.md b/api.md index d04c76960e..496e5548b3 100644 --- a/api.md +++ b/api.md @@ -225,6 +225,21 @@ Methods: # FineTuning +## Methods + +Types: + +```python +from openai.types.fine_tuning import ( + DpoHyperparameters, + DpoMethod, + ReinforcementHyperparameters, + ReinforcementMethod, + SupervisedHyperparameters, + SupervisedMethod, +) +``` + ## Jobs Types: @@ -246,6 +261,8 @@ Methods: - client.fine_tuning.jobs.list(\*\*params) -> SyncCursorPage[FineTuningJob] - client.fine_tuning.jobs.cancel(fine_tuning_job_id) -> FineTuningJob - client.fine_tuning.jobs.list_events(fine_tuning_job_id, \*\*params) -> SyncCursorPage[FineTuningJobEvent] +- client.fine_tuning.jobs.pause(fine_tuning_job_id) -> FineTuningJob +- client.fine_tuning.jobs.resume(fine_tuning_job_id) -> FineTuningJob ### Checkpoints @@ -279,6 +296,38 @@ Methods: - client.fine_tuning.checkpoints.permissions.retrieve(fine_tuned_model_checkpoint, \*\*params) -> PermissionRetrieveResponse - client.fine_tuning.checkpoints.permissions.delete(permission_id, \*, fine_tuned_model_checkpoint) -> PermissionDeleteResponse +## Alpha + +### Graders + +Types: + +```python +from openai.types.fine_tuning.alpha import GraderRunResponse, GraderValidateResponse +``` + +Methods: + +- client.fine_tuning.alpha.graders.run(\*\*params) -> GraderRunResponse +- client.fine_tuning.alpha.graders.validate(\*\*params) -> GraderValidateResponse + +# Graders + +## GraderModels + +Types: + +```python +from openai.types.graders import ( + LabelModelGrader, + MultiGrader, + PythonGrader, + ScoreModelGrader, + StringCheckGrader, + TextSimilarityGrader, +) +``` + # VectorStores Types: @@ -738,10 +787,7 @@ Types: ```python from openai.types import ( EvalCustomDataSourceConfig, - EvalLabelModelGrader, EvalStoredCompletionsDataSourceConfig, - EvalStringCheckGrader, - EvalTextSimilarityGrader, EvalCreateResponse, EvalRetrieveResponse, EvalUpdateResponse, diff --git a/pyproject.toml b/pyproject.toml index 4b854b05e5..3d5af260cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openai" -version = "1.77.0" +version = "1.78.0" description = "The official Python library for the openai API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/src/openai/_client.py b/src/openai/_client.py index 3aca6cb124..b251ab0917 100644 --- a/src/openai/_client.py +++ b/src/openai/_client.py @@ -3,7 +3,7 @@ from __future__ import annotations import os -from typing import Any, Union, Mapping +from typing import TYPE_CHECKING, Any, Union, Mapping from typing_extensions import Self, override import httpx @@ -24,8 +24,8 @@ is_mapping, get_async_library, ) +from ._compat import cached_property from ._version import __version__ -from .resources import files, images, models, batches, embeddings, completions, moderations from ._streaming import Stream as Stream, AsyncStream as AsyncStream from ._exceptions import OpenAIError, APIStatusError from ._base_client import ( @@ -33,37 +33,45 @@ SyncAPIClient, AsyncAPIClient, ) -from .resources.beta import beta -from .resources.chat import chat -from .resources.audio import audio -from .resources.evals import evals -from .resources.uploads import uploads -from .resources.responses import responses -from .resources.fine_tuning import fine_tuning -from .resources.vector_stores import vector_stores + +if TYPE_CHECKING: + from .resources import ( + beta, + chat, + audio, + evals, + files, + images, + models, + batches, + uploads, + responses, + embeddings, + completions, + fine_tuning, + moderations, + vector_stores, + ) + from .resources.files import Files, AsyncFiles + from .resources.images import Images, AsyncImages + from .resources.models import Models, AsyncModels + from .resources.batches import Batches, AsyncBatches + from .resources.beta.beta import Beta, AsyncBeta + from .resources.chat.chat import Chat, AsyncChat + from .resources.embeddings import Embeddings, AsyncEmbeddings + from .resources.audio.audio import Audio, AsyncAudio + from .resources.completions import Completions, AsyncCompletions + from .resources.evals.evals import Evals, AsyncEvals + from .resources.moderations import Moderations, AsyncModerations + from .resources.uploads.uploads import Uploads, AsyncUploads + from .resources.responses.responses import Responses, AsyncResponses + from .resources.fine_tuning.fine_tuning import FineTuning, AsyncFineTuning + from .resources.vector_stores.vector_stores import VectorStores, AsyncVectorStores __all__ = ["Timeout", "Transport", "ProxiesTypes", "RequestOptions", "OpenAI", "AsyncOpenAI", "Client", "AsyncClient"] class OpenAI(SyncAPIClient): - completions: completions.Completions - chat: chat.Chat - embeddings: embeddings.Embeddings - files: files.Files - images: images.Images - audio: audio.Audio - moderations: moderations.Moderations - models: models.Models - fine_tuning: fine_tuning.FineTuning - vector_stores: vector_stores.VectorStores - beta: beta.Beta - batches: batches.Batches - uploads: uploads.Uploads - responses: responses.Responses - evals: evals.Evals - with_raw_response: OpenAIWithRawResponse - with_streaming_response: OpenAIWithStreamedResponse - # client options api_key: str organization: str | None @@ -146,23 +154,103 @@ def __init__( self._default_stream_cls = Stream - self.completions = completions.Completions(self) - self.chat = chat.Chat(self) - self.embeddings = embeddings.Embeddings(self) - self.files = files.Files(self) - self.images = images.Images(self) - self.audio = audio.Audio(self) - self.moderations = moderations.Moderations(self) - self.models = models.Models(self) - self.fine_tuning = fine_tuning.FineTuning(self) - self.vector_stores = vector_stores.VectorStores(self) - self.beta = beta.Beta(self) - self.batches = batches.Batches(self) - self.uploads = uploads.Uploads(self) - self.responses = responses.Responses(self) - self.evals = evals.Evals(self) - self.with_raw_response = OpenAIWithRawResponse(self) - self.with_streaming_response = OpenAIWithStreamedResponse(self) + @cached_property + def completions(self) -> Completions: + from .resources.completions import Completions + + return Completions(self) + + @cached_property + def chat(self) -> Chat: + from .resources.chat import Chat + + return Chat(self) + + @cached_property + def embeddings(self) -> Embeddings: + from .resources.embeddings import Embeddings + + return Embeddings(self) + + @cached_property + def files(self) -> Files: + from .resources.files import Files + + return Files(self) + + @cached_property + def images(self) -> Images: + from .resources.images import Images + + return Images(self) + + @cached_property + def audio(self) -> Audio: + from .resources.audio import Audio + + return Audio(self) + + @cached_property + def moderations(self) -> Moderations: + from .resources.moderations import Moderations + + return Moderations(self) + + @cached_property + def models(self) -> Models: + from .resources.models import Models + + return Models(self) + + @cached_property + def fine_tuning(self) -> FineTuning: + from .resources.fine_tuning import FineTuning + + return FineTuning(self) + + @cached_property + def vector_stores(self) -> VectorStores: + from .resources.vector_stores import VectorStores + + return VectorStores(self) + + @cached_property + def beta(self) -> Beta: + from .resources.beta import Beta + + return Beta(self) + + @cached_property + def batches(self) -> Batches: + from .resources.batches import Batches + + return Batches(self) + + @cached_property + def uploads(self) -> Uploads: + from .resources.uploads import Uploads + + return Uploads(self) + + @cached_property + def responses(self) -> Responses: + from .resources.responses import Responses + + return Responses(self) + + @cached_property + def evals(self) -> Evals: + from .resources.evals import Evals + + return Evals(self) + + @cached_property + def with_raw_response(self) -> OpenAIWithRawResponse: + return OpenAIWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> OpenAIWithStreamedResponse: + return OpenAIWithStreamedResponse(self) @property @override @@ -279,24 +367,6 @@ def _make_status_error( class AsyncOpenAI(AsyncAPIClient): - completions: completions.AsyncCompletions - chat: chat.AsyncChat - embeddings: embeddings.AsyncEmbeddings - files: files.AsyncFiles - images: images.AsyncImages - audio: audio.AsyncAudio - moderations: moderations.AsyncModerations - models: models.AsyncModels - fine_tuning: fine_tuning.AsyncFineTuning - vector_stores: vector_stores.AsyncVectorStores - beta: beta.AsyncBeta - batches: batches.AsyncBatches - uploads: uploads.AsyncUploads - responses: responses.AsyncResponses - evals: evals.AsyncEvals - with_raw_response: AsyncOpenAIWithRawResponse - with_streaming_response: AsyncOpenAIWithStreamedResponse - # client options api_key: str organization: str | None @@ -379,23 +449,103 @@ def __init__( self._default_stream_cls = AsyncStream - self.completions = completions.AsyncCompletions(self) - self.chat = chat.AsyncChat(self) - self.embeddings = embeddings.AsyncEmbeddings(self) - self.files = files.AsyncFiles(self) - self.images = images.AsyncImages(self) - self.audio = audio.AsyncAudio(self) - self.moderations = moderations.AsyncModerations(self) - self.models = models.AsyncModels(self) - self.fine_tuning = fine_tuning.AsyncFineTuning(self) - self.vector_stores = vector_stores.AsyncVectorStores(self) - self.beta = beta.AsyncBeta(self) - self.batches = batches.AsyncBatches(self) - self.uploads = uploads.AsyncUploads(self) - self.responses = responses.AsyncResponses(self) - self.evals = evals.AsyncEvals(self) - self.with_raw_response = AsyncOpenAIWithRawResponse(self) - self.with_streaming_response = AsyncOpenAIWithStreamedResponse(self) + @cached_property + def completions(self) -> AsyncCompletions: + from .resources.completions import AsyncCompletions + + return AsyncCompletions(self) + + @cached_property + def chat(self) -> AsyncChat: + from .resources.chat import AsyncChat + + return AsyncChat(self) + + @cached_property + def embeddings(self) -> AsyncEmbeddings: + from .resources.embeddings import AsyncEmbeddings + + return AsyncEmbeddings(self) + + @cached_property + def files(self) -> AsyncFiles: + from .resources.files import AsyncFiles + + return AsyncFiles(self) + + @cached_property + def images(self) -> AsyncImages: + from .resources.images import AsyncImages + + return AsyncImages(self) + + @cached_property + def audio(self) -> AsyncAudio: + from .resources.audio import AsyncAudio + + return AsyncAudio(self) + + @cached_property + def moderations(self) -> AsyncModerations: + from .resources.moderations import AsyncModerations + + return AsyncModerations(self) + + @cached_property + def models(self) -> AsyncModels: + from .resources.models import AsyncModels + + return AsyncModels(self) + + @cached_property + def fine_tuning(self) -> AsyncFineTuning: + from .resources.fine_tuning import AsyncFineTuning + + return AsyncFineTuning(self) + + @cached_property + def vector_stores(self) -> AsyncVectorStores: + from .resources.vector_stores import AsyncVectorStores + + return AsyncVectorStores(self) + + @cached_property + def beta(self) -> AsyncBeta: + from .resources.beta import AsyncBeta + + return AsyncBeta(self) + + @cached_property + def batches(self) -> AsyncBatches: + from .resources.batches import AsyncBatches + + return AsyncBatches(self) + + @cached_property + def uploads(self) -> AsyncUploads: + from .resources.uploads import AsyncUploads + + return AsyncUploads(self) + + @cached_property + def responses(self) -> AsyncResponses: + from .resources.responses import AsyncResponses + + return AsyncResponses(self) + + @cached_property + def evals(self) -> AsyncEvals: + from .resources.evals import AsyncEvals + + return AsyncEvals(self) + + @cached_property + def with_raw_response(self) -> AsyncOpenAIWithRawResponse: + return AsyncOpenAIWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncOpenAIWithStreamedResponse: + return AsyncOpenAIWithStreamedResponse(self) @property @override @@ -512,79 +662,391 @@ def _make_status_error( class OpenAIWithRawResponse: + _client: OpenAI + def __init__(self, client: OpenAI) -> None: - self.completions = completions.CompletionsWithRawResponse(client.completions) - self.chat = chat.ChatWithRawResponse(client.chat) - self.embeddings = embeddings.EmbeddingsWithRawResponse(client.embeddings) - self.files = files.FilesWithRawResponse(client.files) - self.images = images.ImagesWithRawResponse(client.images) - self.audio = audio.AudioWithRawResponse(client.audio) - self.moderations = moderations.ModerationsWithRawResponse(client.moderations) - self.models = models.ModelsWithRawResponse(client.models) - self.fine_tuning = fine_tuning.FineTuningWithRawResponse(client.fine_tuning) - self.vector_stores = vector_stores.VectorStoresWithRawResponse(client.vector_stores) - self.beta = beta.BetaWithRawResponse(client.beta) - self.batches = batches.BatchesWithRawResponse(client.batches) - self.uploads = uploads.UploadsWithRawResponse(client.uploads) - self.responses = responses.ResponsesWithRawResponse(client.responses) - self.evals = evals.EvalsWithRawResponse(client.evals) + self._client = client + + @cached_property + def completions(self) -> completions.CompletionsWithRawResponse: + from .resources.completions import CompletionsWithRawResponse + + return CompletionsWithRawResponse(self._client.completions) + + @cached_property + def chat(self) -> chat.ChatWithRawResponse: + from .resources.chat import ChatWithRawResponse + + return ChatWithRawResponse(self._client.chat) + + @cached_property + def embeddings(self) -> embeddings.EmbeddingsWithRawResponse: + from .resources.embeddings import EmbeddingsWithRawResponse + + return EmbeddingsWithRawResponse(self._client.embeddings) + + @cached_property + def files(self) -> files.FilesWithRawResponse: + from .resources.files import FilesWithRawResponse + + return FilesWithRawResponse(self._client.files) + + @cached_property + def images(self) -> images.ImagesWithRawResponse: + from .resources.images import ImagesWithRawResponse + + return ImagesWithRawResponse(self._client.images) + + @cached_property + def audio(self) -> audio.AudioWithRawResponse: + from .resources.audio import AudioWithRawResponse + + return AudioWithRawResponse(self._client.audio) + + @cached_property + def moderations(self) -> moderations.ModerationsWithRawResponse: + from .resources.moderations import ModerationsWithRawResponse + + return ModerationsWithRawResponse(self._client.moderations) + + @cached_property + def models(self) -> models.ModelsWithRawResponse: + from .resources.models import ModelsWithRawResponse + + return ModelsWithRawResponse(self._client.models) + + @cached_property + def fine_tuning(self) -> fine_tuning.FineTuningWithRawResponse: + from .resources.fine_tuning import FineTuningWithRawResponse + + return FineTuningWithRawResponse(self._client.fine_tuning) + + @cached_property + def vector_stores(self) -> vector_stores.VectorStoresWithRawResponse: + from .resources.vector_stores import VectorStoresWithRawResponse + + return VectorStoresWithRawResponse(self._client.vector_stores) + + @cached_property + def beta(self) -> beta.BetaWithRawResponse: + from .resources.beta import BetaWithRawResponse + + return BetaWithRawResponse(self._client.beta) + + @cached_property + def batches(self) -> batches.BatchesWithRawResponse: + from .resources.batches import BatchesWithRawResponse + + return BatchesWithRawResponse(self._client.batches) + + @cached_property + def uploads(self) -> uploads.UploadsWithRawResponse: + from .resources.uploads import UploadsWithRawResponse + + return UploadsWithRawResponse(self._client.uploads) + + @cached_property + def responses(self) -> responses.ResponsesWithRawResponse: + from .resources.responses import ResponsesWithRawResponse + + return ResponsesWithRawResponse(self._client.responses) + + @cached_property + def evals(self) -> evals.EvalsWithRawResponse: + from .resources.evals import EvalsWithRawResponse + + return EvalsWithRawResponse(self._client.evals) class AsyncOpenAIWithRawResponse: + _client: AsyncOpenAI + def __init__(self, client: AsyncOpenAI) -> None: - self.completions = completions.AsyncCompletionsWithRawResponse(client.completions) - self.chat = chat.AsyncChatWithRawResponse(client.chat) - self.embeddings = embeddings.AsyncEmbeddingsWithRawResponse(client.embeddings) - self.files = files.AsyncFilesWithRawResponse(client.files) - self.images = images.AsyncImagesWithRawResponse(client.images) - self.audio = audio.AsyncAudioWithRawResponse(client.audio) - self.moderations = moderations.AsyncModerationsWithRawResponse(client.moderations) - self.models = models.AsyncModelsWithRawResponse(client.models) - self.fine_tuning = fine_tuning.AsyncFineTuningWithRawResponse(client.fine_tuning) - self.vector_stores = vector_stores.AsyncVectorStoresWithRawResponse(client.vector_stores) - self.beta = beta.AsyncBetaWithRawResponse(client.beta) - self.batches = batches.AsyncBatchesWithRawResponse(client.batches) - self.uploads = uploads.AsyncUploadsWithRawResponse(client.uploads) - self.responses = responses.AsyncResponsesWithRawResponse(client.responses) - self.evals = evals.AsyncEvalsWithRawResponse(client.evals) + self._client = client + + @cached_property + def completions(self) -> completions.AsyncCompletionsWithRawResponse: + from .resources.completions import AsyncCompletionsWithRawResponse + + return AsyncCompletionsWithRawResponse(self._client.completions) + + @cached_property + def chat(self) -> chat.AsyncChatWithRawResponse: + from .resources.chat import AsyncChatWithRawResponse + + return AsyncChatWithRawResponse(self._client.chat) + + @cached_property + def embeddings(self) -> embeddings.AsyncEmbeddingsWithRawResponse: + from .resources.embeddings import AsyncEmbeddingsWithRawResponse + + return AsyncEmbeddingsWithRawResponse(self._client.embeddings) + + @cached_property + def files(self) -> files.AsyncFilesWithRawResponse: + from .resources.files import AsyncFilesWithRawResponse + + return AsyncFilesWithRawResponse(self._client.files) + + @cached_property + def images(self) -> images.AsyncImagesWithRawResponse: + from .resources.images import AsyncImagesWithRawResponse + + return AsyncImagesWithRawResponse(self._client.images) + + @cached_property + def audio(self) -> audio.AsyncAudioWithRawResponse: + from .resources.audio import AsyncAudioWithRawResponse + + return AsyncAudioWithRawResponse(self._client.audio) + + @cached_property + def moderations(self) -> moderations.AsyncModerationsWithRawResponse: + from .resources.moderations import AsyncModerationsWithRawResponse + + return AsyncModerationsWithRawResponse(self._client.moderations) + + @cached_property + def models(self) -> models.AsyncModelsWithRawResponse: + from .resources.models import AsyncModelsWithRawResponse + + return AsyncModelsWithRawResponse(self._client.models) + + @cached_property + def fine_tuning(self) -> fine_tuning.AsyncFineTuningWithRawResponse: + from .resources.fine_tuning import AsyncFineTuningWithRawResponse + + return AsyncFineTuningWithRawResponse(self._client.fine_tuning) + + @cached_property + def vector_stores(self) -> vector_stores.AsyncVectorStoresWithRawResponse: + from .resources.vector_stores import AsyncVectorStoresWithRawResponse + + return AsyncVectorStoresWithRawResponse(self._client.vector_stores) + + @cached_property + def beta(self) -> beta.AsyncBetaWithRawResponse: + from .resources.beta import AsyncBetaWithRawResponse + + return AsyncBetaWithRawResponse(self._client.beta) + + @cached_property + def batches(self) -> batches.AsyncBatchesWithRawResponse: + from .resources.batches import AsyncBatchesWithRawResponse + + return AsyncBatchesWithRawResponse(self._client.batches) + + @cached_property + def uploads(self) -> uploads.AsyncUploadsWithRawResponse: + from .resources.uploads import AsyncUploadsWithRawResponse + + return AsyncUploadsWithRawResponse(self._client.uploads) + + @cached_property + def responses(self) -> responses.AsyncResponsesWithRawResponse: + from .resources.responses import AsyncResponsesWithRawResponse + + return AsyncResponsesWithRawResponse(self._client.responses) + + @cached_property + def evals(self) -> evals.AsyncEvalsWithRawResponse: + from .resources.evals import AsyncEvalsWithRawResponse + + return AsyncEvalsWithRawResponse(self._client.evals) class OpenAIWithStreamedResponse: + _client: OpenAI + def __init__(self, client: OpenAI) -> None: - self.completions = completions.CompletionsWithStreamingResponse(client.completions) - self.chat = chat.ChatWithStreamingResponse(client.chat) - self.embeddings = embeddings.EmbeddingsWithStreamingResponse(client.embeddings) - self.files = files.FilesWithStreamingResponse(client.files) - self.images = images.ImagesWithStreamingResponse(client.images) - self.audio = audio.AudioWithStreamingResponse(client.audio) - self.moderations = moderations.ModerationsWithStreamingResponse(client.moderations) - self.models = models.ModelsWithStreamingResponse(client.models) - self.fine_tuning = fine_tuning.FineTuningWithStreamingResponse(client.fine_tuning) - self.vector_stores = vector_stores.VectorStoresWithStreamingResponse(client.vector_stores) - self.beta = beta.BetaWithStreamingResponse(client.beta) - self.batches = batches.BatchesWithStreamingResponse(client.batches) - self.uploads = uploads.UploadsWithStreamingResponse(client.uploads) - self.responses = responses.ResponsesWithStreamingResponse(client.responses) - self.evals = evals.EvalsWithStreamingResponse(client.evals) + self._client = client + + @cached_property + def completions(self) -> completions.CompletionsWithStreamingResponse: + from .resources.completions import CompletionsWithStreamingResponse + + return CompletionsWithStreamingResponse(self._client.completions) + + @cached_property + def chat(self) -> chat.ChatWithStreamingResponse: + from .resources.chat import ChatWithStreamingResponse + + return ChatWithStreamingResponse(self._client.chat) + + @cached_property + def embeddings(self) -> embeddings.EmbeddingsWithStreamingResponse: + from .resources.embeddings import EmbeddingsWithStreamingResponse + + return EmbeddingsWithStreamingResponse(self._client.embeddings) + + @cached_property + def files(self) -> files.FilesWithStreamingResponse: + from .resources.files import FilesWithStreamingResponse + + return FilesWithStreamingResponse(self._client.files) + + @cached_property + def images(self) -> images.ImagesWithStreamingResponse: + from .resources.images import ImagesWithStreamingResponse + + return ImagesWithStreamingResponse(self._client.images) + + @cached_property + def audio(self) -> audio.AudioWithStreamingResponse: + from .resources.audio import AudioWithStreamingResponse + + return AudioWithStreamingResponse(self._client.audio) + + @cached_property + def moderations(self) -> moderations.ModerationsWithStreamingResponse: + from .resources.moderations import ModerationsWithStreamingResponse + + return ModerationsWithStreamingResponse(self._client.moderations) + + @cached_property + def models(self) -> models.ModelsWithStreamingResponse: + from .resources.models import ModelsWithStreamingResponse + + return ModelsWithStreamingResponse(self._client.models) + + @cached_property + def fine_tuning(self) -> fine_tuning.FineTuningWithStreamingResponse: + from .resources.fine_tuning import FineTuningWithStreamingResponse + + return FineTuningWithStreamingResponse(self._client.fine_tuning) + + @cached_property + def vector_stores(self) -> vector_stores.VectorStoresWithStreamingResponse: + from .resources.vector_stores import VectorStoresWithStreamingResponse + + return VectorStoresWithStreamingResponse(self._client.vector_stores) + + @cached_property + def beta(self) -> beta.BetaWithStreamingResponse: + from .resources.beta import BetaWithStreamingResponse + + return BetaWithStreamingResponse(self._client.beta) + + @cached_property + def batches(self) -> batches.BatchesWithStreamingResponse: + from .resources.batches import BatchesWithStreamingResponse + + return BatchesWithStreamingResponse(self._client.batches) + + @cached_property + def uploads(self) -> uploads.UploadsWithStreamingResponse: + from .resources.uploads import UploadsWithStreamingResponse + + return UploadsWithStreamingResponse(self._client.uploads) + + @cached_property + def responses(self) -> responses.ResponsesWithStreamingResponse: + from .resources.responses import ResponsesWithStreamingResponse + + return ResponsesWithStreamingResponse(self._client.responses) + + @cached_property + def evals(self) -> evals.EvalsWithStreamingResponse: + from .resources.evals import EvalsWithStreamingResponse + + return EvalsWithStreamingResponse(self._client.evals) class AsyncOpenAIWithStreamedResponse: + _client: AsyncOpenAI + def __init__(self, client: AsyncOpenAI) -> None: - self.completions = completions.AsyncCompletionsWithStreamingResponse(client.completions) - self.chat = chat.AsyncChatWithStreamingResponse(client.chat) - self.embeddings = embeddings.AsyncEmbeddingsWithStreamingResponse(client.embeddings) - self.files = files.AsyncFilesWithStreamingResponse(client.files) - self.images = images.AsyncImagesWithStreamingResponse(client.images) - self.audio = audio.AsyncAudioWithStreamingResponse(client.audio) - self.moderations = moderations.AsyncModerationsWithStreamingResponse(client.moderations) - self.models = models.AsyncModelsWithStreamingResponse(client.models) - self.fine_tuning = fine_tuning.AsyncFineTuningWithStreamingResponse(client.fine_tuning) - self.vector_stores = vector_stores.AsyncVectorStoresWithStreamingResponse(client.vector_stores) - self.beta = beta.AsyncBetaWithStreamingResponse(client.beta) - self.batches = batches.AsyncBatchesWithStreamingResponse(client.batches) - self.uploads = uploads.AsyncUploadsWithStreamingResponse(client.uploads) - self.responses = responses.AsyncResponsesWithStreamingResponse(client.responses) - self.evals = evals.AsyncEvalsWithStreamingResponse(client.evals) + self._client = client + + @cached_property + def completions(self) -> completions.AsyncCompletionsWithStreamingResponse: + from .resources.completions import AsyncCompletionsWithStreamingResponse + + return AsyncCompletionsWithStreamingResponse(self._client.completions) + + @cached_property + def chat(self) -> chat.AsyncChatWithStreamingResponse: + from .resources.chat import AsyncChatWithStreamingResponse + + return AsyncChatWithStreamingResponse(self._client.chat) + + @cached_property + def embeddings(self) -> embeddings.AsyncEmbeddingsWithStreamingResponse: + from .resources.embeddings import AsyncEmbeddingsWithStreamingResponse + + return AsyncEmbeddingsWithStreamingResponse(self._client.embeddings) + + @cached_property + def files(self) -> files.AsyncFilesWithStreamingResponse: + from .resources.files import AsyncFilesWithStreamingResponse + + return AsyncFilesWithStreamingResponse(self._client.files) + + @cached_property + def images(self) -> images.AsyncImagesWithStreamingResponse: + from .resources.images import AsyncImagesWithStreamingResponse + + return AsyncImagesWithStreamingResponse(self._client.images) + + @cached_property + def audio(self) -> audio.AsyncAudioWithStreamingResponse: + from .resources.audio import AsyncAudioWithStreamingResponse + + return AsyncAudioWithStreamingResponse(self._client.audio) + + @cached_property + def moderations(self) -> moderations.AsyncModerationsWithStreamingResponse: + from .resources.moderations import AsyncModerationsWithStreamingResponse + + return AsyncModerationsWithStreamingResponse(self._client.moderations) + + @cached_property + def models(self) -> models.AsyncModelsWithStreamingResponse: + from .resources.models import AsyncModelsWithStreamingResponse + + return AsyncModelsWithStreamingResponse(self._client.models) + + @cached_property + def fine_tuning(self) -> fine_tuning.AsyncFineTuningWithStreamingResponse: + from .resources.fine_tuning import AsyncFineTuningWithStreamingResponse + + return AsyncFineTuningWithStreamingResponse(self._client.fine_tuning) + + @cached_property + def vector_stores(self) -> vector_stores.AsyncVectorStoresWithStreamingResponse: + from .resources.vector_stores import AsyncVectorStoresWithStreamingResponse + + return AsyncVectorStoresWithStreamingResponse(self._client.vector_stores) + + @cached_property + def beta(self) -> beta.AsyncBetaWithStreamingResponse: + from .resources.beta import AsyncBetaWithStreamingResponse + + return AsyncBetaWithStreamingResponse(self._client.beta) + + @cached_property + def batches(self) -> batches.AsyncBatchesWithStreamingResponse: + from .resources.batches import AsyncBatchesWithStreamingResponse + + return AsyncBatchesWithStreamingResponse(self._client.batches) + + @cached_property + def uploads(self) -> uploads.AsyncUploadsWithStreamingResponse: + from .resources.uploads import AsyncUploadsWithStreamingResponse + + return AsyncUploadsWithStreamingResponse(self._client.uploads) + + @cached_property + def responses(self) -> responses.AsyncResponsesWithStreamingResponse: + from .resources.responses import AsyncResponsesWithStreamingResponse + + return AsyncResponsesWithStreamingResponse(self._client.responses) + + @cached_property + def evals(self) -> evals.AsyncEvalsWithStreamingResponse: + from .resources.evals import AsyncEvalsWithStreamingResponse + + return AsyncEvalsWithStreamingResponse(self._client.evals) Client = OpenAI diff --git a/src/openai/_module_client.py b/src/openai/_module_client.py index cf12f7a31e..dd601f9be9 100644 --- a/src/openai/_module_client.py +++ b/src/openai/_module_client.py @@ -1,113 +1,133 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. +from __future__ import annotations + +from typing import TYPE_CHECKING from typing_extensions import override -from . import resources, _load_client +if TYPE_CHECKING: + from .resources.files import Files + from .resources.images import Images + from .resources.models import Models + from .resources.batches import Batches + from .resources.beta.beta import Beta + from .resources.chat.chat import Chat + from .resources.embeddings import Embeddings + from .resources.audio.audio import Audio + from .resources.completions import Completions + from .resources.evals.evals import Evals + from .resources.moderations import Moderations + from .resources.uploads.uploads import Uploads + from .resources.responses.responses import Responses + from .resources.fine_tuning.fine_tuning import FineTuning + from .resources.vector_stores.vector_stores import VectorStores + +from . import _load_client from ._utils import LazyProxy -class ChatProxy(LazyProxy[resources.Chat]): +class ChatProxy(LazyProxy["Chat"]): @override - def __load__(self) -> resources.Chat: + def __load__(self) -> Chat: return _load_client().chat -class BetaProxy(LazyProxy[resources.Beta]): +class BetaProxy(LazyProxy["Beta"]): @override - def __load__(self) -> resources.Beta: + def __load__(self) -> Beta: return _load_client().beta -class FilesProxy(LazyProxy[resources.Files]): +class FilesProxy(LazyProxy["Files"]): @override - def __load__(self) -> resources.Files: + def __load__(self) -> Files: return _load_client().files -class AudioProxy(LazyProxy[resources.Audio]): +class AudioProxy(LazyProxy["Audio"]): @override - def __load__(self) -> resources.Audio: + def __load__(self) -> Audio: return _load_client().audio -class EvalsProxy(LazyProxy[resources.Evals]): +class EvalsProxy(LazyProxy["Evals"]): @override - def __load__(self) -> resources.Evals: + def __load__(self) -> Evals: return _load_client().evals -class ImagesProxy(LazyProxy[resources.Images]): +class ImagesProxy(LazyProxy["Images"]): @override - def __load__(self) -> resources.Images: + def __load__(self) -> Images: return _load_client().images -class ModelsProxy(LazyProxy[resources.Models]): +class ModelsProxy(LazyProxy["Models"]): @override - def __load__(self) -> resources.Models: + def __load__(self) -> Models: return _load_client().models -class BatchesProxy(LazyProxy[resources.Batches]): +class BatchesProxy(LazyProxy["Batches"]): @override - def __load__(self) -> resources.Batches: + def __load__(self) -> Batches: return _load_client().batches -class UploadsProxy(LazyProxy[resources.Uploads]): +class UploadsProxy(LazyProxy["Uploads"]): @override - def __load__(self) -> resources.Uploads: + def __load__(self) -> Uploads: return _load_client().uploads -class ResponsesProxy(LazyProxy[resources.Responses]): +class ResponsesProxy(LazyProxy["Responses"]): @override - def __load__(self) -> resources.Responses: + def __load__(self) -> Responses: return _load_client().responses -class EmbeddingsProxy(LazyProxy[resources.Embeddings]): +class EmbeddingsProxy(LazyProxy["Embeddings"]): @override - def __load__(self) -> resources.Embeddings: + def __load__(self) -> Embeddings: return _load_client().embeddings -class CompletionsProxy(LazyProxy[resources.Completions]): +class CompletionsProxy(LazyProxy["Completions"]): @override - def __load__(self) -> resources.Completions: + def __load__(self) -> Completions: return _load_client().completions -class ModerationsProxy(LazyProxy[resources.Moderations]): +class ModerationsProxy(LazyProxy["Moderations"]): @override - def __load__(self) -> resources.Moderations: + def __load__(self) -> Moderations: return _load_client().moderations -class FineTuningProxy(LazyProxy[resources.FineTuning]): +class FineTuningProxy(LazyProxy["FineTuning"]): @override - def __load__(self) -> resources.FineTuning: + def __load__(self) -> FineTuning: return _load_client().fine_tuning -class VectorStoresProxy(LazyProxy[resources.VectorStores]): +class VectorStoresProxy(LazyProxy["VectorStores"]): @override - def __load__(self) -> resources.VectorStores: + def __load__(self) -> VectorStores: return _load_client().vector_stores -chat: resources.Chat = ChatProxy().__as_proxied__() -beta: resources.Beta = BetaProxy().__as_proxied__() -files: resources.Files = FilesProxy().__as_proxied__() -audio: resources.Audio = AudioProxy().__as_proxied__() -evals: resources.Evals = EvalsProxy().__as_proxied__() -images: resources.Images = ImagesProxy().__as_proxied__() -models: resources.Models = ModelsProxy().__as_proxied__() -batches: resources.Batches = BatchesProxy().__as_proxied__() -uploads: resources.Uploads = UploadsProxy().__as_proxied__() -responses: resources.Responses = ResponsesProxy().__as_proxied__() -embeddings: resources.Embeddings = EmbeddingsProxy().__as_proxied__() -completions: resources.Completions = CompletionsProxy().__as_proxied__() -moderations: resources.Moderations = ModerationsProxy().__as_proxied__() -fine_tuning: resources.FineTuning = FineTuningProxy().__as_proxied__() -vector_stores: resources.VectorStores = VectorStoresProxy().__as_proxied__() +chat: Chat = ChatProxy().__as_proxied__() +beta: Beta = BetaProxy().__as_proxied__() +files: Files = FilesProxy().__as_proxied__() +audio: Audio = AudioProxy().__as_proxied__() +evals: Evals = EvalsProxy().__as_proxied__() +images: Images = ImagesProxy().__as_proxied__() +models: Models = ModelsProxy().__as_proxied__() +batches: Batches = BatchesProxy().__as_proxied__() +uploads: Uploads = UploadsProxy().__as_proxied__() +responses: Responses = ResponsesProxy().__as_proxied__() +embeddings: Embeddings = EmbeddingsProxy().__as_proxied__() +completions: Completions = CompletionsProxy().__as_proxied__() +moderations: Moderations = ModerationsProxy().__as_proxied__() +fine_tuning: FineTuning = FineTuningProxy().__as_proxied__() +vector_stores: VectorStores = VectorStoresProxy().__as_proxied__() diff --git a/src/openai/_utils/_proxy.py b/src/openai/_utils/_proxy.py index ffd883e9dd..0f239a33c6 100644 --- a/src/openai/_utils/_proxy.py +++ b/src/openai/_utils/_proxy.py @@ -46,7 +46,10 @@ def __dir__(self) -> Iterable[str]: @property # type: ignore @override def __class__(self) -> type: # pyright: ignore - proxied = self.__get_proxied__() + try: + proxied = self.__get_proxied__() + except Exception: + return type(self) if issubclass(type(proxied), LazyProxy): return type(proxied) return proxied.__class__ diff --git a/src/openai/_version.py b/src/openai/_version.py index 9d8ba015e1..495a094581 100644 --- a/src/openai/_version.py +++ b/src/openai/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "openai" -__version__ = "1.77.0" # x-release-please-version +__version__ = "1.78.0" # x-release-please-version diff --git a/src/openai/resources/__init__.py b/src/openai/resources/__init__.py index ab9cd73e81..8612dec797 100644 --- a/src/openai/resources/__init__.py +++ b/src/openai/resources/__init__.py @@ -72,14 +72,6 @@ UploadsWithStreamingResponse, AsyncUploadsWithStreamingResponse, ) -from .responses import ( - Responses, - AsyncResponses, - ResponsesWithRawResponse, - AsyncResponsesWithRawResponse, - ResponsesWithStreamingResponse, - AsyncResponsesWithStreamingResponse, -) from .embeddings import ( Embeddings, AsyncEmbeddings, @@ -200,12 +192,6 @@ "AsyncUploadsWithRawResponse", "UploadsWithStreamingResponse", "AsyncUploadsWithStreamingResponse", - "Responses", - "AsyncResponses", - "ResponsesWithRawResponse", - "AsyncResponsesWithRawResponse", - "ResponsesWithStreamingResponse", - "AsyncResponsesWithStreamingResponse", "Evals", "AsyncEvals", "EvalsWithRawResponse", diff --git a/src/openai/resources/fine_tuning/__init__.py b/src/openai/resources/fine_tuning/__init__.py index ed7db4f4e0..c76af83deb 100644 --- a/src/openai/resources/fine_tuning/__init__.py +++ b/src/openai/resources/fine_tuning/__init__.py @@ -8,6 +8,14 @@ JobsWithStreamingResponse, AsyncJobsWithStreamingResponse, ) +from .alpha import ( + Alpha, + AsyncAlpha, + AlphaWithRawResponse, + AsyncAlphaWithRawResponse, + AlphaWithStreamingResponse, + AsyncAlphaWithStreamingResponse, +) from .checkpoints import ( Checkpoints, AsyncCheckpoints, @@ -38,6 +46,12 @@ "AsyncCheckpointsWithRawResponse", "CheckpointsWithStreamingResponse", "AsyncCheckpointsWithStreamingResponse", + "Alpha", + "AsyncAlpha", + "AlphaWithRawResponse", + "AsyncAlphaWithRawResponse", + "AlphaWithStreamingResponse", + "AsyncAlphaWithStreamingResponse", "FineTuning", "AsyncFineTuning", "FineTuningWithRawResponse", diff --git a/src/openai/resources/fine_tuning/alpha/__init__.py b/src/openai/resources/fine_tuning/alpha/__init__.py new file mode 100644 index 0000000000..8bed8af4fd --- /dev/null +++ b/src/openai/resources/fine_tuning/alpha/__init__.py @@ -0,0 +1,33 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from .alpha import ( + Alpha, + AsyncAlpha, + AlphaWithRawResponse, + AsyncAlphaWithRawResponse, + AlphaWithStreamingResponse, + AsyncAlphaWithStreamingResponse, +) +from .graders import ( + Graders, + AsyncGraders, + GradersWithRawResponse, + AsyncGradersWithRawResponse, + GradersWithStreamingResponse, + AsyncGradersWithStreamingResponse, +) + +__all__ = [ + "Graders", + "AsyncGraders", + "GradersWithRawResponse", + "AsyncGradersWithRawResponse", + "GradersWithStreamingResponse", + "AsyncGradersWithStreamingResponse", + "Alpha", + "AsyncAlpha", + "AlphaWithRawResponse", + "AsyncAlphaWithRawResponse", + "AlphaWithStreamingResponse", + "AsyncAlphaWithStreamingResponse", +] diff --git a/src/openai/resources/fine_tuning/alpha/alpha.py b/src/openai/resources/fine_tuning/alpha/alpha.py new file mode 100644 index 0000000000..54c05fab69 --- /dev/null +++ b/src/openai/resources/fine_tuning/alpha/alpha.py @@ -0,0 +1,102 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from .graders import ( + Graders, + AsyncGraders, + GradersWithRawResponse, + AsyncGradersWithRawResponse, + GradersWithStreamingResponse, + AsyncGradersWithStreamingResponse, +) +from ...._compat import cached_property +from ...._resource import SyncAPIResource, AsyncAPIResource + +__all__ = ["Alpha", "AsyncAlpha"] + + +class Alpha(SyncAPIResource): + @cached_property + def graders(self) -> Graders: + return Graders(self._client) + + @cached_property + def with_raw_response(self) -> AlphaWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AlphaWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AlphaWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AlphaWithStreamingResponse(self) + + +class AsyncAlpha(AsyncAPIResource): + @cached_property + def graders(self) -> AsyncGraders: + return AsyncGraders(self._client) + + @cached_property + def with_raw_response(self) -> AsyncAlphaWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncAlphaWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncAlphaWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncAlphaWithStreamingResponse(self) + + +class AlphaWithRawResponse: + def __init__(self, alpha: Alpha) -> None: + self._alpha = alpha + + @cached_property + def graders(self) -> GradersWithRawResponse: + return GradersWithRawResponse(self._alpha.graders) + + +class AsyncAlphaWithRawResponse: + def __init__(self, alpha: AsyncAlpha) -> None: + self._alpha = alpha + + @cached_property + def graders(self) -> AsyncGradersWithRawResponse: + return AsyncGradersWithRawResponse(self._alpha.graders) + + +class AlphaWithStreamingResponse: + def __init__(self, alpha: Alpha) -> None: + self._alpha = alpha + + @cached_property + def graders(self) -> GradersWithStreamingResponse: + return GradersWithStreamingResponse(self._alpha.graders) + + +class AsyncAlphaWithStreamingResponse: + def __init__(self, alpha: AsyncAlpha) -> None: + self._alpha = alpha + + @cached_property + def graders(self) -> AsyncGradersWithStreamingResponse: + return AsyncGradersWithStreamingResponse(self._alpha.graders) diff --git a/src/openai/resources/fine_tuning/alpha/graders.py b/src/openai/resources/fine_tuning/alpha/graders.py new file mode 100644 index 0000000000..f27acdfd9c --- /dev/null +++ b/src/openai/resources/fine_tuning/alpha/graders.py @@ -0,0 +1,272 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union, Iterable + +import httpx + +from .... import _legacy_response +from ...._types import NOT_GIVEN, Body, Query, Headers, NotGiven +from ...._utils import maybe_transform, async_maybe_transform +from ...._compat import cached_property +from ...._resource import SyncAPIResource, AsyncAPIResource +from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper +from ...._base_client import make_request_options +from ....types.fine_tuning.alpha import grader_run_params, grader_validate_params +from ....types.fine_tuning.alpha.grader_run_response import GraderRunResponse +from ....types.fine_tuning.alpha.grader_validate_response import GraderValidateResponse + +__all__ = ["Graders", "AsyncGraders"] + + +class Graders(SyncAPIResource): + @cached_property + def with_raw_response(self) -> GradersWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return GradersWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> GradersWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return GradersWithStreamingResponse(self) + + def run( + self, + *, + grader: grader_run_params.Grader, + model_sample: str, + reference_answer: Union[str, Iterable[object], float, object], + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> GraderRunResponse: + """ + Run a grader. + + Args: + grader: The grader used for the fine-tuning job. + + model_sample: The model sample to be evaluated. + + reference_answer: The reference answer for the evaluation. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._post( + "/fine_tuning/alpha/graders/run", + body=maybe_transform( + { + "grader": grader, + "model_sample": model_sample, + "reference_answer": reference_answer, + }, + grader_run_params.GraderRunParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=GraderRunResponse, + ) + + def validate( + self, + *, + grader: grader_validate_params.Grader, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> GraderValidateResponse: + """ + Validate a grader. + + Args: + grader: The grader used for the fine-tuning job. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._post( + "/fine_tuning/alpha/graders/validate", + body=maybe_transform({"grader": grader}, grader_validate_params.GraderValidateParams), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=GraderValidateResponse, + ) + + +class AsyncGraders(AsyncAPIResource): + @cached_property + def with_raw_response(self) -> AsyncGradersWithRawResponse: + """ + This property can be used as a prefix for any HTTP method call to return + the raw response object instead of the parsed content. + + For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers + """ + return AsyncGradersWithRawResponse(self) + + @cached_property + def with_streaming_response(self) -> AsyncGradersWithStreamingResponse: + """ + An alternative to `.with_raw_response` that doesn't eagerly read the response body. + + For more information, see https://www.github.com/openai/openai-python#with_streaming_response + """ + return AsyncGradersWithStreamingResponse(self) + + async def run( + self, + *, + grader: grader_run_params.Grader, + model_sample: str, + reference_answer: Union[str, Iterable[object], float, object], + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> GraderRunResponse: + """ + Run a grader. + + Args: + grader: The grader used for the fine-tuning job. + + model_sample: The model sample to be evaluated. + + reference_answer: The reference answer for the evaluation. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return await self._post( + "/fine_tuning/alpha/graders/run", + body=await async_maybe_transform( + { + "grader": grader, + "model_sample": model_sample, + "reference_answer": reference_answer, + }, + grader_run_params.GraderRunParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=GraderRunResponse, + ) + + async def validate( + self, + *, + grader: grader_validate_params.Grader, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> GraderValidateResponse: + """ + Validate a grader. + + Args: + grader: The grader used for the fine-tuning job. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return await self._post( + "/fine_tuning/alpha/graders/validate", + body=await async_maybe_transform({"grader": grader}, grader_validate_params.GraderValidateParams), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=GraderValidateResponse, + ) + + +class GradersWithRawResponse: + def __init__(self, graders: Graders) -> None: + self._graders = graders + + self.run = _legacy_response.to_raw_response_wrapper( + graders.run, + ) + self.validate = _legacy_response.to_raw_response_wrapper( + graders.validate, + ) + + +class AsyncGradersWithRawResponse: + def __init__(self, graders: AsyncGraders) -> None: + self._graders = graders + + self.run = _legacy_response.async_to_raw_response_wrapper( + graders.run, + ) + self.validate = _legacy_response.async_to_raw_response_wrapper( + graders.validate, + ) + + +class GradersWithStreamingResponse: + def __init__(self, graders: Graders) -> None: + self._graders = graders + + self.run = to_streamed_response_wrapper( + graders.run, + ) + self.validate = to_streamed_response_wrapper( + graders.validate, + ) + + +class AsyncGradersWithStreamingResponse: + def __init__(self, graders: AsyncGraders) -> None: + self._graders = graders + + self.run = async_to_streamed_response_wrapper( + graders.run, + ) + self.validate = async_to_streamed_response_wrapper( + graders.validate, + ) diff --git a/src/openai/resources/fine_tuning/fine_tuning.py b/src/openai/resources/fine_tuning/fine_tuning.py index 1388c8230c..25ae3e8cf4 100644 --- a/src/openai/resources/fine_tuning/fine_tuning.py +++ b/src/openai/resources/fine_tuning/fine_tuning.py @@ -12,6 +12,14 @@ AsyncJobsWithStreamingResponse, ) from ..._resource import SyncAPIResource, AsyncAPIResource +from .alpha.alpha import ( + Alpha, + AsyncAlpha, + AlphaWithRawResponse, + AsyncAlphaWithRawResponse, + AlphaWithStreamingResponse, + AsyncAlphaWithStreamingResponse, +) from .checkpoints.checkpoints import ( Checkpoints, AsyncCheckpoints, @@ -33,6 +41,10 @@ def jobs(self) -> Jobs: def checkpoints(self) -> Checkpoints: return Checkpoints(self._client) + @cached_property + def alpha(self) -> Alpha: + return Alpha(self._client) + @cached_property def with_raw_response(self) -> FineTuningWithRawResponse: """ @@ -62,6 +74,10 @@ def jobs(self) -> AsyncJobs: def checkpoints(self) -> AsyncCheckpoints: return AsyncCheckpoints(self._client) + @cached_property + def alpha(self) -> AsyncAlpha: + return AsyncAlpha(self._client) + @cached_property def with_raw_response(self) -> AsyncFineTuningWithRawResponse: """ @@ -94,6 +110,10 @@ def jobs(self) -> JobsWithRawResponse: def checkpoints(self) -> CheckpointsWithRawResponse: return CheckpointsWithRawResponse(self._fine_tuning.checkpoints) + @cached_property + def alpha(self) -> AlphaWithRawResponse: + return AlphaWithRawResponse(self._fine_tuning.alpha) + class AsyncFineTuningWithRawResponse: def __init__(self, fine_tuning: AsyncFineTuning) -> None: @@ -107,6 +127,10 @@ def jobs(self) -> AsyncJobsWithRawResponse: def checkpoints(self) -> AsyncCheckpointsWithRawResponse: return AsyncCheckpointsWithRawResponse(self._fine_tuning.checkpoints) + @cached_property + def alpha(self) -> AsyncAlphaWithRawResponse: + return AsyncAlphaWithRawResponse(self._fine_tuning.alpha) + class FineTuningWithStreamingResponse: def __init__(self, fine_tuning: FineTuning) -> None: @@ -120,6 +144,10 @@ def jobs(self) -> JobsWithStreamingResponse: def checkpoints(self) -> CheckpointsWithStreamingResponse: return CheckpointsWithStreamingResponse(self._fine_tuning.checkpoints) + @cached_property + def alpha(self) -> AlphaWithStreamingResponse: + return AlphaWithStreamingResponse(self._fine_tuning.alpha) + class AsyncFineTuningWithStreamingResponse: def __init__(self, fine_tuning: AsyncFineTuning) -> None: @@ -132,3 +160,7 @@ def jobs(self) -> AsyncJobsWithStreamingResponse: @cached_property def checkpoints(self) -> AsyncCheckpointsWithStreamingResponse: return AsyncCheckpointsWithStreamingResponse(self._fine_tuning.checkpoints) + + @cached_property + def alpha(self) -> AsyncAlphaWithStreamingResponse: + return AsyncAlphaWithStreamingResponse(self._fine_tuning.alpha) diff --git a/src/openai/resources/fine_tuning/jobs/jobs.py b/src/openai/resources/fine_tuning/jobs/jobs.py index 90619c8609..5cca219172 100644 --- a/src/openai/resources/fine_tuning/jobs/jobs.py +++ b/src/openai/resources/fine_tuning/jobs/jobs.py @@ -345,6 +345,72 @@ def list_events( model=FineTuningJobEvent, ) + def pause( + self, + fine_tuning_job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Pause a fine-tune job. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return self._post( + f"/fine_tuning/jobs/{fine_tuning_job_id}/pause", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + + def resume( + self, + fine_tuning_job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Resume a fine-tune job. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return self._post( + f"/fine_tuning/jobs/{fine_tuning_job_id}/resume", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + class AsyncJobs(AsyncAPIResource): @cached_property @@ -657,6 +723,72 @@ def list_events( model=FineTuningJobEvent, ) + async def pause( + self, + fine_tuning_job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Pause a fine-tune job. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return await self._post( + f"/fine_tuning/jobs/{fine_tuning_job_id}/pause", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + + async def resume( + self, + fine_tuning_job_id: str, + *, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN, + ) -> FineTuningJob: + """ + Resume a fine-tune job. + + Args: + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + if not fine_tuning_job_id: + raise ValueError(f"Expected a non-empty value for `fine_tuning_job_id` but received {fine_tuning_job_id!r}") + return await self._post( + f"/fine_tuning/jobs/{fine_tuning_job_id}/resume", + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=FineTuningJob, + ) + class JobsWithRawResponse: def __init__(self, jobs: Jobs) -> None: @@ -677,6 +809,12 @@ def __init__(self, jobs: Jobs) -> None: self.list_events = _legacy_response.to_raw_response_wrapper( jobs.list_events, ) + self.pause = _legacy_response.to_raw_response_wrapper( + jobs.pause, + ) + self.resume = _legacy_response.to_raw_response_wrapper( + jobs.resume, + ) @cached_property def checkpoints(self) -> CheckpointsWithRawResponse: @@ -702,6 +840,12 @@ def __init__(self, jobs: AsyncJobs) -> None: self.list_events = _legacy_response.async_to_raw_response_wrapper( jobs.list_events, ) + self.pause = _legacy_response.async_to_raw_response_wrapper( + jobs.pause, + ) + self.resume = _legacy_response.async_to_raw_response_wrapper( + jobs.resume, + ) @cached_property def checkpoints(self) -> AsyncCheckpointsWithRawResponse: @@ -727,6 +871,12 @@ def __init__(self, jobs: Jobs) -> None: self.list_events = to_streamed_response_wrapper( jobs.list_events, ) + self.pause = to_streamed_response_wrapper( + jobs.pause, + ) + self.resume = to_streamed_response_wrapper( + jobs.resume, + ) @cached_property def checkpoints(self) -> CheckpointsWithStreamingResponse: @@ -752,6 +902,12 @@ def __init__(self, jobs: AsyncJobs) -> None: self.list_events = async_to_streamed_response_wrapper( jobs.list_events, ) + self.pause = async_to_streamed_response_wrapper( + jobs.pause, + ) + self.resume = async_to_streamed_response_wrapper( + jobs.resume, + ) @cached_property def checkpoints(self) -> AsyncCheckpointsWithStreamingResponse: diff --git a/src/openai/types/__init__.py b/src/openai/types/__init__.py index 57c91811b9..bf5493fd62 100644 --- a/src/openai/types/__init__.py +++ b/src/openai/types/__init__.py @@ -61,9 +61,7 @@ from .file_chunking_strategy import FileChunkingStrategy as FileChunkingStrategy from .upload_complete_params import UploadCompleteParams as UploadCompleteParams from .embedding_create_params import EmbeddingCreateParams as EmbeddingCreateParams -from .eval_label_model_grader import EvalLabelModelGrader as EvalLabelModelGrader from .completion_create_params import CompletionCreateParams as CompletionCreateParams -from .eval_string_check_grader import EvalStringCheckGrader as EvalStringCheckGrader from .moderation_create_params import ModerationCreateParams as ModerationCreateParams from .vector_store_list_params import VectorStoreListParams as VectorStoreListParams from .create_embedding_response import CreateEmbeddingResponse as CreateEmbeddingResponse @@ -71,7 +69,6 @@ from .vector_store_create_params import VectorStoreCreateParams as VectorStoreCreateParams from .vector_store_search_params import VectorStoreSearchParams as VectorStoreSearchParams from .vector_store_update_params import VectorStoreUpdateParams as VectorStoreUpdateParams -from .eval_text_similarity_grader import EvalTextSimilarityGrader as EvalTextSimilarityGrader from .moderation_text_input_param import ModerationTextInputParam as ModerationTextInputParam from .file_chunking_strategy_param import FileChunkingStrategyParam as FileChunkingStrategyParam from .vector_store_search_response import VectorStoreSearchResponse as VectorStoreSearchResponse @@ -79,10 +76,8 @@ from .image_create_variation_params import ImageCreateVariationParams as ImageCreateVariationParams from .static_file_chunking_strategy import StaticFileChunkingStrategy as StaticFileChunkingStrategy from .eval_custom_data_source_config import EvalCustomDataSourceConfig as EvalCustomDataSourceConfig -from .eval_string_check_grader_param import EvalStringCheckGraderParam as EvalStringCheckGraderParam from .moderation_image_url_input_param import ModerationImageURLInputParam as ModerationImageURLInputParam from .auto_file_chunking_strategy_param import AutoFileChunkingStrategyParam as AutoFileChunkingStrategyParam -from .eval_text_similarity_grader_param import EvalTextSimilarityGraderParam as EvalTextSimilarityGraderParam from .moderation_multi_modal_input_param import ModerationMultiModalInputParam as ModerationMultiModalInputParam from .other_file_chunking_strategy_object import OtherFileChunkingStrategyObject as OtherFileChunkingStrategyObject from .static_file_chunking_strategy_param import StaticFileChunkingStrategyParam as StaticFileChunkingStrategyParam diff --git a/src/openai/types/eval_create_params.py b/src/openai/types/eval_create_params.py index 03f44f2c8c..66178287e4 100644 --- a/src/openai/types/eval_create_params.py +++ b/src/openai/types/eval_create_params.py @@ -6,15 +6,17 @@ from typing_extensions import Literal, Required, TypeAlias, TypedDict from .shared_params.metadata import Metadata -from .eval_string_check_grader_param import EvalStringCheckGraderParam -from .eval_text_similarity_grader_param import EvalTextSimilarityGraderParam +from .graders.python_grader_param import PythonGraderParam +from .graders.score_model_grader_param import ScoreModelGraderParam +from .graders.string_check_grader_param import StringCheckGraderParam from .responses.response_input_text_param import ResponseInputTextParam +from .graders.text_similarity_grader_param import TextSimilarityGraderParam __all__ = [ "EvalCreateParams", "DataSourceConfig", "DataSourceConfigCustom", - "DataSourceConfigLogs", + "DataSourceConfigStoredCompletions", "TestingCriterion", "TestingCriterionLabelModel", "TestingCriterionLabelModelInput", @@ -22,11 +24,9 @@ "TestingCriterionLabelModelInputEvalItem", "TestingCriterionLabelModelInputEvalItemContent", "TestingCriterionLabelModelInputEvalItemContentOutputText", + "TestingCriterionTextSimilarity", "TestingCriterionPython", "TestingCriterionScoreModel", - "TestingCriterionScoreModelInput", - "TestingCriterionScoreModelInputContent", - "TestingCriterionScoreModelInputContentOutputText", ] @@ -65,15 +65,15 @@ class DataSourceConfigCustom(TypedDict, total=False): """ -class DataSourceConfigLogs(TypedDict, total=False): - type: Required[Literal["logs"]] - """The type of data source. Always `logs`.""" +class DataSourceConfigStoredCompletions(TypedDict, total=False): + type: Required[Literal["stored_completions"]] + """The type of data source. Always `stored_completions`.""" metadata: Dict[str, object] - """Metadata filters for the logs data source.""" + """Metadata filters for the stored completions data source.""" -DataSourceConfig: TypeAlias = Union[DataSourceConfigCustom, DataSourceConfigLogs] +DataSourceConfig: TypeAlias = Union[DataSourceConfigCustom, DataSourceConfigStoredCompletions] class TestingCriterionLabelModelInputSimpleInputMessage(TypedDict, total=False): @@ -139,77 +139,28 @@ class TestingCriterionLabelModel(TypedDict, total=False): """The object type, which is always `label_model`.""" -class TestingCriterionPython(TypedDict, total=False): - name: Required[str] - """The name of the grader.""" - - source: Required[str] - """The source code of the python script.""" - - type: Required[Literal["python"]] - """The object type, which is always `python`.""" +class TestingCriterionTextSimilarity(TextSimilarityGraderParam, total=False): + __test__ = False + pass_threshold: Required[float] + """The threshold for the score.""" - image_tag: str - """The image tag to use for the python script.""" +class TestingCriterionPython(PythonGraderParam, total=False): + __test__ = False pass_threshold: float """The threshold for the score.""" -class TestingCriterionScoreModelInputContentOutputText(TypedDict, total=False): - text: Required[str] - """The text output from the model.""" - - type: Required[Literal["output_text"]] - """The type of the output text. Always `output_text`.""" - - -TestingCriterionScoreModelInputContent: TypeAlias = Union[ - str, ResponseInputTextParam, TestingCriterionScoreModelInputContentOutputText -] - - -class TestingCriterionScoreModelInput(TypedDict, total=False): - content: Required[TestingCriterionScoreModelInputContent] - """Text inputs to the model - can contain template strings.""" - - role: Required[Literal["user", "assistant", "system", "developer"]] - """The role of the message input. - - One of `user`, `assistant`, `system`, or `developer`. - """ - - type: Literal["message"] - """The type of the message input. Always `message`.""" - - -class TestingCriterionScoreModel(TypedDict, total=False): - input: Required[Iterable[TestingCriterionScoreModelInput]] - """The input text. This may include template strings.""" - - model: Required[str] - """The model to use for the evaluation.""" - - name: Required[str] - """The name of the grader.""" - - type: Required[Literal["score_model"]] - """The object type, which is always `score_model`.""" - +class TestingCriterionScoreModel(ScoreModelGraderParam, total=False): + __test__ = False pass_threshold: float """The threshold for the score.""" - range: Iterable[float] - """The range of the score. Defaults to `[0, 1]`.""" - - sampling_params: object - """The sampling parameters for the model.""" - TestingCriterion: TypeAlias = Union[ TestingCriterionLabelModel, - EvalStringCheckGraderParam, - EvalTextSimilarityGraderParam, + StringCheckGraderParam, + TestingCriterionTextSimilarity, TestingCriterionPython, TestingCriterionScoreModel, ] diff --git a/src/openai/types/eval_create_response.py b/src/openai/types/eval_create_response.py index 6d77a81870..d5f158ad29 100644 --- a/src/openai/types/eval_create_response.py +++ b/src/openai/types/eval_create_response.py @@ -6,22 +6,21 @@ from .._utils import PropertyInfo from .._models import BaseModel from .shared.metadata import Metadata -from .eval_label_model_grader import EvalLabelModelGrader -from .eval_string_check_grader import EvalStringCheckGrader -from .eval_text_similarity_grader import EvalTextSimilarityGrader -from .responses.response_input_text import ResponseInputText +from .graders.python_grader import PythonGrader +from .graders.label_model_grader import LabelModelGrader +from .graders.score_model_grader import ScoreModelGrader +from .graders.string_check_grader import StringCheckGrader from .eval_custom_data_source_config import EvalCustomDataSourceConfig +from .graders.text_similarity_grader import TextSimilarityGrader from .eval_stored_completions_data_source_config import EvalStoredCompletionsDataSourceConfig __all__ = [ "EvalCreateResponse", "DataSourceConfig", "TestingCriterion", - "TestingCriterionPython", - "TestingCriterionScoreModel", - "TestingCriterionScoreModelInput", - "TestingCriterionScoreModelInputContent", - "TestingCriterionScoreModelInputContentOutputText", + "TestingCriterionEvalGraderTextSimilarity", + "TestingCriterionEvalGraderPython", + "TestingCriterionEvalGraderScoreModel", ] DataSourceConfig: TypeAlias = Annotated[ @@ -29,86 +28,30 @@ ] -class TestingCriterionPython(BaseModel): +class TestingCriterionEvalGraderTextSimilarity(TextSimilarityGrader): __test__ = False - name: str - """The name of the grader.""" - - source: str - """The source code of the python script.""" - - type: Literal["python"] - """The object type, which is always `python`.""" - - image_tag: Optional[str] = None - """The image tag to use for the python script.""" - - pass_threshold: Optional[float] = None + pass_threshold: float """The threshold for the score.""" -class TestingCriterionScoreModelInputContentOutputText(BaseModel): - __test__ = False - text: str - """The text output from the model.""" - - type: Literal["output_text"] - """The type of the output text. Always `output_text`.""" - - -TestingCriterionScoreModelInputContent: TypeAlias = Union[ - str, ResponseInputText, TestingCriterionScoreModelInputContentOutputText -] - - -class TestingCriterionScoreModelInput(BaseModel): +class TestingCriterionEvalGraderPython(PythonGrader): __test__ = False - content: TestingCriterionScoreModelInputContent - """Text inputs to the model - can contain template strings.""" - - role: Literal["user", "assistant", "system", "developer"] - """The role of the message input. - - One of `user`, `assistant`, `system`, or `developer`. - """ - - type: Optional[Literal["message"]] = None - """The type of the message input. Always `message`.""" + pass_threshold: Optional[float] = None + """The threshold for the score.""" -class TestingCriterionScoreModel(BaseModel): +class TestingCriterionEvalGraderScoreModel(ScoreModelGrader): __test__ = False - input: List[TestingCriterionScoreModelInput] - """The input text. This may include template strings.""" - - model: str - """The model to use for the evaluation.""" - - name: str - """The name of the grader.""" - - type: Literal["score_model"] - """The object type, which is always `score_model`.""" - pass_threshold: Optional[float] = None """The threshold for the score.""" - range: Optional[List[float]] = None - """The range of the score. Defaults to `[0, 1]`.""" - - sampling_params: Optional[object] = None - """The sampling parameters for the model.""" - -TestingCriterion: TypeAlias = Annotated[ - Union[ - EvalLabelModelGrader, - EvalStringCheckGrader, - EvalTextSimilarityGrader, - TestingCriterionPython, - TestingCriterionScoreModel, - ], - PropertyInfo(discriminator="type"), +TestingCriterion: TypeAlias = Union[ + LabelModelGrader, + StringCheckGrader, + TestingCriterionEvalGraderTextSimilarity, + TestingCriterionEvalGraderPython, + TestingCriterionEvalGraderScoreModel, ] diff --git a/src/openai/types/eval_list_response.py b/src/openai/types/eval_list_response.py index 8c7e9c5588..b743f57f6a 100644 --- a/src/openai/types/eval_list_response.py +++ b/src/openai/types/eval_list_response.py @@ -6,22 +6,21 @@ from .._utils import PropertyInfo from .._models import BaseModel from .shared.metadata import Metadata -from .eval_label_model_grader import EvalLabelModelGrader -from .eval_string_check_grader import EvalStringCheckGrader -from .eval_text_similarity_grader import EvalTextSimilarityGrader -from .responses.response_input_text import ResponseInputText +from .graders.python_grader import PythonGrader +from .graders.label_model_grader import LabelModelGrader +from .graders.score_model_grader import ScoreModelGrader +from .graders.string_check_grader import StringCheckGrader from .eval_custom_data_source_config import EvalCustomDataSourceConfig +from .graders.text_similarity_grader import TextSimilarityGrader from .eval_stored_completions_data_source_config import EvalStoredCompletionsDataSourceConfig __all__ = [ "EvalListResponse", "DataSourceConfig", "TestingCriterion", - "TestingCriterionPython", - "TestingCriterionScoreModel", - "TestingCriterionScoreModelInput", - "TestingCriterionScoreModelInputContent", - "TestingCriterionScoreModelInputContentOutputText", + "TestingCriterionEvalGraderTextSimilarity", + "TestingCriterionEvalGraderPython", + "TestingCriterionEvalGraderScoreModel", ] DataSourceConfig: TypeAlias = Annotated[ @@ -29,86 +28,30 @@ ] -class TestingCriterionPython(BaseModel): +class TestingCriterionEvalGraderTextSimilarity(TextSimilarityGrader): __test__ = False - name: str - """The name of the grader.""" - - source: str - """The source code of the python script.""" - - type: Literal["python"] - """The object type, which is always `python`.""" - - image_tag: Optional[str] = None - """The image tag to use for the python script.""" - - pass_threshold: Optional[float] = None + pass_threshold: float """The threshold for the score.""" -class TestingCriterionScoreModelInputContentOutputText(BaseModel): - __test__ = False - text: str - """The text output from the model.""" - - type: Literal["output_text"] - """The type of the output text. Always `output_text`.""" - - -TestingCriterionScoreModelInputContent: TypeAlias = Union[ - str, ResponseInputText, TestingCriterionScoreModelInputContentOutputText -] - - -class TestingCriterionScoreModelInput(BaseModel): +class TestingCriterionEvalGraderPython(PythonGrader): __test__ = False - content: TestingCriterionScoreModelInputContent - """Text inputs to the model - can contain template strings.""" - - role: Literal["user", "assistant", "system", "developer"] - """The role of the message input. - - One of `user`, `assistant`, `system`, or `developer`. - """ - - type: Optional[Literal["message"]] = None - """The type of the message input. Always `message`.""" + pass_threshold: Optional[float] = None + """The threshold for the score.""" -class TestingCriterionScoreModel(BaseModel): +class TestingCriterionEvalGraderScoreModel(ScoreModelGrader): __test__ = False - input: List[TestingCriterionScoreModelInput] - """The input text. This may include template strings.""" - - model: str - """The model to use for the evaluation.""" - - name: str - """The name of the grader.""" - - type: Literal["score_model"] - """The object type, which is always `score_model`.""" - pass_threshold: Optional[float] = None """The threshold for the score.""" - range: Optional[List[float]] = None - """The range of the score. Defaults to `[0, 1]`.""" - - sampling_params: Optional[object] = None - """The sampling parameters for the model.""" - -TestingCriterion: TypeAlias = Annotated[ - Union[ - EvalLabelModelGrader, - EvalStringCheckGrader, - EvalTextSimilarityGrader, - TestingCriterionPython, - TestingCriterionScoreModel, - ], - PropertyInfo(discriminator="type"), +TestingCriterion: TypeAlias = Union[ + LabelModelGrader, + StringCheckGrader, + TestingCriterionEvalGraderTextSimilarity, + TestingCriterionEvalGraderPython, + TestingCriterionEvalGraderScoreModel, ] diff --git a/src/openai/types/eval_retrieve_response.py b/src/openai/types/eval_retrieve_response.py index 625bae80f4..dabb20674e 100644 --- a/src/openai/types/eval_retrieve_response.py +++ b/src/openai/types/eval_retrieve_response.py @@ -6,22 +6,21 @@ from .._utils import PropertyInfo from .._models import BaseModel from .shared.metadata import Metadata -from .eval_label_model_grader import EvalLabelModelGrader -from .eval_string_check_grader import EvalStringCheckGrader -from .eval_text_similarity_grader import EvalTextSimilarityGrader -from .responses.response_input_text import ResponseInputText +from .graders.python_grader import PythonGrader +from .graders.label_model_grader import LabelModelGrader +from .graders.score_model_grader import ScoreModelGrader +from .graders.string_check_grader import StringCheckGrader from .eval_custom_data_source_config import EvalCustomDataSourceConfig +from .graders.text_similarity_grader import TextSimilarityGrader from .eval_stored_completions_data_source_config import EvalStoredCompletionsDataSourceConfig __all__ = [ "EvalRetrieveResponse", "DataSourceConfig", "TestingCriterion", - "TestingCriterionPython", - "TestingCriterionScoreModel", - "TestingCriterionScoreModelInput", - "TestingCriterionScoreModelInputContent", - "TestingCriterionScoreModelInputContentOutputText", + "TestingCriterionEvalGraderTextSimilarity", + "TestingCriterionEvalGraderPython", + "TestingCriterionEvalGraderScoreModel", ] DataSourceConfig: TypeAlias = Annotated[ @@ -29,86 +28,30 @@ ] -class TestingCriterionPython(BaseModel): +class TestingCriterionEvalGraderTextSimilarity(TextSimilarityGrader): __test__ = False - name: str - """The name of the grader.""" - - source: str - """The source code of the python script.""" - - type: Literal["python"] - """The object type, which is always `python`.""" - - image_tag: Optional[str] = None - """The image tag to use for the python script.""" - - pass_threshold: Optional[float] = None + pass_threshold: float """The threshold for the score.""" -class TestingCriterionScoreModelInputContentOutputText(BaseModel): - __test__ = False - text: str - """The text output from the model.""" - - type: Literal["output_text"] - """The type of the output text. Always `output_text`.""" - - -TestingCriterionScoreModelInputContent: TypeAlias = Union[ - str, ResponseInputText, TestingCriterionScoreModelInputContentOutputText -] - - -class TestingCriterionScoreModelInput(BaseModel): +class TestingCriterionEvalGraderPython(PythonGrader): __test__ = False - content: TestingCriterionScoreModelInputContent - """Text inputs to the model - can contain template strings.""" - - role: Literal["user", "assistant", "system", "developer"] - """The role of the message input. - - One of `user`, `assistant`, `system`, or `developer`. - """ - - type: Optional[Literal["message"]] = None - """The type of the message input. Always `message`.""" + pass_threshold: Optional[float] = None + """The threshold for the score.""" -class TestingCriterionScoreModel(BaseModel): +class TestingCriterionEvalGraderScoreModel(ScoreModelGrader): __test__ = False - input: List[TestingCriterionScoreModelInput] - """The input text. This may include template strings.""" - - model: str - """The model to use for the evaluation.""" - - name: str - """The name of the grader.""" - - type: Literal["score_model"] - """The object type, which is always `score_model`.""" - pass_threshold: Optional[float] = None """The threshold for the score.""" - range: Optional[List[float]] = None - """The range of the score. Defaults to `[0, 1]`.""" - - sampling_params: Optional[object] = None - """The sampling parameters for the model.""" - -TestingCriterion: TypeAlias = Annotated[ - Union[ - EvalLabelModelGrader, - EvalStringCheckGrader, - EvalTextSimilarityGrader, - TestingCriterionPython, - TestingCriterionScoreModel, - ], - PropertyInfo(discriminator="type"), +TestingCriterion: TypeAlias = Union[ + LabelModelGrader, + StringCheckGrader, + TestingCriterionEvalGraderTextSimilarity, + TestingCriterionEvalGraderPython, + TestingCriterionEvalGraderScoreModel, ] diff --git a/src/openai/types/eval_update_response.py b/src/openai/types/eval_update_response.py index 2c280977a1..c5cb2622ea 100644 --- a/src/openai/types/eval_update_response.py +++ b/src/openai/types/eval_update_response.py @@ -6,22 +6,21 @@ from .._utils import PropertyInfo from .._models import BaseModel from .shared.metadata import Metadata -from .eval_label_model_grader import EvalLabelModelGrader -from .eval_string_check_grader import EvalStringCheckGrader -from .eval_text_similarity_grader import EvalTextSimilarityGrader -from .responses.response_input_text import ResponseInputText +from .graders.python_grader import PythonGrader +from .graders.label_model_grader import LabelModelGrader +from .graders.score_model_grader import ScoreModelGrader +from .graders.string_check_grader import StringCheckGrader from .eval_custom_data_source_config import EvalCustomDataSourceConfig +from .graders.text_similarity_grader import TextSimilarityGrader from .eval_stored_completions_data_source_config import EvalStoredCompletionsDataSourceConfig __all__ = [ "EvalUpdateResponse", "DataSourceConfig", "TestingCriterion", - "TestingCriterionPython", - "TestingCriterionScoreModel", - "TestingCriterionScoreModelInput", - "TestingCriterionScoreModelInputContent", - "TestingCriterionScoreModelInputContentOutputText", + "TestingCriterionEvalGraderTextSimilarity", + "TestingCriterionEvalGraderPython", + "TestingCriterionEvalGraderScoreModel", ] DataSourceConfig: TypeAlias = Annotated[ @@ -29,86 +28,30 @@ ] -class TestingCriterionPython(BaseModel): +class TestingCriterionEvalGraderTextSimilarity(TextSimilarityGrader): __test__ = False - name: str - """The name of the grader.""" - - source: str - """The source code of the python script.""" - - type: Literal["python"] - """The object type, which is always `python`.""" - - image_tag: Optional[str] = None - """The image tag to use for the python script.""" - - pass_threshold: Optional[float] = None + pass_threshold: float """The threshold for the score.""" -class TestingCriterionScoreModelInputContentOutputText(BaseModel): - __test__ = False - text: str - """The text output from the model.""" - - type: Literal["output_text"] - """The type of the output text. Always `output_text`.""" - - -TestingCriterionScoreModelInputContent: TypeAlias = Union[ - str, ResponseInputText, TestingCriterionScoreModelInputContentOutputText -] - - -class TestingCriterionScoreModelInput(BaseModel): +class TestingCriterionEvalGraderPython(PythonGrader): __test__ = False - content: TestingCriterionScoreModelInputContent - """Text inputs to the model - can contain template strings.""" - - role: Literal["user", "assistant", "system", "developer"] - """The role of the message input. - - One of `user`, `assistant`, `system`, or `developer`. - """ - - type: Optional[Literal["message"]] = None - """The type of the message input. Always `message`.""" + pass_threshold: Optional[float] = None + """The threshold for the score.""" -class TestingCriterionScoreModel(BaseModel): +class TestingCriterionEvalGraderScoreModel(ScoreModelGrader): __test__ = False - input: List[TestingCriterionScoreModelInput] - """The input text. This may include template strings.""" - - model: str - """The model to use for the evaluation.""" - - name: str - """The name of the grader.""" - - type: Literal["score_model"] - """The object type, which is always `score_model`.""" - pass_threshold: Optional[float] = None """The threshold for the score.""" - range: Optional[List[float]] = None - """The range of the score. Defaults to `[0, 1]`.""" - - sampling_params: Optional[object] = None - """The sampling parameters for the model.""" - -TestingCriterion: TypeAlias = Annotated[ - Union[ - EvalLabelModelGrader, - EvalStringCheckGrader, - EvalTextSimilarityGrader, - TestingCriterionPython, - TestingCriterionScoreModel, - ], - PropertyInfo(discriminator="type"), +TestingCriterion: TypeAlias = Union[ + LabelModelGrader, + StringCheckGrader, + TestingCriterionEvalGraderTextSimilarity, + TestingCriterionEvalGraderPython, + TestingCriterionEvalGraderScoreModel, ] diff --git a/src/openai/types/fine_tuning/__init__.py b/src/openai/types/fine_tuning/__init__.py index 92b81329b1..cc664eacea 100644 --- a/src/openai/types/fine_tuning/__init__.py +++ b/src/openai/types/fine_tuning/__init__.py @@ -2,13 +2,25 @@ from __future__ import annotations +from .dpo_method import DpoMethod as DpoMethod from .fine_tuning_job import FineTuningJob as FineTuningJob from .job_list_params import JobListParams as JobListParams +from .dpo_method_param import DpoMethodParam as DpoMethodParam from .job_create_params import JobCreateParams as JobCreateParams +from .supervised_method import SupervisedMethod as SupervisedMethod +from .dpo_hyperparameters import DpoHyperparameters as DpoHyperparameters +from .reinforcement_method import ReinforcementMethod as ReinforcementMethod from .fine_tuning_job_event import FineTuningJobEvent as FineTuningJobEvent from .job_list_events_params import JobListEventsParams as JobListEventsParams +from .supervised_method_param import SupervisedMethodParam as SupervisedMethodParam +from .dpo_hyperparameters_param import DpoHyperparametersParam as DpoHyperparametersParam +from .reinforcement_method_param import ReinforcementMethodParam as ReinforcementMethodParam +from .supervised_hyperparameters import SupervisedHyperparameters as SupervisedHyperparameters from .fine_tuning_job_integration import FineTuningJobIntegration as FineTuningJobIntegration +from .reinforcement_hyperparameters import ReinforcementHyperparameters as ReinforcementHyperparameters +from .supervised_hyperparameters_param import SupervisedHyperparametersParam as SupervisedHyperparametersParam from .fine_tuning_job_wandb_integration import FineTuningJobWandbIntegration as FineTuningJobWandbIntegration +from .reinforcement_hyperparameters_param import ReinforcementHyperparametersParam as ReinforcementHyperparametersParam from .fine_tuning_job_wandb_integration_object import ( FineTuningJobWandbIntegrationObject as FineTuningJobWandbIntegrationObject, ) diff --git a/src/openai/types/fine_tuning/alpha/__init__.py b/src/openai/types/fine_tuning/alpha/__init__.py new file mode 100644 index 0000000000..6394961b0b --- /dev/null +++ b/src/openai/types/fine_tuning/alpha/__init__.py @@ -0,0 +1,8 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from .grader_run_params import GraderRunParams as GraderRunParams +from .grader_run_response import GraderRunResponse as GraderRunResponse +from .grader_validate_params import GraderValidateParams as GraderValidateParams +from .grader_validate_response import GraderValidateResponse as GraderValidateResponse diff --git a/src/openai/types/fine_tuning/alpha/grader_run_params.py b/src/openai/types/fine_tuning/alpha/grader_run_params.py new file mode 100644 index 0000000000..fa729f55ba --- /dev/null +++ b/src/openai/types/fine_tuning/alpha/grader_run_params.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union, Iterable +from typing_extensions import Required, TypeAlias, TypedDict + +from ...graders.multi_grader_param import MultiGraderParam +from ...graders.python_grader_param import PythonGraderParam +from ...graders.score_model_grader_param import ScoreModelGraderParam +from ...graders.string_check_grader_param import StringCheckGraderParam +from ...graders.text_similarity_grader_param import TextSimilarityGraderParam + +__all__ = ["GraderRunParams", "Grader"] + + +class GraderRunParams(TypedDict, total=False): + grader: Required[Grader] + """The grader used for the fine-tuning job.""" + + model_sample: Required[str] + """The model sample to be evaluated.""" + + reference_answer: Required[Union[str, Iterable[object], float, object]] + """The reference answer for the evaluation.""" + + +Grader: TypeAlias = Union[ + StringCheckGraderParam, TextSimilarityGraderParam, PythonGraderParam, ScoreModelGraderParam, MultiGraderParam +] diff --git a/src/openai/types/fine_tuning/alpha/grader_run_response.py b/src/openai/types/fine_tuning/alpha/grader_run_response.py new file mode 100644 index 0000000000..8ef046d133 --- /dev/null +++ b/src/openai/types/fine_tuning/alpha/grader_run_response.py @@ -0,0 +1,67 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, Optional + +from pydantic import Field as FieldInfo + +from ...._models import BaseModel + +__all__ = ["GraderRunResponse", "Metadata", "MetadataErrors"] + + +class MetadataErrors(BaseModel): + formula_parse_error: bool + + invalid_variable_error: bool + + api_model_grader_parse_error: bool = FieldInfo(alias="model_grader_parse_error") + + api_model_grader_refusal_error: bool = FieldInfo(alias="model_grader_refusal_error") + + api_model_grader_server_error: bool = FieldInfo(alias="model_grader_server_error") + + api_model_grader_server_error_details: Optional[str] = FieldInfo( + alias="model_grader_server_error_details", default=None + ) + + other_error: bool + + python_grader_runtime_error: bool + + python_grader_runtime_error_details: Optional[str] = None + + python_grader_server_error: bool + + python_grader_server_error_type: Optional[str] = None + + sample_parse_error: bool + + truncated_observation_error: bool + + unresponsive_reward_error: bool + + +class Metadata(BaseModel): + errors: MetadataErrors + + execution_time: float + + name: str + + sampled_model_name: Optional[str] = None + + scores: Dict[str, object] + + token_usage: Optional[int] = None + + type: str + + +class GraderRunResponse(BaseModel): + metadata: Metadata + + api_model_grader_token_usage_per_model: Dict[str, object] = FieldInfo(alias="model_grader_token_usage_per_model") + + reward: float + + sub_rewards: Dict[str, object] diff --git a/src/openai/types/fine_tuning/alpha/grader_validate_params.py b/src/openai/types/fine_tuning/alpha/grader_validate_params.py new file mode 100644 index 0000000000..fe9eb44e32 --- /dev/null +++ b/src/openai/types/fine_tuning/alpha/grader_validate_params.py @@ -0,0 +1,24 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union +from typing_extensions import Required, TypeAlias, TypedDict + +from ...graders.multi_grader_param import MultiGraderParam +from ...graders.python_grader_param import PythonGraderParam +from ...graders.score_model_grader_param import ScoreModelGraderParam +from ...graders.string_check_grader_param import StringCheckGraderParam +from ...graders.text_similarity_grader_param import TextSimilarityGraderParam + +__all__ = ["GraderValidateParams", "Grader"] + + +class GraderValidateParams(TypedDict, total=False): + grader: Required[Grader] + """The grader used for the fine-tuning job.""" + + +Grader: TypeAlias = Union[ + StringCheckGraderParam, TextSimilarityGraderParam, PythonGraderParam, ScoreModelGraderParam, MultiGraderParam +] diff --git a/src/openai/types/fine_tuning/alpha/grader_validate_response.py b/src/openai/types/fine_tuning/alpha/grader_validate_response.py new file mode 100644 index 0000000000..b373292d80 --- /dev/null +++ b/src/openai/types/fine_tuning/alpha/grader_validate_response.py @@ -0,0 +1,20 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Union, Optional +from typing_extensions import TypeAlias + +from ...._models import BaseModel +from ...graders.multi_grader import MultiGrader +from ...graders.python_grader import PythonGrader +from ...graders.score_model_grader import ScoreModelGrader +from ...graders.string_check_grader import StringCheckGrader +from ...graders.text_similarity_grader import TextSimilarityGrader + +__all__ = ["GraderValidateResponse", "Grader"] + +Grader: TypeAlias = Union[StringCheckGrader, TextSimilarityGrader, PythonGrader, ScoreModelGrader, MultiGrader] + + +class GraderValidateResponse(BaseModel): + grader: Optional[Grader] = None + """The grader used for the fine-tuning job.""" diff --git a/src/openai/types/fine_tuning/dpo_hyperparameters.py b/src/openai/types/fine_tuning/dpo_hyperparameters.py new file mode 100644 index 0000000000..b0b3f0581b --- /dev/null +++ b/src/openai/types/fine_tuning/dpo_hyperparameters.py @@ -0,0 +1,36 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Union +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["DpoHyperparameters"] + + +class DpoHyperparameters(BaseModel): + batch_size: Union[Literal["auto"], int, None] = None + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + beta: Union[Literal["auto"], float, None] = None + """The beta value for the DPO method. + + A higher beta value will increase the weight of the penalty between the policy + and reference model. + """ + + learning_rate_multiplier: Union[Literal["auto"], float, None] = None + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int, None] = None + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ diff --git a/src/openai/types/fine_tuning/dpo_hyperparameters_param.py b/src/openai/types/fine_tuning/dpo_hyperparameters_param.py new file mode 100644 index 0000000000..87c6ee80a5 --- /dev/null +++ b/src/openai/types/fine_tuning/dpo_hyperparameters_param.py @@ -0,0 +1,36 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union +from typing_extensions import Literal, TypedDict + +__all__ = ["DpoHyperparametersParam"] + + +class DpoHyperparametersParam(TypedDict, total=False): + batch_size: Union[Literal["auto"], int] + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + beta: Union[Literal["auto"], float] + """The beta value for the DPO method. + + A higher beta value will increase the weight of the penalty between the policy + and reference model. + """ + + learning_rate_multiplier: Union[Literal["auto"], float] + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int] + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ diff --git a/src/openai/types/fine_tuning/dpo_method.py b/src/openai/types/fine_tuning/dpo_method.py new file mode 100644 index 0000000000..3e20f360dd --- /dev/null +++ b/src/openai/types/fine_tuning/dpo_method.py @@ -0,0 +1,13 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel +from .dpo_hyperparameters import DpoHyperparameters + +__all__ = ["DpoMethod"] + + +class DpoMethod(BaseModel): + hyperparameters: Optional[DpoHyperparameters] = None + """The hyperparameters used for the DPO fine-tuning job.""" diff --git a/src/openai/types/fine_tuning/dpo_method_param.py b/src/openai/types/fine_tuning/dpo_method_param.py new file mode 100644 index 0000000000..ce6b6510f6 --- /dev/null +++ b/src/openai/types/fine_tuning/dpo_method_param.py @@ -0,0 +1,14 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +from .dpo_hyperparameters_param import DpoHyperparametersParam + +__all__ = ["DpoMethodParam"] + + +class DpoMethodParam(TypedDict, total=False): + hyperparameters: DpoHyperparametersParam + """The hyperparameters used for the DPO fine-tuning job.""" diff --git a/src/openai/types/fine_tuning/fine_tuning_job.py b/src/openai/types/fine_tuning/fine_tuning_job.py index c7fff2b7b1..f626fbba64 100644 --- a/src/openai/types/fine_tuning/fine_tuning_job.py +++ b/src/openai/types/fine_tuning/fine_tuning_job.py @@ -4,19 +4,13 @@ from typing_extensions import Literal from ..._models import BaseModel +from .dpo_method import DpoMethod from ..shared.metadata import Metadata +from .supervised_method import SupervisedMethod +from .reinforcement_method import ReinforcementMethod from .fine_tuning_job_wandb_integration_object import FineTuningJobWandbIntegrationObject -__all__ = [ - "FineTuningJob", - "Error", - "Hyperparameters", - "Method", - "MethodDpo", - "MethodDpoHyperparameters", - "MethodSupervised", - "MethodSupervisedHyperparameters", -] +__all__ = ["FineTuningJob", "Error", "Hyperparameters", "Method"] class Error(BaseModel): @@ -54,74 +48,18 @@ class Hyperparameters(BaseModel): """ -class MethodDpoHyperparameters(BaseModel): - batch_size: Union[Literal["auto"], int, None] = None - """Number of examples in each batch. - - A larger batch size means that model parameters are updated less frequently, but - with lower variance. - """ - - beta: Union[Literal["auto"], float, None] = None - """The beta value for the DPO method. - - A higher beta value will increase the weight of the penalty between the policy - and reference model. - """ - - learning_rate_multiplier: Union[Literal["auto"], float, None] = None - """Scaling factor for the learning rate. - - A smaller learning rate may be useful to avoid overfitting. - """ - - n_epochs: Union[Literal["auto"], int, None] = None - """The number of epochs to train the model for. - - An epoch refers to one full cycle through the training dataset. - """ - - -class MethodDpo(BaseModel): - hyperparameters: Optional[MethodDpoHyperparameters] = None - """The hyperparameters used for the fine-tuning job.""" - - -class MethodSupervisedHyperparameters(BaseModel): - batch_size: Union[Literal["auto"], int, None] = None - """Number of examples in each batch. - - A larger batch size means that model parameters are updated less frequently, but - with lower variance. - """ - - learning_rate_multiplier: Union[Literal["auto"], float, None] = None - """Scaling factor for the learning rate. - - A smaller learning rate may be useful to avoid overfitting. - """ - - n_epochs: Union[Literal["auto"], int, None] = None - """The number of epochs to train the model for. - - An epoch refers to one full cycle through the training dataset. - """ - - -class MethodSupervised(BaseModel): - hyperparameters: Optional[MethodSupervisedHyperparameters] = None - """The hyperparameters used for the fine-tuning job.""" - - class Method(BaseModel): - dpo: Optional[MethodDpo] = None + type: Literal["supervised", "dpo", "reinforcement"] + """The type of method. Is either `supervised`, `dpo`, or `reinforcement`.""" + + dpo: Optional[DpoMethod] = None """Configuration for the DPO fine-tuning method.""" - supervised: Optional[MethodSupervised] = None - """Configuration for the supervised fine-tuning method.""" + reinforcement: Optional[ReinforcementMethod] = None + """Configuration for the reinforcement fine-tuning method.""" - type: Optional[Literal["supervised", "dpo"]] = None - """The type of method. Is either `supervised` or `dpo`.""" + supervised: Optional[SupervisedMethod] = None + """Configuration for the supervised fine-tuning method.""" class FineTuningJob(BaseModel): diff --git a/src/openai/types/fine_tuning/job_create_params.py b/src/openai/types/fine_tuning/job_create_params.py index f4cf980b08..6b2f41cb71 100644 --- a/src/openai/types/fine_tuning/job_create_params.py +++ b/src/openai/types/fine_tuning/job_create_params.py @@ -5,19 +5,12 @@ from typing import List, Union, Iterable, Optional from typing_extensions import Literal, Required, TypedDict +from .dpo_method_param import DpoMethodParam from ..shared_params.metadata import Metadata +from .supervised_method_param import SupervisedMethodParam +from .reinforcement_method_param import ReinforcementMethodParam -__all__ = [ - "JobCreateParams", - "Hyperparameters", - "Integration", - "IntegrationWandb", - "Method", - "MethodDpo", - "MethodDpoHyperparameters", - "MethodSupervised", - "MethodSupervisedHyperparameters", -] +__all__ = ["JobCreateParams", "Hyperparameters", "Integration", "IntegrationWandb", "Method"] class JobCreateParams(TypedDict, total=False): @@ -166,71 +159,15 @@ class Integration(TypedDict, total=False): """ -class MethodDpoHyperparameters(TypedDict, total=False): - batch_size: Union[Literal["auto"], int] - """Number of examples in each batch. - - A larger batch size means that model parameters are updated less frequently, but - with lower variance. - """ - - beta: Union[Literal["auto"], float] - """The beta value for the DPO method. - - A higher beta value will increase the weight of the penalty between the policy - and reference model. - """ - - learning_rate_multiplier: Union[Literal["auto"], float] - """Scaling factor for the learning rate. - - A smaller learning rate may be useful to avoid overfitting. - """ - - n_epochs: Union[Literal["auto"], int] - """The number of epochs to train the model for. - - An epoch refers to one full cycle through the training dataset. - """ - - -class MethodDpo(TypedDict, total=False): - hyperparameters: MethodDpoHyperparameters - """The hyperparameters used for the fine-tuning job.""" - - -class MethodSupervisedHyperparameters(TypedDict, total=False): - batch_size: Union[Literal["auto"], int] - """Number of examples in each batch. - - A larger batch size means that model parameters are updated less frequently, but - with lower variance. - """ - - learning_rate_multiplier: Union[Literal["auto"], float] - """Scaling factor for the learning rate. - - A smaller learning rate may be useful to avoid overfitting. - """ - - n_epochs: Union[Literal["auto"], int] - """The number of epochs to train the model for. - - An epoch refers to one full cycle through the training dataset. - """ - - -class MethodSupervised(TypedDict, total=False): - hyperparameters: MethodSupervisedHyperparameters - """The hyperparameters used for the fine-tuning job.""" - - class Method(TypedDict, total=False): - dpo: MethodDpo + type: Required[Literal["supervised", "dpo", "reinforcement"]] + """The type of method. Is either `supervised`, `dpo`, or `reinforcement`.""" + + dpo: DpoMethodParam """Configuration for the DPO fine-tuning method.""" - supervised: MethodSupervised - """Configuration for the supervised fine-tuning method.""" + reinforcement: ReinforcementMethodParam + """Configuration for the reinforcement fine-tuning method.""" - type: Literal["supervised", "dpo"] - """The type of method. Is either `supervised` or `dpo`.""" + supervised: SupervisedMethodParam + """Configuration for the supervised fine-tuning method.""" diff --git a/src/openai/types/fine_tuning/reinforcement_hyperparameters.py b/src/openai/types/fine_tuning/reinforcement_hyperparameters.py new file mode 100644 index 0000000000..7c1762d38c --- /dev/null +++ b/src/openai/types/fine_tuning/reinforcement_hyperparameters.py @@ -0,0 +1,43 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Union, Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["ReinforcementHyperparameters"] + + +class ReinforcementHyperparameters(BaseModel): + batch_size: Union[Literal["auto"], int, None] = None + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + compute_multiplier: Union[Literal["auto"], float, None] = None + """ + Multiplier on amount of compute used for exploring search space during training. + """ + + eval_interval: Union[Literal["auto"], int, None] = None + """The number of training steps between evaluation runs.""" + + eval_samples: Union[Literal["auto"], int, None] = None + """Number of evaluation samples to generate per training step.""" + + learning_rate_multiplier: Union[Literal["auto"], float, None] = None + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int, None] = None + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ + + reasoning_effort: Optional[Literal["default", "low", "medium", "high"]] = None + """Level of reasoning effort.""" diff --git a/src/openai/types/fine_tuning/reinforcement_hyperparameters_param.py b/src/openai/types/fine_tuning/reinforcement_hyperparameters_param.py new file mode 100644 index 0000000000..0cc12fcb17 --- /dev/null +++ b/src/openai/types/fine_tuning/reinforcement_hyperparameters_param.py @@ -0,0 +1,43 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union +from typing_extensions import Literal, TypedDict + +__all__ = ["ReinforcementHyperparametersParam"] + + +class ReinforcementHyperparametersParam(TypedDict, total=False): + batch_size: Union[Literal["auto"], int] + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + compute_multiplier: Union[Literal["auto"], float] + """ + Multiplier on amount of compute used for exploring search space during training. + """ + + eval_interval: Union[Literal["auto"], int] + """The number of training steps between evaluation runs.""" + + eval_samples: Union[Literal["auto"], int] + """Number of evaluation samples to generate per training step.""" + + learning_rate_multiplier: Union[Literal["auto"], float] + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int] + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ + + reasoning_effort: Literal["default", "low", "medium", "high"] + """Level of reasoning effort.""" diff --git a/src/openai/types/fine_tuning/reinforcement_method.py b/src/openai/types/fine_tuning/reinforcement_method.py new file mode 100644 index 0000000000..9b65c41033 --- /dev/null +++ b/src/openai/types/fine_tuning/reinforcement_method.py @@ -0,0 +1,24 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Union, Optional +from typing_extensions import TypeAlias + +from ..._models import BaseModel +from ..graders.multi_grader import MultiGrader +from ..graders.python_grader import PythonGrader +from ..graders.score_model_grader import ScoreModelGrader +from ..graders.string_check_grader import StringCheckGrader +from .reinforcement_hyperparameters import ReinforcementHyperparameters +from ..graders.text_similarity_grader import TextSimilarityGrader + +__all__ = ["ReinforcementMethod", "Grader"] + +Grader: TypeAlias = Union[StringCheckGrader, TextSimilarityGrader, PythonGrader, ScoreModelGrader, MultiGrader] + + +class ReinforcementMethod(BaseModel): + grader: Grader + """The grader used for the fine-tuning job.""" + + hyperparameters: Optional[ReinforcementHyperparameters] = None + """The hyperparameters used for the reinforcement fine-tuning job.""" diff --git a/src/openai/types/fine_tuning/reinforcement_method_param.py b/src/openai/types/fine_tuning/reinforcement_method_param.py new file mode 100644 index 0000000000..00d5060536 --- /dev/null +++ b/src/openai/types/fine_tuning/reinforcement_method_param.py @@ -0,0 +1,27 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union +from typing_extensions import Required, TypeAlias, TypedDict + +from ..graders.multi_grader_param import MultiGraderParam +from ..graders.python_grader_param import PythonGraderParam +from ..graders.score_model_grader_param import ScoreModelGraderParam +from ..graders.string_check_grader_param import StringCheckGraderParam +from .reinforcement_hyperparameters_param import ReinforcementHyperparametersParam +from ..graders.text_similarity_grader_param import TextSimilarityGraderParam + +__all__ = ["ReinforcementMethodParam", "Grader"] + +Grader: TypeAlias = Union[ + StringCheckGraderParam, TextSimilarityGraderParam, PythonGraderParam, ScoreModelGraderParam, MultiGraderParam +] + + +class ReinforcementMethodParam(TypedDict, total=False): + grader: Required[Grader] + """The grader used for the fine-tuning job.""" + + hyperparameters: ReinforcementHyperparametersParam + """The hyperparameters used for the reinforcement fine-tuning job.""" diff --git a/src/openai/types/fine_tuning/supervised_hyperparameters.py b/src/openai/types/fine_tuning/supervised_hyperparameters.py new file mode 100644 index 0000000000..3955ecf437 --- /dev/null +++ b/src/openai/types/fine_tuning/supervised_hyperparameters.py @@ -0,0 +1,29 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Union +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["SupervisedHyperparameters"] + + +class SupervisedHyperparameters(BaseModel): + batch_size: Union[Literal["auto"], int, None] = None + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + learning_rate_multiplier: Union[Literal["auto"], float, None] = None + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int, None] = None + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ diff --git a/src/openai/types/fine_tuning/supervised_hyperparameters_param.py b/src/openai/types/fine_tuning/supervised_hyperparameters_param.py new file mode 100644 index 0000000000..bd37d9b239 --- /dev/null +++ b/src/openai/types/fine_tuning/supervised_hyperparameters_param.py @@ -0,0 +1,29 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union +from typing_extensions import Literal, TypedDict + +__all__ = ["SupervisedHyperparametersParam"] + + +class SupervisedHyperparametersParam(TypedDict, total=False): + batch_size: Union[Literal["auto"], int] + """Number of examples in each batch. + + A larger batch size means that model parameters are updated less frequently, but + with lower variance. + """ + + learning_rate_multiplier: Union[Literal["auto"], float] + """Scaling factor for the learning rate. + + A smaller learning rate may be useful to avoid overfitting. + """ + + n_epochs: Union[Literal["auto"], int] + """The number of epochs to train the model for. + + An epoch refers to one full cycle through the training dataset. + """ diff --git a/src/openai/types/fine_tuning/supervised_method.py b/src/openai/types/fine_tuning/supervised_method.py new file mode 100644 index 0000000000..3a32bf27a0 --- /dev/null +++ b/src/openai/types/fine_tuning/supervised_method.py @@ -0,0 +1,13 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel +from .supervised_hyperparameters import SupervisedHyperparameters + +__all__ = ["SupervisedMethod"] + + +class SupervisedMethod(BaseModel): + hyperparameters: Optional[SupervisedHyperparameters] = None + """The hyperparameters used for the fine-tuning job.""" diff --git a/src/openai/types/fine_tuning/supervised_method_param.py b/src/openai/types/fine_tuning/supervised_method_param.py new file mode 100644 index 0000000000..ba277853d7 --- /dev/null +++ b/src/openai/types/fine_tuning/supervised_method_param.py @@ -0,0 +1,14 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import TypedDict + +from .supervised_hyperparameters_param import SupervisedHyperparametersParam + +__all__ = ["SupervisedMethodParam"] + + +class SupervisedMethodParam(TypedDict, total=False): + hyperparameters: SupervisedHyperparametersParam + """The hyperparameters used for the fine-tuning job.""" diff --git a/src/openai/types/graders/__init__.py b/src/openai/types/graders/__init__.py new file mode 100644 index 0000000000..e0a909125e --- /dev/null +++ b/src/openai/types/graders/__init__.py @@ -0,0 +1,16 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from .multi_grader import MultiGrader as MultiGrader +from .python_grader import PythonGrader as PythonGrader +from .label_model_grader import LabelModelGrader as LabelModelGrader +from .multi_grader_param import MultiGraderParam as MultiGraderParam +from .score_model_grader import ScoreModelGrader as ScoreModelGrader +from .python_grader_param import PythonGraderParam as PythonGraderParam +from .string_check_grader import StringCheckGrader as StringCheckGrader +from .text_similarity_grader import TextSimilarityGrader as TextSimilarityGrader +from .label_model_grader_param import LabelModelGraderParam as LabelModelGraderParam +from .score_model_grader_param import ScoreModelGraderParam as ScoreModelGraderParam +from .string_check_grader_param import StringCheckGraderParam as StringCheckGraderParam +from .text_similarity_grader_param import TextSimilarityGraderParam as TextSimilarityGraderParam diff --git a/src/openai/types/eval_label_model_grader.py b/src/openai/types/graders/label_model_grader.py similarity index 85% rename from src/openai/types/eval_label_model_grader.py rename to src/openai/types/graders/label_model_grader.py index 40e6bda140..d95ccc6df6 100644 --- a/src/openai/types/eval_label_model_grader.py +++ b/src/openai/types/graders/label_model_grader.py @@ -3,10 +3,10 @@ from typing import List, Union, Optional from typing_extensions import Literal, TypeAlias -from .._models import BaseModel -from .responses.response_input_text import ResponseInputText +from ..._models import BaseModel +from ..responses.response_input_text import ResponseInputText -__all__ = ["EvalLabelModelGrader", "Input", "InputContent", "InputContentOutputText"] +__all__ = ["LabelModelGrader", "Input", "InputContent", "InputContentOutputText"] class InputContentOutputText(BaseModel): @@ -34,7 +34,7 @@ class Input(BaseModel): """The type of the message input. Always `message`.""" -class EvalLabelModelGrader(BaseModel): +class LabelModelGrader(BaseModel): input: List[Input] labels: List[str] diff --git a/src/openai/types/graders/label_model_grader_param.py b/src/openai/types/graders/label_model_grader_param.py new file mode 100644 index 0000000000..76d01421ee --- /dev/null +++ b/src/openai/types/graders/label_model_grader_param.py @@ -0,0 +1,54 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import List, Union, Iterable +from typing_extensions import Literal, Required, TypeAlias, TypedDict + +from ..responses.response_input_text_param import ResponseInputTextParam + +__all__ = ["LabelModelGraderParam", "Input", "InputContent", "InputContentOutputText"] + + +class InputContentOutputText(TypedDict, total=False): + text: Required[str] + """The text output from the model.""" + + type: Required[Literal["output_text"]] + """The type of the output text. Always `output_text`.""" + + +InputContent: TypeAlias = Union[str, ResponseInputTextParam, InputContentOutputText] + + +class Input(TypedDict, total=False): + content: Required[InputContent] + """Text inputs to the model - can contain template strings.""" + + role: Required[Literal["user", "assistant", "system", "developer"]] + """The role of the message input. + + One of `user`, `assistant`, `system`, or `developer`. + """ + + type: Literal["message"] + """The type of the message input. Always `message`.""" + + +class LabelModelGraderParam(TypedDict, total=False): + input: Required[Iterable[Input]] + + labels: Required[List[str]] + """The labels to assign to each item in the evaluation.""" + + model: Required[str] + """The model to use for the evaluation. Must support structured outputs.""" + + name: Required[str] + """The name of the grader.""" + + passing_labels: Required[List[str]] + """The labels that indicate a passing result. Must be a subset of labels.""" + + type: Required[Literal["label_model"]] + """The object type, which is always `label_model`.""" diff --git a/src/openai/types/graders/multi_grader.py b/src/openai/types/graders/multi_grader.py new file mode 100644 index 0000000000..ee9b31d2b0 --- /dev/null +++ b/src/openai/types/graders/multi_grader.py @@ -0,0 +1,28 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, Union +from typing_extensions import Literal, TypeAlias + +from ..._models import BaseModel +from .python_grader import PythonGrader +from .label_model_grader import LabelModelGrader +from .score_model_grader import ScoreModelGrader +from .string_check_grader import StringCheckGrader +from .text_similarity_grader import TextSimilarityGrader + +__all__ = ["MultiGrader", "Graders"] + +Graders: TypeAlias = Union[StringCheckGrader, TextSimilarityGrader, PythonGrader, ScoreModelGrader, LabelModelGrader] + + +class MultiGrader(BaseModel): + calculate_output: str + """A formula to calculate the output based on grader results.""" + + graders: Dict[str, Graders] + + name: str + """The name of the grader.""" + + type: Literal["multi"] + """The type of grader.""" diff --git a/src/openai/types/graders/multi_grader_param.py b/src/openai/types/graders/multi_grader_param.py new file mode 100644 index 0000000000..4dd1a48530 --- /dev/null +++ b/src/openai/types/graders/multi_grader_param.py @@ -0,0 +1,31 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict, Union +from typing_extensions import Literal, Required, TypeAlias, TypedDict + +from .python_grader_param import PythonGraderParam +from .label_model_grader_param import LabelModelGraderParam +from .score_model_grader_param import ScoreModelGraderParam +from .string_check_grader_param import StringCheckGraderParam +from .text_similarity_grader_param import TextSimilarityGraderParam + +__all__ = ["MultiGraderParam", "Graders"] + +Graders: TypeAlias = Union[ + StringCheckGraderParam, TextSimilarityGraderParam, PythonGraderParam, ScoreModelGraderParam, LabelModelGraderParam +] + + +class MultiGraderParam(TypedDict, total=False): + calculate_output: Required[str] + """A formula to calculate the output based on grader results.""" + + graders: Required[Dict[str, Graders]] + + name: Required[str] + """The name of the grader.""" + + type: Required[Literal["multi"]] + """The type of grader.""" diff --git a/src/openai/types/graders/python_grader.py b/src/openai/types/graders/python_grader.py new file mode 100644 index 0000000000..faa10b1ef9 --- /dev/null +++ b/src/openai/types/graders/python_grader.py @@ -0,0 +1,22 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["PythonGrader"] + + +class PythonGrader(BaseModel): + name: str + """The name of the grader.""" + + source: str + """The source code of the python script.""" + + type: Literal["python"] + """The object type, which is always `python`.""" + + image_tag: Optional[str] = None + """The image tag to use for the python script.""" diff --git a/src/openai/types/graders/python_grader_param.py b/src/openai/types/graders/python_grader_param.py new file mode 100644 index 0000000000..efb923751e --- /dev/null +++ b/src/openai/types/graders/python_grader_param.py @@ -0,0 +1,21 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["PythonGraderParam"] + + +class PythonGraderParam(TypedDict, total=False): + name: Required[str] + """The name of the grader.""" + + source: Required[str] + """The source code of the python script.""" + + type: Required[Literal["python"]] + """The object type, which is always `python`.""" + + image_tag: str + """The image tag to use for the python script.""" diff --git a/src/openai/types/graders/score_model_grader.py b/src/openai/types/graders/score_model_grader.py new file mode 100644 index 0000000000..1349f75a58 --- /dev/null +++ b/src/openai/types/graders/score_model_grader.py @@ -0,0 +1,54 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal, TypeAlias + +from ..._models import BaseModel +from ..responses.response_input_text import ResponseInputText + +__all__ = ["ScoreModelGrader", "Input", "InputContent", "InputContentOutputText"] + + +class InputContentOutputText(BaseModel): + text: str + """The text output from the model.""" + + type: Literal["output_text"] + """The type of the output text. Always `output_text`.""" + + +InputContent: TypeAlias = Union[str, ResponseInputText, InputContentOutputText] + + +class Input(BaseModel): + content: InputContent + """Text inputs to the model - can contain template strings.""" + + role: Literal["user", "assistant", "system", "developer"] + """The role of the message input. + + One of `user`, `assistant`, `system`, or `developer`. + """ + + type: Optional[Literal["message"]] = None + """The type of the message input. Always `message`.""" + + +class ScoreModelGrader(BaseModel): + input: List[Input] + """The input text. This may include template strings.""" + + model: str + """The model to use for the evaluation.""" + + name: str + """The name of the grader.""" + + type: Literal["score_model"] + """The object type, which is always `score_model`.""" + + range: Optional[List[float]] = None + """The range of the score. Defaults to `[0, 1]`.""" + + sampling_params: Optional[object] = None + """The sampling parameters for the model.""" diff --git a/src/openai/types/graders/score_model_grader_param.py b/src/openai/types/graders/score_model_grader_param.py new file mode 100644 index 0000000000..673f14e47d --- /dev/null +++ b/src/openai/types/graders/score_model_grader_param.py @@ -0,0 +1,55 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union, Iterable +from typing_extensions import Literal, Required, TypeAlias, TypedDict + +from ..responses.response_input_text_param import ResponseInputTextParam + +__all__ = ["ScoreModelGraderParam", "Input", "InputContent", "InputContentOutputText"] + + +class InputContentOutputText(TypedDict, total=False): + text: Required[str] + """The text output from the model.""" + + type: Required[Literal["output_text"]] + """The type of the output text. Always `output_text`.""" + + +InputContent: TypeAlias = Union[str, ResponseInputTextParam, InputContentOutputText] + + +class Input(TypedDict, total=False): + content: Required[InputContent] + """Text inputs to the model - can contain template strings.""" + + role: Required[Literal["user", "assistant", "system", "developer"]] + """The role of the message input. + + One of `user`, `assistant`, `system`, or `developer`. + """ + + type: Literal["message"] + """The type of the message input. Always `message`.""" + + +class ScoreModelGraderParam(TypedDict, total=False): + input: Required[Iterable[Input]] + """The input text. This may include template strings.""" + + model: Required[str] + """The model to use for the evaluation.""" + + name: Required[str] + """The name of the grader.""" + + type: Required[Literal["score_model"]] + """The object type, which is always `score_model`.""" + + range: Iterable[float] + """The range of the score. Defaults to `[0, 1]`.""" + + sampling_params: object + """The sampling parameters for the model.""" diff --git a/src/openai/types/eval_string_check_grader.py b/src/openai/types/graders/string_check_grader.py similarity index 84% rename from src/openai/types/eval_string_check_grader.py rename to src/openai/types/graders/string_check_grader.py index 4dfc8035f9..3bf0b8c868 100644 --- a/src/openai/types/eval_string_check_grader.py +++ b/src/openai/types/graders/string_check_grader.py @@ -2,12 +2,12 @@ from typing_extensions import Literal -from .._models import BaseModel +from ..._models import BaseModel -__all__ = ["EvalStringCheckGrader"] +__all__ = ["StringCheckGrader"] -class EvalStringCheckGrader(BaseModel): +class StringCheckGrader(BaseModel): input: str """The input text. This may include template strings.""" diff --git a/src/openai/types/eval_string_check_grader_param.py b/src/openai/types/graders/string_check_grader_param.py similarity index 87% rename from src/openai/types/eval_string_check_grader_param.py rename to src/openai/types/graders/string_check_grader_param.py index 3511329f8b..27b204cec0 100644 --- a/src/openai/types/eval_string_check_grader_param.py +++ b/src/openai/types/graders/string_check_grader_param.py @@ -4,10 +4,10 @@ from typing_extensions import Literal, Required, TypedDict -__all__ = ["EvalStringCheckGraderParam"] +__all__ = ["StringCheckGraderParam"] -class EvalStringCheckGraderParam(TypedDict, total=False): +class StringCheckGraderParam(TypedDict, total=False): input: Required[str] """The input text. This may include template strings.""" diff --git a/src/openai/types/eval_text_similarity_grader.py b/src/openai/types/graders/text_similarity_grader.py similarity index 69% rename from src/openai/types/eval_text_similarity_grader.py rename to src/openai/types/graders/text_similarity_grader.py index 853c6d4fbf..738d317766 100644 --- a/src/openai/types/eval_text_similarity_grader.py +++ b/src/openai/types/graders/text_similarity_grader.py @@ -1,14 +1,13 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import Optional from typing_extensions import Literal -from .._models import BaseModel +from ..._models import BaseModel -__all__ = ["EvalTextSimilarityGrader"] +__all__ = ["TextSimilarityGrader"] -class EvalTextSimilarityGrader(BaseModel): +class TextSimilarityGrader(BaseModel): evaluation_metric: Literal[ "fuzzy_match", "bleu", "gleu", "meteor", "rouge_1", "rouge_2", "rouge_3", "rouge_4", "rouge_5", "rouge_l" ] @@ -21,14 +20,11 @@ class EvalTextSimilarityGrader(BaseModel): input: str """The text being graded.""" - pass_threshold: float - """A float score where a value greater than or equal indicates a passing grade.""" + name: str + """The name of the grader.""" reference: str """The text being graded against.""" type: Literal["text_similarity"] """The type of grader.""" - - name: Optional[str] = None - """The name of the grader.""" diff --git a/src/openai/types/eval_text_similarity_grader_param.py b/src/openai/types/graders/text_similarity_grader_param.py similarity index 76% rename from src/openai/types/eval_text_similarity_grader_param.py rename to src/openai/types/graders/text_similarity_grader_param.py index f07cc29178..db14553217 100644 --- a/src/openai/types/eval_text_similarity_grader_param.py +++ b/src/openai/types/graders/text_similarity_grader_param.py @@ -4,10 +4,10 @@ from typing_extensions import Literal, Required, TypedDict -__all__ = ["EvalTextSimilarityGraderParam"] +__all__ = ["TextSimilarityGraderParam"] -class EvalTextSimilarityGraderParam(TypedDict, total=False): +class TextSimilarityGraderParam(TypedDict, total=False): evaluation_metric: Required[ Literal[ "fuzzy_match", "bleu", "gleu", "meteor", "rouge_1", "rouge_2", "rouge_3", "rouge_4", "rouge_5", "rouge_l" @@ -22,14 +22,11 @@ class EvalTextSimilarityGraderParam(TypedDict, total=False): input: Required[str] """The text being graded.""" - pass_threshold: Required[float] - """A float score where a value greater than or equal indicates a passing grade.""" + name: Required[str] + """The name of the grader.""" reference: Required[str] """The text being graded against.""" type: Required[Literal["text_similarity"]] """The type of grader.""" - - name: str - """The name of the grader.""" diff --git a/tests/api_resources/fine_tuning/alpha/__init__.py b/tests/api_resources/fine_tuning/alpha/__init__.py new file mode 100644 index 0000000000..fd8019a9a1 --- /dev/null +++ b/tests/api_resources/fine_tuning/alpha/__init__.py @@ -0,0 +1 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. diff --git a/tests/api_resources/fine_tuning/alpha/test_graders.py b/tests/api_resources/fine_tuning/alpha/test_graders.py new file mode 100644 index 0000000000..b144c78c74 --- /dev/null +++ b/tests/api_resources/fine_tuning/alpha/test_graders.py @@ -0,0 +1,289 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +import os +from typing import Any, cast + +import pytest + +from openai import OpenAI, AsyncOpenAI +from tests.utils import assert_matches_type +from openai.types.fine_tuning.alpha import ( + GraderRunResponse, + GraderValidateResponse, +) + +base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") + + +class TestGraders: + parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + def test_method_run(self, client: OpenAI) -> None: + grader = client.fine_tuning.alpha.graders.run( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + model_sample="model_sample", + reference_answer="string", + ) + assert_matches_type(GraderRunResponse, grader, path=["response"]) + + @parametrize + def test_method_run_with_all_params(self, client: OpenAI) -> None: + grader = client.fine_tuning.alpha.graders.run( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + model_sample="model_sample", + reference_answer="string", + ) + assert_matches_type(GraderRunResponse, grader, path=["response"]) + + @parametrize + def test_raw_response_run(self, client: OpenAI) -> None: + response = client.fine_tuning.alpha.graders.with_raw_response.run( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + model_sample="model_sample", + reference_answer="string", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + grader = response.parse() + assert_matches_type(GraderRunResponse, grader, path=["response"]) + + @parametrize + def test_streaming_response_run(self, client: OpenAI) -> None: + with client.fine_tuning.alpha.graders.with_streaming_response.run( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + model_sample="model_sample", + reference_answer="string", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + grader = response.parse() + assert_matches_type(GraderRunResponse, grader, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_method_validate(self, client: OpenAI) -> None: + grader = client.fine_tuning.alpha.graders.validate( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + ) + assert_matches_type(GraderValidateResponse, grader, path=["response"]) + + @parametrize + def test_method_validate_with_all_params(self, client: OpenAI) -> None: + grader = client.fine_tuning.alpha.graders.validate( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + ) + assert_matches_type(GraderValidateResponse, grader, path=["response"]) + + @parametrize + def test_raw_response_validate(self, client: OpenAI) -> None: + response = client.fine_tuning.alpha.graders.with_raw_response.validate( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + grader = response.parse() + assert_matches_type(GraderValidateResponse, grader, path=["response"]) + + @parametrize + def test_streaming_response_validate(self, client: OpenAI) -> None: + with client.fine_tuning.alpha.graders.with_streaming_response.validate( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + grader = response.parse() + assert_matches_type(GraderValidateResponse, grader, path=["response"]) + + assert cast(Any, response.is_closed) is True + + +class TestAsyncGraders: + parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) + + @parametrize + async def test_method_run(self, async_client: AsyncOpenAI) -> None: + grader = await async_client.fine_tuning.alpha.graders.run( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + model_sample="model_sample", + reference_answer="string", + ) + assert_matches_type(GraderRunResponse, grader, path=["response"]) + + @parametrize + async def test_method_run_with_all_params(self, async_client: AsyncOpenAI) -> None: + grader = await async_client.fine_tuning.alpha.graders.run( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + model_sample="model_sample", + reference_answer="string", + ) + assert_matches_type(GraderRunResponse, grader, path=["response"]) + + @parametrize + async def test_raw_response_run(self, async_client: AsyncOpenAI) -> None: + response = await async_client.fine_tuning.alpha.graders.with_raw_response.run( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + model_sample="model_sample", + reference_answer="string", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + grader = response.parse() + assert_matches_type(GraderRunResponse, grader, path=["response"]) + + @parametrize + async def test_streaming_response_run(self, async_client: AsyncOpenAI) -> None: + async with async_client.fine_tuning.alpha.graders.with_streaming_response.run( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + model_sample="model_sample", + reference_answer="string", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + grader = await response.parse() + assert_matches_type(GraderRunResponse, grader, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_method_validate(self, async_client: AsyncOpenAI) -> None: + grader = await async_client.fine_tuning.alpha.graders.validate( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + ) + assert_matches_type(GraderValidateResponse, grader, path=["response"]) + + @parametrize + async def test_method_validate_with_all_params(self, async_client: AsyncOpenAI) -> None: + grader = await async_client.fine_tuning.alpha.graders.validate( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + ) + assert_matches_type(GraderValidateResponse, grader, path=["response"]) + + @parametrize + async def test_raw_response_validate(self, async_client: AsyncOpenAI) -> None: + response = await async_client.fine_tuning.alpha.graders.with_raw_response.validate( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + grader = response.parse() + assert_matches_type(GraderValidateResponse, grader, path=["response"]) + + @parametrize + async def test_streaming_response_validate(self, async_client: AsyncOpenAI) -> None: + async with async_client.fine_tuning.alpha.graders.with_streaming_response.validate( + grader={ + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + grader = await response.parse() + assert_matches_type(GraderValidateResponse, grader, path=["response"]) + + assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/fine_tuning/test_jobs.py b/tests/api_resources/fine_tuning/test_jobs.py index 75f72f9d09..4589f12846 100644 --- a/tests/api_resources/fine_tuning/test_jobs.py +++ b/tests/api_resources/fine_tuning/test_jobs.py @@ -52,6 +52,7 @@ def test_method_create_with_all_params(self, client: OpenAI) -> None: ], metadata={"foo": "string"}, method={ + "type": "supervised", "dpo": { "hyperparameters": { "batch_size": "auto", @@ -60,6 +61,24 @@ def test_method_create_with_all_params(self, client: OpenAI) -> None: "n_epochs": "auto", } }, + "reinforcement": { + "grader": { + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + "hyperparameters": { + "batch_size": "auto", + "compute_multiplier": "auto", + "eval_interval": "auto", + "eval_samples": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + "reasoning_effort": "default", + }, + }, "supervised": { "hyperparameters": { "batch_size": "auto", @@ -67,7 +86,6 @@ def test_method_create_with_all_params(self, client: OpenAI) -> None: "n_epochs": "auto", } }, - "type": "supervised", }, seed=42, suffix="x", @@ -258,6 +276,82 @@ def test_path_params_list_events(self, client: OpenAI) -> None: "", ) + @parametrize + def test_method_pause(self, client: OpenAI) -> None: + job = client.fine_tuning.jobs.pause( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) + assert_matches_type(FineTuningJob, job, path=["response"]) + + @parametrize + def test_raw_response_pause(self, client: OpenAI) -> None: + response = client.fine_tuning.jobs.with_raw_response.pause( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = response.parse() + assert_matches_type(FineTuningJob, job, path=["response"]) + + @parametrize + def test_streaming_response_pause(self, client: OpenAI) -> None: + with client.fine_tuning.jobs.with_streaming_response.pause( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = response.parse() + assert_matches_type(FineTuningJob, job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_pause(self, client: OpenAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `fine_tuning_job_id` but received ''"): + client.fine_tuning.jobs.with_raw_response.pause( + "", + ) + + @parametrize + def test_method_resume(self, client: OpenAI) -> None: + job = client.fine_tuning.jobs.resume( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) + assert_matches_type(FineTuningJob, job, path=["response"]) + + @parametrize + def test_raw_response_resume(self, client: OpenAI) -> None: + response = client.fine_tuning.jobs.with_raw_response.resume( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = response.parse() + assert_matches_type(FineTuningJob, job, path=["response"]) + + @parametrize + def test_streaming_response_resume(self, client: OpenAI) -> None: + with client.fine_tuning.jobs.with_streaming_response.resume( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = response.parse() + assert_matches_type(FineTuningJob, job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + def test_path_params_resume(self, client: OpenAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `fine_tuning_job_id` but received ''"): + client.fine_tuning.jobs.with_raw_response.resume( + "", + ) + class TestAsyncJobs: parametrize = pytest.mark.parametrize("async_client", [False, True], indirect=True, ids=["loose", "strict"]) @@ -293,6 +387,7 @@ async def test_method_create_with_all_params(self, async_client: AsyncOpenAI) -> ], metadata={"foo": "string"}, method={ + "type": "supervised", "dpo": { "hyperparameters": { "batch_size": "auto", @@ -301,6 +396,24 @@ async def test_method_create_with_all_params(self, async_client: AsyncOpenAI) -> "n_epochs": "auto", } }, + "reinforcement": { + "grader": { + "input": "input", + "name": "name", + "operation": "eq", + "reference": "reference", + "type": "string_check", + }, + "hyperparameters": { + "batch_size": "auto", + "compute_multiplier": "auto", + "eval_interval": "auto", + "eval_samples": "auto", + "learning_rate_multiplier": "auto", + "n_epochs": "auto", + "reasoning_effort": "default", + }, + }, "supervised": { "hyperparameters": { "batch_size": "auto", @@ -308,7 +421,6 @@ async def test_method_create_with_all_params(self, async_client: AsyncOpenAI) -> "n_epochs": "auto", } }, - "type": "supervised", }, seed=42, suffix="x", @@ -498,3 +610,79 @@ async def test_path_params_list_events(self, async_client: AsyncOpenAI) -> None: await async_client.fine_tuning.jobs.with_raw_response.list_events( "", ) + + @parametrize + async def test_method_pause(self, async_client: AsyncOpenAI) -> None: + job = await async_client.fine_tuning.jobs.pause( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) + assert_matches_type(FineTuningJob, job, path=["response"]) + + @parametrize + async def test_raw_response_pause(self, async_client: AsyncOpenAI) -> None: + response = await async_client.fine_tuning.jobs.with_raw_response.pause( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = response.parse() + assert_matches_type(FineTuningJob, job, path=["response"]) + + @parametrize + async def test_streaming_response_pause(self, async_client: AsyncOpenAI) -> None: + async with async_client.fine_tuning.jobs.with_streaming_response.pause( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = await response.parse() + assert_matches_type(FineTuningJob, job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_pause(self, async_client: AsyncOpenAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `fine_tuning_job_id` but received ''"): + await async_client.fine_tuning.jobs.with_raw_response.pause( + "", + ) + + @parametrize + async def test_method_resume(self, async_client: AsyncOpenAI) -> None: + job = await async_client.fine_tuning.jobs.resume( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) + assert_matches_type(FineTuningJob, job, path=["response"]) + + @parametrize + async def test_raw_response_resume(self, async_client: AsyncOpenAI) -> None: + response = await async_client.fine_tuning.jobs.with_raw_response.resume( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + job = response.parse() + assert_matches_type(FineTuningJob, job, path=["response"]) + + @parametrize + async def test_streaming_response_resume(self, async_client: AsyncOpenAI) -> None: + async with async_client.fine_tuning.jobs.with_streaming_response.resume( + "ft-AF1WoRqd3aJAHsqc9NY7iL8F", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + job = await response.parse() + assert_matches_type(FineTuningJob, job, path=["response"]) + + assert cast(Any, response.is_closed) is True + + @parametrize + async def test_path_params_resume(self, async_client: AsyncOpenAI) -> None: + with pytest.raises(ValueError, match=r"Expected a non-empty value for `fine_tuning_job_id` but received ''"): + await async_client.fine_tuning.jobs.with_raw_response.resume( + "", + ) diff --git a/tests/test_utils/test_proxy.py b/tests/test_utils/test_proxy.py index aedd3731ee..2b5ff19dab 100644 --- a/tests/test_utils/test_proxy.py +++ b/tests/test_utils/test_proxy.py @@ -3,6 +3,7 @@ from typing_extensions import override from openai._utils import LazyProxy +from openai._extras._common import MissingDependencyError class RecursiveLazyProxy(LazyProxy[Any]): @@ -21,3 +22,14 @@ def test_recursive_proxy() -> None: assert dir(proxy) == [] assert type(proxy).__name__ == "RecursiveLazyProxy" assert type(operator.attrgetter("name.foo.bar.baz")(proxy)).__name__ == "RecursiveLazyProxy" + + +def test_isinstance_does_not_error() -> None: + class MissingDepsProxy(LazyProxy[Any]): + @override + def __load__(self) -> Any: + raise MissingDependencyError("Mocking missing dependency") + + proxy = MissingDepsProxy() + assert not isinstance(proxy, dict) + assert isinstance(proxy, LazyProxy)