From a098bbf06341b14298a7331641e933e6daace9ea Mon Sep 17 00:00:00 2001 From: Justin Hallier Date: Thu, 14 Nov 2024 15:37:44 +1300 Subject: [PATCH 1/4] Add vector store apis Update the assistants beta version to v2 Add functions to create and read vector stores and vector store file batches. Add a separate helper function to handle uploading files, then associating them with the vector store --- src/OpenAi.php | 113 ++++++++++++++++++++++++++++++++++++++++++++++--- src/Url.php | 5 +++ 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/src/OpenAi.php b/src/OpenAi.php index 87bfcbe..21272f9 100644 --- a/src/OpenAi.php +++ b/src/OpenAi.php @@ -2,6 +2,7 @@ namespace Orhanerday\OpenAi; +use CURLFile; use Exception; class OpenAi @@ -9,7 +10,8 @@ class OpenAi private string $engine = "davinci"; private string $model = "text-davinci-002"; private string $chatModel = "gpt-3.5-turbo"; - private string $assistantsBetaVersion = "v1"; + + private string $assistantsBetaVersion = "v2"; private array $headers; private array $contentTypes; private int $timeout = 0; @@ -720,7 +722,7 @@ public function createRun($threadId, $data, $stream = null) $this->stream_method = $stream; } - + $this->addAssistantsBetaHeader(); $url = Url::threadsUrl() . '/' . $threadId . '/runs'; $this->baseUrl($url); @@ -791,7 +793,7 @@ public function submitToolOutputs($threadId, $runId, $outputs, $stream = null) $this->stream_method = $stream; } - + $this->addAssistantsBetaHeader(); $url = Url::threadsUrl() . '/' . $threadId . '/runs/' . $runId . '/submit_tool_outputs'; $this->baseUrl($url); @@ -871,6 +873,105 @@ public function tts($opts) return $this->sendRequest($url, 'POST', $opts); } + /** + * @param string $name + * @return bool|string + * @throws Exception + */ + public function createVectorStore(string $name) + { + $this->addAssistantsBetaHeader(); + $url = Url::vectorStoreUrl(); + $this->baseUrl($url); + + return $this->sendRequest($url, 'POST', ['name' => $name]); + } + + /** + * @param string $id + * @return bool|string + * @throws Exception + */ + public function retrieveVectorStore(string $id) + { + $this->addAssistantsBetaHeader(); + $url = Url::vectorStoreUrl() . '/' . $id; + $this->baseUrl($url); + + return $this->sendRequest($url, 'GET'); + } + + /** + * Perform a file upload for each curl file in the files array, then attach them to the given vector store. + * Poll the file batch every 3s until it is completed, then return the file batch. + * @param string $vectorStoreId + * @param CURLFile[] $files + * @return bool|string + * @throws Exception + */ + public function uploadFilesToVectorStoreAndPoll(string $vectorStoreId, array $files) + { + $fileIds = []; + foreach ($files as $file) { + $file = $this->uploadFile([ + 'file' => $file, + 'purpose' => 'assistants', + ]); + $file = json_decode($file, true); + $fileIds[] = $file['id']; + } + + $fileBatch = $this->createVectorFileBatch($vectorStoreId, $fileIds); + $fileBatch = json_decode($fileBatch, true); + + while (true) { + $fileBatch = $this->retrieveVectorFileBatch($vectorStoreId, $fileBatch['id']); + $fileBatch = json_decode($fileBatch, true); + + switch ($fileBatch['status']) { + case 'processing': + break; + case 'cancelled': + case 'failed': + throw new Exception('File batch failed'); + case 'completed': + return $fileBatch; + } + + sleep(3); + } + } + + /** + * @param string $vectorStoreId + * @param array $fileIds + * @return bool|string + * @throws Exception + */ + public function createVectorFileBatch(string $vectorStoreId, array $fileIds) + { + $this->addAssistantsBetaHeader(); + $url = Url::vectorStoreUrl() . '/' . $vectorStoreId . '/file_batches'; + $this->baseUrl($url); + + return $this->sendRequest($url, 'POST', ['file_ids' => $fileIds]); + } + + /** + * @param string $vectorStoreId + * @param string $fileBatchId + * @return bool|string + * @throws Exception + */ + public function retrieveVectorFileBatch(string $vectorStoreId, string $fileBatchId) + { + $this->addAssistantsBetaHeader(); + $url = Url::vectorStoreUrl() . '/' . $vectorStoreId . '/file_batches/' . $fileBatchId; + $this->baseUrl($url); + + return $this->sendRequest($url, 'GET'); + } + /** * @param int $timeout */ @@ -939,7 +1040,7 @@ public function setORG(string $org) $this->headers[] = "OpenAI-Organization: $org"; } } - + /** * @param string $org */ @@ -953,10 +1054,10 @@ public function setAssistantsBetaVersion(string $version) /** * @return void */ - private function addAssistantsBetaHeader(){ + private function addAssistantsBetaHeader() + { $this->headers[] = 'OpenAI-Beta: assistants='.$this->assistantsBetaVersion; } - /** * @param string $url diff --git a/src/Url.php b/src/Url.php index f3c90ef..85c7070 100644 --- a/src/Url.php +++ b/src/Url.php @@ -187,4 +187,9 @@ public static function ttsUrl(): string { return self::OPEN_AI_URL . "/audio/speech"; } + + public static function vectorStoreUrl(): string + { + return self::OPEN_AI_URL . "/vector_stores"; + } } From af3251e2c7217a962ec0ec3b9ca18be67627ef15 Mon Sep 17 00:00:00 2001 From: Justin Hallier Date: Fri, 15 Nov 2024 17:03:18 +1300 Subject: [PATCH 2/4] Update helper function return type Return an array containing the uploaded file ids as well as the file batch info. This enables the caller to do something with the uploaded file objects if needed --- src/OpenAi.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/OpenAi.php b/src/OpenAi.php index 21272f9..fffba53 100644 --- a/src/OpenAi.php +++ b/src/OpenAi.php @@ -903,13 +903,13 @@ public function retrieveVectorStore(string $id) /** * Perform a file upload for each curl file in the files array, then attach them to the given vector store. - * Poll the file batch every 3s until it is completed, then return the file batch. + * Poll the file batch every 3s until it is completed, then return the file batch and openai file object ids. * @param string $vectorStoreId * @param CURLFile[] $files - * @return bool|string + * @return array{fileIds: array, fileBatch: bool|string} * @throws Exception */ - public function uploadFilesToVectorStoreAndPoll(string $vectorStoreId, array $files) + public function uploadFilesToVectorStoreAndPoll(string $vectorStoreId, array $files): array { $fileIds = []; foreach ($files as $file) { @@ -935,7 +935,7 @@ public function uploadFilesToVectorStoreAndPoll(string $vectorStoreId, array $fi case 'failed': throw new Exception('File batch failed'); case 'completed': - return $fileBatch; + return ['fileIds' => $fileIds, 'fileBatch' => $fileBatch]; } sleep(3); From 39cddf73918ba4685d417a5f6fa2b5dd6c311e29 Mon Sep 17 00:00:00 2001 From: Justin Hallier Date: Tue, 26 Nov 2024 12:34:37 +1300 Subject: [PATCH 3/4] Add 'createVectorStoreFile' function Allows attaching a file to a vector store --- src/OpenAi.php | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/OpenAi.php b/src/OpenAi.php index fffba53..05f4225 100644 --- a/src/OpenAi.php +++ b/src/OpenAi.php @@ -901,6 +901,21 @@ public function retrieveVectorStore(string $id) return $this->sendRequest($url, 'GET'); } + /** + * @param string $vectorStoreId + * @param string $fileId + * @return bool|string + * @throws Exception + */ + public function createVectorStoreFile(string $vectorStoreId, string $fileId) + { + $this->addAssistantsBetaHeader(); + $url = Url::vectorStoreUrl() . '/' . $vectorStoreId . '/files'; + $this->baseUrl($url); + + return $this->sendRequest($url, 'POST', ['file_id' => $fileId]); + } + /** * Perform a file upload for each curl file in the files array, then attach them to the given vector store. * Poll the file batch every 3s until it is completed, then return the file batch and openai file object ids. From c9ab621405a05c5201c66f1a436f3cdf1ff0f102 Mon Sep 17 00:00:00 2001 From: Justin Hallier Date: Mon, 16 Dec 2024 11:32:48 +1300 Subject: [PATCH 4/4] Remove polling function and update header function Remove the polling function since it doesn't really fit with the rest of the lib. Add retrieveVectorStoreFile function to allowing checking status of individual vector store files. Update the assistants beta headers function to check if the header has already been added before trying to add it. This avoids issues with the headers growing too large if the header function is called repeatedly, as it would add the same header multiple times --- src/OpenAi.php | 46 +++++++++++----------------------------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/src/OpenAi.php b/src/OpenAi.php index 05f4225..de70cac 100644 --- a/src/OpenAi.php +++ b/src/OpenAi.php @@ -2,7 +2,6 @@ namespace Orhanerday\OpenAi; -use CURLFile; use Exception; class OpenAi @@ -917,44 +916,18 @@ public function createVectorStoreFile(string $vectorStoreId, string $fileId) } /** - * Perform a file upload for each curl file in the files array, then attach them to the given vector store. - * Poll the file batch every 3s until it is completed, then return the file batch and openai file object ids. * @param string $vectorStoreId - * @param CURLFile[] $files - * @return array{fileIds: array, fileBatch: bool|string} + * @param string $fileId + * @return bool|string * @throws Exception */ - public function uploadFilesToVectorStoreAndPoll(string $vectorStoreId, array $files): array + public function retrieveVectorStoreFile(string $vectorStoreId, string $fileId) { - $fileIds = []; - foreach ($files as $file) { - $file = $this->uploadFile([ - 'file' => $file, - 'purpose' => 'assistants', - ]); - $file = json_decode($file, true); - $fileIds[] = $file['id']; - } - - $fileBatch = $this->createVectorFileBatch($vectorStoreId, $fileIds); - $fileBatch = json_decode($fileBatch, true); - - while (true) { - $fileBatch = $this->retrieveVectorFileBatch($vectorStoreId, $fileBatch['id']); - $fileBatch = json_decode($fileBatch, true); - - switch ($fileBatch['status']) { - case 'processing': - break; - case 'cancelled': - case 'failed': - throw new Exception('File batch failed'); - case 'completed': - return ['fileIds' => $fileIds, 'fileBatch' => $fileBatch]; - } + $this->addAssistantsBetaHeader(); + $url = Url::vectorStoreUrl() . '/' . $vectorStoreId . '/files/' . $fileId; + $this->baseUrl($url); - sleep(3); - } + return $this->sendRequest($url, 'GET'); } /** @@ -1071,7 +1044,10 @@ public function setAssistantsBetaVersion(string $version) */ private function addAssistantsBetaHeader() { - $this->headers[] = 'OpenAI-Beta: assistants='.$this->assistantsBetaVersion; + $assistantsBetaHeader = 'OpenAI-Beta: assistants=' . $this->assistantsBetaVersion; + if (!in_array($assistantsBetaHeader, $this->headers)) { + $this->headers[] = $assistantsBetaHeader; + } } /**