Skip to content

Commit 3be54cb

Browse files
jamesrochabrunjamesrochabrun
authored
Vision support for GPT4o (#44)
* Vector store * improving examples * Fixing vision
1 parent 4d90fcb commit 3be54cb

File tree

12 files changed

+524
-30
lines changed

12 files changed

+524
-30
lines changed

‎Examples/SwiftOpenAIExample/SwiftOpenAIExample.xcodeproj/project.pbxproj

+24
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@
4343
7B7239AB2AF6294C00646679 /* URLImageView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B7239AA2AF6294C00646679 /* URLImageView.swift */; };
4444
7B7239AE2AF9FF0000646679 /* ChatFunctionsCallStreamProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B7239AD2AF9FF0000646679 /* ChatFunctionsCallStreamProvider.swift */; };
4545
7B7239B12AF9FF3C00646679 /* ChatFunctionsCalllStreamDemoView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B7239B02AF9FF3C00646679 /* ChatFunctionsCalllStreamDemoView.swift */; };
46+
7B99C2E72C0718DE00E701B3 /* FilesPicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B99C2E62C0718DE00E701B3 /* FilesPicker.swift */; };
47+
7B99C2E92C0718FF00E701B3 /* FileAttachmentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B99C2E82C0718FF00E701B3 /* FileAttachmentView.swift */; };
48+
7B99C2EB2C07191200E701B3 /* AttachmentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B99C2EA2C07191200E701B3 /* AttachmentView.swift */; };
49+
7B99C2ED2C071B1600E701B3 /* FilesPickerProvider.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7B99C2EC2C071B1600E701B3 /* FilesPickerProvider.swift */; };
4650
7BA788CD2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BA788CC2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift */; };
4751
7BA788CF2AE23A48008825D5 /* ApiKeyIntroView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BA788CE2AE23A48008825D5 /* ApiKeyIntroView.swift */; };
4852
7BA788D12AE23A49008825D5 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 7BA788D02AE23A49008825D5 /* Assets.xcassets */; };
@@ -115,6 +119,10 @@
115119
7B7239AA2AF6294C00646679 /* URLImageView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = URLImageView.swift; sourceTree = "<group>"; };
116120
7B7239AD2AF9FF0000646679 /* ChatFunctionsCallStreamProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatFunctionsCallStreamProvider.swift; sourceTree = "<group>"; };
117121
7B7239B02AF9FF3C00646679 /* ChatFunctionsCalllStreamDemoView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ChatFunctionsCalllStreamDemoView.swift; sourceTree = "<group>"; };
122+
7B99C2E62C0718DE00E701B3 /* FilesPicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FilesPicker.swift; sourceTree = "<group>"; };
123+
7B99C2E82C0718FF00E701B3 /* FileAttachmentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileAttachmentView.swift; sourceTree = "<group>"; };
124+
7B99C2EA2C07191200E701B3 /* AttachmentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AttachmentView.swift; sourceTree = "<group>"; };
125+
7B99C2EC2C071B1600E701B3 /* FilesPickerProvider.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FilesPickerProvider.swift; sourceTree = "<group>"; };
118126
7BA788C92AE23A48008825D5 /* SwiftOpenAIExample.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = SwiftOpenAIExample.app; sourceTree = BUILT_PRODUCTS_DIR; };
119127
7BA788CC2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SwiftOpenAIExampleApp.swift; sourceTree = "<group>"; };
120128
7BA788CE2AE23A48008825D5 /* ApiKeyIntroView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ApiKeyIntroView.swift; sourceTree = "<group>"; };
@@ -297,6 +305,17 @@
297305
path = SharedModels;
298306
sourceTree = "<group>";
299307
};
308+
7B99C2E52C0718CD00E701B3 /* Files */ = {
309+
isa = PBXGroup;
310+
children = (
311+
7B99C2E62C0718DE00E701B3 /* FilesPicker.swift */,
312+
7B99C2E82C0718FF00E701B3 /* FileAttachmentView.swift */,
313+
7B99C2EA2C07191200E701B3 /* AttachmentView.swift */,
314+
7B99C2EC2C071B1600E701B3 /* FilesPickerProvider.swift */,
315+
);
316+
path = Files;
317+
sourceTree = "<group>";
318+
};
300319
7BA788C02AE23A48008825D5 = {
301320
isa = PBXGroup;
302321
children = (
@@ -321,6 +340,7 @@
321340
isa = PBXGroup;
322341
children = (
323342
7BA788CC2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift */,
343+
7B99C2E52C0718CD00E701B3 /* Files */,
324344
7B7239AF2AF9FF1D00646679 /* SharedModels */,
325345
7B7239A92AF6294200646679 /* SharedUI */,
326346
7B1268032B08241200400694 /* Assistants */,
@@ -564,6 +584,7 @@
564584
7B7239AB2AF6294C00646679 /* URLImageView.swift in Sources */,
565585
7B7239B12AF9FF3C00646679 /* ChatFunctionsCalllStreamDemoView.swift in Sources */,
566586
7BBE7EAB2B02E8FC0096A693 /* ChatMessageDisplayModel.swift in Sources */,
587+
7B99C2E92C0718FF00E701B3 /* FileAttachmentView.swift in Sources */,
567588
7BBE7EA52B02E8A70096A693 /* Sizes.swift in Sources */,
568589
7B7239A22AF6260D00646679 /* ChatDisplayMessage.swift in Sources */,
569590
0DF957862BB543F100DD2013 /* AIProxyIntroView.swift in Sources */,
@@ -572,6 +593,7 @@
572593
7B436B962AE24A04003CE281 /* OptionsListView.swift in Sources */,
573594
7BBE7EDE2B03718E0096A693 /* ChatFunctionCallProvider.swift in Sources */,
574595
7B7239A62AF628F800646679 /* ChatDisplayMessageView.swift in Sources */,
596+
7B99C2ED2C071B1600E701B3 /* FilesPickerProvider.swift in Sources */,
575597
7B7239A02AF625F200646679 /* ChatFluidConversationProvider.swift in Sources */,
576598
7BA788CF2AE23A48008825D5 /* ApiKeyIntroView.swift in Sources */,
577599
7BA788CD2AE23A48008825D5 /* SwiftOpenAIExampleApp.swift in Sources */,
@@ -586,9 +608,11 @@
586608
7B436BAD2AE788FB003CE281 /* FineTuningJobDemoView.swift in Sources */,
587609
7B436BB02AE79369003CE281 /* FilesDemoView.swift in Sources */,
588610
7BBE7E912AFCA52A0096A693 /* ChatVisionDemoView.swift in Sources */,
611+
7B99C2EB2C07191200E701B3 /* AttachmentView.swift in Sources */,
589612
7B436BAB2AE788F1003CE281 /* FineTuningJobProvider.swift in Sources */,
590613
7B7239A42AF6289900646679 /* ChatStreamFluidConversationDemoView.swift in Sources */,
591614
7BA788FC2AE23B42008825D5 /* AudioDemoView.swift in Sources */,
615+
7B99C2E72C0718DE00E701B3 /* FilesPicker.swift in Sources */,
592616
7B1268072B08247C00400694 /* AssistantConfigurationProvider.swift in Sources */,
593617
7B436BBE2AE7ABDA003CE281 /* ModelsDemoView.swift in Sources */,
594618
7B436BA32AE25962003CE281 /* ChatDemoView.swift in Sources */,

‎Examples/SwiftOpenAIExample/SwiftOpenAIExample/Assistants/AssistantConfigurationDemoView.swift

+33-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,10 @@ struct AssistantConfigurationDemoView: View {
3737
@State private var isAvatarLoading = false
3838
@State private var showAvatarFlow = false
3939
private let service: OpenAIService
40-
40+
@State private var fileIDS: [String] = []
41+
/// Used mostly to display already uploaded files if any.
42+
@State private var filePickerInitialActions: [FilePickerAction] = []
43+
4144
var isCodeInterpreterOn: Binding<Bool> {
4245
Binding(
4346
get: {
@@ -72,6 +75,23 @@ struct AssistantConfigurationDemoView: View {
7275
)
7376
}
7477

78+
var isFileSearchOn: Binding<Bool> {
79+
Binding(
80+
get: {
81+
let contains =
82+
self.parameters.tools.contains { $0.displayToolType == .fileSearch } == true
83+
return contains
84+
},
85+
set: { newValue in
86+
if newValue {
87+
self.parameters.tools.append(AssistantObject.Tool(type: .fileSearch))
88+
} else {
89+
self.parameters.tools.removeAll { $0.displayToolType == .fileSearch }
90+
}
91+
}
92+
)
93+
}
94+
7595
init(service: OpenAIService) {
7696
self.service = service
7797
_provider = State(initialValue: AssistantConfigurationProvider(service: service))
@@ -84,6 +104,7 @@ struct AssistantConfigurationDemoView: View {
84104
inputViews
85105
capabilities
86106
footerActions
107+
knowledge
87108
}
88109
.padding()
89110
}.sheet(isPresented: $showAvatarFlow) {
@@ -186,11 +207,22 @@ struct AssistantConfigurationDemoView: View {
186207
InputView(title: "Capabilities") {
187208
VStack(spacing: 16) {
188209
CheckboxRow(title: "Code interpreter", isChecked: isCodeInterpreterOn)
210+
CheckboxRow(title: "File Search", isChecked: isFileSearchOn)
189211
CheckboxRow(title: "DALL·E Image Generation", isChecked: isDalleToolOn)
190212
}
191213
}
192214
.inputViewStyle(.init(verticalPadding: 16.0))
193215
}
216+
217+
// TODO: Add a demo to create a vector store and add files in to it.
218+
var knowledge: some View {
219+
FilesPicker(
220+
service: service,
221+
sectionTitle: "Knowledge",
222+
actionTitle: "Upload files",
223+
fileIDS: $fileIDS,
224+
actions: $filePickerInitialActions)
225+
}
194226
}
195227

196228
extension Binding where Value == String? {

‎Examples/SwiftOpenAIExample/SwiftOpenAIExample/Assistants/AssistantConfigurationProvider.swift

+5
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,9 @@ import SwiftOpenAI
6767
debugPrint("\(error)")
6868
}
6969
}
70+
71+
// TODO: Create demo for this.
72+
func createVStore ()async throws {
73+
let _ = try await service.createVectorStore(parameters: .init(name: "Personal Data"))
74+
}
7075
}

‎Examples/SwiftOpenAIExample/SwiftOpenAIExample/Files/AttachmentView.swift

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
//
2+
// AttachmentView.swift
3+
// SwiftOpenAIExample
4+
//
5+
// Created by James Rochabrun on 5/29/24.
6+
//
7+
8+
import SwiftUI
9+
10+
struct AttachmentView: View {
11+
12+
let fileName: String
13+
@Binding var actionTrigger: Bool
14+
let isLoading: Bool
15+
16+
var body: some View {
17+
HStack(spacing: Sizes.spacingExtraSmall) {
18+
HStack {
19+
if isLoading == true {
20+
ProgressView()
21+
.frame(width: 10, height: 10)
22+
.padding(.horizontal, Sizes.spacingExtraSmall)
23+
} else {
24+
Image(systemName: "doc")
25+
.resizable()
26+
.aspectRatio(contentMode: .fit)
27+
.frame(width: 10)
28+
.foregroundColor(.secondary)
29+
}
30+
Text(fileName)
31+
.font(.caption2)
32+
}
33+
Button {
34+
actionTrigger = true
35+
36+
} label: {
37+
Image(systemName: "xmark.circle.fill")
38+
}
39+
.disabled(isLoading)
40+
}
41+
.padding(.leading, Sizes.spacingMedium)
42+
.background(
43+
RoundedRectangle(cornerRadius: 8)
44+
.stroke(.gray.opacity(0.5), lineWidth: 0.5)
45+
)
46+
}
47+
}
48+
49+
#Preview {
50+
AttachmentView(fileName: "Mydocument.pdf", actionTrigger: .constant(true), isLoading: true)
51+
}

‎Examples/SwiftOpenAIExample/SwiftOpenAIExample/Files/FileAttachmentView.swift

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
//
2+
// FileAttachmentView.swift
3+
// SwiftOpenAIExample
4+
//
5+
// Created by James Rochabrun on 5/29/24.
6+
//
7+
8+
import SwiftUI
9+
import SwiftOpenAI
10+
11+
struct FileAttachmentView: View {
12+
13+
init(
14+
service: OpenAIService,
15+
action: FilePickerAction,
16+
fileUploadedCompletion: @escaping (_ file: FileObject) -> Void,
17+
fileDeletedCompletion: @escaping (_ parameters: FilePickerAction, _ id: String) -> Void)
18+
{
19+
self.fileProvider = FilesPickerProvider(service: service)
20+
self.action = action
21+
self.fileUploadedCompletion = fileUploadedCompletion
22+
self.fileDeletedCompletion = fileDeletedCompletion
23+
}
24+
25+
func newUploadedFileView(
26+
parameters: FileParameters)
27+
-> some View
28+
{
29+
AttachmentView(fileName: fileObject?.filename ?? parameters.fileName, actionTrigger: $deleted, isLoading: fileObject == nil || deleted)
30+
.disabled(fileObject == nil)
31+
.opacity(fileObject == nil ? 0.3 : 1)
32+
.onFirstAppear {
33+
Task {
34+
fileObject = try await fileProvider.uploadFile(parameters: parameters)
35+
}
36+
}
37+
.onChange(of: fileObject) { oldValue, newValue in
38+
if oldValue != newValue, let newValue {
39+
fileUploadedCompletion(newValue)
40+
}
41+
}
42+
}
43+
44+
func previousUploadedFileView(
45+
id: String)
46+
-> some View
47+
{
48+
AttachmentView(fileName: fileObject?.filename ?? "Document", actionTrigger: $deleted, isLoading: fileObject == nil || deleted)
49+
.onFirstAppear {
50+
Task {
51+
fileObject = try await fileProvider.retrieveFileWith(id: id)
52+
}
53+
}
54+
}
55+
56+
var body: some View {
57+
Group {
58+
switch action {
59+
case .request(let parameters):
60+
newUploadedFileView(parameters: parameters)
61+
case .retrieveAndDisplay(let id):
62+
previousUploadedFileView(id: id)
63+
}
64+
}
65+
.onChange(of: deleted) { oldValue, newValue in
66+
if oldValue != newValue, newValue {
67+
Task {
68+
if let fileObject {
69+
fileDeleteStatus = try await fileProvider.deleteFileWith(id: fileObject.id)
70+
}
71+
}
72+
}
73+
}
74+
.onChange(of: fileDeleteStatus) { oldValue, newValue in
75+
if oldValue != newValue, let newValue, newValue.deleted {
76+
fileDeletedCompletion(action, newValue.id)
77+
}
78+
}
79+
}
80+
81+
// MARK: Private
82+
83+
private let fileProvider: FilesPickerProvider
84+
private let fileUploadedCompletion: (_ file: FileObject) -> Void
85+
private let fileDeletedCompletion: (_ action: FilePickerAction, _ id: String) -> Void
86+
private let action: FilePickerAction
87+
@State private var fileObject: FileObject?
88+
@State private var fileDeleteStatus: DeletionStatus?
89+
@State private var deleted: Bool = false
90+
}
91+
92+
93+
private struct OnFirstAppear: ViewModifier {
94+
let perform: () -> Void
95+
96+
@State private var firstTime = true
97+
98+
func body(content: Content) -> some View {
99+
content.onAppear {
100+
if firstTime {
101+
firstTime = false
102+
perform()
103+
}
104+
}
105+
}
106+
}
107+
108+
extension View {
109+
func onFirstAppear(perform: @escaping () -> Void) -> some View {
110+
modifier(OnFirstAppear(perform: perform))
111+
}
112+
}
113+
114+
extension DeletionStatus: Equatable {
115+
public static func == (lhs: DeletionStatus, rhs: DeletionStatus) -> Bool {
116+
lhs.id == rhs.id
117+
}
118+
}

0 commit comments

Comments
 (0)