Skip to content

Commit 7040ab8

Browse files
vivekk16vivekk16
committed
fix: accurate apiFormats declarations based on verified provider capabilities
Signed-off-by: Vivek Karunai Kiri Ragavan <vkarunai@redhat.com>
1 parent 421f057 commit 7040ab8

10 files changed

Lines changed: 97 additions & 40 deletions

File tree

‎backend/src/test/fixtures.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ export const mockInferenceProviderConfig = {
167167
spec: {
168168
capabilities: {
169169
engines: [
170-
{ name: 'vllm', servingModes: ['aggregated'], apiFormats: ['openai-chat'], gpuSupport: true, cpuSupport: false },
170+
{ name: 'vllm', servingModes: ['aggregated'], apiFormats: ['openai-chat', 'openai-responses', 'anthropic-messages'], gpuSupport: true, cpuSupport: false },
171171
{ name: 'llamacpp', servingModes: ['aggregated'], apiFormats: ['openai-chat'], gpuSupport: true, cpuSupport: true },
172172
],
173173
},

‎controller/config/samples/airunway_v1alpha1_inferenceproviderconfig.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ spec:
1414
engines:
1515
- name: vllm
1616
servingModes: [aggregated]
17-
apiFormats: [openai-chat]
17+
apiFormats: [openai-chat, openai-responses, anthropic-messages]
1818
gpuSupport: true
1919
- name: llamacpp
2020
servingModes: [aggregated]
@@ -42,7 +42,7 @@ spec:
4242
engines:
4343
- name: vllm
4444
servingModes: [aggregated, disaggregated]
45-
apiFormats: [openai-chat, anthropic-messages]
45+
apiFormats: [openai-chat, openai-responses, anthropic-messages]
4646
gpuSupport: true
4747
gateway:
4848
managesInferencePool: true
@@ -58,7 +58,7 @@ spec:
5858
inferencePoolNamespace: "{namespace}"
5959
- name: trtllm
6060
servingModes: [aggregated]
61-
apiFormats: [openai-chat]
61+
apiFormats: [openai-chat, openai-responses]
6262
gpuSupport: true
6363
gateway:
6464
managesInferencePool: true
@@ -110,7 +110,7 @@ spec:
110110
engines:
111111
- name: vllm
112112
servingModes: [aggregated, disaggregated]
113-
apiFormats: [openai-chat]
113+
apiFormats: [openai-chat, openai-responses, anthropic-messages]
114114
gpuSupport: true
115115
requiresCRD: false
116116
selectionRules: []

‎providers/dynamo/config.go‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -111,15 +111,16 @@ func GetProviderConfigSpec() airunwayv1alpha1.InferenceProviderConfigSpec {
111111
airunwayv1alpha1.ServingModeAggregated,
112112
airunwayv1alpha1.ServingModeDisaggregated,
113113
},
114-
APIFormats: []airunwayv1alpha1.APIFormat{
115-
airunwayv1alpha1.APIFormatOpenAIChat,
116-
airunwayv1alpha1.APIFormatAnthropicMessages,
117-
},
118-
GPUSupport: true,
119-
Gateway: dynamoGatewayCapabilities(),
114+
APIFormats: []airunwayv1alpha1.APIFormat{
115+
airunwayv1alpha1.APIFormatOpenAIChat,
116+
airunwayv1alpha1.APIFormatOpenAIResponses,
117+
airunwayv1alpha1.APIFormatAnthropicMessages,
120118
},
121-
{
122-
Name: airunwayv1alpha1.EngineTypeSGLang,
119+
GPUSupport: true,
120+
Gateway: dynamoGatewayCapabilities(),
121+
},
122+
{
123+
Name: airunwayv1alpha1.EngineTypeSGLang,
123124
ServingModes: []airunwayv1alpha1.ServingMode{
124125
airunwayv1alpha1.ServingModeAggregated,
125126
airunwayv1alpha1.ServingModeDisaggregated,
@@ -132,13 +133,14 @@ func GetProviderConfigSpec() airunwayv1alpha1.InferenceProviderConfigSpec {
132133
Gateway: dynamoGatewayCapabilities(),
133134
},
134135
{
135-
Name: airunwayv1alpha1.EngineTypeTRTLLM,
136-
ServingModes: []airunwayv1alpha1.ServingMode{
137-
airunwayv1alpha1.ServingModeAggregated,
138-
},
139-
APIFormats: []airunwayv1alpha1.APIFormat{
140-
airunwayv1alpha1.APIFormatOpenAIChat,
141-
},
136+
Name: airunwayv1alpha1.EngineTypeTRTLLM,
137+
ServingModes: []airunwayv1alpha1.ServingMode{
138+
airunwayv1alpha1.ServingModeAggregated,
139+
},
140+
APIFormats: []airunwayv1alpha1.APIFormat{
141+
airunwayv1alpha1.APIFormatOpenAIChat,
142+
airunwayv1alpha1.APIFormatOpenAIResponses,
143+
},
142144
GPUSupport: true,
143145
Gateway: dynamoGatewayCapabilities(),
144146
},

‎providers/dynamo/config_test.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ func TestGetProviderConfigSpec(t *testing.T) {
4646
}
4747
expectedVLLMFormats := []airunwayv1alpha1.APIFormat{
4848
airunwayv1alpha1.APIFormatOpenAIChat,
49+
airunwayv1alpha1.APIFormatOpenAIResponses,
4950
airunwayv1alpha1.APIFormatAnthropicMessages,
5051
}
5152
if len(vllmCap.APIFormats) != len(expectedVLLMFormats) {
@@ -104,8 +105,24 @@ func TestGetProviderConfigSpec(t *testing.T) {
104105
if len(trtllmCap.ServingModes) != 1 || trtllmCap.ServingModes[0] != airunwayv1alpha1.ServingModeAggregated {
105106
t.Errorf("expected trtllm to support only aggregated serving mode")
106107
}
107-
if len(trtllmCap.APIFormats) != 1 || trtllmCap.APIFormats[0] != airunwayv1alpha1.APIFormatOpenAIChat {
108-
t.Errorf("expected trtllm to support only openai-chat API format")
108+
expectedTRTLLMFormats := []airunwayv1alpha1.APIFormat{
109+
airunwayv1alpha1.APIFormatOpenAIChat,
110+
airunwayv1alpha1.APIFormatOpenAIResponses,
111+
}
112+
if len(trtllmCap.APIFormats) != len(expectedTRTLLMFormats) {
113+
t.Fatalf("expected trtllm to support %d API formats, got %d", len(expectedTRTLLMFormats), len(trtllmCap.APIFormats))
114+
}
115+
for _, expected := range expectedTRTLLMFormats {
116+
found := false
117+
for _, actual := range trtllmCap.APIFormats {
118+
if actual == expected {
119+
found = true
120+
break
121+
}
122+
}
123+
if !found {
124+
t.Errorf("expected trtllm to support API format %s", expected)
125+
}
109126
}
110127

111128
if len(spec.SelectionRules) != 4 {

‎providers/kaito/config.go‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -70,13 +70,15 @@ func GetProviderConfigSpec() airunwayv1alpha1.InferenceProviderConfigSpec {
7070
ServingModes: []airunwayv1alpha1.ServingMode{
7171
airunwayv1alpha1.ServingModeAggregated,
7272
},
73-
APIFormats: []airunwayv1alpha1.APIFormat{
74-
airunwayv1alpha1.APIFormatOpenAIChat,
75-
},
76-
GPUSupport: true,
73+
APIFormats: []airunwayv1alpha1.APIFormat{
74+
airunwayv1alpha1.APIFormatOpenAIChat,
75+
airunwayv1alpha1.APIFormatOpenAIResponses,
76+
airunwayv1alpha1.APIFormatAnthropicMessages,
7777
},
78-
{
79-
Name: airunwayv1alpha1.EngineTypeLlamaCpp,
78+
GPUSupport: true,
79+
},
80+
{
81+
Name: airunwayv1alpha1.EngineTypeLlamaCpp,
8082
ServingModes: []airunwayv1alpha1.ServingMode{
8183
airunwayv1alpha1.ServingModeAggregated,
8284
},

‎providers/kaito/config_test.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,25 @@ func TestGetProviderConfigSpec(t *testing.T) {
5050
if len(vllmCap.ServingModes) != 1 || vllmCap.ServingModes[0] != airunwayv1alpha1.ServingModeAggregated {
5151
t.Errorf("expected vllm to support only aggregated serving mode")
5252
}
53-
if len(vllmCap.APIFormats) != 1 || vllmCap.APIFormats[0] != airunwayv1alpha1.APIFormatOpenAIChat {
54-
t.Errorf("expected vllm to support openai-chat API format")
53+
expectedVLLMFormats := []airunwayv1alpha1.APIFormat{
54+
airunwayv1alpha1.APIFormatOpenAIChat,
55+
airunwayv1alpha1.APIFormatOpenAIResponses,
56+
airunwayv1alpha1.APIFormatAnthropicMessages,
57+
}
58+
if len(vllmCap.APIFormats) != len(expectedVLLMFormats) {
59+
t.Fatalf("expected vllm to support %d API formats, got %d", len(expectedVLLMFormats), len(vllmCap.APIFormats))
60+
}
61+
for _, expected := range expectedVLLMFormats {
62+
found := false
63+
for _, actual := range vllmCap.APIFormats {
64+
if actual == expected {
65+
found = true
66+
break
67+
}
68+
}
69+
if !found {
70+
t.Errorf("expected vllm to support API format %s", expected)
71+
}
5572
}
5673

5774
llamaCap := spec.Capabilities.GetEngineCapability(airunwayv1alpha1.EngineTypeLlamaCpp)

‎providers/kuberay/config.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,10 @@ func GetProviderConfigSpec() airunwayv1alpha1.InferenceProviderConfigSpec {
7777
airunwayv1alpha1.ServingModeAggregated,
7878
airunwayv1alpha1.ServingModeDisaggregated,
7979
},
80-
APIFormats: []airunwayv1alpha1.APIFormat{
81-
airunwayv1alpha1.APIFormatOpenAIChat,
82-
},
83-
GPUSupport: true,
80+
APIFormats: []airunwayv1alpha1.APIFormat{
81+
airunwayv1alpha1.APIFormatOpenAIChat,
82+
},
83+
GPUSupport: true,
8484
},
8585
},
8686
},

‎providers/kuberay/config_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ func TestGetProviderConfigSpec(t *testing.T) {
3838
t.Fatalf("expected vllm to support 2 serving modes, got %d", len(vllmCap.ServingModes))
3939
}
4040
if len(vllmCap.APIFormats) != 1 || vllmCap.APIFormats[0] != airunwayv1alpha1.APIFormatOpenAIChat {
41-
t.Errorf("expected vllm to support openai-chat API format")
41+
t.Errorf("expected vllm to support only openai-chat API format, got %v", vllmCap.APIFormats)
4242
}
4343

4444
if len(spec.SelectionRules) != 1 {

‎providers/llmd/config.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,9 +70,11 @@ func GetProviderConfigSpec() airunwayv1alpha1.InferenceProviderConfigSpec {
7070
airunwayv1alpha1.ServingModeAggregated,
7171
airunwayv1alpha1.ServingModeDisaggregated,
7272
},
73-
APIFormats: []airunwayv1alpha1.APIFormat{
74-
airunwayv1alpha1.APIFormatOpenAIChat,
75-
},
73+
APIFormats: []airunwayv1alpha1.APIFormat{
74+
airunwayv1alpha1.APIFormatOpenAIChat,
75+
airunwayv1alpha1.APIFormatOpenAIResponses,
76+
airunwayv1alpha1.APIFormatAnthropicMessages,
77+
},
7678
GPUSupport: true,
7779
RequiresCRD: &requiresCRD,
7880
},

‎providers/llmd/config_test.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,25 @@ func TestGetProviderConfigSpec(t *testing.T) {
3636
}
3737

3838
// API formats (per-engine)
39-
if len(vllmCap.APIFormats) != 1 || vllmCap.APIFormats[0] != airunwayv1alpha1.APIFormatOpenAIChat {
40-
t.Errorf("expected vllm to support openai-chat API format")
39+
expectedVLLMFormats := []airunwayv1alpha1.APIFormat{
40+
airunwayv1alpha1.APIFormatOpenAIChat,
41+
airunwayv1alpha1.APIFormatOpenAIResponses,
42+
airunwayv1alpha1.APIFormatAnthropicMessages,
43+
}
44+
if len(vllmCap.APIFormats) != len(expectedVLLMFormats) {
45+
t.Fatalf("expected vllm to support %d API formats, got %d", len(expectedVLLMFormats), len(vllmCap.APIFormats))
46+
}
47+
for _, expected := range expectedVLLMFormats {
48+
found := false
49+
for _, actual := range vllmCap.APIFormats {
50+
if actual == expected {
51+
found = true
52+
break
53+
}
54+
}
55+
if !found {
56+
t.Errorf("expected vllm to support API format %s", expected)
57+
}
4158
}
4259

4360
// Serving modes (per-engine)

0 commit comments

Comments
 (0)