Skip to content

Commit 031b477

Browse files
vivekk16vivekk16
committed
feat(api): add SupportsAPIFormat helpers and engine-level format validation
Signed-off-by: Vivek Karunai Kiri Ragavan <vkarunai@redhat.com>
1 parent 3970454 commit 031b477

4 files changed

Lines changed: 274 additions & 0 deletions

File tree

‎controller/api/v1alpha1/inferenceproviderconfig_test.go‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,3 +227,203 @@ func TestEngineNames_Nil(t *testing.T) {
227227
t.Fatalf("expected nil engine names on nil receiver, got %v", names)
228228
}
229229
}
230+
231+
func TestEngineCapability_SupportsAPIFormat(t *testing.T) {
232+
ec := &EngineCapability{
233+
Name: EngineTypeVLLM,
234+
APIFormats: []APIFormat{
235+
APIFormatOpenAIChat,
236+
APIFormatOpenAIResponses,
237+
APIFormatAnthropicMessages,
238+
},
239+
}
240+
if !ec.SupportsAPIFormat(APIFormatOpenAIChat) {
241+
t.Error("expected vllm to support openai-chat")
242+
}
243+
if !ec.SupportsAPIFormat(APIFormatOpenAIResponses) {
244+
t.Error("expected vllm to support openai-responses")
245+
}
246+
if !ec.SupportsAPIFormat(APIFormatAnthropicMessages) {
247+
t.Error("expected vllm to support anthropic-messages")
248+
}
249+
}
250+
251+
func TestEngineCapability_SupportsAPIFormat_EmptyDefaultsToChat(t *testing.T) {
252+
ec := &EngineCapability{
253+
Name: EngineTypeVLLM,
254+
APIFormats: []APIFormat{},
255+
}
256+
if !ec.SupportsAPIFormat(APIFormatOpenAIChat) {
257+
t.Error("expected empty APIFormats to default to supporting openai-chat")
258+
}
259+
if ec.SupportsAPIFormat(APIFormatOpenAIResponses) {
260+
t.Error("expected empty APIFormats to NOT support openai-responses")
261+
}
262+
if ec.SupportsAPIFormat(APIFormatAnthropicMessages) {
263+
t.Error("expected empty APIFormats to NOT support anthropic-messages")
264+
}
265+
}
266+
267+
func TestEngineCapability_SupportsAPIFormat_NilFormats(t *testing.T) {
268+
ec := &EngineCapability{Name: EngineTypeVLLM}
269+
if !ec.SupportsAPIFormat(APIFormatOpenAIChat) {
270+
t.Error("expected nil APIFormats to default to supporting openai-chat")
271+
}
272+
if ec.SupportsAPIFormat(APIFormatAnthropicMessages) {
273+
t.Error("expected nil APIFormats to NOT support anthropic-messages")
274+
}
275+
}
276+
277+
func TestEngineCapability_SupportsAPIFormat_Nil(t *testing.T) {
278+
var ec *EngineCapability
279+
if ec.SupportsAPIFormat(APIFormatOpenAIChat) {
280+
t.Error("expected nil receiver to return false")
281+
}
282+
}
283+
284+
func TestEngineCapability_EffectiveAPIFormats(t *testing.T) {
285+
ec := &EngineCapability{
286+
Name: EngineTypeVLLM,
287+
APIFormats: []APIFormat{
288+
APIFormatOpenAIChat,
289+
APIFormatAnthropicMessages,
290+
},
291+
}
292+
formats := ec.EffectiveAPIFormats()
293+
if len(formats) != 2 {
294+
t.Fatalf("expected 2 formats, got %d", len(formats))
295+
}
296+
if formats[0] != APIFormatOpenAIChat {
297+
t.Errorf("expected first format openai-chat, got %s", formats[0])
298+
}
299+
if formats[1] != APIFormatAnthropicMessages {
300+
t.Errorf("expected second format anthropic-messages, got %s", formats[1])
301+
}
302+
}
303+
304+
func TestEngineCapability_EffectiveAPIFormats_EmptyMaterializesChat(t *testing.T) {
305+
ec := &EngineCapability{Name: EngineTypeVLLM}
306+
formats := ec.EffectiveAPIFormats()
307+
if len(formats) != 1 {
308+
t.Fatalf("expected 1 default format, got %d", len(formats))
309+
}
310+
if formats[0] != APIFormatOpenAIChat {
311+
t.Errorf("expected default format openai-chat, got %s", formats[0])
312+
}
313+
}
314+
315+
func TestEngineCapability_EffectiveAPIFormats_Nil(t *testing.T) {
316+
var ec *EngineCapability
317+
formats := ec.EffectiveAPIFormats()
318+
if formats != nil {
319+
t.Fatalf("expected nil on nil receiver, got %v", formats)
320+
}
321+
}
322+
323+
func TestProviderCapabilities_SupportsAPIFormat(t *testing.T) {
324+
caps := &ProviderCapabilities{
325+
Engines: []EngineCapability{
326+
{
327+
Name: EngineTypeVLLM,
328+
APIFormats: []APIFormat{
329+
APIFormatOpenAIChat,
330+
APIFormatOpenAIResponses,
331+
APIFormatAnthropicMessages,
332+
},
333+
},
334+
{
335+
Name: EngineTypeLlamaCpp,
336+
APIFormats: []APIFormat{
337+
APIFormatOpenAIChat,
338+
},
339+
},
340+
},
341+
}
342+
343+
if !caps.SupportsAPIFormat(EngineTypeVLLM, APIFormatAnthropicMessages) {
344+
t.Error("expected vllm to support anthropic-messages via ProviderCapabilities")
345+
}
346+
if caps.SupportsAPIFormat(EngineTypeLlamaCpp, APIFormatAnthropicMessages) {
347+
t.Error("expected llamacpp to NOT support anthropic-messages")
348+
}
349+
if caps.SupportsAPIFormat(EngineTypeSGLang, APIFormatOpenAIChat) {
350+
t.Error("expected missing engine to not support any format")
351+
}
352+
}
353+
354+
func TestProviderCapabilities_EffectiveAPIFormats(t *testing.T) {
355+
caps := &ProviderCapabilities{
356+
Engines: []EngineCapability{
357+
{Name: EngineTypeVLLM},
358+
},
359+
}
360+
formats := caps.EffectiveAPIFormats(EngineTypeVLLM)
361+
if len(formats) != 1 || formats[0] != APIFormatOpenAIChat {
362+
t.Errorf("expected [openai-chat] default, got %v", formats)
363+
}
364+
formats = caps.EffectiveAPIFormats(EngineTypeSGLang)
365+
if formats != nil {
366+
t.Errorf("expected nil for missing engine, got %v", formats)
367+
}
368+
}
369+
370+
func TestValidateAPIFormatsForEngine(t *testing.T) {
371+
if err := ValidateAPIFormatsForEngine(EngineTypeVLLM, []APIFormat{
372+
APIFormatOpenAIChat, APIFormatOpenAIResponses, APIFormatAnthropicMessages,
373+
}); err != nil {
374+
t.Errorf("expected no error for valid vllm formats, got %v", err)
375+
}
376+
377+
if err := ValidateAPIFormatsForEngine(EngineTypeSGLang, []APIFormat{
378+
APIFormatOpenAIChat, APIFormatAnthropicMessages,
379+
}); err != nil {
380+
t.Errorf("expected no error for valid sglang formats, got %v", err)
381+
}
382+
383+
if err := ValidateAPIFormatsForEngine(EngineTypeLlamaCpp, []APIFormat{
384+
APIFormatOpenAIChat,
385+
}); err != nil {
386+
t.Errorf("expected no error for valid llamacpp formats, got %v", err)
387+
}
388+
389+
if err := ValidateAPIFormatsForEngine(EngineTypeTRTLLM, []APIFormat{
390+
APIFormatOpenAIChat, APIFormatOpenAIResponses,
391+
}); err != nil {
392+
t.Errorf("expected no error for valid trtllm formats, got %v", err)
393+
}
394+
395+
// Empty formats should pass (no invalid entries)
396+
if err := ValidateAPIFormatsForEngine(EngineTypeVLLM, []APIFormat{}); err != nil {
397+
t.Errorf("expected no error for empty formats, got %v", err)
398+
}
399+
}
400+
401+
func TestValidateAPIFormatsForEngine_Invalid(t *testing.T) {
402+
err := ValidateAPIFormatsForEngine(EngineTypeTRTLLM, []APIFormat{
403+
APIFormatOpenAIChat, APIFormatAnthropicMessages,
404+
})
405+
if err == nil {
406+
t.Error("expected error for trtllm with anthropic-messages")
407+
}
408+
409+
err = ValidateAPIFormatsForEngine(EngineTypeSGLang, []APIFormat{
410+
APIFormatOpenAIResponses,
411+
})
412+
if err == nil {
413+
t.Error("expected error for sglang with openai-responses")
414+
}
415+
416+
err = ValidateAPIFormatsForEngine(EngineTypeLlamaCpp, []APIFormat{
417+
APIFormatOpenAIChat, APIFormatAnthropicMessages,
418+
})
419+
if err == nil {
420+
t.Error("expected error for llamacpp with anthropic-messages")
421+
}
422+
}
423+
424+
func TestValidateAPIFormatsForEngine_UnknownEngine(t *testing.T) {
425+
err := ValidateAPIFormatsForEngine(EngineType("unknown"), []APIFormat{APIFormatOpenAIChat})
426+
if err == nil {
427+
t.Error("expected error for unknown engine type")
428+
}
429+
}

‎controller/api/v1alpha1/inferenceproviderconfig_types.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,35 @@ func (c *ProviderCapabilities) GetEngineCapability(engine EngineType) *EngineCap
196196
return nil
197197
}
198198

199+
// SupportsAPIFormat returns true if this engine supports the specified API format.
200+
// When the APIFormats list is empty, only openai-chat is assumed (backward compatibility).
201+
func (e *EngineCapability) SupportsAPIFormat(f APIFormat) bool {
202+
if e == nil {
203+
return false
204+
}
205+
if len(e.APIFormats) == 0 {
206+
return f == APIFormatOpenAIChat
207+
}
208+
for _, a := range e.APIFormats {
209+
if a == f {
210+
return true
211+
}
212+
}
213+
return false
214+
}
215+
216+
// EffectiveAPIFormats returns the API formats this engine supports.
217+
// When the APIFormats list is empty, it materializes [openai-chat] for backward compatibility.
218+
func (e *EngineCapability) EffectiveAPIFormats() []APIFormat {
219+
if e == nil {
220+
return nil
221+
}
222+
if len(e.APIFormats) == 0 {
223+
return []APIFormat{APIFormatOpenAIChat}
224+
}
225+
return e.APIFormats
226+
}
227+
199228
// SupportsServingMode returns true if this engine supports the specified serving mode.
200229
func (e *EngineCapability) SupportsServingMode(mode ServingMode) bool {
201230
if e == nil {
@@ -240,6 +269,16 @@ func (c *ProviderCapabilities) SupportsCPU(engine EngineType) bool {
240269
return c.GetEngineCapability(engine).SupportsCPU()
241270
}
242271

272+
// SupportsAPIFormat returns true if the given engine supports the specified API format
273+
func (c *ProviderCapabilities) SupportsAPIFormat(engine EngineType, f APIFormat) bool {
274+
return c.GetEngineCapability(engine).SupportsAPIFormat(f)
275+
}
276+
277+
// EffectiveAPIFormats returns the API formats the given engine supports
278+
func (c *ProviderCapabilities) EffectiveAPIFormats(engine EngineType) []APIFormat {
279+
return c.GetEngineCapability(engine).EffectiveAPIFormats()
280+
}
281+
243282
// EngineNames returns a list of all engine types supported by this provider
244283
func (c *ProviderCapabilities) EngineNames() []EngineType {
245284
if c == nil {

‎controller/api/v1alpha1/modeldeployment_types.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,37 @@ const (
7070
APIFormatAnthropicMessages APIFormat = "anthropic-messages"
7171
)
7272

73+
// ValidAPIFormatsForEngine defines the maximum set of API formats each engine
74+
// type can natively support. Providers may declare a subset (e.g., KubeRay
75+
// vLLM only exposes openai-chat because Ray Serve does not pass through
76+
// /v1/responses or /v1/messages), but must never declare formats beyond
77+
// this map.
78+
var ValidAPIFormatsForEngine = map[EngineType][]APIFormat{
79+
EngineTypeVLLM: {APIFormatOpenAIChat, APIFormatOpenAIResponses, APIFormatAnthropicMessages},
80+
EngineTypeSGLang: {APIFormatOpenAIChat, APIFormatAnthropicMessages},
81+
EngineTypeTRTLLM: {APIFormatOpenAIChat, APIFormatOpenAIResponses},
82+
EngineTypeLlamaCpp: {APIFormatOpenAIChat},
83+
}
84+
85+
// ValidateAPIFormatsForEngine checks that every format in the list is valid
86+
// for the given engine type. Returns an error describing the first invalid format.
87+
func ValidateAPIFormatsForEngine(engine EngineType, formats []APIFormat) error {
88+
valid, ok := ValidAPIFormatsForEngine[engine]
89+
if !ok {
90+
return fmt.Errorf("unknown engine type %q", engine)
91+
}
92+
validSet := make(map[APIFormat]bool, len(valid))
93+
for _, f := range valid {
94+
validSet[f] = true
95+
}
96+
for _, f := range formats {
97+
if !validSet[f] {
98+
return fmt.Errorf("engine %q does not support API format %q", engine, f)
99+
}
100+
}
101+
return nil
102+
}
103+
73104
// DeploymentPhase defines the phase of the deployment
74105
// +kubebuilder:validation:Enum=Pending;Deploying;Running;Failed;Terminating
75106
type DeploymentPhase string

‎providers/kuberay/config.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,10 @@ func GetProviderConfigSpec() airunwayv1alpha1.InferenceProviderConfigSpec {
7777
airunwayv1alpha1.ServingModeAggregated,
7878
airunwayv1alpha1.ServingModeDisaggregated,
7979
},
80+
// Ray Serve LLM (build_openai_app) only exposes /v1/chat/completions,
81+
// /v1/completions, /v1/embeddings, and /v1/models — the /v1/responses
82+
// and /v1/messages endpoints are not passed through to the underlying
83+
// vLLM engine.
8084
APIFormats: []airunwayv1alpha1.APIFormat{
8185
airunwayv1alpha1.APIFormatOpenAIChat,
8286
},

0 commit comments

Comments
 (0)