enhancement: list model status tracking (#1537)

## Summary

Add key-level model discovery status tracking to improve visibility into API key health and model availability.

## Changes

- Added model discovery status tracking for individual API keys
- Enhanced `ListAllModels` to collect and return key status information
- Added database columns to store model discovery status and error messages
- Updated UI to display key status with visual indicators and tooltips
- Improved error handling in model discovery process
- Added automatic model discovery when adding or updating providers

## Type of change

- [x] Feature
- [x] Refactor

## Affected areas

- [x] Core (Go)
- [x] Transports (HTTP)
- [x] UI (Next.js)

## How to test

1. Add a new provider with valid and invalid API keys
2. Observe the status indicators in the UI for each key
3. Hover over status icons to see detailed error messages for failed keys

```sh
# Core/Transports
go version
go test ./...

# UI
cd ui
pnpm i
pnpm build
```

## Screenshots/Recordings

N/A

## Breaking changes

- [x] No

## Related issues

N/A

## Security considerations

This change only exposes error information about API keys to authenticated users with appropriate permissions.

## Checklist

- [x] I read `docs/contributing/README.md` and followed the guidelines
- [x] I added/updated tests where appropriate
- [x] I updated documentation where needed
- [x] I verified builds succeed (Go and UI)
- [x] I verified the CI pipeline passes locally if applicable

Author: TejasGhatte

core/bifrost.go

@@ -437,11 +437,10 @@ func (bifrost *Bifrost) ListModelsRequest(ctx *schemas.BifrostContext, req *sche
 		return provider.ListModels(ctx, keys, request)
 	}, schemas.ListModelsRequest, req.Provider, "", nil, bifrost.logger)
 	if bifrostErr != nil {
-		bifrostErr.ExtraFields = schemas.BifrostErrorExtraFields{
-			RequestType: schemas.ListModelsRequest,
-			Provider:    req.Provider,
-		}
-		return nil, bifrostErr
+		bifrostErr.ExtraFields.RequestType = schemas.ListModelsRequest
+		bifrostErr.ExtraFields.Provider = req.Provider
+		// Return response even on error if it contains KeyStatuses for tracking
+		return response, bifrostErr
 	}
 	return response, nil
 }
@@ -471,8 +470,10 @@ func (bifrost *Bifrost) ListAllModels(ctx *schemas.BifrostContext, request *sche
 
 	// Result structure for collecting provider responses
 	type providerResult struct {
-		models []schemas.Model
-		err    *schemas.BifrostError
+		provider    schemas.ModelProvider
+		models      []schemas.Model
+		keyStatuses []schemas.KeyStatus
+		err         *schemas.BifrostError
 	}
 
 	results := make(chan providerResult, len(providerKeys))
@@ -489,6 +490,7 @@ func (bifrost *Bifrost) ListAllModels(ctx *schemas.BifrostContext, request *sche
 			defer wg.Done()
 
 			providerModels := make([]schemas.Model, 0)
+			var providerKeyStatuses []schemas.KeyStatus
 			var providerErr *schemas.BifrostError
 
 			// Create request for this provider with limit of 1000
@@ -521,6 +523,10 @@ func (bifrost *Bifrost) ListAllModels(ctx *schemas.BifrostContext, request *sche
 						providerErr = bifrostErr
 						bifrost.logger.Warn("failed to list models for provider %s: %s", providerKey, GetErrorMessage(bifrostErr))
 					}
+					// Collect key statuses from error (failure case)
+					if len(bifrostErr.ExtraFields.KeyStatuses) > 0 {
+						providerKeyStatuses = append(providerKeyStatuses, bifrostErr.ExtraFields.KeyStatuses...)
+					}
 					break
 				}
 
@@ -530,6 +536,10 @@ func (bifrost *Bifrost) ListAllModels(ctx *schemas.BifrostContext, request *sche
 
 				providerModels = append(providerModels, response.Data...)
 
+				if len(response.KeyStatuses) > 0 {
+					providerKeyStatuses = append(providerKeyStatuses, response.KeyStatuses...)
+				}
+
 				// Check if there are more pages
 				if response.NextPageToken == "" {
 					break
@@ -539,29 +549,40 @@ func (bifrost *Bifrost) ListAllModels(ctx *schemas.BifrostContext, request *sche
 				providerRequest.PageToken = response.NextPageToken
 			}
 
-			results <- providerResult{models: providerModels, err: providerErr}
+			results <- providerResult{
+				provider:    providerKey,
+				models:      providerModels,
+				keyStatuses: providerKeyStatuses,
+				err:         providerErr,
+			}
 		}(providerKey)
 	}
 
 	// Wait for all goroutines to complete
 	wg.Wait()
 	close(results)
 
-	// Accumulate all models from all providers
+	// Accumulate all models and key statuses from all providers
 	allModels := make([]schemas.Model, 0)
+	allKeyStatuses := make([]schemas.KeyStatus, 0)
 	var firstError *schemas.BifrostError
 
 	for result := range results {
 		if len(result.models) > 0 {
 			allModels = append(allModels, result.models...)
 		}
+		if len(result.keyStatuses) > 0 {
+			allKeyStatuses = append(allKeyStatuses, result.keyStatuses...)
+		}
 		if result.err != nil && firstError == nil {
 			firstError = result.err
 		}
 	}
 
 	// If we couldn't get any models from any provider, return the first error
 	if len(allModels) == 0 && firstError != nil {
+		// Attach all key statuses to the error
+		firstError.ExtraFields.KeyStatuses = allKeyStatuses
 		return nil, firstError
 	}
 
@@ -570,9 +591,10 @@ func (bifrost *Bifrost) ListAllModels(ctx *schemas.BifrostContext, request *sche
 		return allModels[i].ID < allModels[j].ID
 	})
 
-	// Return aggregated response with accumulated latency
+	// Return aggregated response with accumulated latency and key statuses
 	response := &schemas.BifrostListModelsResponse{
-		Data: allModels,
+		Data:        allModels,
+		KeyStatuses: allKeyStatuses,
 		ExtraFields: schemas.BifrostResponseExtraFields{
 			RequestType: schemas.ListModelsRequest,
 			Latency:     time.Since(startTime).Milliseconds(),

core/providers/anthropic/anthropic.go

@@ -245,7 +245,9 @@ func (provider *AnthropicProvider) ListModels(ctx *schemas.BifrostContext, keys
 		return nil, err
 	}
 	if provider.customProviderConfig != nil && provider.customProviderConfig.IsKeyLess {
-		return provider.listModelsByKey(ctx, schemas.Key{}, request)
+		return providerUtils.HandleKeylessListModelsRequest(schemas.Anthropic, func() (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
+			return provider.listModelsByKey(ctx, schemas.Key{}, request)
+		})
 	}
 	return providerUtils.HandleMultipleListModelsRequests(
 		ctx,

core/providers/anthropic/utils.go

@@ -1265,6 +1265,13 @@ func convertAnthropicOutputFormatToResponsesTextConfig(outputFormat interface{})
 		Type: formatType,
 	}
 
+	// Extract name if present
+	if name, ok := formatMap["name"].(string); ok && strings.TrimSpace(name) != "" {
+		format.Name = schemas.Ptr(strings.TrimSpace(name))
+	} else {
+		format.Name = schemas.Ptr("output_format")
+	}
+
 	// Extract schema if present
 	if schemaMap, ok := formatMap["schema"].(map[string]interface{}); ok {
 		jsonSchema := &schemas.ResponsesTextConfigFormatJSONSchema{}

core/providers/cohere/cohere.go

@@ -265,7 +265,9 @@ func (provider *CohereProvider) ListModels(ctx *schemas.BifrostContext, keys []s
 		return nil, err
 	}
 	if provider.customProviderConfig != nil && provider.customProviderConfig.IsKeyLess {
-		return provider.listModelsByKey(ctx, schemas.Key{}, request)
+		return providerUtils.HandleKeylessListModelsRequest(provider.GetProviderKey(), func() (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
+			return provider.listModelsByKey(ctx, schemas.Key{}, request)
+		})
 	}
 	return providerUtils.HandleMultipleListModelsRequests(
 		ctx,

core/providers/gemini/gemini.go

@@ -196,7 +196,9 @@ func (provider *GeminiProvider) ListModels(ctx *schemas.BifrostContext, keys []s
 		return nil, err
 	}
 	if provider.customProviderConfig != nil && provider.customProviderConfig.IsKeyLess {
-		return provider.listModelsByKey(ctx, schemas.Key{}, request)
+		return providerUtils.HandleKeylessListModelsRequest(provider.GetProviderKey(), func() (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
+			return provider.listModelsByKey(ctx, schemas.Key{}, request)
+		})
 	}
 	return providerUtils.HandleMultipleListModelsRequests(
 		ctx,

core/providers/huggingface/huggingface.go

@@ -423,7 +423,9 @@ func (provider *HuggingFaceProvider) ListModels(ctx *schemas.BifrostContext, key
 		return nil, err
 	}
 	if provider.customProviderConfig != nil && provider.customProviderConfig.IsKeyLess {
-		return provider.listModelsByKey(ctx, schemas.Key{}, request)
+		return providerUtils.HandleKeylessListModelsRequest(provider.GetProviderKey(), func() (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
+			return provider.listModelsByKey(ctx, schemas.Key{}, request)
+		})
 	}
 	return providerUtils.HandleMultipleListModelsRequests(
 		ctx,

core/providers/openai/openai.go

@@ -90,16 +90,18 @@ func (provider *OpenAIProvider) ListModels(ctx *schemas.BifrostContext, keys []s
 	providerName := provider.GetProviderKey()
 
 	if provider.customProviderConfig != nil && provider.customProviderConfig.IsKeyLess {
-		return listModelsByKey(
-			ctx,
-			provider.client,
-			provider.buildRequestURL(ctx, "/v1/models", schemas.ListModelsRequest),
-			schemas.Key{},
-			provider.networkConfig.ExtraHeaders,
-			providerName,
-			providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest),
-			providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse),
-		)
+		return providerUtils.HandleKeylessListModelsRequest(providerName, func() (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
+			return listModelsByKey(
+				ctx,
+				provider.client,
+				provider.buildRequestURL(ctx, "/v1/models", schemas.ListModelsRequest),
+				schemas.Key{},
+				provider.networkConfig.ExtraHeaders,
+				providerName,
+				providerUtils.ShouldSendBackRawRequest(ctx, provider.sendBackRawRequest),
+				providerUtils.ShouldSendBackRawResponse(ctx, provider.sendBackRawResponse),
+			)
+		})
 	}
 
 	return HandleOpenAIListModelsRequest(ctx,

core/providers/utils/utils.go

@@ -1599,30 +1599,50 @@ func aggregateListModelsResponses(responses []*schemas.BifrostListModelsResponse
 }
 
 // extractSuccessfulListModelsResponses extracts successful responses from a results channel
-// and tracks the last error encountered. This utility reduces code duplication across providers
+// and tracks per-key status information. This utility reduces code duplication across providers
 // for handling multi-key ListModels requests.
 func extractSuccessfulListModelsResponses(
 	results chan schemas.ListModelsByKeyResult,
 	providerName schemas.ModelProvider,
-) ([]*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
+) ([]*schemas.BifrostListModelsResponse, []schemas.KeyStatus, *schemas.BifrostError) {
 	var successfulResponses []*schemas.BifrostListModelsResponse
+	var keyStatuses []schemas.KeyStatus
 	var lastError *schemas.BifrostError
 
 	for result := range results {
 		if result.Err != nil {
-			getLogger().Debug(fmt.Sprintf("failed to list models with key %s: %s", result.KeyID, result.Err.Error.Message))
+			errMsg := "unknown error"
+			if errorField := result.Err.Error; errorField != nil {
+				if errorField.Message != "" {
+					errMsg = errorField.Message
+				} else if errorField.Error != nil {
+					errMsg = errorField.Error.Error()
+				}
+			}
+			getLogger().Warn(fmt.Sprintf("failed to list models with key %s: %s", result.KeyID, errMsg))
+			keyStatuses = append(keyStatuses, schemas.KeyStatus{
+				KeyID:    result.KeyID,
+				Provider: providerName,
+				Status:   schemas.KeyStatusListModelsFailed,
+				Error:    result.Err,
+			})
 			lastError = result.Err
 			continue
 		}
 
+		keyStatuses = append(keyStatuses, schemas.KeyStatus{
+			KeyID:    result.KeyID,
+			Provider: providerName,
+			Status:   schemas.KeyStatusSuccess,
+		})
 		successfulResponses = append(successfulResponses, result.Response)
 	}
 
 	if len(successfulResponses) == 0 {
 		if lastError != nil {
-			return nil, lastError
+			return nil, keyStatuses, lastError
 		}
-		return nil, &schemas.BifrostError{
+		return nil, keyStatuses, &schemas.BifrostError{
 			IsBifrostError: false,
 			Error: &schemas.ErrorField{
 				Message: "all keys failed to list models",
@@ -1634,12 +1654,44 @@ func extractSuccessfulListModelsResponses(
 		}
 	}
 
-	return successfulResponses, nil
+	return successfulResponses, keyStatuses, nil
+}
+
+// HandleKeylessListModelsRequest wraps a list models request for keyless providers
+// and automatically populates the KeyStatuses field with provider-level status tracking.
+// This centralizes the status tracking logic for keyless providers.
+func HandleKeylessListModelsRequest(
+	provider schemas.ModelProvider,
+	listFunc func() (*schemas.BifrostListModelsResponse, *schemas.BifrostError),
+) (*schemas.BifrostListModelsResponse, *schemas.BifrostError) {
+	resp, bifrostErr := listFunc()
+
+	keyStatus := schemas.KeyStatus{
+		KeyID:    "", // Empty for keyless providers
+		Provider: provider,
+	}
+
+	// If request failed, attach status to error
+	if bifrostErr != nil {
+		keyStatus.Status = schemas.KeyStatusListModelsFailed
+		keyStatus.Error = bifrostErr
+		bifrostErr.ExtraFields.KeyStatuses = []schemas.KeyStatus{keyStatus}
+		return nil, bifrostErr
+	}
+
+	// Success case
+	if resp != nil {
+		keyStatus.Status = schemas.KeyStatusSuccess
+		resp.KeyStatuses = []schemas.KeyStatus{keyStatus}
+		return resp, nil
+	}
+
+	return resp, bifrostErr
 }
 
 // HandleMultipleListModelsRequests handles multiple list models requests concurrently for different keys.
 // It launches concurrent requests for all keys and waits for all goroutines to complete.
-// It returns the aggregated response or an error if the request fails.
+// It returns the aggregated response with per-key status information or an error if the request fails.
 func HandleMultipleListModelsRequests(
 	ctx *schemas.BifrostContext,
 	keys []schemas.Key,
@@ -1665,15 +1717,20 @@ func HandleMultipleListModelsRequests(
 	wg.Wait()
 	close(results)
 
-	successfulResponses, err := extractSuccessfulListModelsResponses(results, request.Provider)
+	successfulResponses, keyStatuses, err := extractSuccessfulListModelsResponses(results, request.Provider)
 	if err != nil {
+		// Attach key statuses to error's ExtraFields
+		err.ExtraFields.KeyStatuses = keyStatuses
 		return nil, err
 	}
 
 	// Aggregate all successful responses
 	response := aggregateListModelsResponses(successfulResponses)
 	response = response.ApplyPagination(request.PageSize, request.PageToken)
 
+	// Attach key statuses to response
+	response.KeyStatuses = keyStatuses
+
 	// Set ExtraFields
 	latency := time.Since(startTime)
 	response.ExtraFields.Provider = request.Provider

core/schemas/account.go

@@ -3,6 +3,13 @@ package schemas
 
 import "context"
 
+type KeyStatusType string
+
+const (
+	KeyStatusSuccess          KeyStatusType = "success"
+	KeyStatusListModelsFailed KeyStatusType = "list_models_failed"
+)
+
 // Key represents an API key and its associated configuration for a provider.
 // It contains the key value, supported models, and a weight for load balancing.
 type Key struct {
@@ -19,6 +26,8 @@ type Key struct {
 	Enabled              *bool                 `json:"enabled,omitempty"`                // Whether the key is active (default:true)
 	UseForBatchAPI       *bool                 `json:"use_for_batch_api,omitempty"`      // Whether this key can be used for batch API operations (default:false for new keys, migrated keys default to true)
 	ConfigHash           string                `json:"config_hash,omitempty"`            // Hash of config.json version, used for change detection
+	Status               KeyStatusType         `json:"status,omitempty"`                 // Status of key
+	Description          string                `json:"description,omitempty"`            // Description of key
 }
 
 // AzureKeyConfig represents the Azure-specific configuration.

core/schemas/bifrost.go

@@ -758,4 +758,5 @@ type BifrostErrorExtraFields struct {
 	RawRequest     interface{}   `json:"raw_request,omitempty"`
 	RawResponse    interface{}   `json:"raw_response,omitempty"`
 	LiteLLMCompat  bool          `json:"litellm_compat,omitempty"`
+	KeyStatuses    []KeyStatus   `json:"-"`
 }