make hf api key optional (kubeflow#1963)

Signed-off-by: Adysen Rothman <85646824+adysenrothman@users.noreply.github.com>

Author: adysenrothman

catalog/README.md

@@ -82,7 +82,7 @@ The HuggingFace catalog source allows you to discover and import models from the
 
 #### 1. Set Your API Key
 
-The HuggingFace provider requires an API key for authentication. By default, the service reads the API key from the `HF_API_KEY` environment variable:
+Setting a Hugging Face API key is optional. Hugging Face  requires an API key for authentication for full access to data of models that are private and/or gated. If an API key is NOT set, private models will be entirely unavailable and gated models will have limited metadata. By default, the service reads the API key from the `HF_API_KEY` environment variable:
 
 ```bash
 export HF_API_KEY="your-huggingface-api-key-here"

catalog/internal/catalog/hf_catalog.go

@@ -633,21 +633,31 @@ func newHFModelProvider(ctx context.Context, source *Source, reldir string) (<-c
 	}
 	apiKey := os.Getenv(apiKeyEnvVar)
 	if apiKey == "" {
-		return nil, fmt.Errorf("missing %s environment variable for HuggingFace catalog", apiKeyEnvVar)
+		glog.Infof("No API key configured for Hugging Face. Only public models and limited data for gated models will be available.")
 	}
 	p.apiKey = apiKey
 
 	// Parse base URL (optional, defaults to huggingface.co)
 	// This allows tests to use mock servers by providing a custom URL
 	p.baseURL = defaultHuggingFaceURL
 	if url, ok := source.Properties[urlKey].(string); ok && url != "" {
-		p.baseURL = strings.TrimSuffix(url, "/") // Remove trailing slash if present
+		p.baseURL = strings.TrimSuffix(url, "/")
 	}
 
-	// Validate credentials before proceeding
-	if err := p.validateCredentials(ctx); err != nil {
-		glog.Errorf("HuggingFace catalog credential validation failed: %v", err)
-		return nil, fmt.Errorf("failed to validate HuggingFace catalog credentials: %w", err)
+	if p.apiKey != "" {
+		hasValidPrefix := strings.HasPrefix(p.apiKey, "hf_")
+		if !hasValidPrefix {
+			// API key is set but doesn't have expected prefix, warn and continue without authentication
+			glog.Infof("API key does not have expected 'hf_' prefix. Only public models and limited data for gated models will be available.")
+			p.apiKey = "" // Clear invalid key to prevent its use
+		}
+		if hasValidPrefix {
+			// Validate credentials only if API key has correct format
+			if err := p.validateCredentials(ctx); err != nil {
+				glog.Errorf("HuggingFace catalog credential validation failed: %v", err)
+				return nil, fmt.Errorf("failed to validate HuggingFace catalog credentials: %w", err)
+			}
+		}
 	}
 
 	// Use top-level IncludedModels from Source as the list of models to fetch