feat: add tlsInsecureSkipVerify to ExternalModel spec

  Adds an optional spec.tlsInsecureSkipVerify field to the
  ExternalModel CRD. When true, the reconciler generates the
  DestinationRule with insecureSkipVerify: true, allowing
  connections to endpoints with self-signed certificates
  without manual patching that gets overwritten on
  reconciliation. Default is false.

  Closes #627

Author: Brent Salisbury

deployment/base/maas-controller/crd/bases/maas.opendatahub.io_externalmodels.yaml

@@ -101,6 +101,18 @@ spec:
                 maxLength: 253
                 minLength: 1
                 type: string
+              tlsInsecureSkipVerify:
+                description: |-
+                  TLSInsecureSkipVerify disables TLS certificate verification on the
+                  DestinationRule created for this external model. When true, the generated
+                  Istio DestinationRule includes insecureSkipVerify: true under
+                  trafficPolicy.tls, allowing connections to endpoints with self-signed or
+                  untrusted certificates (e.g., simulators, dev environments).
+
+                  WARNING: Do not enable in production. This is intended for testing only.
+
+                  Default: false (certificates are verified).
+                type: boolean
             required:
             - credentialRef
             - endpoint

maas-api/cmd/main.go

@@ -148,7 +148,7 @@ func registerHandlers(ctx context.Context, log *logger.Logger, router *gin.Engin
 
 	subscriptionSelector := subscription.NewSelector(log, cluster.MaaSSubscriptionLister)
 
-	modelManager, err := models.NewManager(log)
+	modelManager, err := models.NewManager(log, cfg.AccessCheckTimeoutSeconds)
 	if err != nil {
 		log.Fatal("Failed to create model manager", "error", err)
 	}

maas-api/internal/config/config.go

@@ -46,6 +46,12 @@ type Config struct {
 	// Default: 30 days. Minimum: 1 day.
 	APIKeyMaxExpirationDays int
 
+	// AccessCheckTimeoutSeconds bounds the total duration of model access validation.
+	// This limits the staleness window between when access is checked and when the
+	// response reaches the client. Models whose probes don't complete within this
+	// window are excluded (fail-closed). Default: 15 seconds. Minimum: 1 second.
+	AccessCheckTimeoutSeconds int
+
 	// Deprecated flag (backward compatibility with pre-TLS version)
 	deprecatedHTTPPort string
 }
@@ -56,6 +62,7 @@ func Load() *Config {
 	gatewayName := env.GetString("GATEWAY_NAME", constant.DefaultGatewayName)
 	secure, _ := env.GetBool("SECURE", false)
 	maxExpirationDays, _ := env.GetInt("API_KEY_MAX_EXPIRATION_DAYS", constant.DefaultAPIKeyMaxExpirationDays)
+	accessCheckTimeoutSeconds, _ := env.GetInt("ACCESS_CHECK_TIMEOUT_SECONDS", 15)
 
 	c := &Config{
 		Name:                      env.GetString("INSTANCE_NAME", gatewayName),
@@ -69,6 +76,7 @@ func Load() *Config {
 		DebugMode:                 debugMode,
 		DBConnectionURL:           "", // Loaded from K8s secret via LoadDatabaseURL()
 		APIKeyMaxExpirationDays:   maxExpirationDays,
+		AccessCheckTimeoutSeconds: accessCheckTimeoutSeconds,
 		// Deprecated env var (backward compatibility with pre-TLS version)
 		deprecatedHTTPPort: env.GetString("PORT", ""),
 	}
@@ -141,6 +149,10 @@ func (c *Config) Validate() error {
 		return errors.New("API_KEY_MAX_EXPIRATION_DAYS must be at least 1")
 	}
 
+	if c.AccessCheckTimeoutSeconds < 1 {
+		return errors.New("ACCESS_CHECK_TIMEOUT_SECONDS must be at least 1")
+	}
+
 	return nil
 }
 

maas-api/internal/config/config_test.go

@@ -120,6 +120,7 @@ func TestValidate(t *testing.T) {
 				DBConnectionURL:           "postgresql://localhost/test",
 				Secure:                    false,
 				APIKeyMaxExpirationDays:   30,
+				AccessCheckTimeoutSeconds: 15,
 				MaaSSubscriptionNamespace: "models-as-a-service",
 			},
 		},
@@ -129,6 +130,7 @@ func TestValidate(t *testing.T) {
 				DBConnectionURL:           "postgresql://localhost/test",
 				TLS:                       TLSConfig{SelfSigned: true, MinVersion: TLSVersion(tls.VersionTLS12)},
 				APIKeyMaxExpirationDays:   30,
+				AccessCheckTimeoutSeconds: 15,
 				MaaSSubscriptionNamespace: "models-as-a-service",
 			},
 		},
@@ -138,6 +140,7 @@ func TestValidate(t *testing.T) {
 				DBConnectionURL:           "postgresql://localhost/test",
 				TLS:                       TLSConfig{Cert: "/cert.pem", Key: "/key.pem", MinVersion: TLSVersion(tls.VersionTLS12)},
 				APIKeyMaxExpirationDays:   30,
+				AccessCheckTimeoutSeconds: 15,
 				MaaSSubscriptionNamespace: "models-as-a-service",
 			},
 		},
@@ -146,6 +149,7 @@ func TestValidate(t *testing.T) {
 			cfg: Config{
 				DBConnectionURL:           "postgresql://localhost/test",
 				APIKeyMaxExpirationDays:   1,
+				AccessCheckTimeoutSeconds: 15,
 				MaaSSubscriptionNamespace: "models-as-a-service",
 			},
 		},
@@ -154,6 +158,7 @@ func TestValidate(t *testing.T) {
 			cfg: Config{
 				DBConnectionURL:           "postgresql://localhost/test",
 				APIKeyMaxExpirationDays:   30,
+				AccessCheckTimeoutSeconds: 15,
 				MaaSSubscriptionNamespace: "models-as-a-service",
 			},
 		},
@@ -162,6 +167,7 @@ func TestValidate(t *testing.T) {
 			cfg: Config{
 				DBConnectionURL:           "postgresql://localhost/test",
 				APIKeyMaxExpirationDays:   365,
+				AccessCheckTimeoutSeconds: 15,
 				MaaSSubscriptionNamespace: "models-as-a-service",
 			},
 		},

maas-api/internal/handlers/models.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/openai/openai-go/v2/packages/pagination"
@@ -260,6 +261,7 @@ func (h *ModelsHandler) ListLLMs(c *gin.Context) {
 
 	// Initialize to empty slice (not nil) so JSON marshals as [] instead of null
 	modelList := []models.Model{}
+	accessCheckedAt := time.Now().UTC()
 	if h.maasModelRefLister != nil {
 		h.logger.Debug("Listing models from MaaSModelRef cache (all namespaces)")
 		list, err := models.ListFromMaaSModelRefLister(h.maasModelRefLister)
@@ -361,11 +363,18 @@ func (h *ModelsHandler) ListLLMs(c *gin.Context) {
 			}
 		}
 
+		accessCheckedAt = time.Now().UTC()
 		h.logger.Debug("Access validation complete", "listed", len(list), "accessible", len(modelList), "subscriptions", len(subscriptionsToUse))
 	} else {
 		h.logger.Debug("MaaSModelRef lister not configured, returning empty model list")
 	}
 
+	// Prevent clients and proxies from caching authorization-checked model listings.
+	// The access check is a point-in-time snapshot; auth policies may change at any moment.
+	// X-Access-Checked-At lets clients assess the freshness of the authorization decision.
+	c.Header("Cache-Control", "no-store")
+	c.Header("X-Access-Checked-At", accessCheckedAt.Format(time.RFC3339))
+
 	h.logger.Debug("GET /v1/models returning models", "count", len(modelList))
 	c.JSON(http.StatusOK, pagination.Page[models.Model]{
 		Object: "list",

maas-api/internal/handlers/models_test.go

@@ -320,7 +320,7 @@ func TestListingModels(t *testing.T) {
 	}
 	router, _ := fixtures.SetupTestServer(t, config)
 
-	modelMgr, errMgr := models.NewManager(testLogger)
+	modelMgr, errMgr := models.NewManager(testLogger, 15)
 	require.NoError(t, errMgr)
 
 	// Set up test fixtures
@@ -350,6 +350,16 @@ func TestListingModels(t *testing.T) {
 
 	require.Equal(t, http.StatusOK, w.Code, "Expected status OK")
 
+	// Verify anti-caching and freshness headers (authorization timing race mitigation)
+	assert.Equal(t, "no-store", w.Header().Get("Cache-Control"),
+		"Expected Cache-Control: no-store to prevent caching of authorization-checked listings")
+	accessCheckedAt := w.Header().Get("X-Access-Checked-At")
+	assert.NotEmpty(t, accessCheckedAt, "Expected X-Access-Checked-At header with RFC3339 timestamp")
+	if accessCheckedAt != "" {
+		_, parseErr := time.Parse(time.RFC3339, accessCheckedAt)
+		require.NoError(t, parseErr, "X-Access-Checked-At should be valid RFC3339")
+	}
+
 	var response pagination.Page[models.Model]
 	err = json.Unmarshal(w.Body.Bytes(), &response)
 	require.NoError(t, err, "Failed to unmarshal response body")
@@ -425,7 +435,7 @@ func TestListingModelsWithSubscriptionHeader(t *testing.T) {
 	}
 	router, _ := fixtures.SetupTestServer(t, config)
 
-	modelMgr, errMgr := models.NewManager(testLogger)
+	modelMgr, errMgr := models.NewManager(testLogger, 15)
 	require.NoError(t, errMgr)
 
 	_, cleanup := fixtures.StubTokenProviderAPIs(t)
@@ -647,7 +657,7 @@ func TestListModels_ReturnAllModels(t *testing.T) {
 		},
 	}
 
-	modelMgr, err := models.NewManager(testLogger)
+	modelMgr, err := models.NewManager(testLogger, 15)
 	require.NoError(t, err)
 
 	subscriptionSelector := subscription.NewSelector(testLogger, subscriptionLister)
@@ -829,7 +839,7 @@ func TestListModels_DeduplicationBySubscription(t *testing.T) {
 		},
 	}
 
-	modelMgr, err := models.NewManager(testLogger)
+	modelMgr, err := models.NewManager(testLogger, 15)
 	require.NoError(t, err)
 
 	subscriptionSelector := subscription.NewSelector(testLogger, subscriptionLister)
@@ -940,7 +950,7 @@ func TestListModels_DifferentModelRefsWithSameModelID(t *testing.T) {
 		},
 	}
 
-	modelMgr, err := models.NewManager(testLogger)
+	modelMgr, err := models.NewManager(testLogger, 15)
 	require.NoError(t, err)
 
 	subscriptionSelector := subscription.NewSelector(testLogger, subscriptionLister)
@@ -1040,7 +1050,7 @@ func TestListModels_DifferentModelRefsWithSameURLAndModelID(t *testing.T) {
 		},
 	}
 
-	modelMgr, err := models.NewManager(testLogger)
+	modelMgr, err := models.NewManager(testLogger, 15)
 	require.NoError(t, err)
 
 	subscriptionSelector := subscription.NewSelector(testLogger, subscriptionLister)
@@ -1139,7 +1149,7 @@ func TestListModels_DifferentModelRefsWithSameModelIDAndDifferentSubscriptions(t
 		},
 	}
 
-	modelMgr, err := models.NewManager(testLogger)
+	modelMgr, err := models.NewManager(testLogger, 15)
 	require.NoError(t, err)
 
 	subscriptionSelector := subscription.NewSelector(testLogger, subscriptionLister)

maas-api/internal/models/discovery.go

@@ -32,28 +32,43 @@ const maxModelsResponseBytes int64 = 4 << 20 // 4 MiB
 
 // HTTP client and concurrency for access-validation probes.
 const (
-	httpClientTimeout       = 5 * time.Second
 	httpMaxIdleConns        = 100
 	httpIdleConnTimeout     = 90 * time.Second
 	maxDiscoveryConcurrency = 10
+
+	// defaultAccessCheckTimeout bounds the total duration of FilterModelsByAccess.
+	// This limits the staleness window between when access is checked and when
+	// the response reaches the client. Models whose probes don't complete within
+	// this window are excluded (fail-closed).
+	defaultAccessCheckTimeout = 15 * time.Second
 )
 
 // Manager runs access validation (probe model endpoints) for models listed from MaaSModelRef.
 type Manager struct {
-	logger     *logger.Logger
-	httpClient *http.Client
+	logger             *logger.Logger
+	httpClient         *http.Client
+	accessCheckTimeout time.Duration
 }
 
 // NewManager creates a Manager for filtering models by access. The client uses InsecureSkipVerify
 // for cluster-internal probes; auth is enforced by the gateway/model server.
-func NewManager(log *logger.Logger) (*Manager, error) {
+// accessCheckTimeoutSeconds controls the total duration bound for access validation;
+// if <= 0, the default of 15 seconds is used.
+func NewManager(log *logger.Logger, accessCheckTimeoutSeconds int) (*Manager, error) {
 	if log == nil {
 		return nil, errors.New("log is required")
 	}
+	timeout := defaultAccessCheckTimeout
+	if accessCheckTimeoutSeconds > 0 {
+		timeout = time.Duration(accessCheckTimeoutSeconds) * time.Second
+	}
 	return &Manager{
-		logger: log,
+		logger:             log,
+		accessCheckTimeout: timeout,
 		httpClient: &http.Client{
-			Timeout: httpClientTimeout,
+			// No per-client Timeout — each request inherits the accessCheckTimeout
+			// deadline via its context. This ensures that configuring a longer
+			// ACCESS_CHECK_TIMEOUT_SECONDS actually allows slower backends to respond.
 			Transport: &http.Transport{
 				TLSClientConfig:     &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // cluster-internal only
 				MaxIdleConns:        httpMaxIdleConns,
@@ -65,12 +80,26 @@ func NewManager(log *logger.Logger) (*Manager, error) {
 }
 
 // FilterModelsByAccess returns only models the user can access by probing each model's
-// /v1/models endpoint with the given Authorization and x-maas-subscription headers (passed through as-is). 2xx or 405 → include, 401/403/404 → exclude.
+// /v1/models endpoint with the given Authorization and x-maas-subscription headers (passed through as-is).
+// 2xx or 405 → include, 401/403/404 → exclude.
 // Models with nil URL are skipped. Concurrency is limited by maxDiscoveryConcurrency.
+//
+// Because authorization policies propagate asynchronously through the gateway, there is an
+// inherent eventual-consistency window: a model listed here may become inaccessible (or vice versa)
+// by the time the client acts on the response. Actual enforcement always happens at the gateway
+// when the model is invoked for inference. Callers should set Cache-Control: no-store and expose
+// a freshness timestamp via response headers so clients can assess freshness.
+//
+// The access check is bounded by accessCheckTimeout to limit the staleness window.
 func (m *Manager) FilterModelsByAccess(ctx context.Context, models []Model, authHeader string, subscriptionHeader string) []Model {
 	if len(models) == 0 {
 		return models
 	}
+
+	// Bound the total access-check duration to limit the staleness window.
+	ctx, cancel := context.WithTimeout(ctx, m.accessCheckTimeout)
+	defer cancel()
+
 	m.logger.Debug("FilterModelsByAccess: validating access for models", "count", len(models), "subscriptionHeaderProvided", subscriptionHeader != "")
 	// Initialize to empty slice (not nil) so JSON marshals as [] instead of null when no models are accessible
 	out := []Model{}
@@ -222,7 +251,11 @@ func (m *Manager) fetchModelsWithRetry(ctx context.Context, authHeader string, s
 		lastResult = authRes
 		return lastResult != authRetry, nil
 	}); err != nil {
-		m.logger.Debug("Access validation failed: model fetch backoff exhausted", "service", meta.ServiceName, "endpoint", meta.Endpoint, "error", err)
+		if errors.Is(err, context.DeadlineExceeded) || ctx.Err() == context.DeadlineExceeded {
+			m.logger.Debug("Access validation failed: context deadline exceeded", "service", meta.ServiceName, "endpoint", meta.Endpoint, "timeout", m.accessCheckTimeout)
+		} else {
+			m.logger.Debug("Access validation failed: model fetch backoff exhausted", "service", meta.ServiceName, "endpoint", meta.Endpoint, "error", err)
+		}
 		return nil // explicit fail-closed on error
 	}
 
@@ -249,6 +282,10 @@ func (m *Manager) fetchModels(ctx context.Context, authHeader string, subscripti
 	// #nosec G704 -- Intentional HTTP request to probe model endpoint for authorization check
 	resp, err := m.httpClient.Do(req)
 	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) || ctx.Err() == context.DeadlineExceeded {
+			m.logger.Debug("Access validation: request timed out (context deadline exceeded)", "service", meta.ServiceName, "endpoint", meta.Endpoint)
+			return nil, authDenied // fail-closed, no point retrying a deadline
+		}
 		m.logger.Debug("Access validation: GET request failed", "service", meta.ServiceName, "endpoint", meta.Endpoint, "error", err)
 		return nil, authRetry
 	}

maas-controller/api/maas/v1alpha1/externalmodel_types.go

@@ -70,6 +70,18 @@ type ExternalModelSpec struct {
 	// The Secret must contain a data key "api-key" with the credential value.
 	// +kubebuilder:validation:Required
 	CredentialRef CredentialReference `json:"credentialRef"`
+
+	// TLSInsecureSkipVerify disables TLS certificate verification on the
+	// DestinationRule created for this external model. When true, the generated
+	// Istio DestinationRule includes insecureSkipVerify: true under
+	// trafficPolicy.tls, allowing connections to endpoints with self-signed or
+	// untrusted certificates (e.g., simulators, dev environments).
+	//
+	// WARNING: Do not enable in production. This is intended for testing only.
+	//
+	// Default: false (certificates are verified).
+	// +optional
+	TLSInsecureSkipVerify bool `json:"tlsInsecureSkipVerify,omitempty"`
 }
 
 // CredentialReference references a Kubernetes Secret with provider API credentials.

maas-controller/pkg/reconciler/externalmodel/reconciler.go

@@ -284,12 +284,12 @@ func specFromExternalModel(extModel *maasv1alpha1.ExternalModel, model *maasv1al
 	}
 
 	spec := ExternalModelSpec{
-		Provider:   extModel.Spec.Provider,
-		Endpoint:   extModel.Spec.Endpoint,
-		PathPrefix: ann[AnnPathPrefix],
-		TLS:        true,
-		Port:       443,
-		// TLSInsecureSkipVerify: extModel.Spec.TLSInsecureSkipVerify, // requires issue #627 CRD change
+		Provider:              extModel.Spec.Provider,
+		Endpoint:              extModel.Spec.Endpoint,
+		PathPrefix:            ann[AnnPathPrefix],
+		TLS:                   true,
+		Port:                  443,
+		TLSInsecureSkipVerify: extModel.Spec.TLSInsecureSkipVerify,
 	}
 
 	if spec.Provider == "" {