envoy: fix cluster locality handling

Make /cilium-locality-cluster use the selected xDS mode instead of always
using EDS.

For external Envoy, render the locality cluster with ADS config when
envoy.xdsMode is ads or strict-ads, and keep the existing gRPC
apiConfigSource for split mode.

For embedded Envoy, reuse GetXDSConfigSource() when building the locality
cluster so EDS references follow the configured xDS mode consistently.

Signed-off-by: viktor-kurchenko <viktor.kurchenko@isovalent.com>

Author: viktor-kurchenko

install/kubernetes/cilium/files/cilium-envoy/configmap/bootstrap-config.yaml

@@ -277,6 +277,10 @@ staticResources:
     connectTimeout: "{{ .Values.envoy.connectTimeoutSeconds }}s"
     edsClusterConfig:
       edsConfig:
+        {{- if or (eq $envoyXdsMode "ads") (eq $envoyXdsMode "strict-ads") }}
+        ads: {}
+        resourceApiVersion: "V3"
+        {{- else }}
         apiConfigSource:
           apiType: "GRPC"
           transportApiVersion: "V3"
@@ -285,6 +289,7 @@ staticResources:
               clusterName: "xds-grpc-cilium"
           setNodeOnFirstMessageOnly: true
         resourceApiVersion: "V3"
+        {{- end }}
       serviceName: "/cilium-locality-cluster"
     lbPolicy: "ROUND_ROBIN"
   {{- end }}

pkg/envoy/locality.go

@@ -57,23 +57,7 @@ func newLocalityCluster(connectTimeout int64) *envoy_config_cluster.Cluster {
 		ClusterDiscoveryType: &envoy_config_cluster.Cluster_Type{Type: envoy_config_cluster.Cluster_EDS},
 		ConnectTimeout:       &durationpb.Duration{Seconds: connectTimeout},
 		EdsClusterConfig: &envoy_config_cluster.Cluster_EdsClusterConfig{
-			EdsConfig: &envoy_config_core.ConfigSource{
-				ResourceApiVersion: envoy_config_core.ApiVersion_V3,
-				ConfigSourceSpecifier: &envoy_config_core.ConfigSource_ApiConfigSource{
-					ApiConfigSource: &envoy_config_core.ApiConfigSource{
-						ApiType:                   envoy_config_core.ApiConfigSource_GRPC,
-						TransportApiVersion:       envoy_config_core.ApiVersion_V3,
-						SetNodeOnFirstMessageOnly: true,
-						GrpcServices: []*envoy_config_core.GrpcService{{
-							TargetSpecifier: &envoy_config_core.GrpcService_EnvoyGrpc_{
-								EnvoyGrpc: &envoy_config_core.GrpcService_EnvoyGrpc{
-									ClusterName: CiliumXDSClusterName,
-								},
-							},
-						}},
-					},
-				},
-			},
+			EdsConfig:   GetXDSConfigSource(),
 			ServiceName: LocalityClusterName,
 		},
 		LbPolicy: envoy_config_cluster.Cluster_ROUND_ROBIN,

pkg/envoy/locality_test.go

@@ -8,22 +8,68 @@ import (
 
 	envoy_config_bootstrap "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
 	envoy_config_cluster "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
+	corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cilium/cilium/pkg/envoy/config"
 )
 
 func TestAppendEmbeddedLocalityBootstrap(t *testing.T) {
-	bs := &envoy_config_bootstrap.Bootstrap{
-		StaticResources: &envoy_config_bootstrap.Bootstrap_StaticResources{},
+	tests := []struct {
+		name      string
+		xdsMode   string
+		assertEDS func(t *testing.T, edsConfig *corev3.ConfigSource)
+	}{
+		{
+			name:    "split",
+			xdsMode: config.EnvoyXDSModeSplit,
+			assertEDS: func(t *testing.T, edsConfig *corev3.ConfigSource) {
+				apiConfigSource := edsConfig.GetApiConfigSource()
+				require.NotNil(t, apiConfigSource)
+				require.NotEmpty(t, apiConfigSource.GetGrpcServices())
+				require.Equal(t, CiliumXDSClusterName, apiConfigSource.GetGrpcServices()[0].GetEnvoyGrpc().GetClusterName())
+				require.Nil(t, edsConfig.GetAds())
+			},
+		},
+		{
+			name:    "ads",
+			xdsMode: config.EnvoyXDSModeADS,
+			assertEDS: func(t *testing.T, edsConfig *corev3.ConfigSource) {
+				require.NotNil(t, edsConfig.GetAds())
+				require.Nil(t, edsConfig.GetApiConfigSource())
+			},
+		},
+		{
+			name:    "strict-ads",
+			xdsMode: config.EnvoyXDSModeStrictADS,
+			assertEDS: func(t *testing.T, edsConfig *corev3.ConfigSource) {
+				require.NotNil(t, edsConfig.GetAds())
+				require.Nil(t, edsConfig.GetApiConfigSource())
+			},
+		},
 	}
 
-	appendEmbeddedLocalityBootstrap(bs, 7, "zone-a")
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			SetXDSMode(tt.xdsMode)
+			t.Cleanup(func() { SetXDSMode("") })
+
+			bs := &envoy_config_bootstrap.Bootstrap{
+				StaticResources: &envoy_config_bootstrap.Bootstrap_StaticResources{},
+			}
+
+			appendEmbeddedLocalityBootstrap(bs, 7, "zone-a")
 
-	require.Equal(t, LocalityClusterName, bs.GetClusterManager().GetLocalClusterName())
-	require.Equal(t, "zone-a", bs.GetNode().GetLocality().GetZone())
-	require.Len(t, bs.GetStaticResources().GetClusters(), 1)
+			require.Equal(t, LocalityClusterName, bs.GetClusterManager().GetLocalClusterName())
+			require.Equal(t, "zone-a", bs.GetNode().GetLocality().GetZone())
+			require.Len(t, bs.GetStaticResources().GetClusters(), 1)
 
-	cluster := bs.GetStaticResources().GetClusters()[0]
-	require.Equal(t, LocalityClusterName, cluster.GetName())
-	require.Equal(t, envoy_config_cluster.Cluster_EDS, cluster.GetType())
-	require.Equal(t, CiliumXDSClusterName, cluster.GetEdsClusterConfig().GetEdsConfig().GetApiConfigSource().GetGrpcServices()[0].GetEnvoyGrpc().GetClusterName())
+			cluster := bs.GetStaticResources().GetClusters()[0]
+			require.Equal(t, LocalityClusterName, cluster.GetName())
+			require.Equal(t, envoy_config_cluster.Cluster_EDS, cluster.GetType())
+			require.Equal(t, LocalityClusterName, cluster.GetEdsClusterConfig().GetServiceName())
+
+			tt.assertEDS(t, cluster.GetEdsClusterConfig().GetEdsConfig())
+		})
+	}
 }