Note that the Kubernetes service must have a type of NodePort - * or 'LoadBalancer' - * to expose a node port for client side load balancing. + *
Two endpoint discovery modes are supported via {@link KubernetesEndpointMode}: + *
{@link KubernetesEndpointGroup} gets and watches the nodes, services and pods in the Kubernetes cluster - * and updates the endpoints, so the credentials in the {@link Config} used to create {@link KubernetesClient} - * should have permission to {@code get}, {@code list} and {@code watch} {@code services}, {@code nodes} and - * {@code pods}. Otherwise, the {@link KubernetesEndpointGroup} will not be able to fetch the endpoints. + *
{@link KubernetesEndpointGroup} watches the services and pods (and nodes in + * {@link KubernetesEndpointMode#NODE_PORT} mode) in the Kubernetes cluster and updates the endpoints, + * so the credentials in the {@link Config} used to create {@link KubernetesClient} + * should have the appropriate permissions. Otherwise, the {@link KubernetesEndpointGroup} will not be + * able to fetch the endpoints. * *
For instance, the following RBAC - * configuration is required: + * configuration is required for {@link KubernetesEndpointMode#NODE_PORT} mode: *
{@code
* apiVersion: rbac.authorization.k8s.io/v1
* kind: ClusterRole
@@ -84,10 +98,21 @@
* verbs: ["get", "list", "watch"]
* }
*
+ * For {@link KubernetesEndpointMode#POD} mode, only {@code pods} and {@code services} are required: + *
{@code
+ * apiVersion: rbac.authorization.k8s.io/v1
+ * kind: ClusterRole
+ * metadata:
+ * name: my-cluster-role
+ * rules:
+ * - apiGroups: [""]
+ * resources: ["pods", "services"]
+ * verbs: ["get", "list", "watch"]
+ * }
+ *
* Example: *
{@code
- * // Create a KubernetesEndpointGroup that fetches the endpoints of the 'my-service' service in the 'default'
- * // namespace. The Kubernetes client will be created with the default configuration in the $HOME/.kube/config.
+ * // NODE_PORT mode (default): uses nodeIP:nodePort
* KubernetesClient kubernetesClient = new KubernetesClientBuilder().build();
* KubernetesEndpointGroup
* .builder(kubernetesClient)
@@ -95,17 +120,13 @@
* .serviceName("my-service")
* .build();
*
- * // If you want to use a custom configuration, you can create a KubernetesEndpointGroup as follows:
- * // The custom configuration would be useful when you want to access Kubernetes from outside the cluster.
- * Config config =
- * new ConfigBuilder()
- * .withMasterUrl("https://my-k8s-master")
- * .withOauthToken("my-token")
- * .build();
+ * // POD mode: uses podIP:containerPort for true client-side load balancing
* KubernetesEndpointGroup
- * .builder(config)
- * .namespace("my-namespace")
+ * .builder(kubernetesClient)
+ * .namespace("default")
* .serviceName("my-service")
+ * .mode(KubernetesEndpointMode.POD)
+ * .portName("http")
* .build();
* }
*/
@@ -224,13 +245,20 @@ public static KubernetesEndpointGroupBuilder builder(Config kubeConfig) {
@Nullable
private volatile Watch podWatch;
+ private final KubernetesEndpointMode mode;
+
+ // NODE_PORT mode maps
private final MapIn {@link KubernetesEndpointMode#NODE_PORT} mode, the port name is matched against the + * ServicePort + * name to determine the + * NodePort. + * If not set, the first node port will be used. + * + *
In {@link KubernetesEndpointMode#POD} mode, the port name is matched against the + * {@link ContainerPort#getName()} to determine the container port. + * If not set, the first container port will be used. */ public KubernetesEndpointGroupBuilder portName(String portName) { this.portName = requireNonNull(portName, "portName"); return this; } + /** + * Sets the {@link KubernetesEndpointMode} for endpoint discovery. + * If unspecified, {@link KubernetesEndpointMode#NODE_PORT} is used. + */ + public KubernetesEndpointGroupBuilder mode(KubernetesEndpointMode mode) { + this.mode = requireNonNull(mode, "mode"); + return this; + } + /** * Sets the {@link Predicate} to filter the addresses * of a Kubernetes node. * The first selected {@link NodeAddress} of a node will be used to create the {@link Endpoint}. * If unspecified, the default is to select an {@code InternalIP} address that is not empty. + * + *
This option is ignored when {@link KubernetesEndpointMode#POD} is used. */ public KubernetesEndpointGroupBuilder nodeAddressFilter(Predicate nodeAddressFilter) { requireNonNull(nodeAddressFilter, "nodeAddressFilter"); @@ -126,6 +148,8 @@ public KubernetesEndpointGroupBuilder nodeAddressFilter(PredicateNote that this method is mutually exclusive with {@link #nodeAddressFilter(Predicate)}. If both * methods are called, the last one will take precedence. + * + *
This option is ignored when {@link KubernetesEndpointMode#POD} is used. */ public KubernetesEndpointGroupBuilder nodeIpExtractor( Function nodeIpExtractor) { @@ -176,7 +200,7 @@ public KubernetesEndpointGroupBuilder maxWatchAge(Duration maxWatchAge) { public KubernetesEndpointGroup build() { checkState(serviceName != null, "serviceName not set"); return new KubernetesEndpointGroup(kubernetesClient, namespace, serviceName, portName, - nodeIpExtractor, autoClose, + nodeIpExtractor, autoClose, mode, selectionStrategy, shouldAllowEmptyEndpoints(), selectionTimeoutMillis(), maxWatchAgeMillis); } diff --git a/kubernetes/src/main/java/com/linecorp/armeria/client/kubernetes/endpoints/KubernetesEndpointMode.java b/kubernetes/src/main/java/com/linecorp/armeria/client/kubernetes/endpoints/KubernetesEndpointMode.java new file mode 100644 index 00000000000..b831f0ae394 --- /dev/null +++ b/kubernetes/src/main/java/com/linecorp/armeria/client/kubernetes/endpoints/KubernetesEndpointMode.java @@ -0,0 +1,45 @@ +/* + * Copyright 2026 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.client.kubernetes.endpoints; + +import com.linecorp.armeria.common.annotation.UnstableApi; + +/** + * Specifies how {@link KubernetesEndpointGroup} discovers endpoints from Kubernetes. + */ +@UnstableApi +public enum KubernetesEndpointMode { + + /** + * Uses {@code nodeIP:nodePort} for endpoints. This is the default mode that relies on kube-proxy + * for traffic routing. The Kubernetes service must have a type of + * NodePort + * or LoadBalancer. + * + *
This mode requires RBAC permissions for {@code pods}, {@code services}, and {@code nodes}. + */ + NODE_PORT, + + /** + * Uses {@code podIP:containerPort} for endpoints. This mode enables true client-side load balancing + * by connecting directly to pod IPs, bypassing kube-proxy. + * + *
This mode is intended for Armeria clients running inside the Kubernetes cluster and requires + * RBAC permissions for {@code pods} and {@code services} only (no {@code nodes} permission needed). + */ + POD +} diff --git a/kubernetes/src/test/java/com/linecorp/armeria/client/kubernetes/endpoints/KubernetesEndpointGroupMockServerTest.java b/kubernetes/src/test/java/com/linecorp/armeria/client/kubernetes/endpoints/KubernetesEndpointGroupMockServerTest.java index 736015cfba5..6e5ad65b053 100644 --- a/kubernetes/src/test/java/com/linecorp/armeria/client/kubernetes/endpoints/KubernetesEndpointGroupMockServerTest.java +++ b/kubernetes/src/test/java/com/linecorp/armeria/client/kubernetes/endpoints/KubernetesEndpointGroupMockServerTest.java @@ -35,6 +35,7 @@ import io.fabric8.kubernetes.api.model.Container; import io.fabric8.kubernetes.api.model.ContainerBuilder; +import io.fabric8.kubernetes.api.model.ContainerPort; import io.fabric8.kubernetes.api.model.ContainerPortBuilder; import io.fabric8.kubernetes.api.model.LabelSelector; import io.fabric8.kubernetes.api.model.LabelSelectorBuilder; @@ -50,6 +51,8 @@ import io.fabric8.kubernetes.api.model.PodBuilder; import io.fabric8.kubernetes.api.model.PodSpec; import io.fabric8.kubernetes.api.model.PodSpecBuilder; +import io.fabric8.kubernetes.api.model.PodStatus; +import io.fabric8.kubernetes.api.model.PodStatusBuilder; import io.fabric8.kubernetes.api.model.PodTemplateSpec; import io.fabric8.kubernetes.api.model.PodTemplateSpecBuilder; import io.fabric8.kubernetes.api.model.Service; @@ -377,6 +380,193 @@ void shouldUseAnnotationsToGetNodeIp() { } } + @Test + void podModeBasic() { + // In POD mode, endpoints are podIP:containerPort (no nodes needed). + final Deployment deployment = newDeployment(); + final Service service = newService(null, "nginx", false); + // Set ClusterIP type since NodePort is not required in POD mode. + service.getSpec().setType("ClusterIP"); + + final Pod pod1 = newPodWithIp(deployment.getSpec().getTemplate(), "pod-1", "10.0.0.1"); + final Pod pod2 = newPodWithIp(deployment.getSpec().getTemplate(), "pod-2", "10.0.0.2"); + + client.pods().resource(pod1).create(); + client.pods().resource(pod2).create(); + client.apps().deployments().resource(deployment).create(); + client.services().resource(service).create(); + + try (KubernetesEndpointGroup endpointGroup = + KubernetesEndpointGroup.builder(client, false) + .serviceName("nginx-service") + .mode(KubernetesEndpointMode.POD) + .build()) { + endpointGroup.whenReady().join(); + await().untilAsserted(() -> { + assertThat(endpointGroup.endpoints()).containsExactlyInAnyOrder( + Endpoint.of("10.0.0.1", 8080), + Endpoint.of("10.0.0.2", 8080)); + }); + + // Add a new pod + final Pod pod3 = newPodWithIp(deployment.getSpec().getTemplate(), "pod-3", "10.0.0.3"); + client.pods().resource(pod3).create(); + await().untilAsserted(() -> { + assertThat(endpointGroup.endpoints()).containsExactlyInAnyOrder( + Endpoint.of("10.0.0.1", 8080), + Endpoint.of("10.0.0.2", 8080), + Endpoint.of("10.0.0.3", 8080)); + }); + + // Remove a pod + client.pods().resource(pod1).delete(); + await().untilAsserted(() -> { + assertThat(endpointGroup.endpoints()).containsExactlyInAnyOrder( + Endpoint.of("10.0.0.2", 8080), + Endpoint.of("10.0.0.3", 8080)); + }); + } + } + + @Test + void podModeWithPortName() { + // In POD mode, portName matches ContainerPort.name. + final PodTemplateSpec template = newPodTemplateWithNamedPorts("nginx"); + final Deployment deployment = newDeployment(); + // Replace the template with one that has named ports. + deployment.getSpec().setTemplate(template); + + final Service service = newService(null, "nginx", false); + service.getSpec().setType("ClusterIP"); + + final Pod pod1 = newPodWithIpAndNamedPorts("pod-1", "10.0.0.1", "nginx"); + client.pods().resource(pod1).create(); + client.apps().deployments().resource(deployment).create(); + client.services().resource(service).create(); + + // Select the "https" port + try (KubernetesEndpointGroup endpointGroup = + KubernetesEndpointGroup.builder(client, false) + .serviceName("nginx-service") + .mode(KubernetesEndpointMode.POD) + .portName("https") + .build()) { + endpointGroup.whenReady().join(); + await().untilAsserted(() -> { + assertThat(endpointGroup.endpoints()).containsExactlyInAnyOrder( + Endpoint.of("10.0.0.1", 8443)); + }); + } + + // Select the "http" port + try (KubernetesEndpointGroup endpointGroup = + KubernetesEndpointGroup.builder(client, false) + .serviceName("nginx-service") + .mode(KubernetesEndpointMode.POD) + .portName("http") + .build()) { + endpointGroup.whenReady().join(); + await().untilAsserted(() -> { + assertThat(endpointGroup.endpoints()).containsExactlyInAnyOrder( + Endpoint.of("10.0.0.1", 8080)); + }); + } + } + + @Test + void podModeSkipsPodWithNullPodIp() { + // A pending pod without a podIP should be excluded. + final Deployment deployment = newDeployment(); + final Service service = newService(null, "nginx", false); + service.getSpec().setType("ClusterIP"); + + final Pod podWithIp = newPodWithIp(deployment.getSpec().getTemplate(), "pod-ready", "10.0.0.1"); + // Pod without podIP (pending state) + final Pod podWithoutIp = newPod(deployment.getSpec().getTemplate(), "node-1"); + + client.pods().resource(podWithIp).create(); + client.pods().resource(podWithoutIp).create(); + client.apps().deployments().resource(deployment).create(); + client.services().resource(service).create(); + + try (KubernetesEndpointGroup endpointGroup = + KubernetesEndpointGroup.builder(client, false) + .serviceName("nginx-service") + .mode(KubernetesEndpointMode.POD) + .build()) { + endpointGroup.whenReady().join(); + await().untilAsserted(() -> { + assertThat(endpointGroup.endpoints()).containsExactlyInAnyOrder( + Endpoint.of("10.0.0.1", 8080)); + }); + } + } + + private static Pod newPodWithIp(PodTemplateSpec template, String podName, String podIp) { + final PodSpec spec = template.getSpec() + .toBuilder() + .withNodeName("dummy-node") + .build(); + final ObjectMeta metadata = template.getMetadata() + .toBuilder() + .withName(podName) + .build(); + final PodStatus status = new PodStatusBuilder() + .withPodIP(podIp) + .build(); + return new PodBuilder() + .withMetadata(metadata) + .withSpec(spec) + .withStatus(status) + .build(); + } + + private static PodTemplateSpec newPodTemplateWithNamedPorts(String selectorName) { + final ObjectMeta metadata = new ObjectMetaBuilder() + .withLabels(ImmutableMap.of("app", selectorName)) + .build(); + final ContainerPort httpPort = new ContainerPortBuilder() + .withName("http") + .withContainerPort(8080) + .build(); + final ContainerPort httpsPort = new ContainerPortBuilder() + .withName("https") + .withContainerPort(8443) + .build(); + final Container container = new ContainerBuilder() + .withName("nginx") + .withImage("nginx:1.14.2") + .withPorts(httpPort, httpsPort) + .build(); + final PodSpec spec = new PodSpecBuilder() + .withContainers(container) + .build(); + return new PodTemplateSpecBuilder() + .withMetadata(metadata) + .withSpec(spec) + .build(); + } + + private static Pod newPodWithIpAndNamedPorts(String podName, String podIp, String selectorName) { + final PodTemplateSpec template = newPodTemplateWithNamedPorts(selectorName); + final PodSpec spec = template.getSpec() + .toBuilder() + .withNodeName("dummy-node") + .build(); + final ObjectMeta metadata = template.getMetadata() + .toBuilder() + .withName(podName) + .build(); + final PodStatus status = new PodStatusBuilder() + .withPodIP(podIp) + .build(); + return new PodBuilder() + .withMetadata(metadata) + .withSpec(spec) + .withStatus(status) + .build(); + } + private static Node newNode(String ip, String type) { final NodeAddress nodeAddress = new NodeAddressBuilder() .withType(type)