Skip to content

Commit c08ad57

Browse files
moritzkiefer-dacocreature
moritzkiefer-da
and
cocreature
authored
Filter IPs using istio instead of in the load balancer (#1214)
* Filter IPs using istio instead of in the load balancer I tested: 1. Deploy the infra stack on a scratchnet 2. Add a few thousand IPs and test that nothing seems to explode. 3. Validate that access from another scratchnet fails. 4. Add the IP from the other scratchnet at the end of the few thousand IPs. 5. Validate that access works. 6. Remove IP and check that access fails again. [static] Signed-off-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> * expected files [static] Signed-off-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> * issue link fixes DACH-NY/canton-network-internal#626 [static] Signed-off-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> * allow node pool ips [static] Signed-off-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> * fmt or something [static] Signed-off-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> --------- Signed-off-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org> Co-authored-by: Moritz Kiefer <moritz.kiefer@purelyfunctional.org>
1 parent e1f6331 commit c08ad57

File tree

3 files changed

+205
-13
lines changed

3 files changed

+205
-13
lines changed

‎cluster/expected/infra/expected.json‎

Lines changed: 127 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1103,6 +1103,132 @@
11031103
"provider": "",
11041104
"type": "kubernetes:cert-manager.io/v1:Issuer"
11051105
},
1106+
{
1107+
"custom": true,
1108+
"id": "",
1109+
"inputs": {
1110+
"apiVersion": "security.istio.io/v1beta1",
1111+
"kind": "AuthorizationPolicy",
1112+
"metadata": {
1113+
"name": "istio-access-policy-allow-0",
1114+
"namespace": "cluster-ingress"
1115+
},
1116+
"spec": {
1117+
"action": "ALLOW",
1118+
"rules": [
1119+
{
1120+
"from": [
1121+
{
1122+
"source": {
1123+
"remoteIpBlocks": [
1124+
"<internal IPs>",
1125+
"1.2.3.4/32",
1126+
"5.6.7.8/32",
1127+
"11.12.13.14/32",
1128+
"4.3.2.1/32",
1129+
"8.7.6.5/32",
1130+
"8.7.6.4/32",
1131+
"9.8.7.6/32",
1132+
"11.22.33.45/32",
1133+
"12.34.56.78/32",
1134+
"10.160.0.0/16"
1135+
]
1136+
}
1137+
}
1138+
]
1139+
}
1140+
],
1141+
"selector": {
1142+
"matchLabels": {
1143+
"app": "istio-ingress"
1144+
}
1145+
}
1146+
}
1147+
},
1148+
"name": "istio-access-policy-allow-0",
1149+
"provider": "",
1150+
"type": "kubernetes:security.istio.io/v1beta1:AuthorizationPolicy"
1151+
},
1152+
{
1153+
"custom": true,
1154+
"id": "",
1155+
"inputs": {
1156+
"apiVersion": "security.istio.io/v1beta1",
1157+
"kind": "AuthorizationPolicy",
1158+
"metadata": {
1159+
"name": "istio-access-policy-allow-public-0",
1160+
"namespace": "cluster-ingress"
1161+
},
1162+
"spec": {
1163+
"action": "ALLOW",
1164+
"rules": [
1165+
{
1166+
"from": [
1167+
{
1168+
"source": {
1169+
"remoteIpBlocks": [
1170+
"0.0.0.0/0"
1171+
]
1172+
}
1173+
}
1174+
]
1175+
}
1176+
],
1177+
"selector": {
1178+
"matchLabels": {
1179+
"app": "istio-ingress-public"
1180+
}
1181+
}
1182+
}
1183+
},
1184+
"name": "istio-access-policy-allow-public-0",
1185+
"provider": "",
1186+
"type": "kubernetes:security.istio.io/v1beta1:AuthorizationPolicy"
1187+
},
1188+
{
1189+
"custom": true,
1190+
"id": "",
1191+
"inputs": {
1192+
"apiVersion": "security.istio.io/v1beta1",
1193+
"kind": "AuthorizationPolicy",
1194+
"metadata": {
1195+
"name": "istio-access-policy-deny-all-public",
1196+
"namespace": "cluster-ingress"
1197+
},
1198+
"spec": {
1199+
"selector": {
1200+
"matchLabels": {
1201+
"app": "istio-ingress-public"
1202+
}
1203+
}
1204+
}
1205+
},
1206+
"name": "istio-access-policy-deny-all-public",
1207+
"provider": "",
1208+
"type": "kubernetes:security.istio.io/v1beta1:AuthorizationPolicy"
1209+
},
1210+
{
1211+
"custom": true,
1212+
"id": "",
1213+
"inputs": {
1214+
"apiVersion": "security.istio.io/v1beta1",
1215+
"kind": "AuthorizationPolicy",
1216+
"metadata": {
1217+
"name": "istio-access-policy-deny-all",
1218+
"namespace": "cluster-ingress"
1219+
},
1220+
"spec": {
1221+
"selector": {
1222+
"matchLabels": {
1223+
"app": "istio-ingress"
1224+
}
1225+
}
1226+
}
1227+
},
1228+
"name": "istio-access-policy-deny-all",
1229+
"provider": "",
1230+
"type": "kubernetes:security.istio.io/v1beta1:AuthorizationPolicy"
1231+
},
11061232
{
11071233
"custom": true,
11081234
"id": "",
@@ -1321,16 +1447,7 @@
13211447
"service": {
13221448
"externalTrafficPolicy": "Local",
13231449
"loadBalancerSourceRanges": [
1324-
"<internal IPs>",
1325-
"1.2.3.4/32",
1326-
"5.6.7.8/32",
1327-
"11.12.13.14/32",
1328-
"4.3.2.1/32",
1329-
"8.7.6.5/32",
1330-
"8.7.6.4/32",
1331-
"9.8.7.6/32",
1332-
"11.22.33.45/32",
1333-
"12.34.56.78/32"
1450+
"0.0.0.0/0"
13341451
],
13351452
"ports": [
13361453
{

‎cluster/pulumi/common/src/dump-config-common.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export enum PulumiFunction {
1414
GCP_GET_PROJECT = 'gcp:organizations/getProject:getProject',
1515
GCP_GET_SUB_NETWORK = 'gcp:compute/getSubnetwork:getSubnetwork',
1616
GCP_GET_SECRET_VERSION = 'gcp:secretmanager/getSecretVersion:getSecretVersion',
17+
GCP_GET_CLUSTER = 'gcp:container/getCluster:getCluster',
1718
}
1819

1920
export class SecretsFixtureMap extends Map<string, Auth0ClientSecret> {
@@ -176,6 +177,10 @@ export async function initDumpConfig(): Promise<void> {
176177
);
177178
break;
178179
}
180+
case PulumiFunction.GCP_GET_CLUSTER:
181+
return {
182+
nodePools: [{ networkConfigs: [{ podIpv4CidrBlock: '10.160.0.0/16' }] }],
183+
};
179184
case PulumiFunction.GCP_GET_SECRET_VERSION:
180185
if (args.inputs.secret.startsWith('sv') && args.inputs.secret.endsWith('-id')) {
181186
return {

‎cluster/pulumi/infra/src/istio.ts‎

Lines changed: 73 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
// Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
22
// SPDX-License-Identifier: Apache-2.0
3+
import * as gcp from '@pulumi/gcp';
34
import * as k8s from '@pulumi/kubernetes';
45
import * as pulumi from '@pulumi/pulumi';
56
import { local } from '@pulumi/command';
@@ -8,8 +9,10 @@ import { PodMonitor, ServiceMonitor } from 'splice-pulumi-common/src/metrics';
89

910
import {
1011
activeVersion,
12+
CLUSTER_NAME,
1113
DecentralizedSynchronizerUpgradeConfig,
1214
ExactNamespace,
15+
GCP_PROJECT,
1316
getDnsNames,
1417
HELM_MAX_HISTORY_SIZE,
1518
infraAffinityAndTolerations,
@@ -154,6 +157,16 @@ function configureInternalGatewayService(
154157
ingressIp: pulumi.Output<string>,
155158
istiod: k8s.helm.v3.Release
156159
) {
160+
const cluster = gcp.container.getCluster({
161+
name: CLUSTER_NAME,
162+
project: GCP_PROJECT,
163+
});
164+
// The loopback traffic would be prevented by our policy. To still allow it, we
165+
// add the node pool ip ranges to the list.
166+
// eslint-disable-next-line promise/prefer-await-to-then
167+
const internalIPRanges = cluster.then(c =>
168+
c.nodePools.map(p => p.networkConfigs.map(c => c.podIpv4CidrBlock)).flat()
169+
);
157170
const externalIPRanges = loadIPRanges();
158171
// see notes when installing a CometBft node in the full deployment
159172
const cometBftIngressPorts = DecentralizedSynchronizerUpgradeConfig.runningMigrations()
@@ -166,7 +179,7 @@ function configureInternalGatewayService(
166179
return configureGatewayService(
167180
ingressNs,
168181
ingressIp,
169-
externalIPRanges,
182+
pulumi.all([externalIPRanges, internalIPRanges]).apply(([a, b]) => a.concat(b)),
170183
[
171184
ingressPort('grpc-cd-pub-api', 5008),
172185
ingressPort('grpc-cs-p2p-api', 5010),
@@ -286,6 +299,57 @@ function configurePublicGatewayService(
286299
);
287300
}
288301

302+
const istioApiVersion = 'security.istio.io/v1beta1';
303+
304+
function istioAccessPolicies(
305+
ingressNs: k8s.core.v1.Namespace,
306+
externalIPRanges: pulumi.Output<string[]>,
307+
suffix: string
308+
) {
309+
const selector = {
310+
matchLabels: {
311+
app: `istio-ingress${suffix}`,
312+
},
313+
};
314+
const defaultDenyAll = new k8s.apiextensions.CustomResource(
315+
`istio-access-policy-deny-all${suffix}`,
316+
{
317+
apiVersion: istioApiVersion,
318+
kind: 'AuthorizationPolicy',
319+
metadata: {
320+
name: `istio-access-policy-deny-all${suffix}`,
321+
namespace: ingressNs.metadata.name,
322+
},
323+
// empty spec is deny all
324+
spec: { selector },
325+
}
326+
);
327+
return externalIPRanges.apply(ipRanges => {
328+
// There doesn't seem to be an istio-level limit on number of IP lists but at some point we probably hit some k8s limits on the size of a definition so we split it into 100 IP ranges per policy.
329+
const chunkSize = 100;
330+
const chunks = Array.from({ length: Math.ceil(ipRanges.length / chunkSize) }, (_, i) =>
331+
ipRanges.slice(i * chunkSize, i * chunkSize + chunkSize)
332+
);
333+
const policies = chunks.map(
334+
(chunk, i) =>
335+
new k8s.apiextensions.CustomResource(`istio-access-policy-allow${suffix}-${i}`, {
336+
apiVersion: istioApiVersion,
337+
kind: 'AuthorizationPolicy',
338+
metadata: {
339+
name: `istio-access-policy-allow${suffix}-${i}`,
340+
namespace: ingressNs.metadata.name,
341+
},
342+
spec: {
343+
selector,
344+
action: 'ALLOW',
345+
rules: [{ from: [{ source: { remoteIpBlocks: chunk } }] }],
346+
},
347+
})
348+
);
349+
return [defaultDenyAll].concat(policies);
350+
});
351+
}
352+
289353
// Note that despite the helm chart name being "gateway", this does not actually
290354
// deploy an istio "gateway" resource, but rather the istio-ingress LoadBalancer
291355
// service and the istio-ingress pod.
@@ -297,6 +361,7 @@ function configureGatewayService(
297361
istiod: k8s.helm.v3.Release,
298362
suffix: string
299363
) {
364+
const istioPolicies = istioAccessPolicies(ingressNs, externalIPRanges, suffix);
300365
const gateway = new k8s.helm.v3.Release(
301366
`istio-ingress${suffix}`,
302367
{
@@ -326,7 +391,9 @@ function configureGatewayService(
326391
},
327392
service: {
328393
loadBalancerIP: ingressIp,
329-
loadBalancerSourceRanges: externalIPRanges,
394+
// We limit IPs using istio instead of through loadBalancerSourceRanges as the latter has a size limit.
395+
// See https://github.com/DACH-NY/canton-network-internal/issues/626
396+
loadBalancerSourceRanges: ['0.0.0.0/0'],
330397
// See https://istio.io/latest/docs/tasks/security/authorization/authz-ingress/#network
331398
// If you are using a TCP/UDP network load balancer that preserves the client IP address ..
332399
// then you can use the externalTrafficPolicy: Local setting to also preserve the client IP inside Kubernetes by bypassing kube-proxy
@@ -343,7 +410,10 @@ function configureGatewayService(
343410
maxHistory: HELM_MAX_HISTORY_SIZE,
344411
},
345412
{
346-
dependsOn: [ingressNs, istiod],
413+
dependsOn: istioPolicies.apply(policies => {
414+
const base: pulumi.Resource[] = [ingressNs, istiod];
415+
return base.concat(policies);
416+
}),
347417
}
348418
);
349419
// Turn on envoy access logging on the ingress gateway

0 commit comments

Comments
 (0)