Skip to content

Commit 25c3181

Browse files
mkmksmkmks
committed
add TUN interface to enclave + NAT on parent
1 parent 5fa1462 commit 25c3181

File tree

15 files changed

+119
-291
lines changed

15 files changed

+119
-291
lines changed

‎charts/kms-core/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
name: kms-core
22
description: A helm chart to distribute and deploy the Zama KMS core service.
3-
version: 1.6.0-0
3+
version: 1.6.1-0
44
appVersion: 0.13.20 # Minimum kms version to run this chart
55
apiVersion: v2
66
keywords:

‎charts/kms-core/templates/kms-core-configmap.yaml‎

Lines changed: 2 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,7 @@ data:
1919
s3_endpoint = "{{ .Values.minio.endpoint }}"
2020
{{- else }}
2121
role_arn = "${AWS_ROLE_ARN}"
22-
{{- if .Values.kmsCore.nitroEnclave.enabled }}
23-
{{- if .Values.kmsCore.nitroEnclave.ports.imds }}
24-
imds_endpoint = "http://localhost:{{ .Values.kmsCore.nitroEnclave.ports.imds }}"
25-
{{- end }}
26-
{{- if .Values.kmsCore.nitroEnclave.ports.sts }}
27-
sts_endpoint = "https://localhost:{{ .Values.kmsCore.nitroEnclave.ports.sts }}"
28-
{{- end }}
29-
s3_endpoint = "https://localhost:{{ .Values.kmsCore.nitroEnclave.ports.s3 }}"
30-
awskms_endpoint = "https://localhost:{{ .Values.kmsCore.nitroEnclave.ports.awskms }}"
31-
{{- end }}
32-
{{- end }}
22+
{{ end }}
3323
vaults.toml: |
3424
{{- if .Values.kmsCore.publicVault.s3.enabled }}
3525
[public_vault.storage.s3]
@@ -166,11 +156,8 @@ data:
166156
party_id = {{ int .id }}
167157
mpc_identity = {{ .host | quote }}
168158
address = {{ .host | quote }}
169-
{{- if $.Values.kmsCore.nitroEnclave.enabled }}
170-
port = {{ add $.Values.kmsCore.nitroEnclave.ports.peer .id }}
171-
{{- else }}
172159
port = {{ int .port }}
173-
{{- end }}
160+
174161
{{- if $.Values.kmsCore.thresholdMode.tls.enabled }}
175162
{{- if (default dict $.Values.kmsCore.thresholdMode.tls.ca_certificate).path }}
176163
tls_cert.path = {{ $.Values.kmsCore.thresholdMode.tls.ca_certificate.path | quote }}
@@ -185,11 +172,7 @@ data:
185172
party_id = {{ int $i }}
186173
mpc_identity = {{ .host | quote }}
187174
address = {{ (printf "%s-%d" $kmsCoreName $i) | quote }}
188-
{{- if $.Values.kmsCore.nitroEnclave.enabled }}
189-
port = {{ add $.Values.kmsCore.nitroEnclave.ports.peer $i }}
190-
{{- else }}
191175
port = {{ int $.Values.kmsCore.ports.peer }}
192-
{{- end }}
193176
{{- end }}
194177
{{- end }}
195178
@@ -202,11 +185,7 @@ data:
202185
[telemetry]
203186
tracing_service_name = {{ include "kmsCoreName" . | quote }}
204187
{{- if .Values.tracing.enabled }}
205-
{{- if .Values.kmsCore.nitroEnclave.enabled }}
206-
tracing_endpoint = "http://localhost:{{ .Values.kmsCore.nitroEnclave.ports.tracing }}"
207-
{{- else }}
208188
tracing_endpoint = {{ .Values.tracing.endpoint | quote }}
209-
{{- end }}
210189
{{- end }}
211190
212191
tracing_otlp_timeout_ms = 10000

‎charts/kms-core/templates/kms-core-statefulset.yaml‎

Lines changed: 14 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,7 @@ spec:
174174
name: config-files
175175
- mountPath: {{ .Values.kmsCore.workdir }}
176176
name: workdir
177-
{{- if .Values.kmsCore.nitroEnclave.enabled }}
177+
{{- if .Values.kmsCore.nitroEnclave.enabled }}
178178
{{- /* Init Enclave Config Socket, hardcoded to 4000 in https://github.com/zama-ai/kms-core/blob/main/docker/core/service/init_enclave.sh */}}
179179
- {{ include "proxyFromEnclave"
180180
(dict "name" "enclave-config"
@@ -193,63 +193,27 @@ spec:
193193
"vsockPort" 3000
194194
"to" "STDOUT"
195195
) | indent 10 | trim }}
196-
{{- if .Values.kmsCore.nitroEnclave.ports.imds }}
197-
{{- /* Init AWS IMDS Proxy Socket */}}
198-
- {{ include "proxyFromEnclaveTcp"
199-
(dict "name" "aws-imds-proxy"
200-
"image" .Values.kubeUtils.image
201-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.imds
202-
"address" "169.254.169.254"
203-
"port" 80
204-
) | indent 10 | trim }}
205-
{{- end }}
206-
{{- if .Values.kmsCore.nitroEnclave.ports.sts }}
196+
{{- /* Init Enclave Network Tunnel Socket, hardcoded to 2000 in https://github.com/zama-ai/kms-core/blob/main/docker/core/service/init_enclave.sh */}}
197+
- {{ include "proxyFromEnclave"
198+
(dict "name" "enclave-network-tunnel"
199+
"image" .Values.kubeUtils.image
200+
"vsockPort" 2000
201+
"to" "TUN:10.118.0.1/24,iff-up"
202+
) | indent 10 | trim }}
207203
{{- /* Init Enclave Web Identity Token Socket, hardcoded to 4100 in https://github.com/zama-ai/kms-core/blob/main/docker/core/service/init_enclave.sh */}}
208204
- {{ include "proxyFromEnclave"
209205
(dict "name" "enclave-web-identity-token"
210206
"image" .Values.kubeUtils.image
211207
"vsockPort" 4100
212-
"to" "OPEN:/var/run/secrets/eks.amazonaws.com/serviceaccount/token,rdonly"
213-
) | indent 10 | trim }}
214-
{{- /* Init AWS STS Proxy Socket */}}
215-
- {{ include "proxyFromEnclaveTcp"
216-
(dict "name" "aws-sts-proxy"
217-
"image" .Values.kubeUtils.image
218-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.sts
219-
"address" (printf "sts.%s.amazonaws.com" .Values.kmsCore.aws.region)
220-
"port" 443
221-
) | indent 10 | trim }}
222-
{{- end }}
223-
{{- /* Init AWS S3 Proxy Socket */}}
224-
- {{ include "proxyFromEnclaveTcp"
225-
(dict "name" "aws-s3-proxy"
226-
"image" .Values.kubeUtils.image
227-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.s3
228-
"address" (printf "s3.%s.amazonaws.com" .Values.kmsCore.aws.region)
229-
"port" 443
230-
) | indent 10 | trim }}
231-
{{- /* Init AWS KMS Proxy Socket */}}
232-
- {{ include "proxyFromEnclaveTcp"
233-
(dict "name" "aws-kms-proxy"
234-
"image" .Values.kubeUtils.image
235-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.awskms
236-
"address" (printf "kms.%s.amazonaws.com" .Values.kmsCore.aws.region)
237-
"port" 443
238-
) | indent 10 | trim }}
208+
"to" "OPEN:/var/run/secrets/eks.amazonaws.com/serviceaccount/token,rdonly"
209+
) | indent 10 | trim }}
239210
{{- /* Init Metrics Endpoint Proxy Socket */}}
240211
- {{ include "proxyToEnclaveTcp"
241212
(dict "name" "metrics-endpoint-proxy"
242213
"image" .Values.kubeUtils.image
243214
"cid" .Values.kmsCore.nitroEnclave.cid
244215
"port" .Values.kmsCore.ports.metrics
245216
) | indent 10 | trim }}
246-
{{- /* Init Enclave Tracing Socket */}}
247-
- {{ include "proxyFromEnclave"
248-
(dict "name" "kms-core-enclave-tracing"
249-
"image" .Values.kubeUtils.image
250-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.tracing
251-
"to" (printf "TCP:%s" (urlParse .Values.tracing.endpoint).host)
252-
) | indent 10 | trim }}
253217
{{- /* Init GRPC Client Proxy Socket */}}
254218
- {{ include "proxyToEnclaveTcp"
255219
(dict "name" "grpc-client-proxy"
@@ -270,36 +234,6 @@ spec:
270234
resources:
271235
{{- toYaml . | nindent 12 }}
272236
{{- end }}
273-
{{- /* Init GRPC External Peers Proxy Sockets */}}
274-
{{- if .Values.kmsCore.thresholdMode.peersList }}
275-
{{- range .Values.kmsCore.thresholdMode.peersList }}
276-
- {{ include "proxyFromEnclaveTcp"
277-
(dict "name" (printf "enclave-peer-proxy-%d" (int .id))
278-
"image" $.Values.kubeUtils.image
279-
"vsockPort" (add $.Values.kmsCore.nitroEnclave.ports.peer .id)
280-
"address" .host
281-
"port" .port
282-
) | indent 10 | trim }}
283-
{{- with $.Values.kmsCore.nitroEnclave.enclavePeerProxy.resources }}
284-
resources:
285-
{{- toYaml . | nindent 12 }}
286-
{{- end }}
287-
{{- end }}
288-
{{- else }}
289-
{{- range $i := $peersIDList }}
290-
- {{ include "proxyFromEnclaveTcp"
291-
(dict "name" (printf "enclave-peer-proxy-%d" (int $i))
292-
"image" $.Values.kubeUtils.image
293-
"vsockPort" (add $.Values.kmsCore.nitroEnclave.ports.peer $i)
294-
"address" (printf "%s-%d" $kmsCoreName (int $i))
295-
"port" $.Values.kmsCore.ports.peer
296-
) | indent 10 | trim }}
297-
{{- with $.Values.kmsCore.nitroEnclave.enclavePeerProxy.resources }}
298-
resources:
299-
{{- toYaml . | nindent 12 }}
300-
{{- end }}
301-
{{- end }}
302-
{{- end }}
303237
{{- end }}
304238
{{- else }}
305239
{{- /* Generate init keys for non-enclave kms-core service */}}
@@ -489,6 +423,10 @@ spec:
489423
args:
490424
- -c
491425
- |
426+
sudo sysctl -w net.ipv4.ip_forward=1
427+
sudo iptables -t nat -A POSTROUTING -s 10.118.0.2 -j MASQUERADE
428+
429+
cd
492430
echo "### BEGIN - enclave.json ###"
493431
cat /var/lib/kms-core/config/enclave.json
494432
echo "### END - enclave.json ###"

‎charts/kms-core/templates/kms-gen-cert-and-keys-job.yaml‎

Lines changed: 8 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -107,45 +107,20 @@ spec:
107107
"vsockPort" 3000
108108
"to" "STDOUT"
109109
) | indent 10 | trim }}
110-
{{- /* Init AWS IMDS Proxy Socket */}}
111-
- {{ include "proxyFromEnclaveTcp"
112-
(dict "name" "aws-imds-proxy"
113-
"image" .Values.kubeUtils.image
114-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.imds
115-
"address" "169.254.169.254"
116-
"port" 80
117-
) | indent 10 | trim }}
110+
{{- /* Init Enclave Network Tunnel Socket, hardcoded to 2000 in https://github.com/zama-ai/kms-core/blob/main/docker/core/service/init_enclave.sh */}}
111+
- {{ include "proxyFromEnclave"
112+
(dict "name" "enclave-network-tunnel"
113+
"image" .Values.kubeUtils.image
114+
"vsockPort" 2000
115+
"to" "TUN:10.118.0.1/24,iff-up"
116+
) | indent 10 | trim }}
118117
{{- /* Init Enclave Web Identity Token Socket, hardcoded to 4100 in https://github.com/zama-ai/kms-core/blob/main/docker/core/service/init_enclave.sh */}}
119118
- {{ include "proxyFromEnclave"
120119
(dict "name" "enclave-web-identity-token"
121120
"image" .Values.kubeUtils.image
122121
"vsockPort" 4100
123122
"to" "OPEN:/var/run/secrets/eks.amazonaws.com/serviceaccount/token,rdonly"
124123
) | indent 10 | trim }}
125-
{{- /* Init AWS STS Proxy Socket */}}
126-
- {{ include "proxyFromEnclaveTcp"
127-
(dict "name" "aws-sts-proxy"
128-
"image" .Values.kubeUtils.image
129-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.sts
130-
"address" (printf "sts.%s.amazonaws.com" .Values.kmsCore.aws.region)
131-
"port" 443
132-
) | indent 10 | trim }}
133-
{{- /* Init AWS S3 Proxy Socket */}}
134-
- {{ include "proxyFromEnclaveTcp"
135-
(dict "name" "aws-s3-proxy"
136-
"image" .Values.kubeUtils.image
137-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.s3
138-
"address" (printf "s3.%s.amazonaws.com" .Values.kmsCore.aws.region)
139-
"port" 443
140-
) | indent 10 | trim }}
141-
{{- /* Init AWS KMS Proxy Socket */}}
142-
- {{ include "proxyFromEnclaveTcp"
143-
(dict "name" "aws-kms-proxy"
144-
"image" .Values.kubeUtils.image
145-
"vsockPort" .Values.kmsCore.nitroEnclave.ports.awskms
146-
"address" (printf "kms.%s.amazonaws.com" .Values.kmsCore.aws.region)
147-
"port" 443
148-
) | indent 10 | trim }}
149124
{{- end }}
150125
{{- /* initContainers section for non-enclave */}}
151126
{{- if not .Values.kmsCore.nitroEnclave.enabled }}
@@ -411,4 +386,4 @@ spec:
411386
- name: certs
412387
emptyDir: {}
413388
{{- end }}
414-
{{- end -}}
389+
{{- end -}}

‎charts/kms-core/values.yaml‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -144,16 +144,6 @@ kmsCore:
144144
eifSignKey: 00000000-0000-0000-0000-000000000000
145145
userId: 10003
146146
groupId: 10002
147-
ports:
148-
tracing: 4317
149-
# For AWS authentication, set the `imds` or `sts` ports
150-
# To authenticate using either IMDS (with the Kubernetes node EC2 instance role) or STS (with the IRSA)
151-
imds: 5000
152-
sts: 5500
153-
s3: 6000
154-
awskms: 7000
155-
peer: 10000
156-
# Important, don't bind to port 9000 as it is reserved by Nitro for communicating with the enclave.
157147
publicVault:
158148
s3:
159149
enabled: true

‎core/service/src/bin/kms-server.rs‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -662,17 +662,6 @@ async fn main_exec() -> anyhow::Result<()> {
662662
}
663663
};
664664

665-
#[cfg(not(feature = "insecure"))]
666-
let need_peer_tcp_proxy = need_security_module;
667-
#[cfg(feature = "insecure")]
668-
let need_peer_tcp_proxy =
669-
need_security_module && !core_config.mock_enclave.is_some_and(|m| m);
670-
671-
if need_peer_tcp_proxy {
672-
tracing::warn!("KMS server will connect to peers through vsock proxies");
673-
} else {
674-
tracing::warn!("KMS server will connect to peers directly");
675-
};
676665
let service_config = core_config.service.clone();
677666
let (kms, (health_reporter, health_service), metastore_status_service) =
678667
new_real_threshold_kms(
@@ -684,7 +673,6 @@ async fn main_exec() -> anyhow::Result<()> {
684673
mpc_listener,
685674
base_kms,
686675
tls_identity,
687-
need_peer_tcp_proxy,
688676
false,
689677
std::future::pending(),
690678
)

‎core/service/src/client/test_tools.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,6 @@ pub async fn setup_threshold_no_client<
158158
mpc_listener,
159159
base_kms,
160160
None,
161-
false,
162161
ensure_default_prss,
163162
mpc_core_rx.map(drop),
164163
)
@@ -391,7 +390,6 @@ pub async fn setup_threshold_with_custom_peers<
391390
mpc_listener,
392391
base_kms,
393392
None,
394-
false,
395393
ensure_default_prss,
396394
mpc_core_rx.map(drop),
397395
)

‎core/service/src/engine/threshold/service/kms_impl.rs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,6 @@ pub async fn new_real_threshold_kms<PubS, PrivS, F>(
345345
mpc_listener: TcpListener,
346346
base_kms: BaseKmsStruct,
347347
tls_config: Option<(ServerConfig, ClientConfig, Arc<AttestedVerifier>)>,
348-
peer_tcp_proxy: bool,
349348
ensure_default_prss: bool,
350349
shutdown_signal: F,
351350
) -> anyhow::Result<(
@@ -415,7 +414,6 @@ where
415414
.as_ref()
416415
.map(|(_, client_config, _)| client_config.clone()),
417416
threshold_config.core_to_core_net,
418-
peer_tcp_proxy,
419417
)?));
420418

421419
// the initial MPC node might not accept any peers because initially there's no context

‎core/service/src/engine/threshold/service/session.rs‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -148,9 +148,8 @@ impl SessionMaker {
148148

149149
#[cfg(test)]
150150
pub(crate) fn empty_dummy_session(rng: AesRng) -> Self {
151-
let networking_manager = Arc::new(RwLock::new(
152-
GrpcNetworkingManager::new(None, None, false).unwrap(),
153-
));
151+
let networking_manager =
152+
Arc::new(RwLock::new(GrpcNetworkingManager::new(None, None).unwrap()));
154153
Self {
155154
networking_manager,
156155
context_map: Arc::new(RwLock::new(HashMap::new())),
@@ -175,9 +174,8 @@ impl SessionMaker {
175174
)
176175
})),
177176
};
178-
let networking_manager = Arc::new(RwLock::new(
179-
GrpcNetworkingManager::new(None, None, false).unwrap(),
180-
));
177+
let networking_manager =
178+
Arc::new(RwLock::new(GrpcNetworkingManager::new(None, None).unwrap()));
181179

182180
let default_context_id = *crate::consts::DEFAULT_MPC_CONTEXT;
183181
let default_context = Context {

‎core/threshold-networking/src/grpc.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,6 @@ impl GrpcNetworkingManager {
325325
pub fn new(
326326
tls_conf: Option<tokio_rustls::rustls::client::ClientConfig>,
327327
conf: Option<CoreToCoreNetworkConfig>,
328-
peer_tcp_proxy: bool,
329328
) -> anyhow::Result<Self> {
330329
#[cfg(feature = "testing")]
331330
let force_tls = tls_conf.is_some();
@@ -366,7 +365,7 @@ impl GrpcNetworkingManager {
366365
active_session_count,
367366
opened_sessions_tracker: Arc::new(DashMap::new()),
368367
conf,
369-
sending_service: GrpcSendingService::new(tls_conf, conf, peer_tcp_proxy)?,
368+
sending_service: GrpcSendingService::new(tls_conf, conf)?,
370369
#[cfg(feature = "testing")]
371370
force_tls,
372371
})

0 commit comments

Comments
 (0)