Skip to content

Commit fdc3be3

Browse files
mikeyrcampmikeyrcamp
authored
test(bdd): require TLS rejection diagnostics (#1324)
Signed-off-by: Mike Camp <mcamp@nvidia.com>
1 parent 81d82d0 commit fdc3be3

2 files changed

Lines changed: 36 additions & 15 deletions

File tree

‎tests/bdd/features/multi-cluster-helmfile-llm-registration-tls-fail-closed.feature‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ Feature: Reject insecure or invalid LLM worker registration
3030
| observability.profile | disabled |
3131
And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-fail-closed-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
3232
And I prepare self-managed secrets file "deploy/stacks/self-managed/secrets/local-bdd-registration-tls-invalid-authority-secrets.yaml" from template "deploy/stacks/self-managed/secrets/secrets.yaml.template" using the current NGC registry credential
33+
When I run command "/bin/sh -c 'command -v grpcurl >/dev/null'"
34+
Then the command exit code should be 0
3335
When I run command "k3d cluster get ncp-local"
3436
Then the command exit code should be 1
3537
And multi-cluster ncp-local compute clusters are running:
@@ -65,38 +67,38 @@ Feature: Reject insecure or invalid LLM worker registration
6567
# expected DNS identity, and HTTP/2 application protocol.
6668
When I run command:
6769
"""
68-
/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
70+
/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 -servername llm-request-router.nvcf.svc.cluster.local -verify_hostname llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error -CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'
6971
"""
7072
Then the command exit code should be 0
7173
And the command output should contain "Verify return code: 0 (ok)"
7274
And the command output should contain "ALPN protocol: h2"
7375

7476
When I run command:
7577
"""
76-
/bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1; grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "wrong-root-rejected\n"'
78+
/bin/bash -c 'set -u; cert_dir=$(mktemp -d); trap '\''rm -rf "$cert_dir"'\'' EXIT; openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=wrong-root -keyout "$cert_dir/key.pem" -out "$cert_dir/ca.pem" -days 1 >/dev/null 2>&1 || exit; if diagnostic=$(grpcurl -max-time 5 -cacert "$cert_dir/ca.pem" -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong root was trusted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*) printf "wrong-root-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
7779
"""
7880
Then the command exit code should be 0
7981

8082
When I run command:
8183
"""
82-
/bin/bash -c 'grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "wrong-host-rejected\n"'
84+
/bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -cacert <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf -o jsonpath="{.data.ca\.crt}" | base64 -d) -authority wrong-host.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "wrong hostname was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"not wrong-host.nvcf.svc.cluster.local"*) printf "wrong-host-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
8385
"""
8486
Then the command exit code should be 0
8587

8688
When I run command:
8789
"""
88-
/bin/bash -c 'grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "missing-trust-rejected\n"'
90+
/bin/bash -c 'if diagnostic=$(grpcurl -max-time 5 -authority llm-request-router.nvcf.svc.cluster.local -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "missing trust was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"certificate signed by unknown authority"*) printf "missing-trust-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
8991
"""
9092
Then the command exit code should be 0
9193

9294
When I run command:
9395
"""
94-
/bin/bash -c 'grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "plaintext-rejected\n"'
96+
/bin/bash -c 'if diagnostic=$(grpcurl -plaintext -max-time 5 -import-path src/libraries/rust/stargate/crates/proto/proto -proto stargate.proto 127.0.0.1:50071 stargate.StargateControlPlane/WatchStargates 2>&1); then printf "plaintext was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"context deadline exceeded"*|*"error reading server preface"*) printf "plaintext-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
9597
"""
9698
Then the command exit code should be 0
9799

98100
When I run command:
99101
"""
100-
/bin/sh -c 'make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority >/dev/null 2>&1; rc=$?; [ "$rc" -ne 0 ] && printf "invalid-authority-rejected\n"'
102+
/bin/sh -c 'if diagnostic=$(make -C deploy/stacks/self-managed template HELMFILE_ENV=local-bdd-registration-tls-invalid-authority 2>&1); then printf "invalid authority was accepted\n" >&2; exit 1; fi; case "$diagnostic" in *"global.workerEndpoints.llmRequestRouterAddress must use optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port with port 1-65535"*) printf "invalid-authority-rejected\n" ;; *) printf "%s\n" "$diagnostic" >&2; exit 1 ;; esac'
101103
"""
102104
Then the command exit code should be 0

‎tests/bdd/godog_test.go‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1166,11 +1166,14 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
11661166
t.Setenv("SAMPLE_NGC_TEAM", "test-team")
11671167

11681168
const tlsHandshakeCommand = `/bin/bash -c 'openssl s_client -connect 127.0.0.1:50071 ` +
1169-
`-servername llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
1169+
`-servername llm-request-router.nvcf.svc.cluster.local ` +
1170+
`-verify_hostname llm-request-router.nvcf.svc.cluster.local -alpn h2 -verify_return_error ` +
11701171
`-CAfile <(kubectl --context k3d-ncp-local-cp get secret stargate-quic-tls -n nvcf ` +
11711172
`-o jsonpath="{.data.ca\.crt}" | base64 -d) </dev/null 2>&1'`
1173+
const grpcurlPreflightCommand = `/bin/sh -c 'command -v grpcurl >/dev/null'`
11721174
suite := newWiringSuite(t, newFakeRunner(map[string]harness.Result{
11731175
"k3d cluster get ncp-local": {ExitCode: 1},
1176+
grpcurlPreflightCommand: {ExitCode: 0},
11741177
tlsHandshakeCommand: {
11751178
ExitCode: 0,
11761179
Stdout: "ALPN protocol: h2\nVerify return code: 0 (ok)\n",
@@ -1197,15 +1200,31 @@ func TestMultiClusterHelmfileLLMRegistrationTLSFailClosedFeatureFileWiresToSteps
11971200
if status != 0 {
11981201
t.Fatalf("godog suite status = %d\n%s", status, out.String())
11991202
}
1200-
for _, marker := range []string{
1201-
"wrong-root-rejected",
1202-
"wrong-host-rejected",
1203-
"missing-trust-rejected",
1204-
"plaintext-rejected",
1205-
"invalid-authority-rejected",
1203+
runs := suite.Runner.(*fakeRunner).runs
1204+
if !commandRanExactly(runs, grpcurlPreflightCommand) {
1205+
t.Fatal("grpcurl availability was not checked before the live probes")
1206+
}
1207+
for _, assertion := range []struct {
1208+
marker string
1209+
diagnostic string
1210+
}{
1211+
{marker: "wrong-root-rejected", diagnostic: "certificate signed by unknown authority"},
1212+
{marker: "wrong-host-rejected", diagnostic: "not wrong-host.nvcf.svc.cluster.local"},
1213+
{marker: "missing-trust-rejected", diagnostic: "certificate signed by unknown authority"},
1214+
{marker: "plaintext-rejected", diagnostic: "context deadline exceeded"},
1215+
{
1216+
marker: "invalid-authority-rejected",
1217+
diagnostic: "global.workerEndpoints.llmRequestRouterAddress must use " +
1218+
"optional http:// or https:// followed by DNS-or-IPv4:port or [IPv6]:port " +
1219+
"with port 1-65535",
1220+
},
12061221
} {
1207-
if !commandRanThatContains(suite.Runner.(*fakeRunner).runs, marker) {
1208-
t.Fatalf("negative registration command containing %q was not invoked", marker)
1222+
if !commandRanThatContainsAll(runs, assertion.marker, assertion.diagnostic) {
1223+
t.Fatalf(
1224+
"negative registration command containing %q did not require diagnostic %q",
1225+
assertion.marker,
1226+
assertion.diagnostic,
1227+
)
12091228
}
12101229
}
12111230
validEnvironment := filepath.Join(

0 commit comments

Comments
 (0)