Skip to content

Commit 1309792

Browse files
maugustosilvamaugustosilva
authored
[Standup] Compatibility with the new istio (1.28.1) (llm-d#567)
* [Standup] Compatibility with the new `istio` (1.28.1) Signed-off-by: maugustosilva <maugusto.silva@gmail.com> * Make `istio` the default for `LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME` Signed-off-by: maugustosilva <maugusto.silva@gmail.com> --------- Signed-off-by: maugustosilva <maugusto.silva@gmail.com>
1 parent dac6929 commit 1309792

7 files changed

Lines changed: 29 additions & 25 deletions

File tree

‎.github/actions/markdown-link-checker/action.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ inputs:
77
args:
88
description: Arguments to pass to markdown-link-check
99
required: false
10-
default: "--quiet --retry"
10+
default: "--retry"
1111

1212
runs:
1313
using: "composite"

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ This repository provides an automated workflow for benchmarking LLM inference us
77
Provide a single source of automation for repeatable and reproducible experiments and performance evaluation on `llm-d`.
88

99

10-
## Prerequisites
11-
Please refer to the official [llm-d prerequisites](https://github.com/llm-d/llm-d?tab=readme-ov-file#pre-requisites) for the most up-to-date requirements.
10+
## Prerequisites
11+
Please refer to the official [llm-d prerequisites](https://github.com/llm-d/llm-d/blob/main/README.md#pre-requisites) for the most up-to-date requirements.
1212
For the client setup, the provided `install-deps.sh` will download and install the necessary tools.
1313

1414
### Administrative Requirements

‎setup/env.sh‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,8 @@ export LLMDBENCH_DEPLOY_METHODS=${LLMDBENCH_DEPLOY_METHODS:-"modelservice"}
9191
# Gateway provider specific variables
9292
export LLMDBENCH_GATEWAY_PROVIDER_KGATEWAY_HELM_REPOSITORY_URL=${LLMDBENCH_GATEWAY_PROVIDER_KGATEWAY_HELM_REPOSITORY_URL:-"oci://cr.kgateway.dev/kgateway-dev/charts"}
9393
export LLMDBENCH_GATEWAY_PROVIDER_KGATEWAY_CHART_VERSION=${LLMDBENCH_GATEWAY_PROVIDER_KGATEWAY_CHART_VERSION:-"v2.0.3"}
94-
export LLMDBENCH_GATEWAY_PROVIDER_ISTIO_HELM_REPOSITORY_URL=${LLMDBENCH_GATEWAY_PROVIDER_ISTIO_HELM_REPOSITORY_URL:-"oci://gcr.io/istio-testing/charts"}
95-
export LLMDBENCH_GATEWAY_PROVIDER_ISTIO_CHART_VERSION=${LLMDBENCH_GATEWAY_PROVIDER_ISTIO_CHART_VERSION:-"1.28-alpha.89f30b26ba71bf5e538083a4720d0bc2d8c06401"}
94+
export LLMDBENCH_GATEWAY_PROVIDER_ISTIO_HELM_REPOSITORY_URL=${LLMDBENCH_GATEWAY_PROVIDER_ISTIO_HELM_REPOSITORY_URL:-"https://istio-release.storage.googleapis.com/charts"}
95+
export LLMDBENCH_GATEWAY_PROVIDER_ISTIO_CHART_VERSION=${LLMDBENCH_GATEWAY_PROVIDER_ISTIO_CHART_VERSION:-"1.28.1"}
9696

9797
# Applicable to both standalone and modelservice
9898
export LLMDBENCH_IGNORE_FAILED_VALIDATION="${LLMDBENCH_IGNORE_FAILED_VALIDATION:-true}" # default is to continue deployment if validation fails
@@ -187,7 +187,7 @@ export LLMDBENCH_VLLM_MODELSERVICE_CHART_NAME=${LLMDBENCH_VLLM_MODELSERVICE_CHAR
187187
export LLMDBENCH_VLLM_MODELSERVICE_HELM_REPOSITORY=${LLMDBENCH_VLLM_MODELSERVICE_HELM_REPOSITORY:-"llm-d-modelservice"}
188188
export LLMDBENCH_VLLM_MODELSERVICE_HELM_REPOSITORY_URL=${LLMDBENCH_VLLM_MODELSERVICE_HELM_REPOSITORY_URL:-"https://llm-d-incubation.github.io/llm-d-modelservice/"}
189189
export LLMDBENCH_VLLM_MODELSERVICE_URI_PROTOCOL=${LLMDBENCH_VLLM_MODELSERVICE_URI_PROTOCOL:-"pvc"}
190-
export LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME=${LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME:-kgateway}
190+
export LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME=${LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_CLASS_NAME:-"istio"}
191191
export LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_SERVICE_TYPE=${LLMDBENCH_VLLM_MODELSERVICE_GATEWAY_SERVICE_TYPE:-NodePort}
192192
export LLMDBENCH_VLLM_MODELSERVICE_ROUTE=${LLMDBENCH_VLLM_MODELSERVICE_ROUTE:-false}
193193
# Endpoint Picker Parameters

‎setup/steps/02_ensure_gateway_provider.py‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ def get_latest_chart_version(
113113

114114
if result.returncode != 0:
115115
if verbose:
116-
announce(f"❌ Helm search failed: {result.stderr}")
116+
announce(f"ERROR: Helm search failed: {result.stderr}")
117117
return ""
118118

119119
# Parse output to get version (equivalent to: tail -1 | awk '{print $2}')
@@ -269,7 +269,7 @@ def install_kgateway(
269269
install_cmd = f"helmfile apply -f {helmfile_path}"
270270
ecode = llmdbench_execute_cmd(actual_cmd=install_cmd, dry_run=ev["control_dry_run"], verbose=ev["control_verbose"])
271271
if ecode != 0:
272-
announce(f"ERROR: Failed while running \"{install_cmd}\" (exit code: {result})")
272+
announce(f"ERROR: Failed while running \"{install_cmd}\" (exit code: {ecode})")
273273
announce(f"✅ kgateway ({ev['gateway_provider_kgateway_chart_version']}) installed")
274274
else :
275275
announce(f"✅ kgateway (unknown version) already installed (*.kgateway.dev CRDs found)")
@@ -299,9 +299,12 @@ def install_istio(
299299
helmfile_path = helm_base_dir / f'helmfile-{ev["current_step"]}.yaml'
300300
with open(helmfile_path, 'w') as f:
301301
f.write(f"""
302+
repositories:
303+
- name: istio
304+
url: {ev["gateway_provider_istio_helm_repository_url"]}
302305
releases:
303306
- name: istio-base
304-
chart: {ev["gateway_provider_istio_helm_repository_url"]}/base
307+
chart: istio/base
305308
version: {ev["gateway_provider_istio_chart_version"]}
306309
namespace: istio-system
307310
installed: true
@@ -310,7 +313,7 @@ def install_istio(
310313
kind: gateway-crds
311314
312315
- name: istiod
313-
chart: {ev["gateway_provider_istio_helm_repository_url"]}/istiod
316+
chart: istio/istiod
314317
version: {ev["gateway_provider_istio_chart_version"]}
315318
namespace: istio-system
316319
installed: true
@@ -320,12 +323,12 @@ def install_istio(
320323
- meshConfig:
321324
defaultConfig:
322325
proxyMetadata:
323-
SUPPORT_GATEWAY_API_INFERENCE_EXTENSION: true
326+
ENABLE_GATEWAY_API_INFERENCE_EXTENSION: true
324327
pilot:
325328
env:
326-
SUPPORT_GATEWAY_API_INFERENCE_EXTENSION: true
329+
ENABLE_GATEWAY_API_INFERENCE_EXTENSION: true
327330
tag: {ev["gateway_provider_istio_chart_version"]}
328-
hub: "gcr.io/istio-testing"
331+
hub: "docker.io/istio"
329332
labels:
330333
type: gateway-provider
331334
kind: gateway-control-plane
@@ -342,7 +345,7 @@ def install_istio(
342345
announce(f"🚀 Installing istio helm charts from {ev['gateway_provider_istio_helm_repository_url']} ({ev['gateway_provider_istio_chart_version']})")
343346
ecode = llmdbench_execute_cmd(actual_cmd=install_cmd, dry_run=ev["control_dry_run"], verbose=ev["control_verbose"])
344347
if ecode != 0:
345-
announce(f"ERROR: Failed while running \"{install_cmd}\" (exit code: {result})")
348+
announce(f"ERROR: Failed while running \"{install_cmd}\" (exit code: {ecode})")
346349
announce(f"✅ istio ({ev['gateway_provider_istio_chart_version']}) installed")
347350
else :
348351
announce(f"✅ isto (unknown version) already installed (*.istio.io CRDs found)")
@@ -392,7 +395,8 @@ def install_gateway_control_plane(
392395
"telemetries.telemetry.istio.io", \
393396
"virtualservices.networking.istio.io", \
394397
"wasmplugins.extensions.istio.io", \
395-
"workloadgroups.networking.istio.io" ] :
398+
"workloadgroups.networking.istio.io", \
399+
"telemetry.istio.io/v1" ] :
396400
if i not in crds :
397401
should_install_gateway_control_plane = True
398402

@@ -470,10 +474,10 @@ def ensure_gateway_provider(
470474
return result
471475

472476
should_install_gateway_api_extension_crds = False
473-
for i in [ "inferenceobjectives.inference.networking.x-k8s.io", \
474-
"inferencepoolimports.inference.networking.x-k8s.io", \
477+
for i in [ "inferenceobjectives.inference.networking.k8s.io", \
478+
"inferencepoolimports.inference.networking.k8s.io", \
475479
"inferencepools.inference.networking.k8s.io", \
476-
"inferencepools.inference.networking.x-k8s.io" ] :
480+
"inferencepools.inference.networking.k8s.io" ] :
477481
if i not in crd_names :
478482
should_install_gateway_api_extension_crds = True
479483

‎setup/steps/07_deploy_setup.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,17 +74,17 @@ def auto_detect_version(ev, chart, version_key, repo_key) -> int:
7474
announce(f"📦 Auto-detected chart version: {version}")
7575
return 0
7676
else:
77-
announce("❌ Unable to parse version from helm search output")
77+
announce("ERROR: Unable to parse version from helm search output")
7878
return 1
7979
else:
80-
announce("❌ No charts found in helm search output")
80+
announce("ERROR: No charts found in helm search output")
8181
return 1
8282
else:
83-
announce("❌ Unable to find a version for model service helm chart!")
83+
announce("ERROR: Unable to find a version for model service helm chart!")
8484
return 1
8585

8686
except Exception as e:
87-
announce(f"❌ Error auto-detecting {chart} chart version: {e}")
87+
announce(f"ERROR: Error auto-detecting {chart} chart version: {e}")
8888
return 1
8989
return 0
9090

‎setup/steps/08_deploy_gaie.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ def main():
150150
inferencePool:
151151
targetPortNumber: {ev['vllm_common_inference_port']}
152152
modelServerType: vllm
153-
apiVersion: "inference.networking.x-k8s.io/v1alpha2"
153+
apiVersion: "inference.networking.k8s.io/v1"
154154
modelServers:
155155
matchLabels:
156156
llm-d.ai/inferenceServing: "true"

‎setup/steps/09_deploy_via_modelservice.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ def define_httproute(
393393
name: infra-{release}-inference-gateway
394394
rules:
395395
- backendRefs:
396-
- group: inference.networking.x-k8s.io
396+
- group: inference.networking.k8s.io
397397
kind: InferencePool
398398
name: {model_id_label}-gaie
399399
port: {service_port}
@@ -416,7 +416,7 @@ def define_httproute(
416416
if single_model:
417417
manifest = f"""{manifest}
418418
- backendRefs:
419-
- group: inference.networking.x-k8s.io
419+
- group: inference.networking.k8s.io
420420
kind: InferencePool
421421
name: {model_id_label}-gaie
422422
port: {service_port}

0 commit comments

Comments
 (0)