[Run] reduce the amount of env vars required to run agains a pre-deployed stack (llm-d#264)

* [Run] reduce the amount of env vars required to run agains a
pre-deployed stack

Also, fix a bug on `standalone` deployment for non-admins.

Fix CI/CD.

Added a comprehensive list of environment variables to the "launcher"
pod

Also allow automated extraction of the HF_TOKEN from a cluster
secret

All dependencies on CI/CD should be installed via install_deps.sh

Signed-off-by: maugustosilva <maugusto.silva@gmail.com>

---------

Signed-off-by: maugustosilva <maugusto.silva@gmail.com>

Author: maugustosilva

.github/workflows/benchmark1.yaml

@@ -65,17 +65,6 @@ jobs:
           ./setup/install_deps.sh
         shell: bash
 
-      - name: Populate python deps
-        run: |
-          echo -e "pandas\ngrip>=4.6.0\nmatplotlib>=3.7.0\nnumpy>=1.22.0\nseaborn>=0.12.0\nkubernetes>=28.0.0\npykube\nkubernetes-asyncio\nGitPython" > requirements.txt
-
-      - name: Install python deps
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.13'
-          cache: 'pip'
-      - run: pip install -r requirements.txt
-
       - name: Cleanup target cloud (modelservice)
         env:
           LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}

.github/workflows/ci-pr-benchmark.yaml

@@ -35,17 +35,6 @@ jobs:
           ./setup/install_deps.sh
         shell: bash
 
-      - name: Populate python deps
-        run: |
-          echo -e "pandas\ngrip>=4.6.0\nmatplotlib>=3.7.0\nnumpy>=1.22.0\nseaborn>=0.12.0\nkubernetes>=28.0.0\npykube\nkubernetes-asyncio\nGitPython" > requirements.txt
-
-      - name: Install python deps
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.13'
-          cache: 'pip'
-      - run: pip install -r requirements.txt
-
       - name: Standup a modelservice using llm-d-inference-sim
         env:
           LLMDBENCH_HF_TOKEN: hf-token-placeholder

setup/env.sh

@@ -155,8 +155,8 @@ export LLMDBENCH_CONTROL_CHECK_CLUSTER_AUTHORIZATIONS=${LLMDBENCH_CONTROL_CHECK_
 export LLMDBENCH_CONTROL_RESOURCE_LIST=${LLMDBENCH_CONTROL_RESOURCE_LIST:-deployment,httproute,service,gateway,gatewayparameters,inferencepool,inferencemodel,cm,ing,pod,job}
 export LLMDBENCH_CONTROL_STEP_00_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_00_IMPLEMENTATION:-py}
 export LLMDBENCH_CONTROL_STEP_01_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_01_IMPLEMENTATION:-py}
-export LLMDBENCH_CONTROL_STEP_02_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_02_IMPLEMENTATION:-sh}
-export LLMDBENCH_CONTROL_STEP_03_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_03_IMPLEMENTATION:-sh}
+export LLMDBENCH_CONTROL_STEP_02_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_02_IMPLEMENTATION:-py}
+export LLMDBENCH_CONTROL_STEP_03_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_03_IMPLEMENTATION:-py}
 export LLMDBENCH_CONTROL_STEP_04_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_04_IMPLEMENTATION:-py}
 export LLMDBENCH_CONTROL_STEP_05_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_05_IMPLEMENTATION:-sh}
 export LLMDBENCH_CONTROL_STEP_06_IMPLEMENTATION=${LLMDBENCH_CONTROL_STEP_06_IMPLEMENTATION:-sh}
@@ -323,16 +323,6 @@ if [[ ! -z $LLMDBENCH_HARNESS_EXPERIMENT_TREATMENTS ]]; then
   fi
 fi
 
-required_vars=("LLMDBENCH_HF_TOKEN")
-for var in "${required_vars[@]}"; do
-  if [ -z "${!var:-}" ]; then
-    echo "❌ Environment variable '$var' is not set."
-    exit 1
-  fi
-done
-
-export HF_TOKEN=${HF_TOKEN:-$LLMDBENCH_HF_TOKEN}
-
 export LLMDBENCH_CONTROL_WORK_DIR=${LLMDBENCH_CONTROL_WORK_DIR:-$(mktemp -d -t ${LLMDBENCH_CONTROL_CLUSTER_NAME}-$(echo $0 | rev | cut -d '/' -f 1 | rev | $LLMDBENCH_CONTROL_SCMD -e 's^.sh^^g' -e 's^./^^g')XXX)}
 export LLMDBENCH_CONTROL_WORK_DIR_SET=${LLMDBENCH_CONTROL_WORK_DIR_SET:-0}
 export LLMDBENCH_CONTROL_WORK_DIR_BACKEDUP=${LLMDBENCH_CONTROL_WORK_DIR_BACKEDUP:-0}
@@ -448,4 +438,21 @@ if [[ $LLMDBENCH_CONTROL_PERMISSIONS_CHECKED -eq 0 && ${LLMDBENCH_CONTROL_CHECK_
   done
 fi
 
-export LLMDBENCH_CONTROL_DEPLOY_HOST_SHELL=${SHELL:5}
+if [[ $LLMDBENCH_CONTROL_CALLER == "run.sh" ]]; then
+  if [[ -z $LLMDBENCH_HF_TOKEN ]]; then
+    announce "🔍 Environment variable \"LLMDBENCH_HF_TOKEN\" not set, attempting to extract it from secret \"LLMDBENCH_VLLM_COMMON_HF_TOKEN_NAME=${LLMDBENCH_VLLM_COMMON_HF_TOKEN_NAME}\"..."
+    LLMDBENCH_HF_TOKEN=$($LLMDBENCH_CONTROL_KCMD --namespace $LLMDBENCH_VLLM_COMMON_NAMESPACE get secrets ${LLMDBENCH_VLLM_COMMON_HF_TOKEN_NAME} -o jsonpath='{.data.*}' | base64 -d | grep ^hf_ || true)
+  fi
+fi
+
+required_vars=("LLMDBENCH_HF_TOKEN")
+for var in "${required_vars[@]}"; do
+  if [ -z "${!var:-}" ]; then
+    echo "❌ Environment variable '$var' is not set."
+    exit 1
+  fi
+done
+
+export HF_TOKEN=${HF_TOKEN:-$LLMDBENCH_HF_TOKEN}
+
+export LLMDBENCH_CONTROL_DEPLOY_HOST_SHELL=${SHELL:5}

setup/functions.sh

@@ -744,6 +744,17 @@ function get_harness_list {
 }
 export -f get_harness_list
 
+function add_env_vars_to_pod {
+    local varpattern=$1
+    varlist=$(env | grep -E "$varpattern" | cut -d "=" -f 1)
+    echo "#    "
+    for envvar in $varlist; do
+      echo "    - name: ${envvar}"
+      echo "      value: \"${!envvar}\""
+    done
+}
+export -f add_env_vars_to_pod
+
 function create_harness_pod {
   is_pvc=$(${LLMDBENCH_CONTROL_KCMD} --namespace ${LLMDBENCH_HARNESS_NAMESPACE} get pvc --ignore-not-found | grep ${LLMDBENCH_HARNESS_PVC_NAME} || true)
   if [[ -z ${is_pvc} ]]; then
@@ -806,6 +817,7 @@ spec:
       value: "${LLMDBENCH_HARNESS_STACK_ENDPOINT_URL}"
     - name: LLMDBENCH_HARNESS_STACK_NAME
       value: "$LLMDBENCH_HARNESS_STACK_NAME"
+    $(add_env_vars_to_pod $LLMDBENCH_CONTROL_ENV_VAR_LIST_TO_POD)
     - name: HF_TOKEN_SECRET
       value: "${LLMDBENCH_VLLM_COMMON_HF_TOKEN_NAME}"
     - name: HUGGING_FACE_HUB_TOKEN

setup/run.sh

@@ -178,6 +178,9 @@ export LLMDBENCH_BASE64_CONTEXT_CONTENTS=$(cat $LLMDBENCH_CONTROL_WORK_DIR/envir
 
 set +euo pipefail
 export LLMDBENCH_CURRENT_STEP=05
+if [[ $LLMDBENCH_CONTROL_ENVIRONMENT_TYPE_STANDALONE_ACTIVE -eq 0 && $LLMDBENCH_CONTROL_ENVIRONMENT_TYPE_MODELSERVICE_ACTIVE -eq 0 ]]; then
+  export LLMDBENCH_VLLM_MODELSERVICE_URI_PROTOCOL="NA"
+fi
 source ${LLMDBENCH_STEPS_DIR}/05_ensure_harness_namespace_prepared.sh > /dev/null 2>&1
 if [[ $? -ne 0 ]]; then
   announce "❌ Error while attempting to setup the harness namespace"
@@ -210,18 +213,21 @@ for method in ${LLMDBENCH_DEPLOY_METHODS//,/ }; do
       export LLMDBENCH_VLLM_FQDN=".${LLMDBENCH_VLLM_COMMON_NAMESPACE}${LLMDBENCH_VLLM_COMMON_FQDN}"
 
       if [[ $LLMDBENCH_CONTROL_ENVIRONMENT_TYPE_STANDALONE_ACTIVE -eq 1 ]]; then
+        export LLMDBENCH_CONTROL_ENV_VAR_LIST_TO_POD="^LLMDBENCH_VLLM_COMMON|^LLMDBENCH_VLLM_STANDALONE"
         export LLMDBENCH_HARNESS_STACK_TYPE=vllm-prod
         export LLMDBENCH_HARNESS_STACK_ENDPOINT_NAME=$(${LLMDBENCH_CONTROL_KCMD} --namespace "$LLMDBENCH_VLLM_COMMON_NAMESPACE" get service --no-headers | grep standalone | awk '{print $1}' || true)
         export LLMDBENCH_HARNESS_STACK_ENDPOINT_PORT=80
       fi
 
       if [[ $LLMDBENCH_CONTROL_ENVIRONMENT_TYPE_MODELSERVICE_ACTIVE -eq 1 ]]; then
+        export LLMDBENCH_CONTROL_ENV_VAR_LIST_TO_POD="^LLMDBENCH_VLLM_COMMON|^LLMDBENCH_VLLM_MODELSERVICE|^LLMDBENCH_VLLM_INFRA|^LLMDBENCH_VLLM_GAIE"
         export LLMDBENCH_HARNESS_STACK_TYPE=llm-d
         export LLMDBENCH_HARNESS_STACK_ENDPOINT_NAME=$(${LLMDBENCH_CONTROL_KCMD} --namespace "$LLMDBENCH_VLLM_COMMON_NAMESPACE" get gateway --no-headers | grep ^infra-${LLMDBENCH_VLLM_MODELSERVICE_RELEASE}-inference-gateway | awk '{print $1}')
         export LLMDBENCH_HARNESS_STACK_ENDPOINT_PORT=80
       fi
 
       if [[ $LLMDBENCH_CONTROL_ENVIRONMENT_TYPE_STANDALONE_ACTIVE -eq 0 && $LLMDBENCH_CONTROL_ENVIRONMENT_TYPE_MODELSERVICE_ACTIVE -eq 0 ]]; then
+        export LLMDBENCH_CONTROL_ENV_VAR_LIST_TO_POD="^LLMDBENCH_VLLM_COMMON_NAMESPACE"
         announce "🔍 Deployment method - $LLMDBENCH_DEPLOY_METHODS - is neither \"standalone\" nor \"modelservice\". Trying to find a matching endpoint name..."
         export LLMDBENCH_HARNESS_STACK_TYPE=vllm-prod
         export LLMDBENCH_HARNESS_STACK_ENDPOINT_NAME=$(${LLMDBENCH_CONTROL_KCMD} --namespace "$LLMDBENCH_VLLM_COMMON_NAMESPACE" get service --no-headers | awk '{print $1}' | grep ${LLMDBENCH_DEPLOY_METHODS} || true)
@@ -325,7 +331,7 @@ for method in ${LLMDBENCH_DEPLOY_METHODS//,/ }; do
         fi
 
         if [[ $LLMDBENCH_CONTROL_DRY_RUN -eq 1 ]]; then
-          announce "ℹ️ Skipping \"${LLMDBENCH_RUN_HARNESS_LAUNCHER_NAME}\ creation"
+          announce "ℹ️ Skipping \"${LLMDBENCH_RUN_HARNESS_LAUNCHER_NAME}\" creation"
         else
           create_harness_pod
 

setup/steps/04_ensure_model_namespace_prepared.py

@@ -100,6 +100,8 @@ def main():
 
     llmdbench_execute_cmd(actual_cmd=f'source "{ev["control_dir"]}/env.sh"', dry_run=ev["control_dry_run"] == '1', verbose=ev["control_verbose"] == '1')
 
+
+
     api = kube_connect(f'{ev["control_work_dir"]}/environment/context.ctx')
     if ev["control_dry_run"] == '1':
         announce("DRY RUN enabled. No actual changes will be made.")
@@ -162,7 +164,7 @@ def main():
                 dry_run=ev["control_dry_run"]
             ))
 
-    if is_openshift(api):
+    if is_openshift(api) and ev["deploy_methods"] == "modelservice" :
         # vllm workloads may need to run as a specific non-root UID , the  default SA needs anyuid
         # some setups might also require privileged access for GPU resources
         add_scc_to_service_account(api, "anyuid", ev["vllm_common_service_account"], ev["vllm_common_namespace"], ev["control_dry_run"]=='1')