Assorted bug fixes (llm-d#206)

Signed-off-by: Nick Masluk <nick@randombytes.net>

Author: namasl

build/llm-d-benchmark.sh

@@ -4,7 +4,7 @@ export LLMDBENCH_RUN_EXPERIMENT_HARNESS_EC=1
   export LLMDBENCH_HARNESS_NAME=${1}
   export LLMDBENCH_RUN_EXPERIMENT_HARNESS=$(find /usr/local/bin | grep ${1}.*-llm-d-benchmark | rev | cut -d '/' -f 1 | rev)
   export LLMDBENCH_RUN_EXPERIMENT_ANALYZER=$(find /usr/local/bin | grep ${1}.*-analyze_results | rev | cut -d '/' -f 1 | rev)
-  export LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR=/requests/$(echo $LLMDBENCH_RUN_EXPERIMENT_HARNESS | sed "s^-llm-d-benchmark^^g" | cut -d '.' -f 1)_${LLMDBENCH_RUN_EXPERIMENT_ID}_${LLMDBENCH_HARNESS_STACK_NAME}_${date -u +%Y-%m-%d_%H.%M.%S}
+  export LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR=/requests/$(echo $LLMDBENCH_RUN_EXPERIMENT_HARNESS | sed "s^-llm-d-benchmark^^g" | cut -d '.' -f 1)_${LLMDBENCH_RUN_EXPERIMENT_ID}_${LLMDBENCH_HARNESS_STACK_NAME}
   export LLMDBENCH_CONTROL_WORK_DIR=$LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR
 fi
 if [[ ! -z $2 ]]; then
@@ -30,66 +30,33 @@ if [[ -f ~/.bashrc ]]; then
   mv -f ~/.bashrc ~/fixbashrc
 fi
 
-#if [[ -d $LLMDBENCH_RUN_EXPERIMENT_HARNESS_DIR && ! -z $LLMDBENCH_HARNESS_GIT_REPO ]]; then
-#  pushd /workspace/$LLMDBENCH_RUN_EXPERIMENT_HARNESS_DIR
-#  current_repo=$(git remote -v | grep \(fetch\) | awk '{ print $2 }')
-#  if [[ $current_repo == $LLMDBENCH_HARNESS_GIT_REPO ]]; then
-#    export LLMDBENCH_RUN_EXPERIMENT_HARNESS_CURRENT_COMMIT=$(git rev-parse --short HEAD)
-#    git fetch
-#  else
-#    popd
-#    rm -rf /workspace/$LLMDBENCH_RUN_EXPERIMENT_HARNESS_DIR
-#    git clone $LLMDBENCH_HARNESS_GIT_REPO
-#    pushd /workspace/$LLMDBENCH_RUN_EXPERIMENT_HARNESS_DIR
-#  fi
-#  git checkout $LLMDBENCH_HARNESS_GIT_BRANCH
-#  if [[ $(git rev-parse --short HEAD) != ${LLMDBENCH_RUN_EXPERIMENT_HARNESS_CURRENT_COMMIT} ]]; then
-#    case ${LLMDBENCH_RUN_EXPERIMENT_HARNESS_DIR} in
-#      fmperf*)
-#        pip install --no-cache-dir -r requirements.txt && pip install  .
-#        ;;
-#      inference-perf*)
-#        pip install  .
-#        ;;
-#      vllm-benchmark*)
-#        VLLM_USE_PRECOMPILED=1 pip install  .
-#        pushd ..
-#        if [[ ! -d vllm ]]; then
-#          mv -f vllm vllm-benchmark
-#        fi
-#        popd
-#        ;;
-#      guidellm*)
-#        pip install  .
-#        ;;
-#    esac
-#  fi
-#  popd
-#fi
-
 env | grep ^LLMDBENCH | grep -v BASE64 | sort
 
-if [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_EC -ne 0 ]]; then
+# Repeat run until success
+while [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_EC -ne 0 ]]; do
   /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_HARNESS}
   ec=$?
   if [[ $ec -ne 0 ]]; then
-    echo "execution of /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_HARNESS} failed, wating 120 seconds and trying again"
-    sleep 120
+    echo "execution of /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_HARNESS} failed, wating 30 seconds and trying again"
+    sleep 30
     set -x
   else
     export LLMDBENCH_RUN_EXPERIMENT_HARNESS_EC=0
   fi
-fi
+done
 
 if [[ -f ~/fixbashrc ]]; then
   mv -f ~/fixbashrc ~/.bashrc
 fi
 
+# Try to run analysis twice then give up
 /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_ANALYZER}
 ec=$?
 if [[ $ec -ne 0 ]]; then
   echo "execution of /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_ANALYZER} failed, wating 120 seconds and trying again"
   sleep 120
   set -x
+  /usr/local/bin/${LLMDBENCH_RUN_EXPERIMENT_ANALYZER}
 fi
+# Return with error code of first iteration of experiment analyzer
 exit $ec

setup/env.sh

@@ -13,7 +13,7 @@ export LLMDBENCH_IMAGE_NAME=${LLMDBENCH_IMAGE_NAME:-llm-d-benchmark}
 export LLMDBENCH_IMAGE_TAG=${LLMDBENCH_IMAGE_TAG:-auto}
 export LLMDBENCH_LLMD_IMAGE_REGISTRY=${LLMDBENCH_LLMD_IMAGE_REGISTRY:-ghcr.io}
 export LLMDBENCH_LLMD_IMAGE_REPO=${LLMDBENCH_LLMD_IMAGE_REPO:-llm-d}
-export LLMDBENCH_LLMD_IMAGE_NAME=${LLMDBENCH_LLMD_IMAGE_REPO:-llm-d}
+export LLMDBENCH_LLMD_IMAGE_NAME=${LLMDBENCH_LLMD_IMAGE_NAME:-llm-d}
 export LLMDBENCH_LLMD_IMAGE_TAG=${LLMDBENCH_LLMD_IMAGE_TAG:-auto}
 export LLMDBENCH_LLMD_MODELSERVICE_IMAGE_REGISTRY=${LLMDBENCH_LLMD_MODELSERVICE_IMAGE_REGISTRY:-ghcr.io}
 export LLMDBENCH_LLMD_MODELSERVICE_IMAGE_REPO=${LLMDBENCH_LLMD_MODELSERVICE_IMAGE_REPO:-llm-d}

workload/harnesses/vllm-benchmark-llm-d-benchmark.sh

@@ -10,10 +10,16 @@ find /workspace/vllm-benchmark -maxdepth 1 -mindepth 1 -name '*.json' -exec mv -
 
 # If benchmark harness returned with an error, exit here
 if [[ $LLMDBENCH_RUN_EXPERIMENT_HARNESS_RC -ne 0 ]]; then
+  echo "Harness returned with error $LLMDBENCH_RUN_EXPERIMENT_HARNESS_RC"
   exit $LLMDBENCH_RUN_EXPERIMENT_HARNESS_RC
 fi
+echo "Harness completed successfully."
 
 # Convert results into universal format
-convert.py $LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR -w vllm-benchmark > $LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR/results_${date -u +%Y-%m-%d_%H.%M.%S}.yaml 2> >(tee -a $LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR/stderr.log >&2)
+convert.py $LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR -w vllm-benchmark > $LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR/results_$(date -u +%Y-%m-%d_%H.%M.%S).yaml 2> >(tee -a $LLMDBENCH_RUN_EXPERIMENT_RESULTS_DIR/stderr.log >&2)
 export LLMDBENCH_RUN_EXPERIMENT_CONVERT_RC=$?
-exit $LLMDBENCH_RUN_EXPERIMENT_CONVERT_RC
+if [[ $LLMDBENCH_RUN_EXPERIMENT_CONVERT_RC -ne 0 ]]; then
+  echo "convert.py returned with error $LLMDBENCH_RUN_EXPERIMENT_CONVERT_RC"
+  exit $LLMDBENCH_RUN_EXPERIMENT_CONVERT_RC
+fi
+echo "Results data conversion completed successfully."