fix issue

Signed-off-by: YanxuanLiu <[email protected]>

Author: nvauto

python/benchmark/databricks/benchmark_utils.sh

@@ -1,4 +1,5 @@
-# Copyright (c) 2024, NVIDIA CORPORATION.
+#!/bin/bash
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -129,13 +130,13 @@ EOF
     duration=0
     while [[ $status != "TERMINATED" ]] && [[ $status != "ERROR" ]]; do
         echo -n "."
-        if [[ $TIME_LIMIT != "" ]] && (( $duration > $TIME_LIMIT ))
+        if [[ $TIME_LIMIT != "" ]] && (( duration > TIME_LIMIT ))
         then
             echo "\ntime limit of $TIME_LIMIT minutes exceeded, canceling run"
             databricks jobs cancel-run $run_id --profile $DB_PROFILE
         fi
         sleep 10
-        duration=$(( $duration + 10 ))
+        duration=$(( duration + 10 ))
         get_run_status
     done
 

python/benchmark/databricks/cpu_cluster_spec.sh

@@ -1,4 +1,5 @@
-# Copyright (c) 2024, NVIDIA CORPORATION.
+#!/bin/bash
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +16,7 @@
 # needed for bm script arguments
 cat <<EOF
 {
-    "num_workers": $(( $num_cpus / 8)),
+    "num_workers": $(( num_cpus / 8)),
     "cluster_name": "$cluster_name",
     "spark_version": "${db_version}.x-cpu-ml-scala2.12",
     "spark_conf": {},

python/benchmark/databricks/gpu_cluster_spec.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Copyright (c) 2024-2025, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

python/benchmark/databricks/gpu_etl_cluster_spec.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Copyright (c) 2025, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

python/run_test.sh

@@ -54,9 +54,9 @@ if [ ${CUDA_VISIBLE_DEVICES} ]; then
 fi
 GPU_MEM_PARALLEL=`nvidia-smi ${NVIDIA_SMI_ARGS} --query-gpu=memory.free --format=csv,noheader | awk 'NR == 1 { MIN = $1 } { if ($1 < MIN) { MIN = $1 } } END { print int((MIN - 2 * 1024) / ((3 * 1024) + 750)) }'`
 CPU_CORES=`nproc`
-TMP_PARALLEL=$(( $GPU_MEM_PARALLEL > $CPU_CORES ? $CPU_CORES : $GPU_MEM_PARALLEL ))
-TMP_PARALLEL=$(( $TMP_PARALLEL > $MAX_PARALLEL ? $MAX_PARALLEL : $TMP_PARALLEL ))
-if  (( $TMP_PARALLEL <= 1 )); then
+TMP_PARALLEL=$(( GPU_MEM_PARALLEL > CPU_CORES ? CPU_CORES : GPU_MEM_PARALLEL ))
+TMP_PARALLEL=$(( TMP_PARALLEL > MAX_PARALLEL ? MAX_PARALLEL : TMP_PARALLEL ))
+if  (( TMP_PARALLEL <= 1 )); then
         TEST_PARALLEL=1
     else
         TEST_PARALLEL=$TMP_PARALLEL