update tpsize

deepspeedai · Mar 6, 2024 · 34388ae · 34388ae
1 parent ebf31f3
commit 34388ae
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/benchmarks/inference/mii/run_fp6.sh b/benchmarks/inference/mii/run_fp6.sh
@@ -3,8 +3,8 @@
 
 # DeepSpeed Team
 
-MODELS=(NousResearch/Llama-2-13b-hf)
+MODELS=(NousResearch/Llama-2-70b-hf)
 
 for MODEL in ${MODELS[@]}; do
-    python ./run_benchmark.py --model ${MODEL} --num_requests 128 --stream --backend fastgen --fp6
+    python ./run_benchmark.py --model ${MODEL} --num_requests 128 --stream --backend fastgen --fp6  --tp_size 1
 done