generated from me/template-mit
ollama-test, enhance to me like pv-test with the parameter looping, also fix some variables being set via env and add QUIET to suppress all but the table.
This commit is contained in:
me committed
1 parent
933e00556c
commit
f3da185fb6
1 file changed
+135
-134
+135
-134
@@ -1566,10 +1566,21 @@ alias ollama-cpu="ollama_threads "
|
||||
|
||||
function ollama_benchmark_usage() {
|
||||
cat <<- EOF
|
||||
USAGE: ollama-benchmark CONTEXT [THREADS] [iGPU] [MODEL] [-h | --help]
|
||||
USAGE: ollama-benchmark CONTEXTS [THREADS] [IGPUS] [MODELS] [-h | --help]
|
||||
|
||||
Tests installed models at a certain context, thread count, and iGPU setting.
|
||||
|
||||
PARAMETERS
|
||||
|
||||
CONTEXTS
|
||||
Sent as num_ctx to ollama. No default.
|
||||
THREADS
|
||||
Sent as num_threads to ollama. Defaults to 0.
|
||||
IGPUS
|
||||
Values to set OLLAMA_IGPU_ENABLE to. Defaults to 1.
|
||||
MODELS
|
||||
Model names to test. Defaults to all of `ollama ls`.
|
||||
|
||||
EXAMPLES:
|
||||
|
||||
Test models with at a context of 32000, default threads, and iGPU enabled.
|
||||
@@ -1580,10 +1591,15 @@ function ollama_benchmark_usage() {
|
||||
|
||||
Variables can also be specified through variables:
|
||||
|
||||
context=64000 threads=7 igpu=0 ollama-benchmark
|
||||
context=64000 threads=7 ollama-benchmark
|
||||
context=64000 igpu=0 ollama-benchmark
|
||||
context=64000 model=qwen3.5:4b ollama-benchmark
|
||||
context=64000 threads=7 igpu=0 ollama-benchmark
|
||||
context=64000 threads=7 ollama-benchmark
|
||||
context=64000 igpu=0 ollama-benchmark
|
||||
context=64000 model=qwen3.5:4b ollama-benchmark
|
||||
|
||||
The parameters can also act as arrays to cause extra looping.
|
||||
|
||||
ollama-benchmark "32000 64000 128000 256000" "4 7 0" "0 1" \
|
||||
"qwen3.8:27b qwen3.6:35b qwen3.6:27b qwen3.5:35b qwen3.5:27b"
|
||||
|
||||
Aliases:
|
||||
|
||||
@@ -1599,50 +1615,40 @@ function ollama_benchmark() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
typeset -i context
|
||||
if [[ -z $context || $context == 0 ]]; then
|
||||
context="$1"
|
||||
fi
|
||||
if [[ -z "$context" ]]; then
|
||||
contexts="${1:-$contexts}"
|
||||
if [[ -z "$contexts" ]]; then
|
||||
echo " ERROR: Context is required."
|
||||
ollama_benchmark_usage
|
||||
return 1
|
||||
fi
|
||||
|
||||
typeset -i threads
|
||||
if [[ -z $threads ]]; then
|
||||
threads="$2"
|
||||
fi
|
||||
threads="${2:-$threads}"
|
||||
if [[ -z $threads ]]; then
|
||||
threads=0
|
||||
fi
|
||||
|
||||
declare igpu
|
||||
if [[ -z "$igpu" ]] ; then
|
||||
igpu="$3"
|
||||
fi
|
||||
if [[ -z "$igpu" ]] ; then
|
||||
igpu=1
|
||||
fi
|
||||
export OLLAMA_IGPU_ENABLE=$igpu
|
||||
export OLLAMA_VULKAN=$igpu
|
||||
|
||||
declare model
|
||||
if [[ -z "$model" ]] ; then
|
||||
model="$4"
|
||||
igpus="${3:-$igpus}"
|
||||
if [[ -z "$igpus" ]] ; then
|
||||
igpus=1
|
||||
fi
|
||||
|
||||
echo -e "\n Sharing run parameters..."
|
||||
cat <<- EOF
|
||||
models="${4:-$models}"
|
||||
if [[ -z "$models" ]]; then
|
||||
models="`ollama ls | grep -v NAME | awk '{print $1}'`"
|
||||
fi
|
||||
|
||||
DEBUG="${DEBUG}"
|
||||
QUIET="${QUIET}"
|
||||
[[ -z "$QUIET" ]] && echo -e "\n Sharing run parameters..."
|
||||
[[ -z "$QUIET" ]] && cat <<- EOF
|
||||
DEBUG=$DEBUG
|
||||
QUIET=$QUIET
|
||||
context=$context
|
||||
threads=$threads
|
||||
igpu=$igpu
|
||||
OLLAMA_IGPU_ENABLE=$OLLAMA_IGPU_ENABLE
|
||||
OLLAMA_VULKAN=$OLLAMA_IGPU_ENABLE
|
||||
model=$model
|
||||
EOF
|
||||
echo " Done!"
|
||||
[[ -z "$QUIET" ]] && echo " Done!"
|
||||
|
||||
# Variables
|
||||
|
||||
@@ -1651,11 +1657,13 @@ function ollama_benchmark() {
|
||||
col_ctx=8
|
||||
col_ram=6
|
||||
col_thr=7
|
||||
col_gpu=4
|
||||
col_cpu=9
|
||||
col_spd=11 col_spd_round=2
|
||||
col_tim=8
|
||||
|
||||
log_file="ollama_benchmark.log.txt"
|
||||
[[ -z "$QUIET" ]] && echo -e "\n Logging ollama to '$log_file'..."
|
||||
|
||||
declare champ_{speed,time}_{model,num}
|
||||
|
||||
@@ -1670,141 +1678,134 @@ function ollama_benchmark() {
|
||||
# Main
|
||||
|
||||
if [[ ! -d /sdcard && -d /etc/sv ]]; then
|
||||
echo -e "\n Stopping ollama runit service..."
|
||||
sudo sv disable ollama
|
||||
sudo sv stop ollama
|
||||
[[ -z "$QUIET" ]] && echo -e "\n Stopping ollama runit service..."
|
||||
sudo sv disable ollama >/dev/null 2>&1
|
||||
sudo sv stop ollama >/dev/null 2>&1
|
||||
sleep 5
|
||||
echo " Done!"
|
||||
[[ -z "$QUIET" ]] && echo " Done!"
|
||||
fi
|
||||
|
||||
echo -e "\n Stopping any ollama processes..."
|
||||
killall ollama
|
||||
sleep 5
|
||||
echo " Done!"
|
||||
[[ -z "$QUIET" ]] && echo -e "\n Starting main loop... \n"
|
||||
|
||||
echo -e "\n Starting background ollama service, logging to '$log_file'..."
|
||||
(ollama serve >$log_file 2>&1 &)
|
||||
wait_for_ollama
|
||||
echo " Done!"
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
||||
"Model" "HDD" "Context" "RAM" "Threads" "iGPU" "Processor" "Speed (t/s)" "Time (s)"
|
||||
|
||||
echo -e "\n Starting main loop... \n"
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
||||
":------------------------" "-----:" "-------:" "-----:" "---:" "------:" "--------:" "----------:" "-------:"
|
||||
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
||||
"Model @ CTX=$context" "HDD" "Context" "RAM" "Threads" "Processor" "Speed (t/s)" "Time (s)"
|
||||
for model in $models; do
|
||||
for context in $contexts; do
|
||||
for thread in $threads; do
|
||||
for igpu in $igpus; do
|
||||
(killall ollama >/dev/null 2>&1)
|
||||
sleep 5
|
||||
export OLLAMA_IGPU_ENABLE=$igpu
|
||||
export OLLAMA_VULKAN=$igpu
|
||||
(ollama serve >$log_file 2>&1 &)
|
||||
wait_for_ollama
|
||||
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
|
||||
":------------------------" "-----:" "-------:" "-----:" "------:" "--------:" "----------:" "-------:"
|
||||
hdd="`ollama ls | grep $model | awk '{print $3 $4}'`"
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd"
|
||||
|
||||
model_focus="$model"
|
||||
unset model
|
||||
for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do
|
||||
# If a model was specified then only focus on it.
|
||||
if [[ -n "$model_focus" ]]; then
|
||||
if [[ "$model" != "$model_focus" ]]; then
|
||||
continue;
|
||||
fi
|
||||
fi
|
||||
# Start the model and store the response.
|
||||
start=$SECONDS
|
||||
speed_json=$(curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$thread'}}' \
|
||||
2>/dev/null
|
||||
)
|
||||
status_ollama=$?
|
||||
finish=$SECONDS
|
||||
|
||||
hdd="`ollama ls | grep $model | awk '{print $3 $4}'`"
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama"
|
||||
|
||||
printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd"
|
||||
time=$(( $finish - $start ))
|
||||
|
||||
# Start the model and store the response.
|
||||
start=$SECONDS
|
||||
speed_json=$(curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$threads'}}' \
|
||||
2>/dev/null
|
||||
)
|
||||
status_ollama=$?
|
||||
finish=$SECONDS
|
||||
# Calculate the values if we got a successful response.
|
||||
declare ram cpu ctx speed status_jq=0
|
||||
if [[ "$status_ollama" == 0 ]]; then
|
||||
speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null)
|
||||
status_jq=$?
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq"
|
||||
if [[ "$status_jq" != 0 ]]; then
|
||||
speed=0
|
||||
fi
|
||||
else
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)"
|
||||
ctx="OLLAMA"
|
||||
ram="DIED"
|
||||
cpu="FAIL"
|
||||
speed=0
|
||||
killall ollama
|
||||
(ollama serve >>$log_file 2>&1 &)
|
||||
wait_for_ollama
|
||||
fi
|
||||
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama"
|
||||
if [[ "$status_jq" != 0 ]]; then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)"
|
||||
ctx="RESPONSE"
|
||||
ram="MLFRMD"
|
||||
cpu="FAIL"
|
||||
speed=0
|
||||
fi
|
||||
|
||||
time=$(( $finish - $start ))
|
||||
if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values."
|
||||
ram="`ollama ps | grep $model | awk '{print $3 $4}'`"
|
||||
cpu="`ollama ps | grep $model | awk '{print $5 $6}'`"
|
||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||
|
||||
# Calculate the values if we got a successful response.
|
||||
declare ram cpu ctx speed status_jq=0
|
||||
if [[ "$status_ollama" == 0 ]]; then
|
||||
speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null)
|
||||
status_jq=$?
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq"
|
||||
if [[ "$status_jq" != 0 ]]; then
|
||||
speed=0
|
||||
fi
|
||||
else
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)"
|
||||
ctx="OLLAMA"
|
||||
ram="DIED"
|
||||
cpu="FAIL"
|
||||
speed=0
|
||||
killall ollama
|
||||
(ollama serve >>$log_file 2>&1 &)
|
||||
wait_for_ollama
|
||||
fi
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed"
|
||||
if [[ -z "$champ_speed_num" ]] \
|
||||
|| awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }'
|
||||
then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!"
|
||||
champ_speed_num="$speed"
|
||||
champ_speed_model="$model"
|
||||
fi
|
||||
|
||||
if [[ "$status_jq" != 0 ]]; then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)"
|
||||
ctx="RESPONSE"
|
||||
ram="MLFRMD"
|
||||
cpu="FAIL"
|
||||
speed=0
|
||||
fi
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time"
|
||||
if [[ -z "$champ_time_num" ]] \
|
||||
|| awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a < b) }'
|
||||
then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!"
|
||||
champ_time_num="$time"
|
||||
champ_time_model="$model"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values."
|
||||
ram="`ollama ps | grep $model | awk '{print $3 $4}'`"
|
||||
cpu="`ollama ps | grep $model | awk '{print $5 $6}'`"
|
||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||
printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_gpu}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \
|
||||
"$ctx" "$ram" "$thread" "$igpu" "$cpu" "$speed" "$time"
|
||||
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed"
|
||||
if [[ -z "$champ_speed_num" ]] \
|
||||
|| awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }'
|
||||
then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!"
|
||||
champ_speed_num="$speed"
|
||||
champ_speed_model="$model"
|
||||
fi
|
||||
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time"
|
||||
if [[ -z "$champ_time_num" ]] \
|
||||
|| awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a < b) }'
|
||||
then
|
||||
[[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!"
|
||||
champ_time_num="$time"
|
||||
champ_time_model="$model"
|
||||
fi
|
||||
fi
|
||||
|
||||
printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \
|
||||
"$ctx" "$ram" "$threads" "$cpu" "$speed" "$time"
|
||||
|
||||
ollama stop $model >/dev/null 2>&1
|
||||
while ollama ps | grep "$model"; do
|
||||
[[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..."
|
||||
sleep 1
|
||||
ollama stop $model >/dev/null 2>&1
|
||||
while ollama ps | grep "$model"; do
|
||||
[[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..."
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
echo -e "\n Done!! :D \n"
|
||||
cat <<- EOF
|
||||
[[ -z "$QUIET" ]] && echo -e "\n Done!! :D \n"
|
||||
[[ -z "$QUIET" ]] && cat <<- EOF
|
||||
Speed Champion: $champ_speed_model with a tokens/seconds of ${champ_speed_num}!
|
||||
Time Champion: $champ_time_model with a time of ${champ_time_num}!
|
||||
EOF
|
||||
|
||||
# Finish
|
||||
|
||||
echo -e "\n Killing local ollama serve..."
|
||||
[[ -z "$QUIET" ]] && echo -e "\n Killing local ollama serve..."
|
||||
killall ollama
|
||||
echo " Done!"
|
||||
[[ -z "$QUIET" ]] && echo " Done!"
|
||||
|
||||
if [[ ! -d /sdcard && -d /etc/sv ]]; then
|
||||
echo -e "\n Restarting runit service..."
|
||||
[[ -z "$QUIET" ]] && echo -e "\n Restarting runit service..."
|
||||
sudo sv enable ollama
|
||||
sudo sv start ollama
|
||||
echo " Done!"
|
||||
[[ -z "$QUIET" ]] && echo " Done!"
|
||||
fi
|
||||
|
||||
echo -e "\n Done!!! :)"
|
||||
[[ -z "$QUIET" ]] && echo -e "\n Done!!! :)"
|
||||
}
|
||||
alias ollama-benchmark="ollama_benchmark "
|
||||
alias ollama-test="ollama_benchmark "
|
||||
|
||||
Reference in new issue
Block a user