ollama_benchmark, add a much more capable function for testing ollama models.

rc_shared.sh

@@ -1475,7 +1475,7 @@ function ollama_sizes() {
 	for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do
 		#ollama run $model "" --format json >/dev/null 2>&1
 		curl -s http://localhost:11434/api/generate \
-			-d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":1'$context'}' \
+			-d '{"model":"'$model'","prompt":"","stream":false,"keep_alive":"1m"'$context'}' \
 			>/dev/null 2>&1
 		status=$?
 
@@ -1501,6 +1501,8 @@ function ollama_sizes() {
 }
 alias ollama-sizes="ollama_sizes "
 alias model-sizes="ollama_sizes "
+alias model-memory="ollama_sizes "
+alias model-mem="ollama_sizes "
 
 function ollama_threads() {
 	# VAR1 - Model
@@ -1545,7 +1547,7 @@ function ollama_threads() {
 		echo -n "`date` - $model w/ $t Thread(s): "
 
 		speed=$(curl -s http://localhost:11434/api/generate \
-			-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t$context'}}' \
+			-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_thread":'$t$context'}}' \
 			| jq '.eval_count * 1000000000 / .eval_duration'
 		)
 		finish=$SECONDS
@@ -1562,6 +1564,251 @@ alias ollama-threads="ollama_threads "
 alias ollama-cpus="ollama_threads "
 alias ollama-cpu="ollama_threads "
 
+function ollama_benchmark_usage() {
+	cat <<- EOF
+		USAGE: ollama-benchmark CONTEXT [THREADS] [iGPU] [MODEL] [-h | --help]
+
+		Tests installed models at a certain context, thread count, and iGPU setting.
+
+		EXAMPLES:
+
+		   Test models with at a context of 32000, default threads, and iGPU enabled.
+		      ollama-benchmark 32000 0 1
+
+		   Test qwen3.5:4b with at a context of 128k, 4 threads, and iGPU disabled.
+		      ollama-benchmark 128000 4 0 qwen3.5:4b
+
+		Variables can also be specified through variables:
+
+		      context=64000 threads=7 igpu=0 ollama-benchmark
+		      context=64000 threads=7 ollama-benchmark
+		      context=64000 igpu=0 ollama-benchmark
+		      context=64000 model=qwen3.5:4b ollama-benchmark
+
+		Aliases:
+
+		   ollama-test
+	EOF
+}
+
+function ollama_benchmark() {
+	# Parameters
+
+	if [[ "$@" == *"-h"* || "$@" == *"--help"* ]]; then
+		ollama_benchmark_usage
+		return 0
+	fi
+
+	typeset -i context
+	if [[ -z $context || $context == 0 ]]; then
+		context="$1"
+	fi
+	if [[ -z "$context" ]]; then
+		echo " ERROR: Context is required."
+		ollama_benchmark_usage
+		return 1
+	fi
+
+	typeset -i threads
+	if [[ -z $threads ]]; then
+		threads="$2"
+	fi
+	if [[ -z $threads ]]; then
+		threads=0
+	fi
+
+	declare igpu
+	if [[ -z "$igpu" ]] ; then
+		igpu="$3"
+	fi
+	if [[ -z "$igpu" ]] ; then
+		igpu=1
+	fi
+	export OLLAMA_IGPU_ENABLE=$igpu
+	export OLLAMA_VULKAN=$igpu
+
+	declare model
+	if [[ -z "$model" ]] ; then
+		model="$4"
+	fi
+
+	echo -e "\n Sharing run parameters..."
+	cat <<- EOF
+		DEBUG=$DEBUG
+		context=$context
+		threads=$threads
+		igpu=$igpu
+		OLLAMA_IGPU_ENABLE=$OLLAMA_IGPU_ENABLE
+		OLLAMA_VULKAN=$OLLAMA_IGPU_ENABLE
+		model=$model
+	EOF
+	echo " Done!"
+
+	# Variables
+
+	col_mdl=25
+	col_hdd=6
+	col_ctx=8
+	col_ram=6
+	col_thr=7
+	col_cpu=9
+	col_spd=11 col_spd_round=2
+	col_tim=8
+
+	log_file="ollama_benchmark.log.txt"
+
+	declare champ_{speed,time}_{model,num}
+
+	# Functions
+
+	function wait_for_ollama() {
+		while ! ollama ps >/dev/null 2>&1; do
+			sleep 1
+		done
+	}
+
+	# Main
+
+	if [[ ! -d /sdcard && -d /etc/sv ]]; then
+		echo -e "\n Stopping ollama runit service..."
+		sudo sv disable ollama
+		sudo sv stop ollama
+		sleep 5
+		echo " Done!"
+	fi
+
+	echo -e "\n Stopping any ollama processes..."
+	killall ollama
+	sleep 5
+	echo " Done!"
+
+	echo -e "\n Starting background ollama service, logging to '$log_file'..."
+	(ollama serve >$log_file 2>&1 &)
+	wait_for_ollama
+	echo " Done!"
+
+	echo -e "\n Starting main loop... \n"
+
+	printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
+		"Model @ CTX=$context" "HDD" "Context" "RAM" "Threads" "Processor" "Speed (t/s)" "Time (s)"
+
+	printf "| %-${col_mdl}s | %${col_hdd}s | %${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}s | %${col_tim}s |\n" \
+		":------------------------" "-----:" "-------:" "-----:" "------:" "--------:" "----------:" "-------:"
+
+	model_focus="$model"
+	unset model
+	for model in `ollama ls | grep -v NAME | awk '{print $1}'`; do
+		# If a model was specified then only focus on it.
+		if [[ -n "$model_focus" ]]; then
+			if [[ "$model" != "$model_focus" ]]; then
+				continue;
+			fi
+		fi
+
+		hdd="`ollama ls | grep $model | awk '{print $3 $4}'`"
+
+		printf "| %-${col_mdl}s | %${col_hdd}s | " "$model" "$hdd"
+
+		# Start the model and store the response.
+		start=$SECONDS
+		speed_json=$(curl -s http://localhost:11434/api/generate \
+			-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":"1m","options":{"num_ctx":'$context',"num_thread":'$threads'}}' \
+			2>/dev/null
+		)
+		status_ollama=$?
+		finish=$SECONDS
+
+		[[ -n "$DEBUG" ]] && echo "DEBUG: status_ollama=$status_ollama"
+
+		time=$(( $finish - $start ))
+
+		# Calculate the values if we got a successful response.
+		declare ram cpu ctx speed status_jq=0
+		if [[ "$status_ollama" == 0 ]]; then
+			speed=$(echo "$speed_json" | jq '.eval_count * 1000000000 / .eval_duration' 2>/dev/null)
+			status_jq=$?
+			[[ -n "$DEBUG" ]] && echo "DEBUG: status_jq=$status_jq"
+			if [[ "$status_jq" != 0 ]]; then
+				speed=0
+			fi
+		else
+			[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (ollama)"
+			ctx="OLLAMA"
+			ram="DIED"
+			cpu="FAIL"
+			speed=0
+			killall ollama
+			(ollama serve >>$log_file 2>&1 &)
+			wait_for_ollama
+		fi
+
+		if [[ "$status_jq" != 0 ]]; then
+			[[ -n "$DEBUG" ]] && echo "DEBUG: Using failed values. (jq)"
+			ctx="RESPONSE"
+			ram="MLFRMD"
+			cpu="FAIL"
+			speed=0
+		fi
+
+		if [[ "$status_jq" == 0 && "$status_jq" == 0 ]]; then
+			[[ -n "$DEBUG" ]] && echo "DEBUG: Using successful values."
+			ram="`ollama ps | grep $model | awk '{print $3 $4}'`"
+			cpu="`ollama ps | grep $model | awk '{print $5 $6}'`"
+			ctx="`ollama ps | grep $model | awk '{print $7}'`"
+
+			[[ -n "$DEBUG" ]] && echo "DEBUG: champ_speed_num=$champ_speed_num, speed=$speed"
+			if [[ -z "$champ_speed_num" ]] \
+				|| awk -v a="$speed" -v b="$champ_speed_num" 'BEGIN { exit !(a > b) }'
+			then
+				[[ -n "$DEBUG" ]] && echo "DEBUG: New speed champ!"
+				champ_speed_num="$speed"
+				champ_speed_model="$model"
+			fi
+
+			[[ -n "$DEBUG" ]] && echo "DEBUG: champ_time_num=$champ_time_num, time=$time"
+			if [[ -z "$champ_time_num" ]] \
+				|| awk -v a="$time" -v b="$champ_time_num" 'BEGIN { exit !(a > b) }'
+			then
+				[[ -n "$DEBUG" ]] && echo "DEBUG: New time champ!"
+				champ_time_num="$time"
+				champ_time_model="$model"
+			fi
+		fi
+
+		printf "%${col_ctx}s | %${col_ram}s | %${col_thr}s | %${col_cpu}s | %${col_spd}.${col_spd_round}f | %${col_tim}s |\n" \
+			"$ctx" "$ram" "$threads" "$cpu" "$speed" "$time"
+
+		ollama stop $model >/dev/null 2>&1
+		while ollama ps | grep "$model"; do
+			[[ -n "DEBUG" ]] && echo "DEBUG: Waiting for model to finish..."
+			sleep 1
+		done
+	done
+
+	echo -e "\n Done!! :D \n"
+	cat <<- EOF
+		Speed Champion: $champ_speed_model with a tokens/seconds of ${champ_speed_num}!
+		Time Champion: $champ_time_model with a time of ${champ_time_num}!
+	EOF
+
+	# Finish
+
+	echo -e "\n Killing local ollama serve..."
+	killall ollama
+	echo " Done!"
+
+	if [[ ! -d /sdcard && -d /etc/sv ]]; then
+		echo -e "\n Restarting runit service..."
+		sudo sv enable ollama
+		sudo sv start ollama
+		echo " Done!"
+	fi
+
+	echo -e "\n Done!!! :)"
+}
+alias ollama-benchmark="ollama_benchmark "
+alias ollama-test="ollama_benchmark "
+
 ### Finalize ###
 
 # Export all functions! #