llm, add context param to ollama_threads.
This commit is contained in:
1 parent
5719ec6b60
commit
92be32345a
1 file changed
+10
-2
@@ -909,12 +909,20 @@
|
|||||||
alias model-sizes="ollama_sizes "
|
alias model-sizes="ollama_sizes "
|
||||||
|
|
||||||
function ollama_threads() {
|
function ollama_threads() {
|
||||||
|
# VAR1 - Model
|
||||||
model="$1"
|
model="$1"
|
||||||
if [[ -z $model ]]; then
|
if [[ -z $model ]]; then
|
||||||
model="qwen3.5:4b"
|
model="qwen3.5:4b"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
#echo " Stopping all ollama usage..."
|
# VAR2 - Context
|
||||||
|
typeset -i num_ctx="$2"
|
||||||
|
context=""
|
||||||
|
if [[ -n $num_ctx ]]; then
|
||||||
|
context=',"num_ctx":'$num_ctx''
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " Stopping all ollama usage..."
|
||||||
sudo sv restart ollama >/dev/null 2>&1
|
sudo sv restart ollama >/dev/null 2>&1
|
||||||
|
|
||||||
# Benchmark Thread Count
|
# Benchmark Thread Count
|
||||||
@@ -922,7 +930,7 @@
|
|||||||
echo -n "`date` - Model $model w/ $t Thread(s), tokens/sec: "
|
echo -n "`date` - Model $model w/ $t Thread(s), tokens/sec: "
|
||||||
|
|
||||||
speed=$(curl -s http://localhost:11434/api/generate \
|
speed=$(curl -s http://localhost:11434/api/generate \
|
||||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t'}}' \
|
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t'$context}}' \
|
||||||
| jq '.eval_count * 1000000000 / .eval_duration'
|
| jq '.eval_count * 1000000000 / .eval_duration'
|
||||||
)
|
)
|
||||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||||
|
|||||||
Reference in new issue
Block a user