llm, add optional open-terminal package list, helpful but required Internet, may have to do an Internet check and then choose which docker run command to execute. Also fix and enhance the ollama_thread checker.
This commit is contained in:
1 parent
85e60ef60c
commit
5719ec6b60
1 file changed
+18
-4
@@ -560,6 +560,9 @@
|
||||
$image run --port {{ open_terminal_port }}
|
||||
|
||||
# -p {{ open_terminal_port }}:8000
|
||||
|
||||
# This requires internet access to be successful...
|
||||
#-e OPEN_TERMINAL_PACKAGES="xz-utils libglu1-mesa libglu1-mesa ninja-build libnotify-bin libgtk-3-dev liblzma-dev libstdc++-12-dev libc6:amd64 libstdc++6 lib32z1 libbz2-1.0 default-jdk libncurses-dev android-udev-rules" \
|
||||
mode: "0755"
|
||||
|
||||
- name: Workstation | Linux | Software | LLMs | runit | open-terminal | Script | Finish
|
||||
@@ -910,12 +913,23 @@
|
||||
if [[ -z $model ]]; then
|
||||
model="qwen3.5:4b"
|
||||
fi
|
||||
|
||||
#echo " Stopping all ollama usage..."
|
||||
sudo sv restart ollama >/dev/null 2>&1
|
||||
|
||||
# Benchmark Thread Count
|
||||
for t in {1..{{ ansible_processor_vcpus | int }}}; do
|
||||
echo "`date` - Model $model w/ $t Threads`"
|
||||
curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"$model","prompt":"Hello","stream":false,"options":{"num_thread":$t}}' \
|
||||
| jq .eval_count / .eval_duration
|
||||
echo -n "`date` - Model $model w/ $t Thread(s), tokens/sec: "
|
||||
|
||||
speed=$(curl -s http://localhost:11434/api/generate \
|
||||
-d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t'}}' \
|
||||
| jq '.eval_count * 1000000000 / .eval_duration'
|
||||
)
|
||||
ctx="`ollama ps | grep $model | awk '{print $7}'`"
|
||||
|
||||
echo "$speed @ context=$ctx"
|
||||
|
||||
ollama stop $model >/dev/null 2>&1
|
||||
done
|
||||
}
|
||||
alias ollama-threads="ollama_threads "
|
||||
|
||||
Reference in new issue
Block a user