From d5e3a57f5668a3e8039c7da283e1ba8caf98ea9b Mon Sep 17 00:00:00 2001
From: Hyperling <me@hyperling.com>
Date: Wed, 21 Oct 2026 09:56:14 -0700
Subject: [PATCH] llm, add a few models for mistral-small4 but they are too
 large or not the correct format for ollama, a few more edits to
 ollama_threads.

---
 tasks/workstation/linux/software/llm.yml | 32 ++++++++++++++++++------
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/tasks/workstation/linux/software/llm.yml b/tasks/workstation/linux/software/llm.yml
index 2fc017e..bd02572 100644
--- a/tasks/workstation/linux/software/llm.yml
+++ b/tasks/workstation/linux/software/llm.yml
@@ -144,7 +144,17 @@
           - { "purpose": ["code"], "model": "qwen2.5-coder:7b",   "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef *  4700 }}", "context": 32768 }
           - { "purpose": ["code"], "model": "qwen2.5-coder:14b",  "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef *  9000 }}", "context": 32768 }
           - { "purpose": ["code"], "model": "qwen2.5-coder:32b",  "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32768 }
-          ##### Mistral, Ministral, and Devstral - By a French company? #####
+          ##### Mistral, Ministral, and Devstral - By a French company. #####
+            # mistral-small-4
+              # https://huggingface.co/unsloth/Mistral-Small-4-119B-2603-GGUF
+              # Supposed to be better than Qwen equivalents, it's basically mistral, mixtral, magistral, and devstral all in one.
+              # unsloth version is supposed to be the best quantization. UD-Q3_K_XL is supposed to take 64GB RAM, lol, while Q4_K_M would need 80+ GB.
+              # needs done via 'ollama create', fails with pull. https://github.com/ollama/ollama/issues/5245
+          - { "purpose": ["chat", "lang", "code"], "model": "hf.co/unsloth/Mistral-Small-4-119B-2603-GGUF:UD-Q3_K_XL", "provider": "mistral_small40_1190_unsloth_q3",  "size_mb": "{{ llm_size_coef *   54800 }}", "context": 256000 }
+              # https://huggingface.co/mistralai/Mistral-Small-4-119B-2603 : 242GB unquantized, lol.
+          - { "purpose": ["chat", "lang", "code"], "model": "hf.co/mistralai/Mistral-Small-4-119B-2603",               "provider": "mistral_small40_1190",             "size_mb": "{{ llm_size_coef *  242000 }}", "context": 256000 }
+              # https://huggingface.co/unsloth/Mistral-Small-4-119B-2603 : 121GB, A6B MoE, hmmm!
+          - { "purpose": ["chat", "lang", "code"], "model": "hf.co/unsloth/Mistral-Small-4-119B-2603",                 "provider": "mistral_small40_1190_unsloth_a6b", "size_mb": "{{ llm_size_coef *  121000 }}", "context": 256000 }
             # https://ollama.com/library/devstral-small-2 : Last Updated 2025-12-13
             # Most recommended in Mistral family for agentic coding.
           - { "purpose": ["code"], "model": "devstral-small-2:24b", "provider": "mistral_dev20_0240_small", "size_mb": "{{ llm_size_coef * 15000 }}", "context": 393216 }
@@ -881,7 +891,7 @@
             spacing="$sep %-${col1}s $sep %${col2}s $sep %${col3}s $sep %${col4}s $sep"
 
             #echo " Stopping all ollama usage..."
-            sudo sv restart ollama >/dev/null 2>&1
+            #sudo sv restart ollama >/dev/null 2>&1
 
             #echo -e "\n Checking model disk and RAM sizes...\n"
 
@@ -932,7 +942,11 @@
             # Runnning with 0 causes default behavior, 1 per physical core, 8!
             cores=$(cat /sys/devices/cpu_core/cpus | tr ',' '\n' | sed 's/-/ /' | xargs -L1 seq)
 
-            # Take out useless tests.
+            # Flatten the output (trims newlines and extra spacing).
+            cores=$(echo $cores)
+
+            # Take out useless tests. 1 CPU is silly. If we have 8 physical then
+            # only need to look at 4, 6, 7, and 8 (which is done by 0/default).
             cores=${cores/ 1 / }
             if [[ "$cores" == *" 7" ]]; then
               cores=${cores/ 2 / }
@@ -940,13 +954,16 @@
               cores=${cores/ 5 / }
             fi
 
-            echo " Stopping all ollama usage..."
-            sudo sv restart ollama >/dev/null 2>&1
+            # Do the 0 test last.
+            cores="${cores/0 / } 0"
+
+            #echo " Stopping all ollama usage..."
+            #sudo sv restart ollama >/dev/null 2>&1
 
             # Benchmark Thread Count
             for t in $cores; do
               start=$SECONDS
-              echo -n "`date` - Model $model w/ $t Thread(s): "
+              echo -n "`date` - $model w/ $t Thread(s): "
 
               speed=$(curl -s http://localhost:11434/api/generate \
                 -d '{"model":"'$model'","prompt":"Hello","stream":false,"keep_alive":1,"options":{"num_thread":'$t$context'}}' \
@@ -956,7 +973,8 @@
               ctx="`ollama ps | grep $model | awk '{print $7}'`"
               time=$(( $finish - $start ))
 
-              echo "tokens/sec: $speed @ context=$ctx, time: $time secs"
+              #echo "tokens/sec: $speed @ context=$ctx, time: $time secs"
+              printf "%-32s %18s %16s\n" "tokens/sec: $speed" "@ context=$ctx," "time: $time secs"
 
               ollama stop $model >/dev/null 2>&1
             done