diff --git a/tasks/workstation/linux/software/llm.yml b/tasks/workstation/linux/software/llm.yml index 840fb11..c06536a 100644 --- a/tasks/workstation/linux/software/llm.yml +++ b/tasks/workstation/linux/software/llm.yml @@ -59,7 +59,7 @@ # - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000. llm_size_coef: 1.3 # Take the GB of RAM and use it as the MB of context. - llm_context: "{{ [(ansible_memtotal_mb | round(0, method='ceil')), 32000] | max | int }}" + llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 32000] | max | int }}" - name: Workstation | Linux | Software | LLMs | Facts 02 ansible.builtin.set_fact: @@ -88,33 +88,33 @@ ansible.builtin.set_fact: llm_installs: [] llm_removals: - - { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } - - { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000, "context": 123456 } - - { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 } + - { "purpose": "chat", "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } + - { "purpose": "chat", "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000, "context": 123456 } + - { "purpose": "chat", "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 } llm_choices: ### QWEN >3.5 - Need to try! - - { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000, "context": 123456 } - - { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000, "context": 123456 } + - { "purpose": ["chat", code], "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000, "context": 123456 } + - { "purpose": ["chat", code], "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000, "context": 123456 } ### QWEN 3.5 - Good for generic usage and a little coding. - - { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 } - - { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 } - - { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 } - - { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 } + - { "purpose": "chat", "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 } + - { "purpose": "chat", "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 } + - { "purpose": "chat", "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 } + - { "purpose": "chat", "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 } # skip 27b and use 35b - - { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 } - - { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 } + - { "purpose": ["chat", code], "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 } + - { "purpose": ["chat", code], "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 } ### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it. - - { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 } - - { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 } + - { "purpose": "code", "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 } + - { "purpose": "code", "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 } ### QWEN 2.5 Coder - Ol'reliable! - - { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 } - - { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 } - - { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 } - - { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 } - - { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 } - - { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 } + - { "purpose": "code", "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 } + - { "purpose": "code", "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 } + - { "purpose": "code", "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 } + - { "purpose": "code", "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 } + - { "purpose": "code", "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 } + - { "purpose": "code", "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 } ### Mistral, Ministral, and Devstral - By a French company? - - { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000, "context": 123456 } + - { "purpose": "code", "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000, "context": 123456 } #- mistral:7b #- ministral-3:3b #- ministral-3:8b @@ -213,7 +213,7 @@ export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars) ### Resource Usage ### - export OLLAMA_KEEP_ALIVE=20m # default is 5m + export OLLAMA_KEEP_ALIVE=2h # default is 5m export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model (unconfirmed) export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size @@ -584,6 +584,7 @@ block: | {% for llm in llm_installs %} {% if llm.model in llm_models.stdout %} + {% if "chat" in llm.purpose %} [providers.models.ollama.{{ llm.provider }}] model = "{{ llm.model }}" @@ -591,6 +592,7 @@ context_window = {{ [llm_context, llm.context] | min | int }} max_tokens = {{ [llm_tokens, (llm.context / 4), 16000] | min | int }} + {% endif %} {% endif %} {% endfor %} state: present @@ -615,6 +617,7 @@ {% for llm in llm_installs %} {% if llm.model in llm_models.stdout %} + {% if "code" in llm.purpose %} - name: {{ llm.provider }}_chat-edit provider: ollama @@ -632,6 +635,7 @@ roles: - autocomplete + {% endif %} {% endif %} {% endfor %} mode: "0644"