llm, filter models by purpose, use chat in Zeroclaw and code in Continue.
This commit is contained in:
@@ -59,7 +59,7 @@
|
||||
# - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000.
|
||||
llm_size_coef: 1.3
|
||||
# Take the GB of RAM and use it as the MB of context.
|
||||
llm_context: "{{ [(ansible_memtotal_mb | round(0, method='ceil')), 32000] | max | int }}"
|
||||
llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 32000] | max | int }}"
|
||||
|
||||
- name: Workstation | Linux | Software | LLMs | Facts 02
|
||||
ansible.builtin.set_fact:
|
||||
@@ -88,33 +88,33 @@
|
||||
ansible.builtin.set_fact:
|
||||
llm_installs: []
|
||||
llm_removals:
|
||||
- { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 }
|
||||
- { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000, "context": 123456 }
|
||||
- { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 }
|
||||
- { "purpose": "chat", "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 }
|
||||
- { "purpose": "chat", "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000, "context": 123456 }
|
||||
- { "purpose": "chat", "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}", "context": 123456 }
|
||||
llm_choices:
|
||||
### QWEN >3.5 - Need to try!
|
||||
- { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000, "context": 123456 }
|
||||
- { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000, "context": 123456 }
|
||||
- { "purpose": ["chat", code], "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000, "context": 123456 }
|
||||
- { "purpose": ["chat", code], "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000, "context": 123456 }
|
||||
### QWEN 3.5 - Good for generic usage and a little coding.
|
||||
- { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 }
|
||||
- { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 }
|
||||
- { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 }
|
||||
- { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 }
|
||||
- { "purpose": "chat", "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 256000 }
|
||||
- { "purpose": "chat", "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700 }}", "context": 256000 }
|
||||
- { "purpose": "chat", "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300 }}", "context": 256000 }
|
||||
- { "purpose": "chat", "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600 }}", "context": 256000 }
|
||||
# skip 27b and use 35b
|
||||
- { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 }
|
||||
- { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 }
|
||||
- { "purpose": ["chat", code], "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}", "context": 256000 }
|
||||
- { "purpose": ["chat", code], "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}", "context": 256000 }
|
||||
### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it.
|
||||
- { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 }
|
||||
- { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 }
|
||||
- { "purpose": "code", "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}", "context": 123456 }
|
||||
- { "purpose": "code", "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}", "context": 123456 }
|
||||
### QWEN 2.5 Coder - Ol'reliable!
|
||||
- { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 }
|
||||
- { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 }
|
||||
- { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 }
|
||||
- { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 }
|
||||
- { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 }
|
||||
- { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 }
|
||||
- { "purpose": "code", "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}", "context": 32000 }
|
||||
- { "purpose": "code", "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}", "context": 32000 }
|
||||
- { "purpose": "code", "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}", "context": 32000 }
|
||||
- { "purpose": "code", "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}", "context": 32000 }
|
||||
- { "purpose": "code", "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}", "context": 32000 }
|
||||
- { "purpose": "code", "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}", "context": 32000 }
|
||||
### Mistral, Ministral, and Devstral - By a French company?
|
||||
- { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000, "context": 123456 }
|
||||
- { "purpose": "code", "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000, "context": 123456 }
|
||||
#- mistral:7b
|
||||
#- ministral-3:3b
|
||||
#- ministral-3:8b
|
||||
@@ -213,7 +213,7 @@
|
||||
export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars)
|
||||
|
||||
### Resource Usage ###
|
||||
export OLLAMA_KEEP_ALIVE=20m # default is 5m
|
||||
export OLLAMA_KEEP_ALIVE=2h # default is 5m
|
||||
export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs
|
||||
export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model (unconfirmed)
|
||||
export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size
|
||||
@@ -584,6 +584,7 @@
|
||||
block: |
|
||||
{% for llm in llm_installs %}
|
||||
{% if llm.model in llm_models.stdout %}
|
||||
{% if "chat" in llm.purpose %}
|
||||
|
||||
[providers.models.ollama.{{ llm.provider }}]
|
||||
model = "{{ llm.model }}"
|
||||
@@ -591,6 +592,7 @@
|
||||
context_window = {{ [llm_context, llm.context] | min | int }}
|
||||
max_tokens = {{ [llm_tokens, (llm.context / 4), 16000] | min | int }}
|
||||
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
state: present
|
||||
@@ -615,6 +617,7 @@
|
||||
|
||||
{% for llm in llm_installs %}
|
||||
{% if llm.model in llm_models.stdout %}
|
||||
{% if "code" in llm.purpose %}
|
||||
|
||||
- name: {{ llm.provider }}_chat-edit
|
||||
provider: ollama
|
||||
@@ -632,6 +635,7 @@
|
||||
roles:
|
||||
- autocomplete
|
||||
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
mode: "0644"
|
||||
|
||||
Reference in New Issue
Block a user