diff --git a/tasks/workstation/linux/software/llm.yml b/tasks/workstation/linux/software/llm.yml index d44b0a5..b2a1c09 100644 --- a/tasks/workstation/linux/software/llm.yml +++ b/tasks/workstation/linux/software/llm.yml @@ -70,21 +70,8 @@ # Conversely: # - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000. llm_size_coef: 1.3 - # Use the MB of RAM as the context, with a maximum to prevent swapping. - #llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 34543] | min | int }}" - # Let's just be consistent for now, only having 8000 on the Macbook was not usable. - llm_context: 64000 - runit_finish_sleep: | - #!/bin/sh - exec 2>&1 - # $1 is the exit code, $2 is the exit status byte (signal number if killed) - echo "FINISH: Service returned with exit code: $1 and status: $2" - - sleep=30 - if [[ "$1" != 0 ]]; then - echo "FINISH: Sleeping for $sleep seconds due to error..." - sleep $sleep - fi + # Base the context on MB of RAM, with a minumum 64k for decent thinking. + llm_context: "{{ [(ansible_memtotal_mb * 4) | round(-3, method='ceil'), 64000] | max | int }}" - name: Workstation | Linux | Software | LLMs | Facts 02 ansible.builtin.set_fact: @@ -93,7 +80,7 @@ zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}" zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}" # Amount of output tokens allowed per response. Half of context is ok. - llm_tokens: "{{ (llm_context / 4) | int }}" + llm_tokens: "{{ (llm_context / 8) | int }}" - name: Workstation | Linux | Software | LLMs | Facts 03 ansible.builtin.set_fact: @@ -115,7 +102,6 @@ llm_removals: ##### QWEN ##### - { "purpose": ["chat"], "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } # succeeded - - { "purpose": ["chat", "code"], "model": "qwen3.6:35b", "provider": "qwen_36_0350_moe", "size_mb": "{{ llm_size_coef * 23000 }}", "context": 262144 } # succeeded, not dense so not recommended ##### Mistral ##### - { "purpose": ["code"], "model": "devstral:24b", "provider": "mistral_dev10_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 128000 } # succeeded # https://ollama.com/library/mistral-small : Last Updated 2025-01-30, succeeded @@ -133,7 +119,8 @@ - { "purpose": ["code", "chat", "lang"], "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 } # https://ollama.com/library/qwen3.6 : Last Updated 2026-10-13 (yesterday as of this commit!!) # Has been succeeded but dense and does not overthink so still worthy - - { "purpose": ["code", "chat", "lang"], "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 } + - { "purpose": ["code", "chat", "lang"], "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 } + - { "purpose": ["chat", "code", "lang"], "model": "qwen3.6:35b", "provider": "qwen_36_0350_moe", "size_mb": "{{ llm_size_coef * 23000 }}", "context": 262144 } # succeeded, not dense so not recommended # https://ollama.com/library/qwen3.5 : Last Updated 2026-10-13 (yesterday as of this commit!!) ### QWEN 3.5 - Good for generic usage and a little coding. - { "purpose": ["chat"], "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200 }}", "context": 262144 } @@ -269,7 +256,7 @@ export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars) ### Resource Usage ### - export OLLAMA_KEEP_ALIVE=-1 # default is 5m + export OLLAMA_KEEP_ALIVE=2h # default is 5m export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model (unconfirmed) export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size @@ -918,17 +905,10 @@ ,"envKey": "OLLAMA_API_KEY" ,"id": "{{ llm.model }}" ,"generationConfig": { - "reasoning": { - "effort": "medium" - }, - "timeout": 600000, - "streamIdleTimeoutMs": 600000, - "contextWindowSize": {{ [llm.context] | min | int }}, - "maxRetries": 3, - "samplingParams": { - "max_tokens": {{ [(llm.context / 8), 16000] | min | int }} - ,"temperature": 0.7 - } + "timeout": 600000 + ,"streamIdleTimeoutMs": 600000 + ,"contextWindowSize": {{ [llm.context] | min | int }} + ,"maxRetries": 3 } }{{ ',' if not loop.last else '' }} @@ -952,7 +932,14 @@ #- "{{ user_user.home }}/Code/.qwen/settings.json" when: qwen_code_enabled - # "top_p": 0.8 + #"reasoning": { + # "effort": "medium" + #}, + #"samplingParams": { + # "max_tokens": {{ [(llm.context / 8), 16000] | min | int }} + # ,"temperature": 0.7 + # ,"top_p": 0.8 + #} - name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config | Validate ansible.builtin.shell: |