diff --git a/tasks/workstation/linux/software/llm.yml b/tasks/workstation/linux/software/llm.yml index 9574514..e633acc 100644 --- a/tasks/workstation/linux/software/llm.yml +++ b/tasks/workstation/linux/software/llm.yml @@ -62,8 +62,8 @@ # Conversely: # - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000. llm_size_coef: 1.3 - # Take the GB of RAM and use it as the MB of context. - llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 32000] | max | int }}" + # Use the MB of RAM as the context, with a maximum to prevent swapping. + llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 32000] | min | int }}" - name: Workstation | Linux | Software | LLMs | Facts 02 ansible.builtin.set_fact: @@ -591,7 +591,7 @@ [providers.models.ollama.{{ llm.provider }}] model = "{{ llm.model }}" timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though. - context_window = {{ [llm_context, llm.context, 24000] | min | int }} + context_window = {{ [llm_context, llm.context, 16000] | min | int }} max_tokens = {{ [llm_tokens, (llm.context / 4), 4000] | min | int }} {% endif %} @@ -625,14 +625,14 @@ provider: ollama model: {{ llm.model }} defaultCompletionOptions: - contextLength: {{ [llm_context, llm.context, 32000] | min | int }} + contextLength: {{ [llm_context, llm.context, 24000] | min | int }} maxTokens: {{ [llm_tokens, (llm.context / 4), 8000] | min | int }} - name: {{ llm.provider }}_autocomplete provider: ollama model: {{ llm.model }} defaultCompletionOptions: - contextLength: {{ [llm_context, llm.context, 16000] | min | int }} + contextLength: {{ [llm_context, llm.context, 12000] | min | int }} maxTokens: {{ [llm_tokens, (llm.context / 4), 2000] | min | int }} roles: - autocomplete