me/env-ansible
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm, increase context as to not hinder models from achieving their intended abilities. Keep max_tokens limited where applicable.

Browse Source
This commit is contained in:
me
2026-10-16 09:23:58 -07:00
parent 2706011fa0
commit 0a0b85e66d
1 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tasks/workstation/linux/software/llm.yml
+8 -8
View File
@@ -73,7 +73,7 @@
# Use the MB of RAM as the context, with a maximum to prevent swapping. # Use the MB of RAM as the context, with a maximum to prevent swapping.
#llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 34543] | min | int }}" #llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 34543] | min | int }}"
# Let's just be consistent for now, only having 8000 on the Macbook was not usable. # Let's just be consistent for now, only having 8000 on the Macbook was not usable.
llm_context: 32000 llm_context: 64000
runit_finish_sleep: | runit_finish_sleep: |
#!/bin/sh #!/bin/sh
exec 2>&1 exec 2>&1
@@ -835,8 +835,8 @@
[providers.models.ollama.{{ llm.provider }}] [providers.models.ollama.{{ llm.provider }}]
model = "{{ llm.model }}" model = "{{ llm.model }}"
timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though. timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though.
context_window = {{ [llm_context, llm.context, 16000] | min | int }} context_window = {{ [llm.context] | min | int }}
max_tokens = {{ [llm_tokens, (llm.context / 4), 4000] | min | int }} max_tokens = {{ [(llm.context / 8)] | min | int }}
{% endif %} {% endif %}
{% endif %} {% endif %}
@@ -869,15 +869,15 @@
provider: ollama provider: ollama
model: {{ llm.model }} model: {{ llm.model }}
defaultCompletionOptions: defaultCompletionOptions:
contextLength: {{ [llm_context, llm.context, 32000] | min | int }} contextLength: {{ [llm.context] | min | int }}
maxTokens: {{ [llm_tokens, (llm.context / 4), 8000] | min | int }} maxTokens: {{ [16000] | min | int }}
- name: {{ llm.provider }}_AUTO - name: {{ llm.provider }}_AUTO
provider: ollama provider: ollama
model: {{ llm.model }} model: {{ llm.model }}
defaultCompletionOptions: defaultCompletionOptions:
contextLength: {{ [llm_context, llm.context, 32000] | min | int }} contextLength: {{ [llm.context] | min | int }}
maxTokens: {{ [llm_tokens, (llm.context / 4), 2000] | min | int }} maxTokens: {{ [4000] | min | int }}
roles: roles:
- autocomplete - autocomplete
@@ -918,7 +918,7 @@
}, },
"timeout": 600000, "timeout": 600000,
"streamIdleTimeoutMs": 600000, "streamIdleTimeoutMs": 600000,
"contextWindowSize": 32000, "contextWindowSize": {{ llm.context }},
"maxRetries": 3, "maxRetries": 3,
"samplingParams": { "samplingParams": {
"max_tokens": 8000 "max_tokens": 8000