llm, increase context as to not hinder models from achieving their intended abilities. Keep max_tokens limited where applicable.

tasks/workstation/linux/software/llm.yml

@@ -73,7 +73,7 @@
         # Use the MB of RAM as the context, with a maximum to prevent swapping.
         #llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 34543] | min | int }}"
         # Let's just be consistent for now, only having 8000 on the Macbook was not usable.
-        llm_context: 32000
+        llm_context: 64000
         runit_finish_sleep: |
           #!/bin/sh
           exec 2>&1
@@ -835,8 +835,8 @@
           [providers.models.ollama.{{ llm.provider }}]
           model = "{{ llm.model }}"
           timeout_secs = 3600 # 1 hour, default is 120 (2 minutes), issue happens between 5 and 6 minutes though.
-          context_window = {{ [llm_context, llm.context, 16000] | min | int }}
-          max_tokens = {{ [llm_tokens, (llm.context / 4), 4000] | min | int }}
+          context_window = {{ [llm.context] | min | int }}
+          max_tokens = {{ [(llm.context / 8)] | min | int }}
 
           {% endif %}
           {% endif %}
@@ -869,15 +869,15 @@
               provider: ollama
               model: {{ llm.model }}
               defaultCompletionOptions:
-                contextLength: {{ [llm_context, llm.context, 32000] | min | int }}
-                maxTokens: {{ [llm_tokens, (llm.context / 4), 8000] | min | int }}
+                contextLength: {{ [llm.context] | min | int }}
+                maxTokens: {{ [16000] | min | int }}
 
             - name: {{ llm.provider }}_AUTO
               provider: ollama
               model: {{ llm.model }}
               defaultCompletionOptions:
-                contextLength: {{ [llm_context, llm.context, 32000] | min | int }}
-                maxTokens: {{ [llm_tokens, (llm.context / 4), 2000] | min | int }}
+                contextLength: {{ [llm.context] | min | int }}
+                maxTokens: {{ [4000] | min | int }}
               roles:
                 - autocomplete
 
@@ -918,7 +918,7 @@
                     },
                     "timeout": 600000,
                     "streamIdleTimeoutMs": 600000,
-                    "contextWindowSize": 32000,
+                    "contextWindowSize": {{ llm.context }},
                     "maxRetries": 3,
                     "samplingParams": {
                       "max_tokens": 8000