llm, adjust some more parameters and try reducing complexity of qwen config a bit to see if that's why it's stopped working decently, move runit script to system..

2026-10-17 08:18:18 -07:00
parent c709930bf0
commit be53012c48
1 changed files with 18 additions and 31 deletions
@@ -70,21 +70,8 @@
        # Conversely:
        #   - qwen3.5:4b would be 3.3GB * 1.3 = 4290, rather than 4000.
        llm_size_coef: 1.3
-        # Use the MB of RAM as the context, with a maximum to prevent swapping.
-        #llm_context: "{{ [(ansible_memtotal_mb | round(-3, method='ceil')), 34543] | min | int }}"
-        # Let's just be consistent for now, only having 8000 on the Macbook was not usable.
-        llm_context: 64000
-        runit_finish_sleep: |
-          #!/bin/sh
-          exec 2>&1
-          # $1 is the exit code, $2 is the exit status byte (signal number if killed)
-          echo "FINISH: Service returned with exit code: $1 and status: $2"
-
-          sleep=30
-          if [[ "$1" != 0 ]]; then
-            echo "FINISH: Sleeping for $sleep seconds due to error..."
-            sleep $sleep
-          fi
+        # Base the context on MB of RAM, with a minumum 64k for decent thinking.
+        llm_context: "{{ [(ansible_memtotal_mb * 4) | round(-3, method='ceil'), 64000] | max | int }}"

    - name: Workstation | Linux | Software | LLMs | Facts 02
      ansible.builtin.set_fact:
@@ -93,7 +80,7 @@
        zeroclaw_runit_dir: "{{ runit_script_dir }}/{{ zeroclaw }}"
        zeroclaw_runit_log_dest: "{{ global_log }}/{{ zeroclaw }}"
        # Amount of output tokens allowed per response. Half of context is ok.
-        llm_tokens: "{{ (llm_context / 4) | int }}"
+        llm_tokens: "{{ (llm_context / 8) | int }}"

    - name: Workstation | Linux | Software | LLMs | Facts 03
      ansible.builtin.set_fact:
@@ -115,7 +102,6 @@
        llm_removals:
          ##### QWEN #####
          - { "purpose": ["chat"], "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000, "context": 123456 } # succeeded
-          - { "purpose": ["chat", "code"], "model": "qwen3.6:35b", "provider": "qwen_36_0350_moe", "size_mb": "{{ llm_size_coef * 23000 }}", "context": 262144 } # succeeded, not dense so not recommended
          ##### Mistral #####
          - { "purpose": ["code"], "model": "devstral:24b", "provider": "mistral_dev10_0240", "size_mb": "{{ llm_size_coef * 14000 }}", "context": 128000 } # succeeded
            # https://ollama.com/library/mistral-small : Last Updated 2025-01-30, succeeded
@@ -133,7 +119,8 @@
          - { "purpose": ["code", "chat", "lang"], "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 }
            # https://ollama.com/library/qwen3.6 : Last Updated 2026-10-13 (yesterday as of this commit!!)
            # Has been succeeded but dense and does not overthink so still worthy
-          - { "purpose": ["code", "chat", "lang"], "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 }
+          - { "purpose": ["code", "chat", "lang"], "model": "qwen3.6:27b", "provider": "qwen_36_0270",     "size_mb": "{{ llm_size_coef * 18000 }}", "context": 262144 }
+          - { "purpose": ["chat", "code", "lang"], "model": "qwen3.6:35b", "provider": "qwen_36_0350_moe", "size_mb": "{{ llm_size_coef * 23000 }}", "context": 262144 } # succeeded, not dense so not recommended
            # https://ollama.com/library/qwen3.5 : Last Updated 2026-10-13 (yesterday as of this commit!!)
            ### QWEN 3.5 - Good for generic usage and a little coding.
          - { "purpose": ["chat"], "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef *  1200 }}", "context": 262144 }
@@ -269,7 +256,7 @@
              export OLLAMA_REQUEST_TIMEOUT=3600 # default 300 (unconfirmed, not in the list of --help vars)

              ### Resource Usage ###
-              export OLLAMA_KEEP_ALIVE=-1 # default is 5m
+              export OLLAMA_KEEP_ALIVE=2h # default is 5m
              export OLLAMA_MAX_LOADED_MODELS={{ llm_concurrency }} # default is 3 if using CPUs, or 3 * NumGPUs
              export OLLAMA_NUM_PARALLEL={{ llm_parallel }} # default is 1 and applies per-model (unconfirmed)
              export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size
@@ -918,17 +905,10 @@
                  ,"envKey": "OLLAMA_API_KEY"
                  ,"id": "{{ llm.model }}"
                  ,"generationConfig": {
-                    "reasoning": {
-                      "effort": "medium"
-                    },
-                    "timeout": 600000,
-                    "streamIdleTimeoutMs": 600000,
-                    "contextWindowSize": {{ [llm.context] | min | int }},
-                    "maxRetries": 3,
-                    "samplingParams": {
-                      "max_tokens": {{ [(llm.context / 8), 16000] | min | int }}
-                      ,"temperature": 0.7
-                    }
+                    "timeout": 600000
+                    ,"streamIdleTimeoutMs": 600000
+                    ,"contextWindowSize": {{ [llm.context] | min | int }}
+                    ,"maxRetries": 3
                  }
                }{{ ',' if not loop.last else '' }}

@@ -952,7 +932,14 @@
        #- "{{ user_user.home }}/Code/.qwen/settings.json"
      when: qwen_code_enabled

-                      # "top_p": 0.8
+                    #"reasoning": {
+                    #  "effort": "medium"
+                    #},
+                    #"samplingParams": {
+                    #  "max_tokens": {{ [(llm.context / 8), 16000] | min | int }}
+                      # ,"temperature": 0.7
+                      # ,"top_p": 0.8
+                    #}

    - name: Workstation | Linux | Software | LLMs | Models | qwen-code | Config | Validate
      ansible.builtin.shell: |