diff --git a/tasks/workstation/linux/software/llm.yml b/tasks/workstation/linux/software/llm.yml index adffd46..d7e5f66 100644 --- a/tasks/workstation/linux/software/llm.yml +++ b/tasks/workstation/linux/software/llm.yml @@ -32,10 +32,10 @@ zeroclaw: zeroclaw zerocode: zerocode qwen_code: qwen-code - # Good for preventing swap usage. - llm_concurrency: 1 + # Good to keep low to prevent swap usage. + llm_concurrency: "{{ ansible_memtotal_mb / 64000 | round(0, method='ceil') | int }}" # Allow having multiple chats open without them just waiting each other. - llm_parallel: "{{ ansible_processor_vcpus / 4 | round(0, method='ceil') | int }}" + llm_parallel: "{{ ansible_processor_vcpus / 8 | round(0, method='ceil') | int }}" # Most LLMs take a coefficient of about 1.2 to 1.5 of their file size in # RAM, which is usually less than guessing a usage of weight*1000. # Example: @@ -71,31 +71,31 @@ ansible.builtin.set_fact: llm_installs: [] llm_removals: - - { "model": "qwen3:4b", "provider": "qwen_30_0040" } - - { "model": "qwen3.6:27b", "provider": "qwen_36_0270" } - - { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": llm_size_coef * 17000 } + - { "model": "qwen3:4b", "provider": "qwen_30_0040", "size_mb": 4000 } + - { "model": "qwen3.6:27b", "provider": "qwen_36_0270", "size_mb": 27000 } + - { "model": "qwen3.5:27b", "provider": "qwen_35_0270", "size_mb": "{{ llm_size_coef * 17000 }}" } llm_choices: ### QWEN >3.5 - Need to try! - { "model": "qwen3.8:27b", "provider": "qwen_38_0270", "size_mb": 27000 } - { "model": "qwen3.6:35b", "provider": "qwen_36_0350", "size_mb": 35000 } ### QWEN 3.5 - Good for generic usage and a little coding. - - { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": llm_size_coef * 1200 } - - { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": llm_size_coef * 2700 } - - { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": llm_size_coef * 3300 } - - { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": llm_size_coef * 6600 } + - { "model": "qwen3.5:0.8b", "provider": "qwen_35_0008", "size_mb": "{{ llm_size_coef * 1200}}" } + - { "model": "qwen3.5:2b", "provider": "qwen_35_0020", "size_mb": "{{ llm_size_coef * 2700}}" } + - { "model": "qwen3.5:4b", "provider": "qwen_35_0040", "size_mb": "{{ llm_size_coef * 3300}}" } + - { "model": "qwen3.5:9b", "provider": "qwen_35_0090", "size_mb": "{{ llm_size_coef * 6600}}" } # skip 27b and use 35b - - { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": llm_size_coef * 22000 } - - { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": llm_size_coef * 81000 } + - { "model": "qwen3.5:35b", "provider": "qwen_35_0350", "size_mb": "{{ llm_size_coef * 22000 }}" } + - { "model": "qwen3.5:122b", "provider": "qwen_35_1220", "size_mb": "{{ llm_size_coef * 81000 }}" } ### QWEN 3.0 Coder - Supposed to be better than 2.5 for machines that can handle it. - - { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": llm_size_coef * 19000 } - - { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": llm_size_coef * 290000 } + - { "model": "qwen3-coder:30b", "provider": "qwen_30_coder_0300", "size_mb": "{{ llm_size_coef * 19000 }}" } + - { "model": "qwen3-coder:420b", "provider": "qwen_30_coder_4200", "size_mb": "{{ llm_size_coef * 290000 }}" } ### QWEN 2.5 Coder - Ol'reliable! - - { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": llm_size_coef * 398 } - - { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": llm_size_coef * 986 } - - { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": llm_size_coef * 1900 } - - { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": llm_size_coef * 4700 } - - { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": llm_size_coef * 9000 } - - { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": llm_size_coef * 20000 } + - { "model": "qwen2.5-coder:0.5b", "provider": "qwen_25_coder_0005", "size_mb": "{{ llm_size_coef * 398 }}" } + - { "model": "qwen2.5-coder:1.5b", "provider": "qwen_25_coder_0015", "size_mb": "{{ llm_size_coef * 986 }}" } + - { "model": "qwen2.5-coder:3b", "provider": "qwen_25_coder_0030", "size_mb": "{{ llm_size_coef * 1900 }}" } + - { "model": "qwen2.5-coder:7b", "provider": "qwen_25_coder_0070", "size_mb": "{{ llm_size_coef * 4700 }}" } + - { "model": "qwen2.5-coder:14b", "provider": "qwen_25_coder_0140", "size_mb": "{{ llm_size_coef * 9000 }}" } + - { "model": "qwen2.5-coder:32b", "provider": "qwen_25_coder_0320", "size_mb": "{{ llm_size_coef * 20000 }}" } ### Mistral, Ministral, and Devstral - By a French company? - { "model": "devstral-small-2:24b", "provider": "devstral_20_0240_small", "size_mb": 24000 } #- mistral:7b @@ -118,6 +118,18 @@ #- deepseek-r1:1.5b #- deepseek-r1:7b + - name: Workstation | Linux | Software | LLMs | Facts | Merge | Print + ansible.builtin.debug: + var: "{{ item }}" + loop: + - ansible_processor_vcpus + - ansible_memtotal_mb + - llm_concurrency + - llm_parallel + - llm_installs + - llm_removals + - llm_choices + - name: Workstation | Linux | Software | LLMs | Facts | Choices | Sort ansible.builtin.set_fact: llm_installs: >- @@ -129,14 +141,10 @@ ansible.builtin.debug: var: "{{ item }}" loop: - - ansible_processor_vcpus - - ansible_memtotal_mb - - llm_concurrency - - llm_parallel - llm_installs - llm_removals - - name: Workstation | Linux | Software | LLMs | Install + - name: Workstation | Linux | Software | LLMs | Packages | Install ansible.builtin.package: name: - "{{ zeroclaw }}" @@ -171,6 +179,7 @@ - name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Run ansible.builtin.copy: + dest: "{{ ollama_runit_script }}" content: | #!/bin/sh -e # Allow pulling LLMs for use with ZeroClaw. @@ -186,23 +195,23 @@ export OLLAMA_CONTEXT_LENGTH={{ llm_context }} # default is 4096, https://docs.ollama.com/faq#how-can-i-specify-the-context-window-size export OLLAMA_MAX_QUEUE=999 # default is 512 (unconfirmed) exec chpst -u {{ user }} ollama serve - dest: "{{ ollama_runit_script }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log ansible.builtin.copy: + dest: "{{ ollama_runit_log_script }}" content: | #!/bin/sh -e exec svlogd -tt {{ ollama_runit_log_dest }} - dest: "{{ ollama_runit_log_script }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | ollama | Script | Log Config ansible.builtin.copy: - content: | - s10000000 # rotate at 10 MB - n5 # keep 5 old files dest: "{{ ollama_runit_log_config }}" + content: | + s10000000 # rotate at 10 MB + n5 # keep 5 old files + dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file. owner: "{{ user }}" mode: "0755" @@ -229,6 +238,7 @@ - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | File ansible.builtin.copy: + dest: "{{ zeroclaw_runit_script }}" content: | #!/bin/sh -e # Allow using the ZeroClaw frontend. @@ -236,28 +246,28 @@ export HOME={{ user_user.home }} export SHELL={{ bash_exec.stdout }} exec chpst -u {{ user }} zeroclaw daemon -v --log-level info - dest: "{{ zeroclaw_runit_script }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log ansible.builtin.copy: + dest: "{{ zeroclaw_runit_log_script }}" content: | #!/bin/sh -e exec svlogd -tt {{ zeroclaw_runit_log_dest }} - dest: "{{ zeroclaw_runit_log_script }}" mode: "0755" - name: Workstation | Linux | Software | LLMs | runit | zeroclaw | Script | Log Config ansible.builtin.copy: - content: | - s1000000 # rotate at 1 MB - n5 # keep 5 old files dest: "{{ zeroclaw_runit_log_config }}" + content: | + s1000000 # rotate at 1 MB + n5 # keep 5 old files + dateformat %Y-%m-%d_%H-%M-%S # each run gets its own file. mode: "0755" when: ansible_service_mgr == "runit" - - name: Workstation | Linux | Software | LLMs | Enable + - name: Workstation | Linux | Software | LLMs | Services | Enable ansible.builtin.service: name: "{{ item }}" state: started @@ -266,7 +276,7 @@ - "{{ ollama }}" - "{{ zeroclaw }}" - - name: Workstation | Linux | Software | LLMs | Verify + - name: Workstation | Linux | Software | LLMs | Services | Verify ansible.builtin.uri: url: "{{ item }}" status_code: 200 @@ -372,4 +382,33 @@ create: false backup: true + - name: Workstation | Linux | Software | LLMs | Models | Continue | Config + ansible.builtin.copy: + dest: "{{ user_user.home }}/.continue/config.yml" + content: | + # Maintained by `env-ansible/tasks/workstation/linux/software/llm.yml` + name: Main Config + version: 1.0.0 + schema: v1 + models: + mode: "0644" + + - name: Workstation | Linux | Software | LLMs | Models | Continue | Models + ansible.builtin.blockinfile: + path: "{{ user_user.home }}/.continue/config.yml" + marker: "# {mark} ANSIBLE MANAGED BLOCK | llm.yml | Continue" + insertafter: EOF + block: | + {% for llm in llm_installs %} + - name: llm.provider + provider: ollama + model: llm.model + defaultCompletionOptions: + contextLength: {{ llm_context }} + maxTokens: {{ llm_context }} + {% endfor %} + state: present + create: false + backup: true + when: ansible_pkg_mgr == "pacman"