From 451e76559de0728e439a2c7c8b5b302be9476248 Mon Sep 17 00:00:00 2001 From: Hyperling Date: Fri, 24 Dec 2021 14:55:21 -0600 Subject: [PATCH] Add Git Server To Keep `ansible-pull` Traffic Local (#21) * Fedora Minimal does not come with tar. Need installed for Telegraf. * First attempt at automating HTTP git server setup. * Add cron jobs to keep projects up to date. * Add new git playbook to server. * Add new server variables for git. Put all variables in a dictionary. * Put variables into run file. * Fix the loop variables to be dictionaries, not jinja. * Upgrade nanominer. * Attempt to fix templating error. * Attempt to fix templating error, but in the right spot this time. :) * Attempt another fix for building list of dictionaries. * Change strings to dicts. * Add quotes for jinja variables. * Remove extra curly brackets. * Fix the rest of the file's dictionaries now that General works. * Remove testing code. * Variablize ansible repo. Begin watching personal repos rather than pinging GitHub. * Fix variables to append, not replace. * Fix variable names. * Try to prevent variables from being strings, without getting formatting error. * Try to fix variables, again. * Fixed git.yml. Found a way to test locally and all is working now. --- facts/general/system.yml | 7 + facts/miner/system.yml | 2 +- local.yml | 59 +------ tasks/general/acct_mgmt/provision_config.yml | 122 ++++++++----- tasks/general/cron/ansible.yml | 2 +- tasks/general/scripts/root.yml | 2 +- tasks/general/scripts/user.yml | 2 +- tasks/general/software/packages.yml | 1 + tasks/general/start.yml | 4 + tasks/server/software/git.yml | 171 +++++++++++++++++++ 10 files changed, 267 insertions(+), 105 deletions(-) create mode 100644 tasks/server/software/git.yml diff --git a/facts/general/system.yml b/facts/general/system.yml index d76211a..1afac8e 100644 --- a/facts/general/system.yml +++ b/facts/general/system.yml @@ -10,6 +10,13 @@ st_install_dir: /usr/local/src/st global_bin: /usr/local/bin x_desktops: /usr/share/xsessions + repo_github: https://github.com/Hyperling/ansible.git + repo_local: https://git.hyperling.com/ansible.git + +- name: General | Facts | System | Dev + set_fact: + repo_local: https://git.hyperling.com/ansible-dev.git + when: branch == "dev" - name: General | Facts | System | Linux diff --git a/facts/miner/system.yml b/facts/miner/system.yml index 706a76d..ad29f5e 100644 --- a/facts/miner/system.yml +++ b/facts/miner/system.yml @@ -33,7 +33,7 @@ - name: Miner | Facts | System | nanominer 1 set_fact: nanominer_home: "{{ user_mfn.home }}/nanominer" - nanominer_version: "3.4.3" # Download link: https://github.com/nanopool/nanominer/releases + nanominer_version: "3.5.2" # Download link: https://github.com/nanopool/nanominer/releases nanominer_script: "{{ user_mfn.home }}/nanominer.sh" - name: Miner | Facts | System | nanominer 2 diff --git a/local.yml b/local.yml index 352bbdb..8ddbb1b 100644 --- a/local.yml +++ b/local.yml @@ -47,62 +47,6 @@ when: pentesting == true - - name: FORTESTING | Outside Block | Output Block Variables - debug: - var: "{{ item }}" - loop: - - workstation - - miner - - server - - - name: FORTESTING | Block - block: - - - name: FORTESTING | Inside Block | Output Block Variables - debug: - var: "{{ item }}" - loop: - - workstation - - miner - - server - - - name: FORTESTING | Inside Block | W Test 1 # Did not run on true. :( - shell: echo "test" - when: workstation == "true" - - - name: FORTESTING | Inside Block | W Test 2 # Did run on true! :) - shell: echo "test" - when: workstation == true - - - name: FORTESTING | Inside Block | W Test 3 # Did run on true! :) - shell: echo "test" - when: workstation - - - name: FORTESTING | Inside Block | M Test 1 # Skipped on false. :) - shell: echo "test" - when: miner == "true" - - - name: FORTESTING | Inside Block | M Test 2 # Skipped on false. :) - shell: echo "test" - when: miner == true - - - name: FORTESTING | Inside Block | M Test 3 # Skipped on false. :) - shell: echo "test" - when: miner - - - name: FORTESTING | Inside Block | S Test 1 # Skipped on yolo. :) - shell: echo "test" - when: server == "true" - - - name: FORTESTING | Inside Block | S Test 2 # Skipped on yolo. :) - shell: echo "test" - when: server == true - - - name: FORTESTING | Inside Block | S Test 3 # Ran on yolo. :( - shell: echo "test" - when: server - - ####### Workstations ####### @@ -228,6 +172,9 @@ - include: tasks/server/software/gitlab.yml when: gitlab and ansible_pkg_mgr in ("apt", "dnf") and ansible_distribution not in ("Fedora") + - include: tasks/server/software/git.yml + when: git and ansible_pkg_mgr in ("apt") + when: server == true diff --git a/tasks/general/acct_mgmt/provision_config.yml b/tasks/general/acct_mgmt/provision_config.yml index 7c43675..9ad9c22 100644 --- a/tasks/general/acct_mgmt/provision_config.yml +++ b/tasks/general/acct_mgmt/provision_config.yml @@ -4,6 +4,7 @@ - name: General | Account Management | Provisioning Configuration | Variables 1 set_fact: prov_dir: "/usr/local/etc/hyperling-scm" + provision_variables: [] - name: General | Account Management | Provisioning Configuration | Variables 2 set_fact: @@ -54,15 +55,15 @@ branch: "{{ lookup('ini', 'branch file={{gen_file}} default=main') }}" pentesting: "{{ lookup('ini', 'pentesting file={{gen_file}} default=false') }}" -- name: General | Account Management | Provisioning Configuration | General | Print - debug: - var: "{{ item }}" +- name: General | Account Management | Provisioning Configuration | General | List + set_fact: + provision_variables: "{{ provision_variables | combine(item) }}" loop: - - provision - - user - - user_desc - - branch - - pentesting + - { 'provision': "{{ provision }}" } + - { 'user': "{{ user }}" } + - { 'user_desc': "{{ user_desc }}" } + - { 'branch': "{{ branch }}" } + - { 'pentesting': "{{ pentesting }}" } ## Workstation ## @@ -102,16 +103,16 @@ rdp: "{{ lookup('ini', 'rdp file={{wrk_file}} default=false') }}" vnc: "{{ lookup('ini', 'vnc file={{wrk_file}} default=false') }}" -- name: General | Account Management | Provisioning Configuration | Workstation | Print - debug: - var: "{{ item }}" +- name: General | Account Management | Provisioning Configuration | Workstation | List + set_fact: + provision_variables: "{{ provision_variables | combine(item) }}" loop: - - workstation - - coding - - editing - - gaming - - rdp - - vnc + - { 'workstation': "{{ workstation }}" } + - { 'coding': "{{ coding }}" } + - { 'editing': "{{ editing }}" } + - { 'gaming': "{{ gaming }}" } + - { 'rdp': "{{ rdp }}" } + - { 'vnc': "{{ vnc }}" } ## Miner ## @@ -176,22 +177,22 @@ nvidia: "{{ lookup('ini', 'nvidia file={{mnr_file}} default=false') }}" xmr_stak_cpu: "{{ lookup('ini', 'xmr_stak_cpu file={{mnr_file}} default=false') }}" -- name: General | Account Management | Provisioning Configuration | Miner | Print - debug: - var: "{{ item }}" +- name: General | Account Management | Provisioning Configuration | Miner | List + set_fact: + provision_variables: "{{ provision_variables | combine(item) }}" loop: - - miner - - amdgpu - - nanominer - - nanominer_cpu - - nanominer_cpu_pool - - nanominer_gpu - - nanominer_gpus - - nanominer_gpu_pool - - eth_minhashrate - - ethminer - - nvidia - - xmr_stak_cpu + - { 'miner': "{{ miner }}" } + - { 'amdgpu': "{{ amdgpu }}" } + - { 'nanominer': "{{ nanominer }}" } + - { 'nanominer_cpu': "{{ nanominer_cpu }}" } + - { 'nanominer_cpu_pool': "{{ nanominer_cpu_pool }}" } + - { 'nanominer_gpu': "{{ nanominer_gpu }}" } + - { 'nanominer_gpus': "{{ nanominer_gpus }}" } + - { 'nanominer_gpu_pool': "{{ nanominer_gpu_pool }}" } + - { 'eth_minhashrate': "{{ eth_minhashrate }}" } + - { 'ethminer': "{{ ethminer }}" } + - { 'nvidia': "{{ nvidia }}" } + - { 'xmr_stak_cpu': "{{ xmr_stak_cpu }}" } ## Server ## @@ -227,6 +228,20 @@ ; Regardless of whether you choose http or https, the Certbot config will be skipped since the maintainer uses a reverse proxy. ; Example: https://gitlab ; + ; git : Set to true for this server to be configured as an HTTP git server. + ; + ; git_name : Name(s) of project(s) that the Git server should host. + ; Example: ansible#ansible-dev + ; + ; git_branch : Branch(es) of project(s) corresponding with git_name. + ; Example: main#dev + ; + ; git_url : URL(s) of project(s) corresponding with git_name. + ; Example: https://github.com/Hyperling/ansible#https://github.com/Hyperling/ansible + ; + ; git_sep : Separator for git variables above. + ; Example: # + ; [global] marker: '; {mark} MANAGED BY ANSIBLE | Server Config' state: present @@ -245,21 +260,31 @@ hugo: "{{ lookup('ini', 'hugo file={{srv_file}} default=false') }}" gitlab: "{{ lookup('ini', 'gitlab file={{srv_file}} default=false') }}" gitlab_url: "{{ lookup('ini', 'gitlab_url file={{srv_file}} default=false') }}" + git: "{{ lookup('ini', 'git file={{srv_file}} default=false') }}" + git_name: "{{ lookup('ini', 'git_name file={{srv_file}} default=false') }}" + git_branch: "{{ lookup('ini', 'git_branch file={{srv_file}} default=false') }}" + git_url: "{{ lookup('ini', 'git_url file={{srv_file}} default=false') }}" + git_sep: "{{ lookup('ini', 'git_sep file={{srv_file}} default=false') }}" -- name: General | Account Management | Provisioning Configuration | Server | Print - debug: - var: "{{ item }}" +- name: General | Account Management | Provisioning Configuration | Server | List + set_fact: + provision_variables: "{{ provision_variables | combine(item) }}" loop: - - server - - domain - - onlyoffice - - grafana - - influxdb1 - - influxdb2 - - certbot - - hugo - - gitlab - - gitlab_url + - { 'server': "{{ server }}" } + - { 'domain': "{{ domain }}" } + - { 'onlyoffice': "{{ onlyoffice }}" } + - { 'grafana': "{{ grafana }}" } + - { 'influxdb1': "{{ influxdb1 }}" } + - { 'influxdb2': "{{ influxdb2 }}" } + - { 'certbot': "{{ certbot }}" } + - { 'hugo': "{{ hugo }}" } + - { 'gitlab': "{{ gitlab }}" } + - { 'gitlab_url': "{{ gitlab_url }}" } + - { 'git': "{{ git }}" } + - { 'git_name': "{{ git_name }}" } + - { 'git_branch': "{{ git_branch }}" } + - { 'git_url': "{{ git_url }}" } + - { 'git_sep': "{{ git_sep }}" } ## Exit if not turned on yet ## @@ -267,3 +292,10 @@ - name: General | Account Management | Provisioning Configuration | Exit Check shell: echo "Please configure the config files in {{ prov_dir }} then try again." && exit 1 when: provision != true + + +## Display values ## + +- name: General | Account Management | Provisioning Configuration | Print + debug: + var: provision_variables diff --git a/tasks/general/cron/ansible.yml b/tasks/general/cron/ansible.yml index 33a8e92..5366a36 100644 --- a/tasks/general/cron/ansible.yml +++ b/tasks/general/cron/ansible.yml @@ -6,7 +6,7 @@ user: ansible name: "Ansible Sync" minute: "*/30" - job: "sudo {{ ansible_pull_exec.stdout }} -o -U https://github.com/Hyperling/ansible.git --checkout {{ branch }}" + job: "sudo {{ ansible_pull_exec.stdout }} -o -U {{ repo_local }} --checkout {{ branch }}" state: present disabled: no diff --git a/tasks/general/scripts/root.yml b/tasks/general/scripts/root.yml index 7fc695a..4f7f2ec 100644 --- a/tasks/general/scripts/root.yml +++ b/tasks/general/scripts/root.yml @@ -56,7 +56,7 @@ ## Main ## push.sh - time {{ ansible_pull_exec.stdout }} -U https://github.com/Hyperling/ansible.git --checkout $branch + time {{ ansible_pull_exec.stdout }} -U {{ repo_github }} --checkout $branch marker: '{mark}' marker_begin: "#!{{ bash_exec.stdout }}" diff --git a/tasks/general/scripts/user.yml b/tasks/general/scripts/user.yml index 843f891..6057ef8 100644 --- a/tasks/general/scripts/user.yml +++ b/tasks/general/scripts/user.yml @@ -56,7 +56,7 @@ ## Main ## push.sh - time sudo {{ ansible_pull_exec.stdout }} -U https://github.com/Hyperling/ansible.git --checkout $branch + time sudo {{ ansible_pull_exec.stdout }} -U {{ repo_github }} --checkout $branch marker: '{mark}' marker_begin: "#!{{ bash_exec.stdout }}" diff --git a/tasks/general/software/packages.yml b/tasks/general/software/packages.yml index 77f1c96..02f5f24 100644 --- a/tasks/general/software/packages.yml +++ b/tasks/general/software/packages.yml @@ -48,6 +48,7 @@ - "{{ locate }}" - zsh - "{{ opensshd }}" + - tar state: present - name: General | Software | Services | Install CROND (Looking at you, Fedora) diff --git a/tasks/general/start.yml b/tasks/general/start.yml index 20f45dc..09f0cf5 100644 --- a/tasks/general/start.yml +++ b/tasks/general/start.yml @@ -6,6 +6,10 @@ shell: "echo 'START' > {{ ansible_pull_report }}" when: leet_drive_details.stat.exists +- name: General | Start | Ansible Pull Report | Add Provision Variables + shell: "echo '{{ provision_variables }}' >> {{ ansible_pull_report }}" + when: leet_drive_details.stat.exists + - name: General | Start | Ansible Pull Report | Make Viewable (SSHFS) file: path: "{{ ansible_pull_report }}" diff --git a/tasks/server/software/git.yml b/tasks/server/software/git.yml new file mode 100644 index 0000000..85e49ae --- /dev/null +++ b/tasks/server/software/git.yml @@ -0,0 +1,171 @@ +--- +# Webserver to replace calling GitHub for projects. +# https://www.howtoforge.com/how-to-install-http-git-server-with-nginx-on-debian-11/ + +## Variables ## + +- name: Server | Software | Git | Check Config Variables + debug: + var: "{{ item }}" + loop: + - git_name + - git_branch + - git_url + - git_sep + +- name: Server | Software | Git | Split Config Variables To Array + set_fact: + git_names: "{{ git_name.split(git_sep) }}" + git_branches: "{{ git_branch.split(git_sep) }}" + git_urls: "{{ git_url.split(git_sep) }}" + git_projects: [] + +- name: Server | Software | Git | Check Arrays + debug: + var: "{{ item }}" + loop: + - git_names + - git_branches + - git_urls + +- name: Server | Software | Git | Build Dictionary + set_fact: + git_projects: "{{ git_projects + [{ 'name': git_names[item], 'branch': git_branches[item], 'url': git_urls[item] }] }}" + loop: "{{ range(0, git_names|length) | list }}" + +- name: Server | Software | Git | Display Dictionary + debug: + var: git_projects + +- name: Server | Software | Git | Variables 1 + set_fact: + git_web_root: /var/www/html/git + git_config_file: /etc/nginx/conf.d/git.conf + git_nginx_user: www-data + +- name: Server | Software | Git | Variables 2 + set_fact: + git_cron_commands: "git pull --rebase && cd {{ git_web_root }}/ && chown -R {{ git_nginx_user }} . && chmod -R 775 ." + + +## Pre-reqs ## + +- name: Server | Software | Git | Install + package: + name: + - nginx + - git + - fcgiwrap + - apache2-utils + - unzip + state: present + when: ansible_pkg_mgr == "apt" + + +## Repositories ## + +- name: Server | Software | Git | Delete + file: + path: "{{ git_web_root }}" + state: absent + +- name: Server | Software | Git | Download + git: + repo: "{{ item.url }}" + dest: "{{ git_web_root }}/{{ item.name }}.git" + version: "{{ item.branch }}" + clone: yes + force: yes + #ignore_errors: yes + loop: "{{ git_projects }}" + +- name: Server | Software | Git | Permissions + file: + path: "{{ git_web_root }}/{{ item.name }}.git" + state: directory + mode: '755' + owner: "{{ git_nginx_user }}" + group: "{{ git_nginx_user }}" + recurse: yes + loop: "{{ git_projects }}" + + +## NGINX ## + +- name: Server | Software | Git | Index + blockinfile: + path: "{{ git_web_root }}/index.html" + block: | + {{ item.name }}.git is cloned from {{ item.url }} using branch {{ item.branch }}. + state: present + mode: '755' + owner: "{{ git_nginx_user }}" + group: "{{ git_nginx_user }}" + create: yes + backup: yes + marker: "" + loop: "{{ git_projects }}" + +- name: Server | Software | Git | Config + blockinfile: + path: "{{ git_config_file }}" + block: | + server { + listen 80; + + root {{ git_web_root }}; + + # Add index.php to the list if you are using PHP + index index.html; + #autoindex on; + + server_name {{ ansible_hostname }}.{{ domain }}; + + location / { + # First attempt to serve request as file, then + # as directory, then fall back to displaying a 404. + try_files $uri $uri/ =404; + } + + location ~ (/.*) { + client_max_body_size 0; + #auth_basic "Hyperling's Git Login"; + #auth_basic_user_file "{{ git_web_root }}/htpasswd"; + include /etc/nginx/fastcgi_params; + fastcgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend; + fastcgi_param GIT_HTTP_EXPORT_ALL ""; + fastcgi_param GIT_PROJECT_ROOT {{ git_web_root }}; + fastcgi_param REMOTE_USER $remote_user; + fastcgi_param PATH_INFO $1; + fastcgi_pass unix:/var/run/fcgiwrap.socket; + } + } + state: present + mode: '755' + create: yes + backup: yes + marker: "# {mark} Managed By Ansible Git Server Playbook -->" + +- name: Server | Software | Git | Service + service: + name: nginx + pattern: nginx + enabled: yes + state: restarted + + +## Cron ## + +- name: Server | Software | Git | Cron | Hourly + ansible.builtin.cron: + name: "{{ item.name }} hourly" + special_time: hourly + job: "cd {{ git_web_root }}/{{ item.name }}.git && {{ git_cron_commands }}" + loop: "{{ git_projects }}" + +- name: Server | Software | Git | Cron | Reboot + ansible.builtin.cron: + name: "{{ item.name }} reboot" + special_time: reboot + job: "cd {{ git_web_root }}/{{ item.name }}.git && {{ git_cron_commands }}" + loop: "{{ git_projects }}"