Add Git Server To Keep ansible-pull Traffic Local (#21)

* Fedora Minimal does not come with tar. Need installed for Telegraf.

* First attempt at automating HTTP git server setup.

* Add cron jobs to keep projects up to date.

* Add new git playbook to server.

* Add new server variables for git. Put all variables in a dictionary.

* Put variables into run file.

* Fix the loop variables to be dictionaries, not jinja.

* Upgrade nanominer.

* Attempt to fix templating error.

* Attempt to fix templating error, but in the right spot this time. :)

* Attempt another fix for building list of dictionaries.

* Change strings to dicts.

* Add quotes for jinja variables.

* Remove extra curly brackets.

* Fix the rest of the file's dictionaries now that General works.

* Remove testing code.

* Variablize ansible repo. Begin watching personal repos rather than pinging GitHub.

* Fix variables to append, not replace.

* Fix variable names.

* Try to prevent variables from being strings, without getting formatting error.

* Try to fix variables, again.

* Fixed git.yml. Found a way to test locally and all is working now.

facts/general/system.yml

@@ -10,6 +10,13 @@
     st_install_dir: /usr/local/src/st
     global_bin: /usr/local/bin
     x_desktops: /usr/share/xsessions
+    repo_github: https://github.com/Hyperling/ansible.git
+    repo_local: https://git.hyperling.com/ansible.git
+
+- name: General | Facts | System | Dev
+  set_fact:
+    repo_local: https://git.hyperling.com/ansible-dev.git
+  when: branch == "dev"
 
 
 - name: General | Facts | System | Linux

facts/miner/system.yml

@@ -33,7 +33,7 @@
 - name: Miner | Facts | System | nanominer 1
   set_fact:
     nanominer_home: "{{ user_mfn.home }}/nanominer"
-    nanominer_version: "3.4.3" # Download link: https://github.com/nanopool/nanominer/releases
+    nanominer_version: "3.5.2" # Download link: https://github.com/nanopool/nanominer/releases
     nanominer_script: "{{ user_mfn.home }}/nanominer.sh"
 
 - name: Miner | Facts | System | nanominer 2

local.yml

@@ -47,62 +47,6 @@
       when: pentesting == true
 
 
-    - name: FORTESTING | Outside Block | Output Block Variables
-      debug: 
-        var: "{{ item }}"
-      loop:
-        - workstation
-        - miner
-        - server
-
-    - name: FORTESTING | Block
-      block: 
-      
-      - name: FORTESTING | Inside Block | Output Block Variables
-        debug: 
-          var: "{{ item }}"
-        loop:
-          - workstation
-          - miner
-          - server
-
-      - name: FORTESTING | Inside Block | W Test 1 # Did not run on true. :(
-        shell: echo "test"
-        when: workstation == "true"
-
-      - name: FORTESTING | Inside Block | W Test 2 # Did run on true! :)
-        shell: echo "test"
-        when: workstation == true
-
-      - name: FORTESTING | Inside Block | W Test 3 # Did run on true! :)
-        shell: echo "test"
-        when: workstation
-
-      - name: FORTESTING | Inside Block | M Test 1 # Skipped on false. :)
-        shell: echo "test"
-        when: miner == "true"
-
-      - name: FORTESTING | Inside Block | M Test 2 # Skipped on false. :)
-        shell: echo "test"
-        when: miner == true
-
-      - name: FORTESTING | Inside Block | M Test 3 # Skipped on false. :)
-        shell: echo "test"
-        when: miner
-
-      - name: FORTESTING | Inside Block | S Test 1 # Skipped on yolo. :)
-        shell: echo "test"
-        when: server == "true"
-
-      - name: FORTESTING | Inside Block | S Test 2 # Skipped on yolo. :)
-        shell: echo "test"
-        when: server == true
-
-      - name: FORTESTING | Inside Block | S Test 3 # Ran on yolo. :(
-        shell: echo "test"
-        when: server
-
-
 
     ####### Workstations #######
 
@@ -228,6 +172,9 @@
       - include: tasks/server/software/gitlab.yml
         when: gitlab and ansible_pkg_mgr in ("apt", "dnf") and ansible_distribution not in ("Fedora")
 
+      - include: tasks/server/software/git.yml
+        when: git and ansible_pkg_mgr in ("apt")
+
       when: server == true
 
 

tasks/general/acct_mgmt/provision_config.yml

@@ -4,6 +4,7 @@
 - name: General | Account Management | Provisioning Configuration | Variables 1
   set_fact:
     prov_dir: "/usr/local/etc/hyperling-scm"
+    provision_variables: []
 
 - name: General | Account Management | Provisioning Configuration | Variables 2
   set_fact:
@@ -54,15 +55,15 @@
     branch:     "{{ lookup('ini', 'branch     file={{gen_file}} default=main') }}"
     pentesting: "{{ lookup('ini', 'pentesting file={{gen_file}} default=false') }}"
 
-- name: General | Account Management | Provisioning Configuration | General | Print
+- name: General | Account Management | Provisioning Configuration | General | List
-  debug: 
+  set_fact: 
-    var: "{{ item }}"
+    provision_variables: "{{ provision_variables | combine(item) }}"
   loop:
-    - provision
+    - { 'provision': "{{ provision }}" }
-    - user
+    - { 'user': "{{ user }}" }
-    - user_desc
+    - { 'user_desc': "{{ user_desc }}" }
-    - branch
+    - { 'branch': "{{ branch }}" }
-    - pentesting
+    - { 'pentesting': "{{ pentesting }}" }
 
 
 ## Workstation ##
@@ -102,16 +103,16 @@
     rdp:         "{{ lookup('ini', 'rdp     file={{wrk_file}} default=false') }}"
     vnc:         "{{ lookup('ini', 'vnc     file={{wrk_file}} default=false') }}"
 
-- name: General | Account Management | Provisioning Configuration | Workstation | Print
+- name: General | Account Management | Provisioning Configuration | Workstation | List
-  debug: 
+  set_fact: 
-    var: "{{ item }}"
+    provision_variables: "{{ provision_variables | combine(item) }}"
   loop:
-    - workstation
+    - { 'workstation': "{{ workstation }}" }
-    - coding
+    - { 'coding': "{{ coding }}" }
-    - editing
+    - { 'editing': "{{ editing }}" }
-    - gaming
+    - { 'gaming': "{{ gaming }}" }
-    - rdp
+    - { 'rdp': "{{ rdp }}" }
-    - vnc
+    - { 'vnc': "{{ vnc }}" }
 
 
 ## Miner ##
@@ -176,22 +177,22 @@
     nvidia:             "{{ lookup('ini', 'nvidia             file={{mnr_file}} default=false') }}"
     xmr_stak_cpu:       "{{ lookup('ini', 'xmr_stak_cpu       file={{mnr_file}} default=false') }}"
 
-- name: General | Account Management | Provisioning Configuration | Miner | Print
+- name: General | Account Management | Provisioning Configuration | Miner | List
-  debug: 
+  set_fact: 
-    var: "{{ item }}"
+    provision_variables: "{{ provision_variables | combine(item) }}"
   loop:
-    - miner
+    - { 'miner': "{{ miner }}" }
-    - amdgpu
+    - { 'amdgpu': "{{ amdgpu }}" }
-    - nanominer
+    - { 'nanominer': "{{ nanominer }}" }
-    - nanominer_cpu
+    - { 'nanominer_cpu': "{{ nanominer_cpu }}" }
-    - nanominer_cpu_pool
+    - { 'nanominer_cpu_pool': "{{ nanominer_cpu_pool }}" }
-    - nanominer_gpu
+    - { 'nanominer_gpu': "{{ nanominer_gpu }}" }
-    - nanominer_gpus
+    - { 'nanominer_gpus': "{{ nanominer_gpus }}" }
-    - nanominer_gpu_pool
+    - { 'nanominer_gpu_pool': "{{ nanominer_gpu_pool }}" }
-    - eth_minhashrate
+    - { 'eth_minhashrate': "{{ eth_minhashrate }}" }
-    - ethminer
+    - { 'ethminer': "{{ ethminer }}" }
-    - nvidia
+    - { 'nvidia': "{{ nvidia }}" }
-    - xmr_stak_cpu
+    - { 'xmr_stak_cpu': "{{ xmr_stak_cpu }}" }
 
 
 ## Server ##
@@ -227,6 +228,20 @@
       ;              Regardless of whether you choose http or https, the Certbot config will be skipped since the maintainer uses a reverse proxy.
       ;                Example: https://gitlab
       ;
+      ;        git : Set to true for this server to be configured as an HTTP git server. 
+      ;
+      ;   git_name : Name(s) of project(s) that the Git server should host.
+      ;                Example: ansible#ansible-dev
+      ;
+      ; git_branch : Branch(es) of project(s) corresponding with git_name.
+      ;                Example: main#dev
+      ;
+      ;    git_url : URL(s) of project(s) corresponding with git_name.
+      ;                Example: https://github.com/Hyperling/ansible#https://github.com/Hyperling/ansible
+      ;
+      ;    git_sep : Separator for git variables above.
+      ;                Example: #
+      ;
       [global]
     marker: '; {mark} MANAGED BY ANSIBLE | Server Config'
     state: present
@@ -245,21 +260,31 @@
     hugo:       "{{ lookup('ini', 'hugo       file={{srv_file}} default=false') }}"
     gitlab:     "{{ lookup('ini', 'gitlab     file={{srv_file}} default=false') }}"
     gitlab_url: "{{ lookup('ini', 'gitlab_url file={{srv_file}} default=false') }}"
+    git:        "{{ lookup('ini', 'git        file={{srv_file}} default=false') }}"
+    git_name:   "{{ lookup('ini', 'git_name   file={{srv_file}} default=false') }}"
+    git_branch: "{{ lookup('ini', 'git_branch file={{srv_file}} default=false') }}"
+    git_url:    "{{ lookup('ini', 'git_url    file={{srv_file}} default=false') }}"
+    git_sep:    "{{ lookup('ini', 'git_sep    file={{srv_file}} default=false') }}"
 
-- name: General | Account Management | Provisioning Configuration | Server | Print
+- name: General | Account Management | Provisioning Configuration | Server | List
-  debug: 
+  set_fact: 
-    var: "{{ item }}"
+    provision_variables: "{{ provision_variables | combine(item) }}"
   loop:
-    - server
+    - { 'server': "{{ server }}" }
-    - domain
+    - { 'domain': "{{ domain }}" }
-    - onlyoffice
+    - { 'onlyoffice': "{{ onlyoffice }}" }
-    - grafana
+    - { 'grafana': "{{ grafana }}" }
-    - influxdb1
+    - { 'influxdb1': "{{ influxdb1 }}" }
-    - influxdb2
+    - { 'influxdb2': "{{ influxdb2 }}" }
-    - certbot
+    - { 'certbot': "{{ certbot }}" }
-    - hugo
+    - { 'hugo': "{{ hugo }}" }
-    - gitlab
+    - { 'gitlab': "{{ gitlab }}" }
-    - gitlab_url
+    - { 'gitlab_url': "{{ gitlab_url }}" }
+    - { 'git': "{{ git }}" }
+    - { 'git_name': "{{ git_name }}" }
+    - { 'git_branch': "{{ git_branch }}" }
+    - { 'git_url': "{{ git_url }}" }
+    - { 'git_sep': "{{ git_sep }}" }
 
 
 ## Exit if not turned on yet ##
@@ -267,3 +292,10 @@
 - name: General | Account Management | Provisioning Configuration | Exit Check
   shell: echo "Please configure the config files in {{ prov_dir }} then try again." && exit 1
   when: provision != true
+
+
+## Display values ##
+
+- name: General | Account Management | Provisioning Configuration | Print
+  debug: 
+    var: provision_variables

tasks/general/cron/ansible.yml

@@ -6,7 +6,7 @@
     user: ansible
     name: "Ansible Sync"
     minute: "*/30"
-    job: "sudo {{ ansible_pull_exec.stdout }} -o -U https://github.com/Hyperling/ansible.git --checkout {{ branch }}"
+    job: "sudo {{ ansible_pull_exec.stdout }} -o -U {{ repo_local }} --checkout {{ branch }}"
     state: present
     disabled: no
 

tasks/general/scripts/root.yml

@@ -56,7 +56,7 @@
       ## Main ##
 
       push.sh
-      time {{ ansible_pull_exec.stdout }} -U https://github.com/Hyperling/ansible.git --checkout $branch
+      time {{ ansible_pull_exec.stdout }} -U {{ repo_github }} --checkout $branch
 
     marker: '{mark}'
     marker_begin: "#!{{ bash_exec.stdout }}"

tasks/general/scripts/user.yml

@@ -56,7 +56,7 @@
       ## Main ##
 
       push.sh
-      time sudo {{ ansible_pull_exec.stdout }} -U https://github.com/Hyperling/ansible.git --checkout $branch
+      time sudo {{ ansible_pull_exec.stdout }} -U {{ repo_github }} --checkout $branch
 
     marker: '{mark}'
     marker_begin: "#!{{ bash_exec.stdout }}"

tasks/general/software/packages.yml

@@ -48,6 +48,7 @@
     - "{{ locate }}"
     - zsh
     - "{{ opensshd }}"
+    - tar
     state: present
 
 - name: General | Software | Services | Install CROND (Looking at you, Fedora)

tasks/general/start.yml

@@ -6,6 +6,10 @@
   shell: "echo 'START' > {{ ansible_pull_report }}"
   when: leet_drive_details.stat.exists
 
+- name: General | Start | Ansible Pull Report | Add Provision Variables
+  shell: "echo '{{ provision_variables }}' >> {{ ansible_pull_report }}"
+  when: leet_drive_details.stat.exists
+
 - name: General | Start | Ansible Pull Report | Make Viewable (SSHFS)
   file:
     path: "{{ ansible_pull_report }}"

tasks/server/software/git.yml

@@ -0,0 +1,171 @@
+---
+# Webserver to replace calling GitHub for projects.
+# https://www.howtoforge.com/how-to-install-http-git-server-with-nginx-on-debian-11/
+
+## Variables ##
+
+- name: Server | Software | Git | Check Config Variables
+  debug: 
+    var: "{{ item }}"
+  loop:
+    - git_name
+    - git_branch
+    - git_url
+    - git_sep
+
+- name: Server | Software | Git | Split Config Variables To Array
+  set_fact:
+    git_names: "{{ git_name.split(git_sep) }}"
+    git_branches: "{{ git_branch.split(git_sep) }}"
+    git_urls: "{{ git_url.split(git_sep) }}"
+    git_projects: []
+
+- name: Server | Software | Git | Check Arrays
+  debug: 
+    var: "{{ item }}"
+  loop:
+    - git_names
+    - git_branches
+    - git_urls
+
+- name: Server | Software | Git | Build Dictionary
+  set_fact:
+    git_projects: "{{ git_projects + [{ 'name': git_names[item], 'branch': git_branches[item], 'url': git_urls[item]  }] }}"
+  loop: "{{ range(0, git_names|length) | list }}"
+
+- name: Server | Software | Git | Display Dictionary
+  debug: 
+    var: git_projects
+
+- name: Server | Software | Git | Variables 1
+  set_fact: 
+    git_web_root: /var/www/html/git
+    git_config_file: /etc/nginx/conf.d/git.conf
+    git_nginx_user: www-data
+
+- name: Server | Software | Git | Variables 2
+  set_fact: 
+    git_cron_commands: "git pull --rebase && cd {{ git_web_root }}/ && chown -R {{ git_nginx_user }} . && chmod -R 775 ."
+
+
+## Pre-reqs ##
+
+- name: Server | Software | Git | Install
+  package: 
+    name:
+      - nginx
+      - git
+      - fcgiwrap
+      - apache2-utils
+      - unzip
+    state: present
+  when: ansible_pkg_mgr == "apt"
+
+
+## Repositories ##
+
+- name: Server | Software | Git | Delete
+  file:
+    path: "{{ git_web_root }}"
+    state: absent
+
+- name: Server | Software | Git | Download
+  git: 
+    repo: "{{ item.url }}"
+    dest: "{{ git_web_root }}/{{ item.name }}.git"
+    version: "{{ item.branch }}"
+    clone: yes
+    force: yes
+  #ignore_errors: yes
+  loop: "{{ git_projects }}"
+
+- name: Server | Software | Git | Permissions
+  file:
+    path: "{{ git_web_root }}/{{ item.name }}.git"
+    state: directory
+    mode: '755'
+    owner: "{{ git_nginx_user }}"
+    group: "{{ git_nginx_user }}"
+    recurse: yes
+  loop: "{{ git_projects }}"
+
+
+## NGINX ##
+
+- name: Server | Software | Git | Index
+  blockinfile:
+    path: "{{ git_web_root }}/index.html"
+    block: |
+      {{ item.name }}.git is cloned from {{ item.url }} using branch {{ item.branch }}.
+    state: present
+    mode: '755'
+    owner: "{{ git_nginx_user }}"
+    group: "{{ git_nginx_user }}"
+    create: yes
+    backup: yes
+    marker: "<!-- {mark} {{ item.name }} -->"
+  loop: "{{ git_projects }}"
+
+- name: Server | Software | Git | Config
+  blockinfile:
+    path: "{{ git_config_file }}"
+    block: |
+      server {
+        listen 80;
+      
+        root {{ git_web_root }};
+      
+        # Add index.php to the list if you are using PHP
+        index index.html;
+        #autoindex on;
+      
+        server_name {{ ansible_hostname }}.{{ domain }};
+      
+        location / {
+          # First attempt to serve request as file, then
+          # as directory, then fall back to displaying a 404.
+          try_files $uri $uri/ =404;
+        }
+      
+        location ~ (/.*) {
+          client_max_body_size 0;
+          #auth_basic "Hyperling's Git Login";
+          #auth_basic_user_file "{{ git_web_root }}/htpasswd";
+          include /etc/nginx/fastcgi_params;
+          fastcgi_param SCRIPT_FILENAME /usr/lib/git-core/git-http-backend;
+          fastcgi_param GIT_HTTP_EXPORT_ALL "";
+          fastcgi_param GIT_PROJECT_ROOT {{ git_web_root }};
+          fastcgi_param REMOTE_USER $remote_user;
+          fastcgi_param PATH_INFO $1;
+          fastcgi_pass unix:/var/run/fcgiwrap.socket;
+        }
+      }
+    state: present
+    mode: '755'
+    create: yes
+    backup: yes
+    marker: "# {mark} Managed By Ansible Git Server Playbook -->"
+    
+- name: Server | Software | Git | Service
+  service:
+    name: nginx
+    pattern: nginx
+    enabled: yes
+    state: restarted
+
+
+## Cron ##
+
+- name: Server | Software | Git | Cron | Hourly
+  ansible.builtin.cron:
+    name: "{{ item.name }} hourly"
+    special_time: hourly
+    job: "cd {{ git_web_root }}/{{ item.name }}.git && {{ git_cron_commands }}"
+  loop: "{{ git_projects }}"
+
+- name: Server | Software | Git | Cron | Reboot
+  ansible.builtin.cron:
+    name: "{{ item.name }} reboot"
+    special_time: reboot
+    job: "cd {{ git_web_root }}/{{ item.name }}.git && {{ git_cron_commands }}"
+  loop: "{{ git_projects }}"