From 8602f7ada24b15d735bd3a7233015c123654313a Mon Sep 17 00:00:00 2001 From: Chad Date: Sun, 23 Jul 2023 14:51:48 -0700 Subject: [PATCH] Fix Directory Hardcodes, Documentation Improvements (#10) * Improve the main crontab example. * Remove /opt/Docker hardcodes. * Add periods. * Improve readability. * Begin removing hardcoded path from the bin files. * Update main README to no longer enforce hardcoded path. Other improvements. * Add the load folder with a README. * Add load folder and its README. * Improve reverse proxy text files. * Switch to tabs. * Update all scripts for tabs, DOCKER_HOME, and comments. * Let users know the directory choice is optional. * Fix environment file. * Add more details for the reverse proxy load balancing. * Don't actually listen for postgres. * Fix comments on source file. * Be more explicit on the pathing. --- .gitignore | 4 +- Config/DynamicDNS/README.md | 10 +++-- Config/Nextcloud/docker-compose.yml | 7 ++-- Config/ReverseProxy/config/hosts/README.md | 1 - Config/ReverseProxy/config/hosts/example.com | 1 - Config/ReverseProxy/config/html/README.md | 11 +++-- .../ReverseProxy/config/load.conf.d/README.md | 8 ++++ .../config/load.conf.d/example.com | 15 +++++++ Config/ReverseProxy/config/nginx.conf | 7 +++- README.md | 42 ++++++++++++------- bin/create.sh | 15 +++++-- bin/get_logs.sh | 22 +++++++--- bin/install.sh | 11 ++++- bin/start.sh | 24 +++++++---- bin/stop.sh | 22 +++++++--- bin/uninstall.sh | 1 - bin/update.sh | 4 +- source.env | 22 +++------- 18 files changed, 150 insertions(+), 77 deletions(-) create mode 100644 Config/ReverseProxy/config/load.conf.d/README.md create mode 100644 Config/ReverseProxy/config/load.conf.d/example.com mode change 100644 => 100755 source.env diff --git a/.gitignore b/.gitignore index df0fc48..955bca9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,8 @@ Config/ReverseProxy/config/hosts/* Config/ReverseProxy/config/load.conf.d/* Config/ReverseProxy/config/mail.conf.d/* -# Ignore downloaded mail server files +# Ignore downloaded mail server files. Config/MailServer/setup.sh -# Ignore any private key information +# Ignore any private key information. private.key diff --git a/Config/DynamicDNS/README.md b/Config/DynamicDNS/README.md index 778eaca..8729cb8 100644 --- a/Config/DynamicDNS/README.md +++ b/Config/DynamicDNS/README.md @@ -13,13 +13,15 @@ product's self-built solutions can be found below. git clone https://github.com/Hyperling/docker $PROJECT_DIR ``` -1. Add your Afraid DNS account key to $PROJECT_DIR/Config/DynamicDNS/private.key -Account key can be found [here](https://freedns.afraid.org/dynamic/v2/). +1. Add your user key to `$PROJECT_DIR/Config/DynamicDNS/private.key`. The key can +be found on [this page](https://freedns.afraid.org/dynamic/v2/) after signing in. 1. Add this line to the system's cron scheduling using a command like `crontab -e`. +The sleep waits anywhere from 0 to 55 minutes due to the +[Random/10](https://tldp.org/LDP/abs/html/randomvar.html). ``` - 5 * * * * $PROJECT_DIR/Config/DynamicDNS/update_dns.sh + @hourly sleep $(( $RANDOM / 10 )); $PROJECT_DIR/Config/DynamicDNS/update_dns.sh ``` ### TESTING @@ -61,6 +63,6 @@ and preferably add a 30-45 second sleep so that you do not hit near :00 seconds. ### No-IP.org Instructions -Please see this guide on installing the Dynamic Update Client (DUC). +Please see the official guide on installing the Dynamic Update Client (DUC). https://my.noip.com/dynamic-dns/duc diff --git a/Config/Nextcloud/docker-compose.yml b/Config/Nextcloud/docker-compose.yml index 57d7a37..27796f8 100644 --- a/Config/Nextcloud/docker-compose.yml +++ b/Config/Nextcloud/docker-compose.yml @@ -1,5 +1,5 @@ # Nextcloud configuration. -# This is a revised version of the original work here: +# This is a revised version of the original work here: # https://hub.docker.com/_/nextcloud # Changelog: @@ -13,7 +13,7 @@ services: restart: always command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW volumes: - - /opt/Docker/Volumes/Nextcloud/mariadb:/var/lib/mysql + - ../../Volumes/Nextcloud/mariadb:/var/lib/mysql environment: - MYSQL_ROOT_PASSWORD=ChangeMe - MYSQL_DATABASE=nextcloud @@ -28,11 +28,10 @@ services: links: - db volumes: - - /opt/Docker/Volumes/Nextcloud/nextcloud:/var/www/html + - ../../Volumes/Nextcloud/nextcloud:/var/www/html environment: - MYSQL_DATABASE=nextcloud - MYSQL_USER=nc - MYSQL_PASSWORD=changeme - MYSQL_HOST=db - PHP_UPLOAD_LIMIT=5G - diff --git a/Config/ReverseProxy/config/hosts/README.md b/Config/ReverseProxy/config/hosts/README.md index a33da34..7ce88f2 100644 --- a/Config/ReverseProxy/config/hosts/README.md +++ b/Config/ReverseProxy/config/hosts/README.md @@ -1,3 +1,2 @@ # Upstream Host Configuration For systems which do not resolve well such as ignoring `/etc/hosts`. - diff --git a/Config/ReverseProxy/config/hosts/example.com b/Config/ReverseProxy/config/hosts/example.com index 170de7e..21d9e34 100644 --- a/Config/ReverseProxy/config/hosts/example.com +++ b/Config/ReverseProxy/config/hosts/example.com @@ -5,4 +5,3 @@ upstream example-proxy-site { #server 127.0.0.1:8080; server hyperling.com; } - diff --git a/Config/ReverseProxy/config/html/README.md b/Config/ReverseProxy/config/html/README.md index 4cbc027..97b93b6 100644 --- a/Config/ReverseProxy/config/html/README.md +++ b/Config/ReverseProxy/config/html/README.md @@ -1,7 +1,6 @@ # HTML Sites -If the reverse proxy also serves static HTML sites, - the root directories of each can be placed here. -Then in `../conf.d` add a file which points the domain to the HTML web root, - such as `/etc/nginx/html/www.website.name`. -An example for this exists called `html.example.com`. -It should be fairly easy to recreate for another website. +If the reverse proxy also serves static HTML sites, the root directories of each +can be placed here. Then in `../conf.d` add a file which points the domain to +the HTML web root, such as `/etc/nginx/html/www.website.name`. An example for +this exists called `html.example.com`. It should be fairly easy to recreate for +another website. diff --git a/Config/ReverseProxy/config/load.conf.d/README.md b/Config/ReverseProxy/config/load.conf.d/README.md new file mode 100644 index 0000000..c6d7171 --- /dev/null +++ b/Config/ReverseProxy/config/load.conf.d/README.md @@ -0,0 +1,8 @@ +# Load Balancing Files +Allow requests coming to this server to be spread amongst multiple servers based +on port number. It does not seem possible to spread them based on `server_name` +or other directives like a reverse proxy. The server simply listens on the port +then runs through the upstream list to determine the destination. + +## Official Documentation +http://nginx.org/en/docs/stream/ngx_stream_core_module.html diff --git a/Config/ReverseProxy/config/load.conf.d/example.com b/Config/ReverseProxy/config/load.conf.d/example.com new file mode 100644 index 0000000..6c7b7ca --- /dev/null +++ b/Config/ReverseProxy/config/load.conf.d/example.com @@ -0,0 +1,15 @@ +# Example of how to load balance 4 Postgres servers for example.com. Since this +# does not act under a reverse proxy situation, code is commented so that the +# container does not needlessly start listening on the port. + +#upstream postgres_servers { +# server 1.2.3.1:5432; +# server 1.2.3.2:5432; +# server 1.2.3.3:5432; +# server 1.2.3.4:5432; +#} +# +#server { +# listen 5432; +# proxy_pass postgres_servers; +#} diff --git a/Config/ReverseProxy/config/nginx.conf b/Config/ReverseProxy/config/nginx.conf index 299f8e4..02006e3 100644 --- a/Config/ReverseProxy/config/nginx.conf +++ b/Config/ReverseProxy/config/nginx.conf @@ -66,12 +66,17 @@ http { include /etc/nginx/conf.d/*; } -# TBD, going live with HTTP first. +## TBD. mail { ## Reverse Proxied Mail Server Configurations ## #include /etc/nginx/mail.conf.d/*; } +## Under Experimentation +# So far does not seem like server_name works, only listen, so not useful as a +# reverse proxy. Such as 2 Postgres servers needing traffic from different +# domains, or two SMTP servers on the same IP serving two different domains. +# Those possibilities do not seem to exist here, unfortunately. stream { ## Service Forwarding and Load Balancing ## # If this supports the `listen` and `server_name` directives then this may diff --git a/README.md b/README.md index 55a3836..feb83fe 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,28 @@ # My Docker Setup -Scripting my way into the Docker world. -I was unable to find a good tutorial on using and managing containers so this is what made sense to me based on practice with `docker-compose`. -I am still new to Docker and am likely to make mistakes, but you're welcome to learn with me. ;) +Scripting my way into the Docker world. I was unable to find a good tutorial on +using and managing containers so this is what made sense to me based on practice +with `docker-compose`. I am still new-ish to Docker and am likely to make +mistakes, but you're welcome to learn with me. ;) ## Disclaimer Currently the project only focuses on `apt` based operating systems. +## Other README's +Each `./Config/PROJECT/` folder also contains its own README file with specific +information to running that sub project. This file's job is to cover the general +Docker installation. The others then contain details on their program setup. + ## How To Use -Most of these commands benefit from being root. Something like a `sudo su -` if you feel comfortable with it. -Otherwise be aware that using sudo may cause file permission conflicts when interacting with the configuration files and folders. +Most of these commands benefit from being root. Something like a `sudo su -` if +you feel comfortable with it. Otherwise be aware that using sudo may cause file +permission conflicts when interacting with the configuration files and folders. Install the project dependencies. ``` apt install git bash ``` -Clone the project. +Clone the project. You may choose anywhere, but `/opt/Docker` is recommended. ``` git clone https://github.com/hyperling/docker /opt/Docker ``` @@ -25,7 +32,7 @@ Load the environment variables. source /opt/Docker/source.env ``` -Install docker to the system. +Install docker to the system using the official repos. ``` install.sh ``` @@ -50,19 +57,24 @@ Cross your fingers and hope to profit! ## Folders ### Config -Compose projects are set up here. Each folder should have a `docker-compose.yml` file set up. +Compose projects are set up here. Each folder should have a `docker-compose.yml` +file set up. ### Volumes -The data of the files go here if the Config is done correctly. -I think this should be easier to remember than `/var/lib/docker/volumes` when it comes time for migrations. -Hopefully all that'd be needed is to rsync `/opt/Docker` and run `install.sh` and then `start.sh` on the new server. -That's my opinion though, if someone else uses this then they are welcome to place it where they'd like. +The data of the files go here if the Config is done correctly. I think this +should be easier to remember than `/var/lib/docker/volumes` when it comes time +for migrations. Hopefully all that'd be needed is to rsync `/opt/Docker` and run +`install.sh` and then `start.sh` on the new server. You are welcome to use a +directory other than `/opt/Docker`, this project is location agnostic. ### bin -Scripts to help make life easier. Some are pretty basic, but others do nice things like handle the container IDs. +Scripts to help make life easier. Some are pretty basic, but others do nice +things like handle the container IDs. * `install.sh` : Install dependencies on a new server with apt. * `create.sh` : Create a new folder with the needed yml file. * `start.sh` : Start all compose containers. * `stop.sh` : Stop all compose containers. -* `get_logs.sh` : Create log files rather than using the `docker log` command or searching in /var/whatever. -* `uninstall.sh` : If something goes wrong and you'd like to start from scratch without provisioning a new server then this should do the job. +* `get_logs.sh` : Create log files rather than using the `docker log` command or + searching in /var/whatever. +* `uninstall.sh` : If something goes wrong and you'd like to start from scratch + without provisioning a new server then this should do the job. diff --git a/bin/create.sh b/bin/create.sh index 628a77f..835a41d 100755 --- a/bin/create.sh +++ b/bin/create.sh @@ -3,7 +3,17 @@ # Create new container template. # usage: create.sh PROJECT_NAME -source /opt/Docker/source.env +## Setup ## + +DIR="`dirname $0`" +PROG=`basename $0` +if [[ $DIR == *"."* ]]; then + DIR="`pwd`" +fi + +if [[ -z $DOCKER_HOME ]]; then + DOCKER_HOME="$DIR/.." +fi ## Validation ## @@ -26,10 +36,9 @@ file="$dir/docker-compose.yml" cd $DOCKER_HOME mkdir -pv "$dir" -[[ ! -f "$file" ]] && echo -e "# Comment.\nservices:\n" >> "$file" || +[[ ! -f "$file" ]] && echo -e "# Comment.\nservices:\n" >> "$file" || echo "File already exists, leaving contents alone." echo "${file}:" cat "$file" exit 0 - diff --git a/bin/get_logs.sh b/bin/get_logs.sh index 33ab9a1..75964e4 100755 --- a/bin/get_logs.sh +++ b/bin/get_logs.sh @@ -3,21 +3,31 @@ # Put active logs into files for analysis. # usage: get_logs.sh -source /opt/Docker/source.env +## Setup ## + +DIR="`dirname $0`" +PROG=`basename $0` +if [[ $DIR == *"."* ]]; then + DIR="`pwd`" +fi +if [[ -z $DOCKER_HOME ]]; then + DOCKER_HOME="$DIR/.." +fi dir=logs date_format="+%Y%m%d-%H%M%S" +## Main ## + cd $DOCKER_HOME mkdir -p $dir docker ps | while read container_id image_name other; do - image_name=${image_name##*/} - echo $container_id $image_name - docker inspect $container_id 1>/dev/null 2>&1 && - docker logs $container_id 1>${dir}/${image_name}.log.`date $date_format` 2>&1 + image_name=${image_name##*/} + echo $container_id $image_name + docker inspect $container_id 1>/dev/null 2>&1 && + docker logs $container_id 1>${dir}/${image_name}.log.`date $date_format` 2>&1 done chmod -R 755 $dir exit 0 - diff --git a/bin/install.sh b/bin/install.sh index b8aeba7..02b0a21 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -4,7 +4,16 @@ # Original comands came from here: https://docs.docker.com/engine/install/debian/ # usage: install.sh -## Variables ## +## Setup ## + +DIR="`dirname $0`" +PROG=`basename $0` +if [[ $DIR == *"."* ]]; then + DIR="`pwd`" +fi +if [[ -z $DOCKER_HOME ]]; then + DOCKER_HOME="$DIR/.." +fi os=`grep ^'NAME=' /etc/os-release` pkgmgr="" diff --git a/bin/start.sh b/bin/start.sh index bc73f0e..c24ce37 100755 --- a/bin/start.sh +++ b/bin/start.sh @@ -3,16 +3,26 @@ # Start all containers. # usage: start.sh -source /opt/Docker/source.env +## Setup ## + +DIR="`dirname $0`" +PROG=`basename $0` +if [[ $DIR == *"."* ]]; then + DIR="`pwd`" +fi +if [[ -z $DOCKER_HOME ]]; then + DOCKER_HOME="$DIR/.." +fi + +## Main ## cd $DOCKER_HOME/Config for dir in `ls`; do - [ -d $dir ] && cd $dir || continue - pwd - [ -e Dockerfile ] && docker compose build - [ -e docker-compose.yml ] && docker compose up -d - cd .. + [ -d $dir ] && cd $dir || continue + pwd + [ -e Dockerfile ] && docker compose build + [ -e docker-compose.yml ] && docker compose up -d + cd .. done exit 0 - diff --git a/bin/stop.sh b/bin/stop.sh index b15d87f..0fafdc0 100755 --- a/bin/stop.sh +++ b/bin/stop.sh @@ -3,15 +3,25 @@ # Stop all containers. # usage: stop.sh -source /opt/Docker/source.env +## Setup ## + +DIR="`dirname $0`" +PROG=`basename $0` +if [[ $DIR == *"."* ]]; then + DIR="`pwd`" +fi +if [[ -z $DOCKER_HOME ]]; then + DOCKER_HOME="$DIR/.." +fi + +## Main ## cd $DOCKER_HOME/Config for dir in `ls`; do - [ -d $dir ] && cd $dir || continue - pwd - [ -e docker-compose.yml ] && docker compose down - cd .. + [ -d $dir ] && cd $dir || continue + pwd + [ -e docker-compose.yml ] && docker compose down + cd .. done exit 0 - diff --git a/bin/uninstall.sh b/bin/uninstall.sh index 0a33471..160a926 100755 --- a/bin/uninstall.sh +++ b/bin/uninstall.sh @@ -9,4 +9,3 @@ rm -v /etc/apt/sources.list.d/docker.list && rm -rfv /var/lib/docker exit 0 - diff --git a/bin/update.sh b/bin/update.sh index 583ea33..7ce40ad 100755 --- a/bin/update.sh +++ b/bin/update.sh @@ -2,11 +2,11 @@ # 2022-09-25 Hyperling # Script to update a docker compose image. -docker compose down +docker compose down docker compose pull && +docker compose build && docker compose up -d && exit 0 echo "ERROR: Did not update or start correctly." && exit 1 - diff --git a/source.env b/source.env old mode 100644 new mode 100755 index 8699855..4511153 --- a/source.env +++ b/source.env @@ -1,19 +1,7 @@ -#!/bin/bash # Provide any necessary project variables. -# Script is a shell file so that the paths can be relative. +# Needs run in the current shell environment, such as: +# source /PATH_TO_GIT_PROJECT/source.env -DIR=`dirname $0` -if [[ $DIR == \.* ]]; then - DIR=`pwd` -fi - -# Some projects are hard-coded to use /opt/Docker/Volumes so display a notice -# until they are updated to be directory agnostic. Will help with testing! -PREFERRED_HOME="/opt/Docker" -if [[ $DIR != $PREFERRED_HOME ]]; then - echo "WARNING: Preferred home is $PREFERRED_HOME but using $DIR." >&2 -fi - -DOCKER_HOME=$DIR -DOCKER_PATH=$DOCKER_HOME/bin -PATH=$DOCKER_PATH:$PATH +export DOCKER_HOME="$(dirname -- "${BASH_SOURCE[0]}")" +export DOCKER_PATH="$DOCKER_HOME/bin" +export PATH="$DOCKER_PATH:$PATH"