Merge pull request #5 from Hyperling/reverseproxy

Finish Reverse Proxy Project

.gitignore

@@ -1,4 +1,4 @@
-# Ignore everything in Volumes, it is large and private data.
+# Ignore everything in Volumes, it contains large and private data.
 Volumes/*
 
 # Ignore backed up config files.
@@ -7,5 +7,9 @@ Volumes/*
 # Ignore logs
 logs/*
 
+# Ignore private reverse proxy configurations.
+Config/ReverseProxy/config/conf.d/*.*
+Config/ReverseProxy/config/html/*.*
+
 # Ignore MailServer Files
 Config/MailServer/setup.sh

Config/ReverseProxy/Dockerfile

@@ -0,0 +1,20 @@
+# 2022-10-05 Hyperling
+# Create the nginx environment for a reverse proxy.
+# https://docs.docker.com/engine/reference/builder/
+
+FROM nginx
+
+## Configuration Files ##
+# Clear old configurations.
+RUN rm -rfv /etc/nginx/conf.d
+# Add all the configuration files to the environment.
+COPY ./config/nginx.conf /etc/nginx/nginx.conf
+COPY ./config/conf.d /etc/nginx/conf.d
+RUN rm -rfv /etc/nginx/nginx.conf/README*
+
+## Hosted Sites ##
+# Clear old sites.
+RUN rm -rfv /etc/nginx/html
+# Add any static HTML websites.
+COPY ./config/html /etc/nginx/html
+RUN rm -rfv /etc/nginx/html/README*

Config/ReverseProxy/Dockerfile.disabled

@@ -1,11 +0,0 @@
-# 2022-10-05 Hyperling
-# Move config to nginx container. 
-# This is because nginx image does not play well with Volumes.
-# Nextcloud and MariaDB created files in their folders fine, but nginx stays empty.
-
-FROM nginx
-
-COPY ./config/nginx.conf /etc/nginx/nginx.conf
-
-COPY ./config/conf.d/* /etc/nginx/conf.d/
-

Config/ReverseProxy/README.md

@@ -0,0 +1,65 @@
+# Initial Setup Instructions
+How to first begin using this subproject.
+1. Move to the directory of this README.
+    ```
+    $ cd $DOCKER_HOME/Config/ReverseProxy
+    ```
+1. Add configuration files to `./config/conf.d/` which are named based on the domains and subdomains they point to.
+1. Run the placeholder certificate program.
+    ```
+    # ./create_placeholder_certs.sh
+    ```
+1. Make any personal changes to `./config/nginx.conf`.
+1. Build the project.
+    ```
+    # docker compose build
+    ```
+1. Start the project.
+    ```
+    # docker compose up -d
+    ```
+1. Verify it started correctly, no configuration file errors.
+    ```
+    # docker logs reverseproxy-app-1
+    # docker logs reverseproxy-certbot-1
+    ```
+
+## DO NOT
+* Edit any configurations or website data inside the container. It is destroyed on each build.
+    * Instead, modify the files in `./config/` then use the Update Config commands below.
+* Install any additional software inside of the container. It will not persist a down and up.
+    * Instead, add what is needed to the `docker-compose.yml` or `Dockerfile` to be done on each rebuild.
+    * Alternatively write a script such as `../Nextcloud/fixes.ksh` which is run after every upgrade.
+
+# Other Commands
+Tasks which will also likely come up while using this subproject.
+
+## Stop
+If the proxy needs turned off either stop or down may be used.
+```
+# docker compose stop
+# docker compose down
+```
+
+## Upgrade
+Upgrading the containers should be as easy as this:
+```
+# docker compose down
+# docker compose pull
+# docker compose build
+# docker compose up -d
+```
+
+## Update Config
+Replace the configuration based on any new, updated, or removed files.
+This may be possible to do when the system is up, but the best results have come from going down and back up.
+This is essentially an upgrade but there is no pull.
+```
+# docker compose down
+# docker compose build
+# docker compose up -d
+```
+If wanted as a one-line command:
+```
+# docker compose down && docker compose build && docker compose up -d
+```

Config/ReverseProxy/config/conf.d/html.example.com

@@ -0,0 +1,42 @@
+# 2023-07-08 Hyperling
+# A dummy test file since true scripts are being kept private.
+# This should help anyone understand how the project is being used.
+
+## Instructions ##
+# Add this without the comment to your /etc/hosts to test that it is working,
+#   YOUR_DOCKER_SERVER_IP html.example.com
+# If testing locally on a workstation,
+#   127.0.0.1 html.example.com
+# Then to test, first start the container,
+#   cd $DOCKER_HOME/Config/ReverseProxy && docker compose build && docker compose up -d
+# Then from the system with the modified /etc/hosts,
+#   curl --insecure html.example.com
+# You should see activity in the container log as well as the contents of the
+# proxied website in the terminal, NOT html.example.com. If using a browser then you
+# should notice that the URL is still html.example.com but the website is correct.
+
+# Force HTTPS
+server {
+
+    listen 80;
+    server_name html.example.com;
+
+    # Redirect to a more secure protocol.
+    return 301 https://$host$request_uri;
+
+}
+
+# Serve Resource
+server {
+
+    listen 443 ssl;
+    server_name html.example.com;
+
+    # The certs being used for the website.
+    ssl_certificate /etc/nginx/certs/html.example.com/fullchain.pem;
+    ssl_certificate_key /etc/nginx/certs/html.example.com/privkey.pem;
+
+    # Load the static web content.
+    root /etc/nginx/html/html.example.com;
+
+}

Config/ReverseProxy/config/conf.d/proxy.example.com

@@ -0,0 +1,66 @@
+# 2022-10-05 Hyperling
+# A dummy test file since true scripts are being kept private.
+# This should help anyone understand how the project is being used.
+
+## Instructions ##
+# Add this without the comment to your /etc/hosts to test that it is working,
+#   YOUR_DOCKER_SERVER_IP proxy.example.com
+# If testing locally on a workstation,
+#   127.0.0.1 proxy.example.com
+# Then to test, first start the container,
+#   cd $DOCKER_HOME/Config/ReverseProxy && docker compose build && docker compose up -d
+# Then from the system with the modified /etc/hosts,
+#   curl --insecure proxy.example.com
+# You should see activity in the container log as well as the contents of the
+# proxied website in the terminal, NOT proxy.example.com. If using a browser then you
+# should notice that the URL is still proxy.example.com but the website is correct.
+
+# Force HTTPS
+server {
+
+    listen 80;
+    server_name proxy.example.com;
+
+    # Redirect to a more secure protocol.
+    return 301 https://$host$request_uri;
+
+}
+
+# Serve Resource
+server {
+
+    listen 443 ssl;
+    server_name proxy.example.com;
+
+    # The certs being used for the website.
+    ssl_certificate /etc/nginx/certs/proxy.example.com/fullchain.pem;
+    ssl_certificate_key /etc/nginx/certs/proxy.example.com/privkey.pem;
+
+    # Send traffic to upstream server
+    location / {
+        proxy_set_header X-Forwarded-Proto https;
+
+        # These cause "400 Bad Request Request Header Or Cookie Too Large"?
+        #proxy_set_header Host $host;
+        #proxy_set_header X-Real-IP $remote_addr;
+        #proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+
+        ## General format is PROTOCOL://SERVER:PORT. For example:
+        #
+        # If using a domain name:
+        #proxy_pass http://YOUR_SERVER_NAME:8080;
+        #
+        # If using an IP address:
+        #proxy_pass http://192.168.1.80:8080;
+        #
+        # If forwarding to an external source:
+        #proxy_pass https://website.name;
+        #
+        # Or alternatively, do it like the force of HTTPS if not your server.
+        #return 301 https://website.name/$request_uri;
+
+        # This should forward you from 'proxy.example.com' to a real site:
+        proxy_pass https://hyperling.com;
+    }
+
+}

Config/ReverseProxy/config/conf.d/test.conf

@@ -1,2 +0,0 @@
-# 2022-10-05 Hyperling
-# Just a dummy test file.

Config/ReverseProxy/config/html/README.md

@@ -0,0 +1,7 @@
+# HTML Sites
+If the reverse proxy also serves static HTML sites,
+  the root directories of each can be placed here.
+Then in `../conf.d` add a file which points the domain to the HTML web root,
+  such as `/etc/nginx/html/www.website.name`.
+An example for this exists called `html.example.com`.
+It should be fairly easy to recreate for another website.

Config/ReverseProxy/config/html/html.example.com/index.html

@@ -0,0 +1,8 @@
+<!DOCTYPE html>
+<header>
+	<title>Example HTML Website</title>
+</header>
+<body>
+	<h1>Welcome to the example HTML website!</h1>
+	<p>This means the reverse proxy is successfully serving static HTML content. Yay!</p>
+</body>

Config/ReverseProxy/config/nginx.conf

@@ -1,34 +1,55 @@
 # 2022-10-05 Hyperling
 
-user  nginx;
-worker_processes  auto;
-
-error_log  /var/log/nginx/error.log notice;
-pid        /var/run/nginx.pid;
+user nginx;
+worker_processes auto;
 
+error_log /var/log/nginx/error.log notice;
+pid       /var/run/nginx.pid;
 
 events {
-    worker_connections  1024;
+    worker_connections 1024;
 }
 
-
 http {
-    include       /etc/nginx/mime.types;
-    default_type  application/octet-stream;
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
 
-    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-                      '$status $body_bytes_sent "$http_referer" '
-                      '"$http_user_agent" "$http_x_forwarded_for"';
+    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
+                    '$status $body_bytes_sent "$http_referer" '
+                    '"$http_user_agent" "$http_x_forwarded_for"';
 
-    access_log  /var/log/nginx/access.log  main;
+    access_log /var/log/nginx/access.log main;
 
-    sendfile        on;
-    #tcp_nopush     on;
+    sendfile on;
+    #tcp_nopush on;
 
-    keepalive_timeout  65;
+    keepalive_timeout 65;
 
-    #gzip  on;
+    #gzip on;
 
-    include /etc/nginx/conf.d/*.conf;
+    ## LetsEncrypt Certbot Setup ##
+    # Allow nginx to fulfill LetsEncrypt Certbot challenges.
+    server {
+        location /.well-known/acme-challenge/ {
+            root /etc/nginx/letsencrypt;
+        }
+    }
+
+    ## Reverse Proxied Website Configurations ##
+    include /etc/nginx/conf.d/*;
 }
 
+# TBD, going live with HTTP first.
+mail {
+    ## Reverse Proxied Mail Server Configurations ##
+    #include /etc/nginx/mail.conf.d/*;
+}
+
+# TBD, going live with HTTP first.
+stream {
+    ## Service Forwarding and Load Balancing ##
+    # If this supports the `listen` and `server_name` directives then this may
+    # be a better choice than mail{} since it will not require an auth server.
+    # It could also be useful as a frontend for ssh, databases, APIs, etc.
+    #include /etc/nginx/load.conf.d/*;
+}

Config/ReverseProxy/create_letsencrypt_certs.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+# Create a real cert for each file in config/conf.d/.
+
+## Variables ##
+
+DIR=`dirname $0`
+if [[ $DIR == \.* ]]; then
+	DIR=`pwd`
+fi
+
+# Where the files need to live.
+CERT_DIR=$DIR/../../Volumes/ReverseProxy/letsencrypt-certs
+echo "CERT_DIR=$CERT_DIR"
+
+## Validations ##
+
+# Ensure that fake certs were created at some point, or that the system has been run at least once.
+if [[ ! -d $CERT_DIR ]]; then
+	echo "ERROR: Certificate directory does not exist yet. Run the placeholder script first." >&2
+	exit 1
+fi
+
+# The container needs to be running in order to use the certbot command.
+certbot_running=`docker ps | grep -c reverseproxy-certbot-1`
+if [[ $certbot_running != 1 ]]; then
+	echo "ERROR: Certbot container does not appear to be running, cannot continue." >&2
+	exit 1
+fi
+
+## Input ##
+
+# Gather information from the user.
+echo -n "Please provide the email address you would like the certs bound to: "
+read email
+if [[ -z $email ]]; then
+	echo "ERROR: Email address is mandatory. $email" >&2
+	exit 1
+fi
+
+echo -n "Please double check that '$email' looks correct and provide Yes if so: "
+typeset -u confirm
+read confirm
+if [[ $confirm != "Y"* ]]; then
+	echo "Email address was not confirmed, received '$confirm', aborting."
+	exit 0
+fi
+
+## Main ##
+
+# Loop over the proxy configuration files and ensure they have certs.
+ls $DIR/config/conf.d/*.* | while read file; do
+	filename=`basename $file`
+
+	if [[ $filename == *"example.com"* ]]; then
+		echo "Skipping $filename since it is only an example."
+		continue
+	fi
+
+	echo "*** Checking $filename ***"
+	if [[ -d $CERT_DIR/$filename ]]; then
+		echo "Getting the domains which need the cert."
+		domains=`grep server_name $file`
+
+		# Clean up the data by removing the directive and semi-colon, changing
+		#  spaces to commas, and making sure there are no gaps.
+		domains=${domains//server_name/}
+		domains=${domains//;/}
+		domains=`echo $domains`
+		domains=${domains// /,}
+		echo "Domains='$domains'"
+
+		echo "Attempting to create real certs at $CERT_DIR/$filename."
+		docker exec reverseproxy-certbot-1 certbot certonly -n --standalone \
+				--agree-tos -m $email -d $filename
+
+		ls -lh $CERT_DIR/$filename/*
+	else
+		echo "Website's certificate folder does not exist, skipping."
+		continue
+	fi
+done
+
+exit 0

Config/ReverseProxy/create_placeholder_certs.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Create a fake cert for each file in config/conf.d/.
+
+## Variables ##
+
+DIR=`dirname $0`
+if [[ $DIR == \.* ]]; then
+	DIR=`pwd`
+fi
+
+# Where the files need to live.
+CERT_DIR=$DIR/../../Volumes/ReverseProxy/letsencrypt-certs
+echo "CERT_DIR=$CERT_DIR"
+
+## Main ##
+
+# Create the directory if it does not exist.
+mkdir -pv $CERT_DIR
+
+# Loop over the proxy configuration files and ensure they have certs.
+ls $DIR/config/conf.d/*.* | while read file; do
+	filename=`basename $file`
+	echo "*** Checking $filename ***"
+	if [[ ! -d $CERT_DIR/$filename ]]; then
+		echo "Creating self-signed certs at $CERT_DIR/$filename."
+		mkdir -pv $CERT_DIR/$filename
+		openssl req -new -x509 -days 3 -nodes \
+				-out $CERT_DIR/$filename/fullchain.pem \
+				-keyout $CERT_DIR/$filename/privkey.pem \
+				-subj "/CN=$filename/O=$filename/C=XX"
+		ls -lh $CERT_DIR/$filename/*
+	else
+		echo "Certs already exist!"
+	fi
+done
+
+exit 0

Config/ReverseProxy/docker-compose.yml

@@ -0,0 +1,26 @@
+# 2022-10-05 Hyperling
+# Reverse Proxy with LetsEncrypt Certbot.
+# This is a revised version of these works:
+#   https://phoenixnap.com/kb/docker-nginx-reverse-proxy
+#   https://www.docker.com/blog/how-to-use-the-official-nginx-docker-image/
+#   https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71
+
+version: '3'
+
+services:
+  app:
+    build: .
+    restart: always
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ../../Volumes/ReverseProxy/letsencrypt:/etc/nginx/letsencrypt
+      - ../../Volumes/ReverseProxy/letsencrypt-certs:/etc/nginx/certs
+    command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"
+  certbot:
+    image: certbot/certbot
+    volumes:
+      - ../../Volumes/ReverseProxy/letsencrypt:/etc/letsencrypt
+      - ../../Volumes/ReverseProxy/letsencrypt-certs:/var/www/certbot
+    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"

bin/install.sh

@@ -30,7 +30,7 @@ echo "pkgmgr=$pkgmgr"
 ## Main ##
 
 if [[ "$pkgmgr" == "apt" ]]; then
-	apt purge docker docker-engine docker.io containerd runc
+	apt purge docker docker-engine docker.io containerd runc podman-docker
 
 	apt update &&
 	apt install -y ca-certificates curl gnupg lsb-release &&
@@ -40,13 +40,13 @@ if [[ "$pkgmgr" == "apt" ]]; then
 		"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/$repo \
 		$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null &&
 	apt update &&
-	apt install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin &&
+	apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin &&
 
 	echo "Success!" &&
 	exit 0
 elif [[ $pkgmgr == "pacman" ]]; then
 	pacman -Rcns --noconfirm *docker*
-	
+
 	pacman -Sy --noconfirm docker docker-compose &&
 	echo "Success!" &&
 	exit 0

source.env

@@ -1,3 +1,19 @@
-DOCKER_HOME=/opt/Docker
+#!/bin/bash
+# Provide any necessary project variables.
+# Script is a shell file so that the paths can be relative.
+
+DIR=`dirname $0`
+if [[ $DIR == \.* ]]; then
+	DIR=`pwd`
+fi
+
+# Some projects are hard-coded to use /opt/Docker/Volumes so display a notice
+#  until they are updated to be directory agnostic. Will help with testing!
+PREFERRED_HOME="/opt/Docker"
+if [[ $DIR != $PREFERRED_HOME ]]; then
+	echo "WARNING: Preferred home is $PREFERRED_HOME but using $DIR." >&2
+fi
+
+DOCKER_HOME=$DIR
 DOCKER_PATH=$DOCKER_HOME/bin
-PATH=$PATH:$DOCKER_PATH
+PATH=$DOCKER_PATH:$PATH