From 1ef1903f6b2e91c7f0a34d2a89e67639920d7e5e Mon Sep 17 00:00:00 2001 From: Chad Date: Tue, 30 Jan 2024 06:11:32 -0700 Subject: [PATCH] Major Enhancements (#20) * Fix program for html websites, need to handle errors for non-local sites though. * Add message that the main program is starting since node-modules does not show anything. * Go ahead and give DB a bump. * Fix warning in DB log and NC not having enough power sometimes. * Fix errors in DB log. * This command uses the root account, not nextcloud. * Add example CRON file. * Specify that the host network should be used. * Specify that the host network should be used. * Recommend daily reset of the photos library. * Add another package the container is missing. * Add a backup script for the project directory. * Fix a typo. * Improve example file and add the backup job. * Add an example of pulling the backup file. * Give the zip a temporary filename while being built. * Always get the full path, relative path does not work well when not in `.`. * Remove extra newline. * Prevent doing large uploads on a micro Nextcloud instance. * Change how bz2 gets installed. * More trying to fix the annoying bz2 warning. * Remove "bad" package. * Use a temporary filename during transfer. * Bump CPUs up just a bit more for Nextcloud to avoid sync errors. * Be more explicit with instructions. --- Config/DNS/docker-compose.standard.yml | 4 +- Config/Hyperling.com/Dockerfile | 1 + Config/Hyperling.com/docker-compose.micro.yml | 4 +- .../Hyperling.com/docker-compose.standard.yml | 4 +- Config/Nextcloud/docker-compose.micro.yml | 8 ++-- Config/Nextcloud/env.micro | 2 +- Config/Nextcloud/fixes.sh | 18 +++++++- Config/ReverseProxy/docker-compose.micro.yml | 4 +- .../ReverseProxy/docker-compose.standard.yml | 4 +- bin/backup.sh | 43 +++++++++++++++++++ bin/manage.sh | 8 ++-- bin/pull_backup.example.sh | 34 +++++++++++++++ cron.example | 22 ++++++++++ source.env | 12 +++--- 14 files changed, 146 insertions(+), 22 deletions(-) create mode 100755 bin/backup.sh create mode 100755 bin/pull_backup.example.sh create mode 100644 cron.example diff --git a/Config/DNS/docker-compose.standard.yml b/Config/DNS/docker-compose.standard.yml index 7b6ae6e..8a00d47 100644 --- a/Config/DNS/docker-compose.standard.yml +++ b/Config/DNS/docker-compose.standard.yml @@ -10,7 +10,9 @@ version: '3' services: app: container_name: dns-app - build: ./ + build: + context: ./ + network: host restart: always ports: - "53:53/udp" diff --git a/Config/Hyperling.com/Dockerfile b/Config/Hyperling.com/Dockerfile index d843412..f6c5093 100644 --- a/Config/Hyperling.com/Dockerfile +++ b/Config/Hyperling.com/Dockerfile @@ -15,4 +15,5 @@ CMD cd /var/www/api && \ rm -rfv website/files && \ mv -v website/* ./ && \ rm -rfv website && \ + echo "Starting Website" && \ ./run.sh diff --git a/Config/Hyperling.com/docker-compose.micro.yml b/Config/Hyperling.com/docker-compose.micro.yml index 0a05ae6..31e24fe 100644 --- a/Config/Hyperling.com/docker-compose.micro.yml +++ b/Config/Hyperling.com/docker-compose.micro.yml @@ -7,7 +7,9 @@ version: '2' services: app: container_name: website-app - build: ./ + build: + context: ./ + network: host restart: always ports: - 8317:8080 diff --git a/Config/Hyperling.com/docker-compose.standard.yml b/Config/Hyperling.com/docker-compose.standard.yml index 3571960..bc4b1a2 100644 --- a/Config/Hyperling.com/docker-compose.standard.yml +++ b/Config/Hyperling.com/docker-compose.standard.yml @@ -6,7 +6,9 @@ version: '2' services: app: container_name: website-app - build: ./ + build: + context: ./ + network: host restart: always ports: - 8317:8080 diff --git a/Config/Nextcloud/docker-compose.micro.yml b/Config/Nextcloud/docker-compose.micro.yml index 19c27f9..7daf820 100644 --- a/Config/Nextcloud/docker-compose.micro.yml +++ b/Config/Nextcloud/docker-compose.micro.yml @@ -27,7 +27,7 @@ services: container_name: nc-db image: mariadb:10.6 restart: always - command: --transaction-isolation=READ-COMMITTED --binlog-format=ROW + command: --transaction-isolation=READ-COMMITTED --log-bin=binlog --binlog-format=ROW volumes: - ../../Volumes/Nextcloud/mariadb:/var/lib/mysql environment: @@ -40,7 +40,7 @@ services: resources: limits: cpus: '0.20' - memory: 192M + memory: 256M ## Redis ## nc-redis: @@ -99,5 +99,5 @@ services: mode: global resources: limits: - cpus: '0.30' - memory: 512M + cpus: '0.40' + memory: 640M diff --git a/Config/Nextcloud/env.micro b/Config/Nextcloud/env.micro index 99d3556..0d7b512 100644 --- a/Config/Nextcloud/env.micro +++ b/Config/Nextcloud/env.micro @@ -15,7 +15,7 @@ NEXTCLOUD_ADMIN_USER=nc_admin NEXTCLOUD_ADMIN_PASSWORD=SuperDuperSecretPassword PHP_MEMORY_LIMIT=512M -PHP_UPLOAD_LIMIT=512M +PHP_UPLOAD_LIMIT=32M # ## MariaDB ## diff --git a/Config/Nextcloud/fixes.sh b/Config/Nextcloud/fixes.sh index 10afab3..27f9c5b 100755 --- a/Config/Nextcloud/fixes.sh +++ b/Config/Nextcloud/fixes.sh @@ -6,6 +6,7 @@ DIR="$(dirname -- "${BASH_SOURCE[0]}")" PROG="$(basename -- "${BASH_SOURCE[0]}")" +source $DIR/.env source $DIR/../../source.env ## Main ## @@ -15,9 +16,14 @@ echo -e "\n*** APT ***" echo -e "\n`date` - Update Apt Cache" docker exec -it nc-app apt update -y -echo -e "\n`date` - Install Additonal Software" +echo -e "\n`date` - Install Additional Software" docker exec -it nc-app apt install -y sudo libmagickcore-6.q16-6-extra htop \ - iputils-ping dnsutils vim + iputils-ping dnsutils vim bzip2 libbz2-dev # php-bz2 + +# 20240130 +# https://help.nextcloud.com/t/docker-image-setup-warning-missing-bz2-after-update-to-nc-28-0-0/176605 +echo -e "\n`date` - Compile PHP Modules" +docker exec -it nc-app docker-php-ext-install bz2 # 2023-12-04 Make sure cron and chmod commands get run. echo -e "\n*** CRON ***" @@ -28,6 +34,14 @@ $DOCKER_HOME/Config/Nextcloud/cron.sh && echo "Success!" # 2022-10-30 More additions after moving to Nextcloud version 25. echo -e "\n*** DATABASE ***" +# Clear DB errors by allowing the DB to perform any incremental upgrades. +# https://jira.mariadb.org/browse/MDEV-26342 +echo -e "\n`date` - mysql_upgrade" +docker exec -itu www-data nc-app ./occ maintenance:mode --on +time docker exec -it nc-db mysql_upgrade \ + -user="root" --password="$MYSQL_ROOT_PASSWORD" +docker exec -itu www-data nc-app ./occ maintenance:mode --off + echo -e "\n`date` - Add Missing Columns" docker exec -itu www-data nc-app ./occ db:add-missing-columns diff --git a/Config/ReverseProxy/docker-compose.micro.yml b/Config/ReverseProxy/docker-compose.micro.yml index d350152..136ff50 100644 --- a/Config/ReverseProxy/docker-compose.micro.yml +++ b/Config/ReverseProxy/docker-compose.micro.yml @@ -11,7 +11,9 @@ services: app: container_name: rp-app - build: ./ + build: + context: ./ + network: host restart: always ports: - "80:80" diff --git a/Config/ReverseProxy/docker-compose.standard.yml b/Config/ReverseProxy/docker-compose.standard.yml index 613800e..0b096b5 100644 --- a/Config/ReverseProxy/docker-compose.standard.yml +++ b/Config/ReverseProxy/docker-compose.standard.yml @@ -11,7 +11,9 @@ services: app: container_name: rp-app - build: ./ + build: + context: ./ + network: host restart: always ports: - "80:80" diff --git a/bin/backup.sh b/bin/backup.sh new file mode 100755 index 0000000..17c82e8 --- /dev/null +++ b/bin/backup.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# 2024-01-29 Hyperling +# Create a backup file with a generic name for polling. + +DIR="$(dirname -- "${BASH_SOURCE[0]}")" +PROG="$(basename -- "${BASH_SOURCE[0]}")" +echo "$DIR/$PROG" +source $DIR/../source.env + +## Variables ## + +file="/tmp/Backup.zip" +time="`which time`" + +## Main ## + +# Remove the last backup. +if [[ -e $file ]]; then + echo "`date` - Removing existing file." + rm -fv $file +fi + +echo -e "\n`date` - Take down services for a cold backup." +manage.sh -d + +echo -e "\n`date` - Create the backup for '$DOCKER_HOME'." +cd $DOCKER_HOME +$time zip -r $file.tmp . 1>/dev/null +mv -v $file.tmp $file + +echo -e "\n`date` - Done with zipping, check size." +ls -sh $file + +echo -e "\n`date` - Ensure other users can access the file." +chmod -v 755 $file + +echo -e "\n`date` - Bring services back up." +manage.sh -u + +## Finish ## + +echo -e "\n`date` - Done!" +exit 0 diff --git a/bin/manage.sh b/bin/manage.sh index 20e0b16..edfda8e 100755 --- a/bin/manage.sh +++ b/bin/manage.sh @@ -70,7 +70,7 @@ function check_container() { # Ensure a container which will be accessed is either running or starting. # Parameters: # 1) CONTAINER, either as ID or Name. - # 2) WHy the container is being checked. + # 2) Why the container is being checked. container_to_check="$1" reason_to_check="$2" @@ -111,9 +111,9 @@ fi # Script will behave poorly if not run with admin privileges. if [[ $LOGNAME != "root" ]]; then - echo "*************************************************************" - echo "WARNING: Script is intended for root. Please su or sudo/doas." - echo -e "*************************************************************\n" + echo "*************************************************************" + echo "WARNING: Script is intended for root. Please su or sudo/doas." + echo "*************************************************************" fi # Options which only work if the container exists or is going to be started. diff --git a/bin/pull_backup.example.sh b/bin/pull_backup.example.sh new file mode 100755 index 0000000..c110e7c --- /dev/null +++ b/bin/pull_backup.example.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# 2024-01-29 Hyperling +# Example of how to pull the polled Backip.zip file. This would be placed on +# the machine holding the backups in the directory that it should land. + +DIR="$(dirname -- "${BASH_SOURCE[0]}")" +PROG="$(basename -- "${BASH_SOURCE[0]}")" +echo "$DIR/$PROG" + +## Variables ## + +DATE="`date '+%Y%m%d'`" +NEWFILE="$DIR/Backup_${DATE}.zip" +LATEST="`ls $DIR/Backup_*.zip | sort -r | head -n 1`" + +## Main ## + +echo "`date` - Creating '$NEWFILE'." +scp -P 4022 user@example.com:/tmp/Backup.zip $NEWFILE.tmp +mv -v $NEWFILE.tmp $NEWFILE + +## Validation ## + +# TBD: Can make this fancier, such as doing a real comparison for size growth. + +echo "`date` - New backup's size:" +du -h $NEWFILE + +echo "`date` - Previous backup's size:" +du -h $LATEST + +## Finish ## + +exit 0 diff --git a/cron.example b/cron.example new file mode 100644 index 0000000..77dd8eb --- /dev/null +++ b/cron.example @@ -0,0 +1,22 @@ +# The entries below should be added to the root crontab, taking into account +# where you have placed your Docker Home. This assumes `/opt/Docker`. + +## Dynamic DNS ## +# Ensure the Dynamic DNS matches the current production environment. +@hourly bash -c 'sleep $(( $RANDOM / 10 )); /opt/Docker/Config/DynamicDNS/update_dns.sh' + +## Nextcloud ## +# Run the cron job. +*/5 * * * * /opt/Docker/Config/Nextcloud/cron.sh + +## Certbot ## +# Ensure all domains have up to date certificates. +0 23 * * * docker exec rp-certbot certbot renew + +## Hyperling.com/photos/ ## +# Rebuild the cached index file each night. +0 0 * * * sh -c "rm /opt/Docker/Config/Hyperling.com/files/photos/index.html"; curl localhost:8317/photos >/dev/null 2>&1 + +## Backup ## +# Create a new backup file for any listening pullers. +0 2 * * * /opt/Docker/bin/backup.sh diff --git a/source.env b/source.env index 1cc6b3b..f8b03ea 100755 --- a/source.env +++ b/source.env @@ -5,12 +5,12 @@ DIR="$(dirname -- "${BASH_SOURCE[0]}")" PROG="$(basename -- "${BASH_SOURCE[0]}")" -if [[ "$DIR" == '.'* ]]; then - RETURN="`pwd`" - cd $DIR - DIR="`pwd`" - cd "$RETURN" -fi + +# Get the absolute path. +CURR="`pwd`" +cd $DIR +DIR="`pwd`" +cd "$CURR" ## Variables ##