Nextcloud Purge Script, Hyperling.com Static Media Volume, Memory Tweaks (#19)

* Fix memory; was accidentally done in `docker-compose.yml`.

* Fix bad path.

* Prevent DB from crashing during `fixes.sh`.

* Bump the DB up a little more.

* Add photos directory.

* Log files as they copy.

* Increase certbot by a little, was maxing near 15MB after an image update.

* Add a HUGE space saver to the cleanup crew.

* Add the beginnings of a purge script. Helps try to bring down the DB size.

* Enhance output.

* Further enhance output.

* Fix loading of ENV.

* Enhance output. Purge file previews.

* Fix extra newline.

* Add sourcing of main ENV, use RM instead of MV to reduce steps.

* Fix rm command.

* Ensure maintenance mode is off when starting.

* Add volume rather than duplicating all media.

* Ignore media.

* Check space before as well.

.gitignore

@@ -29,5 +29,8 @@ disabled
 # Ignore any .env files which are not explicitly committed to the project.
 *.env
 
-# 2024-01-23 Ignore true docker-compose files and only show example from now on.
+# 2024-01-23 Ignore true docker-compose files, only show examples from now on.
 docker-compose.yml
+
+# 2024-01-24 Hide static files for Hyperling.com.
+Config/Hyperling.com/files/*

Config/Hyperling.com/Dockerfile

@@ -8,15 +8,11 @@ FROM debian:testing
 # Install Dependencies
 RUN apt update && apt install -y sudo php-fpm nodejs npm
 
-# Copy Static Content
-RUN rm -rfv files
-COPY ./files files
-RUN rm -rfv files/README*
-
 # Run Website
-CMD rm -rfv www && \
+CMD cd /var/www/api && \
-    git clone https://github.com/Hyperling/www www && \
+    rm -rfv pages main.js run.sh && \
-    rm -rfv www/files && \
+    git clone https://github.com/Hyperling/Website website && \
-    mv files www/ && \
+    rm -rfv website/files && \
-    cd www && \
+    mv -v website/* ./ && \
+    rm -rfv website && \
     ./run.sh

Config/Hyperling.com/docker-compose.micro.yml

@@ -1,5 +1,6 @@
 # 2022-10-29
 # Hyperling.com configuration.
+# 2023-01-24 container needs 128M to get started fully then settles to 25M.
 
 version: '2'
 
@@ -10,9 +11,11 @@ services:
     restart: always
     ports:
       - 8317:8080
+    volumes:
+      - ./files:/var/www/api/files
     deploy:
       mode: global
       resources:
         limits:
           cpus: '0.10'
-          memory: 32M
+          memory: 128M

Config/Hyperling.com/docker-compose.standard.yml

@@ -10,6 +10,8 @@ services:
     restart: always
     ports:
       - 8317:8080
+    volumes:
+      - ./files:/var/www/api/files
     deploy:
       mode: global
       resources:

Config/Hyperling.com/files/photos/README.md

@@ -0,0 +1,3 @@
+# Hyperling.com - files/photos/
+
+Used by `http://localhost:8080/photos` to build albums for sharing.

Config/Nextcloud/docker-compose.micro.yml

@@ -40,7 +40,7 @@ services:
       resources:
         limits:
           cpus: '0.20'
-          memory: 64M
+          memory: 192M
 
   ## Redis ##
   nc-redis:

Config/Nextcloud/fixes.sh

@@ -6,38 +6,61 @@
 
 DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 PROG="$(basename -- "${BASH_SOURCE[0]}")"
-source $DIR/../source.env
+source $DIR/../../source.env
 
 ## Main ##
 
-echo -e "\n*** APT ***\n"
+echo -e "\n*** APT ***"
+
+echo -e "\n`date` - Update Apt Cache"
 docker exec -it nc-app apt update -y
+
+echo -e "\n`date` - Install Additonal Software"
 docker exec -it nc-app apt install -y sudo libmagickcore-6.q16-6-extra htop \
 	iputils-ping dnsutils vim
 
 # 2023-12-04 Make sure cron and chmod commands get run.
-echo -e "\n*** CRON ***\n"
+echo -e "\n*** CRON ***"
+
+echo -e "\n`date` - Run Cron Job"
 $DOCKER_HOME/Config/Nextcloud/cron.sh && echo "Success!"
 
 # 2022-10-30 More additions after moving to Nextcloud version 25.
-echo -e "\n*** DATABASE ***\n"
+echo -e "\n*** DATABASE ***"
+
+echo -e "\n`date` - Add Missing Columns"
 docker exec -itu www-data nc-app ./occ db:add-missing-columns
+
+echo -e "\n`date` - Add Missing Indexes"
 docker exec -itu www-data nc-app ./occ db:add-missing-indices
+
+echo -e "\n`date` - Add Missing PKs"
 docker exec -itu www-data nc-app ./occ db:add-missing-primary-keys
+
+echo -e "\n`date` - Convert Filecache BigInt"
 docker exec -itu www-data nc-app ./occ db:convert-filecache-bigint
 
 # 2023-07-02
-echo -e "\n*** FILES ***\n"
+echo -e "\n*** FILES ***"
+
 # This maybe used to exist, but make sure that Files app is correct.
+echo -e "\n`date` - Scanning All Files"
 docker exec -itu www-data nc-app ./occ files:scan --all
+
 # This one takes a while.
+echo -e "\n`date` - Scanning App Data"
 docker exec -itu www-data nc-app ./occ files:scan-app-data
+
 # Extras? Have used the commands in the past and may help in the future.
+echo -e "\n`date` - Theme Update"
 docker exec -itu www-data nc-app ./occ maintenance:theme:update
+echo -e "\n`date` - Repair"
 docker exec -itu www-data nc-app ./occ maintenance:repair
-# May alsp be useful but do not have much experience with them.
+
+# May also be useful but do not have much experience with them.
+echo -e "\n`date` - Clean Versions"
 docker exec -itu www-data nc-app ./occ versions:cleanup
+echo -e "\n`date` - Clean Files"
 docker exec -itu www-data nc-app ./occ files:cleanup
 
 exit 0
-

Config/Nextcloud/purge.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# 2024-01-24 Hyperling
+# Commands to purge space from both Nextcloud and its database.
+
+## Setup ##
+
+DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+PROG="$(basename -- "${BASH_SOURCE[0]}")"
+source $DIR/.env
+source $DIR/../../source.env
+
+## Main ##
+
+echo -e "\n*** Files ***"
+
+# Files commands only work during normal operations.
+echo -e "\n`date` - Disable Maintenance Mode."
+docker exec -itu www-data nc-app ./occ maintenance:mode --off
+
+echo -e "\n`date` - Scanning Files"
+time docker exec -itu www-data nc-app ./occ files:scan --all
+
+echo -e "\n`date` - Clean Files"
+time docker exec -itu www-data nc-app ./occ files:cleanup
+
+echo -e "\n`date` - Clean Versions"
+time docker exec -itu www-data nc-app ./occ files:cleanup
+
+echo -e "\n`date` - Clean Trash"
+time docker exec -itu www-data nc-app ./occ trashbin:cleanup --all-users
+
+echo -e "\n`date` - Trash Previews"
+rm -rfv $DOCKER_HOME/Volumes/Nextcloud/nextcloud/data/appdata_*/preview/*
+
+echo -e "\n*** Database ***"
+
+# Prevent any catastrophic collisions.
+echo -e "\n`date` - Enable Maintenance Mode"
+docker exec -itu www-data nc-app ./occ maintenance:mode --on
+
+echo -e "\n`date` - Delete Preview Records"
+docker exec -itu www-data nc-db /bin/bash -c \
+	"echo 'delete from oc_filecache where path like \"appdata_%/preview/%\";' | \
+	mysql --user=\"$MYSQL_USER\" --password=\"$MYSQL_PASSWORD\" $MYSQL_DATABASE"
+
+# https://mariadb.com/kb/en/mariadb-check/
+
+echo -e "\n`date` - Checking DB Tables"
+time docker exec -it nc-db mariadb-check \
+	-Ac --user="$MYSQL_USER" --password="$MYSQL_PASSWORD"
+
+echo -e "\n`date` - Analyzing DB Tables"
+time docker exec -it nc-db mariadb-check \
+	-Aa --user="$MYSQL_USER" --password="$MYSQL_PASSWORD"
+
+echo -e "\n`date` - Optimizing DB Tables -- May take some time!!"
+time docker exec -itu www-data nc-db mariadb-check \
+	-Ao --user="$MYSQL_USER" --password="$MYSQL_PASSWORD"
+
+# Purge Spreed Messages?
+
+# Purge Anything Else?
+
+# Return to normal.
+echo -e "\n`date` - Disable Maintenance Mode."
+docker exec -itu www-data nc-app ./occ maintenance:mode --off
+
+exit 0

Config/ReverseProxy/docker-compose.micro.yml

@@ -40,4 +40,4 @@ services:
       resources:
         limits:
           cpus: '0.10'
-          memory: 16M
+          memory: 32M

bin/manage.sh

@@ -193,10 +193,13 @@ fi
 # Clean every type of Docker object which can be abandoined by Compose.
 if [[ -n $clean ]]; then
 	echo -e "\n*** Cleaning Abandoned Objects ***"
+	docker system df
 	docker image prune -a
 	docker container prune
 	docker volume prune
 	docker network prune
+	docker builder prune -a
+	docker system df
 fi
 
 # Follow the logs of a container.