Merge pull request 'Nextcloud, Hyperling.com, LibreTranslate, and backup.sh Enhancements' (#11) from dev into main

Reviewed-on: #11

Config/Hyperling.com/Dockerfile

@@ -6,20 +6,28 @@
 FROM node:lts-slim
 
 # Cache System Dependencies
-RUN apt-get update && apt-get install -y git php-cli sudo curl
+RUN apt-get update && apt-get install -y git php-cli sudo curl procps
 
 # Cache Node Dependencies
 RUN mkdir -p /var/www/api
 WORKDIR /var/www/api
-RUN echo '{ "dependencies": { "express": "" } }' > package.json
+RUN echo '{ "dependencies": { "express": ">=4.18.1 < 5.0.0" } }' > package.json
 RUN npm install
 
 # Install + Run Website
 CMD cd /var/www/api && \
-    rm -rfv pages main.js run.sh && \
-    git clone https://git.hyperling.com/me/nodejs-website website && \
-    rm -rfv website/files && \
-    mv -v website/* ./ && \
+    echo "Dockerfile: Ensure symlink is properly deleted if it exists..." && \
+    ls -l website/files; rm -v website/files; sleep 0 && \
+    echo "Dockerfile: Clone the website's Git repo..." && \
     rm -rfv website && \
-    echo "Starting Website's Run Script" && \
+    git clone https://git.hyperling.com/me/nodejs-website $BRANCH website && \
+    echo "Dockerfile: Remove dummy files and replace with symlink..." && \
+    rm -rfv website/files && \
+    cd website && \
+    echo -n "Dockerfile: Using branch " && \
+    git branch && \
+    ln -sv ../files ./files && \
+    echo "Dockerfile: Allow files under files/ to be ignored by Git..." && \
+    find ./files -exec git update-index --assume-unchanged {} \; && \
+    echo "Dockerfile: Start website!" && \
     ./run.sh

Config/Hyperling.com/docker-compose.main.yml

@@ -10,11 +10,15 @@ services:
     restart: always
     ports:
       - 8317:8080
+    environment:
+      - BRANCH=$BRANCH
+      - CPU=$CPU
+      - MEM=$MEM
     volumes:
       - ./files:/var/www/api/files
     deploy:
       mode: global
       resources:
         limits:
-          cpus: '0.25'
-          memory: 256M
+          cpus: $CPU
+          memory: $MEM

Config/Hyperling.com/docker-compose.micro.yml

@@ -1,21 +0,0 @@
-# 2022-10-29
-# Hyperling.com configuration.
-# 2023-01-24 container needs 128M to get started fully then settles to 25M.
-
-services:
-  app:
-    container_name: www-app
-    build:
-      context: ./
-      network: host
-    restart: always
-    ports:
-      - 8317:8080
-    volumes:
-      - ./files:/var/www/api/files
-    deploy:
-      mode: global
-      resources:
-        limits:
-          cpus: '0.10'
-          memory: 128M

Config/Hyperling.com/env.example

@@ -7,3 +7,18 @@
 # Subsequent builds consistently take less than 2 seconds.
 # Testing was done on the micro server, not a workstation.
 COMPOSE_BAKE=true
+
+## Branch ##
+# Allow choosing the branch. By leaving it blank, the main branch will be used.
+# Should be in the full "--branch my-branch" syntax.
+BRANCH=
+#BRANCH="--branch dev"
+
+## Performance ##
+# Standard Server
+CPU=0.25
+MEM=0.25G
+# Micro Server
+# 2023-01-24 container needs 128M to get started fully then settles to 25M.
+#CPU=0.10
+#MEM=128M

Config/LibreTranslate/env.example

@@ -47,3 +47,7 @@ MODELS_DIR=../../Volumes/LibreTranslate/models
 ## them from backup type locations such as Volumes/LibreTranslate/ and /var/.
 ## Please see the README for the CRON command related to this setting.
 # MODELS_DIR=/tmp/LibreTranslate/models
+
+## Can also place them under /opt/ which may be recommended by FHS standards.
+## This keeps them from taking up precious RAM / swap if using tmpfs or similar.
+# MODELS_DIR=/opt/LibreTranslate/models

Config/Nextcloud/Dockerfile

@@ -4,7 +4,7 @@
 FROM nextcloud:stable
 
 # Cache System Dependencies
-RUN apt-get update && apt-get install -y sudo libmagickcore-6.q16-6-extra htop \
+RUN apt-get update && apt-get install -y sudo libmagickcore-7.q16-10-extra htop \
     iputils-ping dnsutils vim bzip2 libbz2-dev
 
 # Configure PHP Dependency

Config/Nextcloud/maintenance.sh

@@ -35,12 +35,27 @@ $DOCKER_HOME/Config/Nextcloud/cron.sh && echo "Success!"
 # 2022-10-30 More additions after moving to Nextcloud version 25.
 echo -e "\n*** DATABASE ***"
 
+echo -e "\n`date` - Enable Maintenance Mode"
+docker exec -itu www-data nc-app ./occ maintenance:mode --on
+
+# 20251004: Disabled since it has not been working.
 # Clear DB errors by allowing the DB to perform any incremental upgrades.
 # https://jira.mariadb.org/browse/MDEV-26342
-echo -e "\n`date` - mysql_upgrade"
-docker exec -itu www-data nc-app ./occ maintenance:mode --on
-time docker exec -it nc-db mysql_upgrade \
-	-user="root" --password="$MYSQL_ROOT_PASSWORD"
+#echo -e "\n`date` - mysql_upgrade"
+#time docker exec -it nc-db mysql_upgrade \
+#	-user="root" --password="$MYSQL_ROOT_PASSWORD"
+
+echo -e "\n`date` - Enable Dynamic Row Formats"
+docker exec -it nc-db mysql -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" \
+    -e "SELECT CONCAT('ALTER TABLE ', TABLE_NAME,' ROW_FORMAT=DYNAMIC; ') FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '$MYSQL_DATABASE' AND ENGINE = 'InnoDB' AND ROW_FORMAT != 'DYNAMIC'; " \
+    -B -N |
+while read -r sql; do
+    echo "SQL being executed: $sql "
+    docker exec nc-db mysql -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "$sql" "$MYSQL_DATABASE";
+    echo " - Done!\ "
+done
+
+echo -e "\n`date` - Disable Maintenance Mode"
 docker exec -itu www-data nc-app ./occ maintenance:mode --off
 
 echo -e "\n`date` - Add Missing Columns"

Config/ProvisionTests/Dockerfiles/arch

@@ -0,0 +1,12 @@
+# Create a VM for testing Ansible provisioning.
+
+FROM archlinux:base
+
+RUN pacman -Syu --noconfirm && pacman -Sy --noconfirm $PACKAGES $EXTRA_ARCH
+
+COPY bin/main.sh /root/main.sh
+RUN chmod +x /root/main.sh
+
+WORKDIR /root/
+USER root
+CMD ./main.sh

Config/ProvisionTests/Dockerfiles/debian

@@ -0,0 +1,12 @@
+# Create a VM for testing Ansible provisioning.
+
+FROM debian:trixie
+
+RUN apt update && apt dist-upgrade -y && apt install -y $PACKAGES $EXTRA_DEBIAN
+
+COPY bin/main.sh /root/main.sh
+RUN chmod +x /root/main.sh
+
+WORKDIR /root/
+USER root
+CMD ./main.sh

Config/ProvisionTests/Dockerfiles/fedora

@@ -0,0 +1,12 @@
+# Create a VM for testing Ansible provisioning.
+
+FROM fedora:latest
+
+RUN dnf upgrade --refresh -y && dnf install -y $PACKAGES $EXTRA_FEDORA
+
+COPY bin/main.sh /root/main.sh
+RUN chmod +x /root/main.sh
+
+WORKDIR /root/
+USER root
+CMD ./main.sh

Config/ProvisionTests/Dockerfiles/opensuse

@@ -0,0 +1,14 @@
+# Create a VM for testing Ansible provisioning.
+
+FROM opensuse/tumbleweed
+
+RUN zypper -n refresh \
+   && zypper -n dist-upgrade -y \
+   && zypper -n install -y $PACKAGES $EXTRA_OPENSUSE
+
+COPY bin/main.sh /root/main.sh
+RUN chmod +x /root/main.sh
+
+WORKDIR /root/
+USER root
+CMD ./main.sh

Config/ProvisionTests/Dockerfiles/ubuntu

@@ -0,0 +1,12 @@
+# Create a VM for testing Ansible provisioning.
+
+FROM ubuntu:rolling
+
+RUN apt update && apt dist-upgrade -y && apt install -y $PACKAGES $EXTRA_UBUNTU
+
+COPY bin/main.sh /root/main.sh
+RUN chmod +x /root/main.sh
+
+WORKDIR /root/
+USER root
+CMD ./main.sh

Config/ProvisionTests/README.md

@@ -0,0 +1,3 @@
+# Provision Test Images
+
+Create containers of popular Linux distributions in order to test Ansible.

Config/ProvisionTests/bin/main.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+sh -c "rm -rfv prov-test"
+git clone $REPO --branch=$BRANCH prov-test
+
+cd prov-test
+
+echo "Starting Executable Script"
+
+tail -F /var/mail/mail &
+tail -F /var/mail/ansible &
+
+logfile="/root/test.log"
+$EXEC > $logfile &
+tail -F $logfile
+
+echo "*** Finished $0 @ `date` ***"
+
+wait -n
+
+exit $?

Config/ProvisionTests/docker-compose.standard.yml

@@ -0,0 +1,142 @@
+# Create containers which each execute a provisioning script and exit.
+
+volumes:
+
+  pt-storage:
+    driver: local
+    driver_opts:
+      type: none
+      device: $PT_STORAGE_DIR
+      o: bind
+
+services:
+
+  pt-arch:
+    container_name: pt-arch
+    build:
+      context: ./
+      dockerfile: Dockerfiles/arch
+      network: host
+    restart: no
+    environment:
+      - PACKAGES=$PACKAGES
+      - EXTRA_ARCH=$EXTRA_ARCH
+      - REPO=$REPO
+      - BRANCH=$BRANCH
+      - EXEC=$EXEC
+    volumes:
+      - type: volume
+        source: pt-storage
+        target: /root
+        volume:
+          subpath: arch
+    deploy:
+      mode: global
+      resources:
+        limits:
+          cpus: $CPU
+          memory: $RAM
+
+  pt-debian:
+    container_name: pt-debian
+    build:
+      context: ./
+      dockerfile: Dockerfiles/debian
+      network: host
+    restart: no
+    environment:
+      - PACKAGES=$PACKAGES
+      - EXTRA_DEBIAN=$EXTRA_DEBIAN
+      - REPO=$REPO
+      - BRANCH=$BRANCH
+      - EXEC=$EXEC
+    volumes:
+      - type: volume
+        source: pt-storage
+        target: /root
+        volume:
+          subpath: debian
+    deploy:
+      mode: global
+      resources:
+        limits:
+          cpus: $CPU
+          memory: $RAM
+
+  pt-fedora:
+    container_name: pt-fedora
+    build:
+      context: ./
+      dockerfile: Dockerfiles/fedora
+      network: host
+    restart: no
+    environment:
+      - PACKAGES=$PACKAGES
+      - EXTRA_FEDORA=$EXTRA_FEDORA
+      - REPO=$REPO
+      - BRANCH=$BRANCH
+      - EXEC=$EXEC
+    volumes:
+      - type: volume
+        source: pt-storage
+        target: /root
+        volume:
+          subpath: fedora
+    deploy:
+      mode: global
+      resources:
+        limits:
+          cpus: $CPU
+          memory: $RAM
+
+  pt-opensuse:
+    container_name: pt-opensuse
+    build:
+      context: ./
+      dockerfile: Dockerfiles/opensuse
+      network: host
+    restart: no
+    environment:
+      - PACKAGES=$PACKAGES
+      - EXTRA_OPENSUSE=$EXTRA_OPENSUSE
+      - REPO=$REPO
+      - BRANCH=$BRANCH
+      - EXEC=$EXEC
+    volumes:
+      - type: volume
+        source: pt-storage
+        target: /root
+        volume:
+          subpath: opensuse
+    deploy:
+      mode: global
+      resources:
+        limits:
+          cpus: $CPU
+          memory: $RAM
+
+  pt-ubuntu:
+    container_name: pt-ubuntu
+    build:
+      context: ./
+      dockerfile: Dockerfiles/ubuntu
+      network: host
+    restart: no
+    environment:
+      - PACKAGES=$PACKAGES
+      - EXTRA_UBUNTU=$EXTRA_UBUNTU
+      - REPO=$REPO
+      - BRANCH=$BRANCH
+      - EXEC=$EXEC
+    volumes:
+      - type: volume
+        source: pt-storage
+        target: /root
+        volume:
+          subpath: ubuntu
+    deploy:
+      mode: global
+      resources:
+        limits:
+          cpus: $CPU
+          memory: $RAM

Config/ProvisionTests/env.standard

@@ -0,0 +1,24 @@
+# This file should be renamed '.env' and have any private values modified.
+
+COMPOSE_BAKE=true
+
+## Performance
+CPU=0.2
+RAM=0.2G
+
+## Storage
+#STORAGE_DIR=../../Volumes/ProvisionTests
+PT_STORAGE_DIR=/tmp/ProvisionTests
+
+## Packages
+PACKAGES="git bash curl sudo which"
+EXTRA_ARCH=""
+EXTRA_DEBIAN=""
+EXTRA_FEDORA=""
+EXTRA_OPENSUSE=""
+EXTRA_UBUNTU=""
+
+## Script
+REPO=https://git.hyperling.com/me/env-ansible
+BRANCH=dev
+EXEC="./setup.sh -l"

Config/ProvisionTests/prep.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# 2025-07-22 Hyperling
+# Create the necessary folders for LibreTranslate's volumes to work.
+# This must be run before the container will start properly.
+
+## Setup ##
+
+DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+source $DIR/.env
+source ../../source.env
+
+## Main ##
+
+# Create folders.
+
+mkdir -pv "$PT_STORAGE_DIR/arch"
+mkdir -pv "$PT_STORAGE_DIR/debian"
+mkdir -pv "$PT_STORAGE_DIR/fedora"
+mkdir -pv "$PT_STORAGE_DIR/opensuse"
+mkdir -pv "$PT_STORAGE_DIR/ubuntu"
+
+# Finish successfully.
+exit 0

bin/backup.sh

@@ -9,23 +9,35 @@ source $DIR/../source.env
 
 ## Variables ##
 
-file="/tmp/Backup.zip"
+TAG="Docker"
+DATE="`date "+%Y%m%d-%H%M%S"`"
+BACKUP_DIR="/srv/backup"
+BASENAME="Backup"
+file="$BACKUP_DIR/$BASENAME.$DATE.$HOSTNAME.$TAG.zip"
 time="`which time`"
 
 ## Main ##
 
+# Ensure backup directory exists with correct permissions.
+mkdir -pv "$BACKUP_DIR"
+mkdir -pv "$BACKUP_DIR/TRASH"
+chmod -Rv 775 "$BACKUP_DIR"
+chown -Rv root:root "$BACKUP_DIR"
+
 # Remove the last backup.
-if [[ -e $file ]]; then
-	echo "`date` - Removing existing file."
-	rm -fv $file
-fi
+echo "`date` - Removing existing files."
+cd "$BACKUP_DIR"
+mv -v "$BASENAME"*"$TAG"* TRASH/
+rm -v TRASH/*
 
 echo -e "\n`date` - Take down services for a cold backup."
 manage.sh -d
 
 echo -e "\n`date` - Create the backup for '$DOCKER_HOME'."
-cd $DOCKER_HOME
-$time zip -r $file.tmp . 1>/dev/null
+$time zip -r $file.tmp \
+	$DOCKER_HOME \
+	/etc/crontab /etc/cron.d /var/spool/cron \
+	/var/{log,mail} 1>/dev/null
 mv -v $file.tmp $file
 
 echo -e "\n`date` - Done with zipping, check size."

bin/manage.sh

@@ -130,7 +130,7 @@ fi
 if [[ -z $up && -z $down && -z $build && -z $pull && -z $clean
 	&& -z $interact && -z $logs && -z $stats
 ]]; then
-	docker ps
+	docker ps --size
 	exit 0
 fi