From a55e94cbd669976e8435a0657245e22c55a0df85 Mon Sep 17 00:00:00 2001 From: localnerve Date: Wed, 24 Dec 2025 00:08:29 -0500 Subject: [PATCH 1/2] @2.8.4 - docker script updates --- .dockerignore | 2 + .gitignore | 3 + docker-build.sh | 3 - docker-db-backup.sh | 4 - docker-db-restore.sh | 7 -- docker-extract-backup.sh | 4 - docker-run-maint.sh | 4 - .env.dev => docker/.env.dev | 0 Dockerfile => docker/Dockerfile | 0 .../authorizer-dev.yml | 0 docker/build-image.sh | 14 ++++ docker/compose-container.sh | 8 ++ docker/db-backup-extract.sh | 7 ++ docker/db-backup-insert.sh | 7 ++ docker/db-backup.sh | 8 ++ docker/db-restore.sh | 12 +++ .../docker-compose.yml | 81 +++++++++++-------- docker/run-maint.sh | 9 +++ {volumes => docker/volumes}/backup/.gitkeep | 0 docs/localsetup.md | 11 +-- docs/stats.md | 2 +- package.json | 2 +- 22 files changed, 127 insertions(+), 61 deletions(-) delete mode 100755 docker-build.sh delete mode 100755 docker-db-backup.sh delete mode 100755 docker-db-restore.sh delete mode 100755 docker-extract-backup.sh delete mode 100755 docker-run-maint.sh rename .env.dev => docker/.env.dev (100%) rename Dockerfile => docker/Dockerfile (100%) rename docker-authorizer-dev.yml => docker/authorizer-dev.yml (100%) create mode 100755 docker/build-image.sh create mode 100755 docker/compose-container.sh create mode 100755 docker/db-backup-extract.sh create mode 100755 docker/db-backup-insert.sh create mode 100755 docker/db-backup.sh create mode 100755 docker/db-restore.sh rename docker-compose.yml => docker/docker-compose.yml (71%) create mode 100755 docker/run-maint.sh rename {volumes => docker/volumes}/backup/.gitkeep (100%) diff --git a/.dockerignore b/.dockerignore index c25c6217..48164d98 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,8 @@ node_modules dist tmp test-results +stats +docker docs art dump diff --git a/.gitignore b/.gitignore index f30132a7..80c5c47b 100644 --- a/.gitignore +++ b/.gitignore @@ -156,5 +156,8 @@ stats # no development docker compose *-dev-docker-compose.yml +# no volume backups +docker/volumes/backup/*.gz + # no insights docs docs/insights \ No newline at end of file diff --git a/docker-build.sh b/docker-build.sh deleted file mode 100755 index a69847e4..00000000 --- a/docker-build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -docker buildx build --tag "jam-build" --secret id=jam-build,src=./private/host-env.json . --build-arg UID=`id -u` --build-arg GID=`id -g` --build-arg AUTHZ_URL --build-arg AUTHZ_CLIENT_ID \ No newline at end of file diff --git a/docker-db-backup.sh b/docker-db-backup.sh deleted file mode 100755 index 90cf6d0d..00000000 --- a/docker-db-backup.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# TODO: update with real environment -docker compose --env-file .env.dev run --rm backup \ No newline at end of file diff --git a/docker-db-restore.sh b/docker-db-restore.sh deleted file mode 100755 index 731e0bef..00000000 --- a/docker-db-restore.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -# TODO: update with real environment -docker compose --env-file .env.dev run --rm hydrate - -# Example: Restore from specific backup -# docker compose --env-file .env.dev run --rm -e BACKUP_FILE=db-20250116-143052.tar.gz hydrate \ No newline at end of file diff --git a/docker-extract-backup.sh b/docker-extract-backup.sh deleted file mode 100755 index 4a4b8b70..00000000 --- a/docker-extract-backup.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# extract sql dumps to local folder -docker compose run --rm extract-backup \ No newline at end of file diff --git a/docker-run-maint.sh b/docker-run-maint.sh deleted file mode 100755 index c16e3600..00000000 --- a/docker-run-maint.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -# 2 hour maintenance window -docker-compose run --service-ports jam-build --MAINTENANCE="`date -v+2H -u +'%a, %d %b %Y %H:%M:%S GMT'`" \ No newline at end of file diff --git a/.env.dev b/docker/.env.dev similarity index 100% rename from .env.dev rename to docker/.env.dev diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile diff --git a/docker-authorizer-dev.yml b/docker/authorizer-dev.yml similarity index 100% rename from docker-authorizer-dev.yml rename to docker/authorizer-dev.yml diff --git a/docker/build-image.sh b/docker/build-image.sh new file mode 100755 index 00000000..a09374f4 --- /dev/null +++ b/docker/build-image.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) +DOCKER_FILE=$SCRIPT_DIR/Dockerfile +PROJECT_DIR=$(readlink -f $SCRIPT_DIR/..) + +# If not exist, falls back to environment +HOSTENV_FILE=$PROJECT_DIR/private/host-env.json + +# Change these if building a non-demo image +export AUTHZ_URL=http://localhost:9010 +export AUTHZ_CLIENT_ID=deadbeef-cafe-babe-feed-baadc0deface + +docker buildx build --tag "jam-build" --secret id=jam-build,src=$HOSTENV_FILE --file $DOCKER_FILE $PROJECT_DIR --build-arg UID=`id -u` --build-arg GID=`id -g` --build-arg AUTHZ_URL --build-arg AUTHZ_CLIENT_ID \ No newline at end of file diff --git a/docker/compose-container.sh b/docker/compose-container.sh new file mode 100755 index 00000000..3f1cfe59 --- /dev/null +++ b/docker/compose-container.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) +PROJECT_DIR=$(readlink -f $SCRIPT_DIR/..) +COMPOSE_FILE=$PROJECT_DIR/docker/docker-compose.yml +ENV_FILE=$PROJECT_DIR/docker/.env.dev + +docker compose --file $COMPOSE_FILE --env-file $ENV_FILE up -d \ No newline at end of file diff --git a/docker/db-backup-extract.sh b/docker/db-backup-extract.sh new file mode 100755 index 00000000..975936da --- /dev/null +++ b/docker/db-backup-extract.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) +COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml + +# extract sql dumps to local folder +docker compose --file $COMPOSE_FILE run --rm backup-extract \ No newline at end of file diff --git a/docker/db-backup-insert.sh b/docker/db-backup-insert.sh new file mode 100755 index 00000000..745271bd --- /dev/null +++ b/docker/db-backup-insert.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) +COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml + +# insert local sql dumps into container volume (for hydration) +docker compose --file $COMPOSE_FILE run --rm backup-insert \ No newline at end of file diff --git a/docker/db-backup.sh b/docker/db-backup.sh new file mode 100755 index 00000000..006d5050 --- /dev/null +++ b/docker/db-backup.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) +# TODO: update with real environment +ENV_FILE=$SCRIPT_DIR/.env.dev +COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml + +docker compose --file $COMPOSE_FILE --env-file $ENV_FILE run --rm backup \ No newline at end of file diff --git a/docker/db-restore.sh b/docker/db-restore.sh new file mode 100755 index 00000000..7bbfb46b --- /dev/null +++ b/docker/db-restore.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) +# TODO: update with real environment +ENV_FILE=$SCRIPT_DIR/.env.dev +COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml + +# Restore from latest backup +# docker compose --file $COMPOSE_FILE --env-file $ENV_FILE run --rm hydrate + +# Restore from specific backup +docker compose --file $COMPOSE_FILE --env-file $ENV_FILE run --rm -e BACKUP_FILE=db-20251224-021601.tar.gz hydrate \ No newline at end of file diff --git a/docker-compose.yml b/docker/docker-compose.yml similarity index 71% rename from docker-compose.yml rename to docker/docker-compose.yml index c6ce3084..4c828fec 100644 --- a/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,7 +3,8 @@ services: # The app jam-build: build: - context: . + context: ../ + dockerfile: ./docker/Dockerfile args: UID: ${UID} GID: ${GID} @@ -99,10 +100,10 @@ services: - 3306:3306 volumes: - dbdata:/var/lib/mysql - - ./data/database/mariadb-ddl-init.sh:/docker-entrypoint-initdb.d/1.sh - - ./data/database/mariadb-ddl-tables.sql:/docker-entrypoint-initdb.d/2.sql - - ./data/database/mariadb-ddl-procedures.sql:/docker-entrypoint-initdb.d/3.sql - - ./data/database/mariadb-ddl-privileges.sql:/docker-entrypoint-initdb.d/4.sql + - ../data/database/mariadb-ddl-init.sh:/docker-entrypoint-initdb.d/1.sh + - ../data/database/mariadb-ddl-tables.sql:/docker-entrypoint-initdb.d/2.sql + - ../data/database/mariadb-ddl-procedures.sql:/docker-entrypoint-initdb.d/3.sql + - ../data/database/mariadb-ddl-privileges.sql:/docker-entrypoint-initdb.d/4.sql healthcheck: test: ["CMD-SHELL", "mariadb-admin -u root --password=$DB_ROOT_PASSWORD ping -h localhost"] timeout: 20s @@ -127,8 +128,8 @@ services: profiles: - tools - # Extract backups to local bind mount - extract-backup: + # Extract backups from container volume to local bind mount + backup-extract: image: alpine:latest volumes: - backup:/source @@ -138,12 +139,23 @@ services: profiles: - tools + # Insert backups from local bind mount to container volume + backup-insert: + image: alpine:latest + volumes: + - backup:/dest + - ./volumes/backup:/source + entrypoint: ["/bin/sh"] + command: ["-c", "cp -av /source/* /dest/ && echo 'Backup files inserted into named backup volume'"] + profiles: + - tools + # The database hydration command hydrate: image: mariadb:12.1.2 environment: DB_ROOT_PASSWORD: ${DB_ROOT_PASSWORD} - BACKUP_FILE: ${BACKUP_FILE:-latest} # Specify backup file or 'latest' + BACKUP_FILE: ${BACKUP_FILE} depends_on: mariadb: condition: service_healthy @@ -151,49 +163,54 @@ services: - backup:/backup networks: - backend - entrypoint: ["/bin/sh"] + entrypoint: ["/bin/bash"] command: - "-c" - | set -e echo "Starting database restore..." - + + # Update a local variable + BACKUP_FILE_="$$BACKUP_FILE" # can be unset or 'latest' + # Find the backup file - if [ "$BACKUP_FILE" = "latest" ]; then - BACKUP_FILE=$(ls -t /backup/db-*.tar.gz 2>/dev/null | head -n1) - if [ -z "$BACKUP_FILE" ]; then + if [ -z "$$BACKUP_FILE_" ] || [ "$$BACKUP_FILE_" = "latest" ]; then + BACKUP_FILE_=$(ls -t /backup/db-*.tar.gz 2>/dev/null | head -n1) + if [ -z "$$BACKUP_FILE_" ]; then echo "Error: No backup files found in /backup/" exit 1 fi - echo "Using latest backup: $BACKUP_FILE" + echo "Using latest backup: $$BACKUP_FILE_" else - BACKUP_FILE="/backup/$BACKUP_FILE" - if [ ! -f "$BACKUP_FILE" ]; then - echo "Error: Backup file not found: $BACKUP_FILE" + BACKUP_FILE_="/backup/$$BACKUP_FILE_" + if [ ! -f "$$BACKUP_FILE_" ]; then + echo "Error: Backup file not found: $$BACKUP_FILE_" exit 1 fi fi - + # Extract the backup echo "Extracting backup..." - tar -xzf "$BACKUP_FILE" -C /backup - SQL_FILE=$(tar -tzf "$BACKUP_FILE" | head -n1) - + tar -xzf "$$BACKUP_FILE_" -C /backup + SQL_FILE=$(tar -tzf "$$BACKUP_FILE_" | head -n1) + # Drop all databases except system databases echo "Dropping existing databases..." - mariadb -h mariadb -u root --password="$DB_ROOT_PASSWORD" \ - -e "SELECT CONCAT('DROP DATABASE IF EXISTS \`', schema_name, '\`;') - FROM information_schema.schemata - WHERE schema_name NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys')" \ - -sN | mariadb -h mariadb -u root --password="$DB_ROOT_PASSWORD" - + { + echo "SET foreign_key_checks = 0;"; + mariadb -h mariadb -u root --password="$DB_ROOT_PASSWORD" -sN \ + -e "SELECT CONCAT('DROP DATABASE IF EXISTS \`', schema_name, '\`;') + FROM information_schema.schemata + WHERE schema_name NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys');" + } | mariadb -h mariadb -u root --password="$DB_ROOT_PASSWORD" + # Restore from backup - echo "Restoring databases from $SQL_FILE..." - mariadb -h mariadb -u root --password="$DB_ROOT_PASSWORD" < "/backup/$SQL_FILE" - + echo "Restoring databases from $$SQL_FILE..." + mariadb -h mariadb -u root --password="$DB_ROOT_PASSWORD" < "/backup/$$SQL_FILE" + # Cleanup extracted SQL file - rm -f "/backup/$SQL_FILE" - + rm -f "/backup/$$SQL_FILE" + echo "Database restore complete!" profiles: - tools diff --git a/docker/run-maint.sh b/docker/run-maint.sh new file mode 100755 index 00000000..03073ac6 --- /dev/null +++ b/docker/run-maint.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) +ENV_FILE=$SCRIPT_DIR/.env.dev +COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml + +# Bring the app down for maintenance +# Start with a 2 hour maintenance window on a running container +docker compose --file $COMPOSE_FILE --env-file $ENV_FILE run --service-ports --name jam-build-maint jam-build --MAINTENANCE="`date -v+2H -u +'%a, %d %b %Y %H:%M:%S GMT'`" \ No newline at end of file diff --git a/volumes/backup/.gitkeep b/docker/volumes/backup/.gitkeep similarity index 100% rename from volumes/backup/.gitkeep rename to docker/volumes/backup/.gitkeep diff --git a/docs/localsetup.md b/docs/localsetup.md index 654ea46b..fda4495a 100644 --- a/docs/localsetup.md +++ b/docs/localsetup.md @@ -1,6 +1,6 @@ --- Author: Alex Grant (https://www.localnerve.com) -Date: December 11, 2025 +Date: December 23, 2025 Title: Getting Started --- @@ -13,8 +13,9 @@ Title: Getting Started ### Installation Steps 1. **Install Docker Desktop**: Download and install Docker Desktop from the official website. 2. **Run Docker Compose**: - - Execute `docker compose --env-file .env.dev up` to build and start the services. Wait for it to complete. - - Optionally, create your own `.env` file if needed. + - Execute `docker compose --file docker/docker-compose.yml --env-file docker/.env.dev up` to build and start the services. + - Or just execute `docker/compose-container.sh` to build and compose the demo. + - Optionally, create your own `.env` file. - Local ports 3306, 5000, 6379, and 9010 must be free prior to service start. ### Configuration Steps @@ -52,9 +53,9 @@ Title: Getting Started ``` #### Authorizer.dev Setup -- **Installation**: Use Docker to run and manage Authorizer.dev. A sample configuration file is provided in [**docker-authorizer-dev.yml**](/docker-authorizer-dev.yml) to run Authorizer.dev standalone using the local MariaDB instance. +- **Installation**: Use Docker to run and manage Authorizer.dev. A sample configuration file is provided in [**authorizer-dev.yml**](/docker/authorizer-dev.yml) to run Authorizer.dev standalone using the local MariaDB instance. -- Use [**docker-authorizer-dev.yml**](/docker-authorizer-dev.yml) to run authorizer.dev locally on port 9010 using the locally installed MariaDB instance. +- Use [**authorizer-dev.yml**](/docker/authorizer-dev.yml) to run authorizer.dev locally on port 9010 using the locally installed MariaDB instance. - Generate `ADMIN_SECRET` and `CLIENT_ID` using local CLI tools like `uuidgen` and `openssl`. - Choose any port, but the default settings in `package.json` scripts are easiest to use. diff --git a/docs/stats.md b/docs/stats.md index 2ed735a3..ee415959 100644 --- a/docs/stats.md +++ b/docs/stats.md @@ -1,6 +1,6 @@ --- Author: Alex Grant (https://www.localnerve.com) -Date: August 30, 2025 +Date: December 15, 2025 Title: Project Statistics --- diff --git a/package.json b/package.json index 583e151f..c9f93745 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jam-build", - "version": "2.8.3", + "version": "2.8.4", "description": "An adventurous, scalable, fullstack web application reference project", "main": "index.js", "type": "module", From 26d630e026e9653ad689dcac6c05af955f10e504 Mon Sep 17 00:00:00 2001 From: localnerve Date: Wed, 24 Dec 2025 17:48:04 -0500 Subject: [PATCH 2/2] @2.8.4 - db restore script updates, docker api accomodations --- docker/Dockerfile => Dockerfile | 0 docker/build-image.sh | 2 +- ...-backup-insert.sh => db-restore-insert.sh} | 2 +- docker/db-restore.sh | 4 +-- docker/docker-compose.yml | 33 ++++++++++--------- 5 files changed, 21 insertions(+), 20 deletions(-) rename docker/Dockerfile => Dockerfile (100%) rename docker/{db-backup-insert.sh => db-restore-insert.sh} (75%) diff --git a/docker/Dockerfile b/Dockerfile similarity index 100% rename from docker/Dockerfile rename to Dockerfile diff --git a/docker/build-image.sh b/docker/build-image.sh index a09374f4..aaf8bb0f 100755 --- a/docker/build-image.sh +++ b/docker/build-image.sh @@ -1,8 +1,8 @@ #!/bin/sh SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) -DOCKER_FILE=$SCRIPT_DIR/Dockerfile PROJECT_DIR=$(readlink -f $SCRIPT_DIR/..) +DOCKER_FILE=$PROJECT_DIR/Dockerfile # If not exist, falls back to environment HOSTENV_FILE=$PROJECT_DIR/private/host-env.json diff --git a/docker/db-backup-insert.sh b/docker/db-restore-insert.sh similarity index 75% rename from docker/db-backup-insert.sh rename to docker/db-restore-insert.sh index 745271bd..0e48ddf4 100755 --- a/docker/db-backup-insert.sh +++ b/docker/db-restore-insert.sh @@ -4,4 +4,4 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" &> /dev/null && pwd) COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml # insert local sql dumps into container volume (for hydration) -docker compose --file $COMPOSE_FILE run --rm backup-insert \ No newline at end of file +docker compose --file $COMPOSE_FILE run --rm restore-insert \ No newline at end of file diff --git a/docker/db-restore.sh b/docker/db-restore.sh index 7bbfb46b..39c4bb4a 100755 --- a/docker/db-restore.sh +++ b/docker/db-restore.sh @@ -6,7 +6,7 @@ ENV_FILE=$SCRIPT_DIR/.env.dev COMPOSE_FILE=$SCRIPT_DIR/docker-compose.yml # Restore from latest backup -# docker compose --file $COMPOSE_FILE --env-file $ENV_FILE run --rm hydrate +# docker compose --file $COMPOSE_FILE --env-file $ENV_FILE run --rm restore # Restore from specific backup -docker compose --file $COMPOSE_FILE --env-file $ENV_FILE run --rm -e BACKUP_FILE=db-20251224-021601.tar.gz hydrate \ No newline at end of file +docker compose --file $COMPOSE_FILE --env-file $ENV_FILE run --rm -e BACKUP_FILE=db-20251224-021601.tar.gz restore \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 4c828fec..1f2849d1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -4,7 +4,7 @@ services: jam-build: build: context: ../ - dockerfile: ./docker/Dockerfile + dockerfile: ./Dockerfile args: UID: ${UID} GID: ${GID} @@ -44,7 +44,7 @@ services: # The cache cache: - image: redis:8.2.3 + image: redis:8.4.0 restart: unless-stopped ports: - 6379:6379 @@ -139,19 +139,8 @@ services: profiles: - tools - # Insert backups from local bind mount to container volume - backup-insert: - image: alpine:latest - volumes: - - backup:/dest - - ./volumes/backup:/source - entrypoint: ["/bin/sh"] - command: ["-c", "cp -av /source/* /dest/ && echo 'Backup files inserted into named backup volume'"] - profiles: - - tools - - # The database hydration command - hydrate: + # The database hydration command: Restore from dump in the backup volume + restore: image: mariadb:12.1.2 environment: DB_ROOT_PASSWORD: ${DB_ROOT_PASSWORD} @@ -195,9 +184,10 @@ services: SQL_FILE=$(tar -tzf "$$BACKUP_FILE_" | head -n1) # Drop all databases except system databases + # 'foreign_key_checks = 0' to ignore cross db fk constraints echo "Dropping existing databases..." { - echo "SET foreign_key_checks = 0;"; + echo "SET foreign_key_checks = 0;"; mariadb -h mariadb -u root --password="$DB_ROOT_PASSWORD" -sN \ -e "SELECT CONCAT('DROP DATABASE IF EXISTS \`', schema_name, '\`;') FROM information_schema.schemata @@ -215,6 +205,17 @@ services: profiles: - tools + # Insert dumps from local bind mount to container volume for restore + restore-insert: + image: alpine:latest + volumes: + - backup:/dest + - ./volumes/backup:/source + entrypoint: ["/bin/sh"] + command: ["-c", "cp -av /source/* /dest/ && echo 'Dump files inserted into named backup volume'"] + profiles: + - tools + volumes: dbdata: backup: