From af5027bd0b385808ab31c49cc591881cae706ab8 Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Fri, 31 Oct 2025 10:27:41 -0500 Subject: [PATCH 01/12] refactor: Incorporate fastdb support into backup/restore scripts Originally, it was not desirable to wrap fastdb logic into the official backup/restore scripts, since fastdb is an unsafe backup approach. However, with the change where the DB restore now occurs immediately as part of the restore (instead of just decompressing a SQL dump file for when the container starts), we lose the ability to utilize fastdb to expedite a restore in development. By incorporating fastdb into the backup/restore scripts, we can reestablish the DX feature of being able to rapidly restore an environment. Example usage: Make a normal backup with a fastdb archive, with no compression: mdl backup mymoodle container -l wow --fastdb -c none Restore with fastdb: mdl restore mymoodle wow --fastdb --- libexec/mdl-backup.sh | 54 +++++++++++++++++--- libexec/mdl-fast-db-backup.sh | 61 ----------------------- libexec/mdl-fast-db-restore.sh | 73 --------------------------- libexec/mdl-restore.sh | 91 ++++++++++++++++++++-------------- 4 files changed, 99 insertions(+), 180 deletions(-) delete mode 100755 libexec/mdl-fast-db-backup.sh delete mode 100755 libexec/mdl-fast-db-restore.sh diff --git a/libexec/mdl-backup.sh b/libexec/mdl-backup.sh index 5300eb2..1fc052a 100755 --- a/libexec/mdl-backup.sh +++ b/libexec/mdl-backup.sh @@ -3,7 +3,7 @@ . "${0%/*}/../lib/mdl-common.sh" # Valid Options -valid_modules='src data db' +valid_modules='src data db fastdb' valid_compress='bzip2 gzip xz none' # Defaults @@ -44,6 +44,7 @@ Options: -l, --label Label for the backup. Default is today's date. (i.e. $ul$default_label$norm) -m, --modules Which module to backup. ($(echo "${valid_modules}" | sed "s/ /, /g" | sed "s/\([^, ]*\)/${ul}\1$norm/g")) -c, --compress Which compression, default is $ul$default_compress_arg$norm. ($(echo "${valid_compress}" | sed "s/ /, /g" | sed "s/\([^, ]*\)/${ul}\1$norm/g")) +-f, --fastdb Perform an unsafe fast database backup instead of a SQL dump. -e, --ssh-args Additional SSH arguments to pass when using ssh. -t, --container-tool Which container tool to use (docker or podman). -n, --dry-run Show what would've happened without executing. @@ -95,7 +96,7 @@ fi # Collect optional arguments. # shellcheck disable=SC2214 # spellchecker: disable-next-line -while getopts hsvnl:m:e:c:-: OPT; do +while getopts hfsvnl:m:e:c:-: OPT; do support_long_options case "$OPT" in h | help) @@ -109,6 +110,14 @@ while getopts hsvnl:m:e:c:-: OPT; do # Convert to lowercase. compress_arg=$(echo "$OPTARG" | tr '[:upper:]' '[:lower:]') ;; + f | fastdb) + # Remove "db" and add "fastdb" to modules list + new_modules='fastdb' + for word in $modules; do + [[ $word == db || $word == fastdb ]] || new_modules="$new_modules $word" + done + modules=$new_modules + ;; e | ssh-args) ssh_args=$OPTARG ;; v | verbose) verbose=true ;; n | dry-run) dry_run=true ;; @@ -146,7 +155,7 @@ fi # Only valid modules for m in $modules; do - if [[ ! "$valid_modules" =~ $m ]]; then + if [[ " $valid_modules " != *" $m "* ]]; then echo -e "${red}Error: Invalid module type: $m.$norm\n" >&2 exit 1 fi @@ -158,6 +167,17 @@ if [[ ! "$valid_compress" =~ $compress_arg ]]; then exit 1 fi +# Fast DB backup is only for container sources +if [[ $modules =~ fastdb ]]; then + if $is_container; then + echo -e "${yellow}Warning: Fast database backups are unsafe for production use.$norm\n" >&2 + else + echo -e "${red}Error: Fast database backups are only supported for container sources.$norm\n" >&2 + exit 1 + fi +fi + + # Check necessary utilities cmds=(tar "${MDL_CONTAINER_TOOL[0]}") [[ $compress_arg != none ]] && cmds+=("$compress_arg") @@ -224,7 +244,7 @@ for mname in $mnames; do # If "container", the container must be active in order to dump the database for the "db" module. source_db_container='' if $is_container; then - if [[ $modules =~ db ]]; then + if [[ " $modules " == *" db "* ]]; then source_db_container_cmd=" ${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} $container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep mariadb | head -1 @@ -233,7 +253,7 @@ for mname in $mnames; do source_db_container=$(eval "$source_db_container_cmd") [[ -z $source_db_container ]] && echo "${red}The database container cannot be found. Start the environment to perform a backup." >&2 && exit 1 fi - if [[ $modules =~ data || $modules =~ src ]]; then + if [[ $modules =~ data || $modules =~ src || $modules =~ fastdb ]]; then vols_cmd=" ${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} $container_tool volume ls -q --filter 'label=com.docker.compose.project=$mname' @@ -242,6 +262,7 @@ for mname in $mnames; do vols=$(eval "$vols_cmd") [[ $modules =~ data ]] && source_data_volume=$(grep data <<< "$vols") [[ $modules =~ src ]] && source_src_volume=$(grep src <<< "$vols") + [[ $modules =~ fastdb ]] && source_db_volume=$(grep db <<< "$vols") fi fi @@ -259,7 +280,8 @@ for mname in $mnames; do # Targets [[ $modules =~ data ]] && data_target="$MDL_BACKUP_DIR/${mname}_${label}_data.tar$compress_ext" [[ $modules =~ src ]] && src_target="$MDL_BACKUP_DIR/${mname}_${label}_src.tar$compress_ext" - [[ $modules =~ db ]] && db_target="$MDL_BACKUP_DIR/${mname}_${label}_db.sql$compress_ext" + [[ $modules =~ fastdb ]] && fastdb_target="$MDL_BACKUP_DIR/${mname}_${label}_dbfiles.tar$compress_ext" + [[ " $modules " == *" db "* ]] && db_target="$MDL_BACKUP_DIR/${mname}_${label}_db.sql$compress_ext" action_word='Backing up' echo -e "$ul$bold$action_word $mname environment$norm" @@ -271,6 +293,7 @@ for mname in $mnames; do [[ -n $source_data_path ]] && echo "$bold 'data' Path:$norm $source_data_path" [[ -n $source_src_volume ]] && echo "$bold 'src' Volume:$norm $source_src_volume" [[ -n $source_data_volume ]] && echo "$bold 'data' Volume:$norm $source_data_volume" + [[ -n $source_db_volume ]] && echo "$bold 'db' Volume:$norm $source_db_volume" [[ -n $source_db_container ]] && echo "$bold DB Container:$norm $source_db_container" [[ -n $source_db_name ]] && echo "$bold DB Name:$norm $source_db_name" [[ -n $source_db_username ]] && echo "$bold DB Username:$norm $source_db_username" @@ -282,6 +305,7 @@ for mname in $mnames; do [[ -n $src_target ]] && echo "$bold 'src' Path:$norm $src_target" [[ -n $data_target ]] && echo "$bold 'data' Path:$norm $data_target" [[ -n $db_target ]] && echo "$bold DB Path:$norm $db_target" + [[ -n $fastdb_target ]] && echo "$bold Fast DB Path:$norm $fastdb_target" echo fi @@ -290,6 +314,7 @@ for mname in $mnames; do exit 1 fi + # MODULE: data # shellcheck disable=SC2034 data_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} $is_container && data_cmd="$data_cmd $container_tool run --rm --name '${mname}_worker_bk_data' -v '$source_data_volume':/data $MDL_SHELL_IMAGE" @@ -304,6 +329,8 @@ for mname in $mnames; do -C $($is_container && echo /data || echo "$source_data_path") . \ " data_cmd=$data_cmd${source_host:+"\\\"\""} + + # MODULE: src # shellcheck disable=SC2034 src_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} $is_container && src_cmd="$src_cmd $container_tool run --rm --name '${mname}_worker_bk_src' -v '$source_src_volume':/src $MDL_SHELL_IMAGE" @@ -312,6 +339,15 @@ for mname in $mnames; do -C $($is_container && echo /src || echo "$source_src_path") . \ " src_cmd=$src_cmd${source_host:+"\\\"\""} + + # MODULE: fastdb + # shellcheck disable=SC2034 + fastdb_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} + $is_container && fastdb_cmd="$fastdb_cmd $container_tool run --rm --name '${mname}_worker_bk_fastdb' -v '$source_db_volume':/db $MDL_SHELL_IMAGE" + $is_container && fastdb_cmd="$fastdb_cmd tar c $compress_flag -C /db ." + fastdb_cmd=$fastdb_cmd${source_host:+"\\\"\""} + + # MODULE: db # TODO: When piping to compression program, a failed status of mysqldump will be lost. # shellcheck disable=SC2034 db_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} @@ -326,9 +362,11 @@ for mname in $mnames; do " db_cmd=$db_cmd${source_host:+"\\\"\""} [[ $compress_arg != none ]] && db_cmd="$db_cmd | $compress_arg -cq9" + + # Actually EXECUTE the commands pids=() - for t in data src db; do - if [[ $modules =~ $t ]]; then + for t in $valid_modules; do + if [[ " $modules " == *" $t "* ]]; then targ_var=${t}_target; targ=${!targ_var} cmd_var=${t}_cmd; cmd=${!cmd_var} echo "$mname $t: $targ" diff --git a/libexec/mdl-fast-db-backup.sh b/libexec/mdl-fast-db-backup.sh deleted file mode 100755 index 6764441..0000000 --- a/libexec/mdl-fast-db-backup.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -. "${0%/*}/../lib/mdl-common.sh" - -display_help() { - cat < [LABEL] - -Makes a fast database backup, which is just a tar archive of the raw database files. The -reason this is fast is because the archive process is faster than a database dump, and -because the restore process directly restores the database files, as opposed to the -traditional restore which saves the dump in the environment and requires the dump to be -processed by the database container on startup. -$bold$red -This is unsafe for production but often works fine for development purposes. -$norm -Options: --h, --help Show this help message and exit. -EOF -} - -[[ $* =~ -h || $* =~ --help ]] && display_help && exit - -requires "${MDL_CONTAINER_TOOL[0]}" - -mnames=$("$scr_dir"/mdl-select-env.sh "$1") - -echo ' -WARNING: This makes a fast database backup, which is just a tar archive of the -filesystem. This should not be used for production purposes. -' - -for mname in $mnames; do - - echo "Fast backup of the $mname database." - - # Abort if the volume can't be found - db_vol_name=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname" | grep db) - if [ -z "$db_vol_name" ]; then - echo "Database volume for $mname could not be found." - exit 1 - fi - - # What label on the backup do they want? (Defaults to "local_branchver_yyyymmdd") - branchver=$("$scr_dir"/mdl-moodle-version.sh "$mname") - defaultlabel="local_${branchver}_$(date +"%Y%m%d")" - label="$2" - if [ "$label" = "" ]; then - echo -n "Enter the label to put on the backup [$defaultlabel]: " - read -r label - label="${label:-$defaultlabel}" - fi - - "$scr_dir/mdl-stop.sh" "$mname" - - db_target="${mname}_${label}_dbfiles.tar" - container_tool run --rm --privileged -v "$db_vol_name":/db -v "$MDL_BACKUP_DIR":/backup "$MDL_SHELL_IMAGE" tar cf "/backup/$db_target" -C /db . - - echo "Fast backup of $mname is done!" - -done diff --git a/libexec/mdl-fast-db-restore.sh b/libexec/mdl-fast-db-restore.sh deleted file mode 100755 index 1040d46..0000000 --- a/libexec/mdl-fast-db-restore.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -. "${0%/*}/../lib/mdl-common.sh" - -display_help() { - cat < [LABEL] - -Restores a fast database backup, just overwriting the raw database files. $bold${red}This is unsafe -for production.$norm However, it can make the development cycle of restoring to a previous -state much faster; whereas you do need to restart the environment, the database will -start up instantaneously since it doesn't have to import a database dump file. - -Options: --h, --help Show this help message and exit. -EOF -} - -[[ $* =~ -h || $* =~ --help ]] && display_help && exit - -requires "${MDL_CONTAINER_TOOL[0]}" - -mnames=$("$scr_dir/mdl-select-env.sh" "$1") - -echo ' -WARNING: This is restoring a fast database backup, which is just restoring the -database filesystem. Whereas this often works in a dev environment, it should -never be used for production purposes. -' - -for mname in $mnames; do - - export_env "$mname" - echo "Preparing to restore a fast database backup of $mname..." - - # Stop the services if they're running - "$scr_dir/mdl-stop.sh" "$mname" - echo - - # What timestamp of backup do they want? (Select from the list if they did not provide) - labels=$(find "$MDL_BACKUP_DIR" -name "${mname}_*_dbfiles.tar" | cut -d"_" -f2- | sed -e "s/_dbfiles.tar//") - [ -z "$labels" ] && echo "There are no fast backup files for $mname." && exit 1 - label="$2" - # Even if they provided a label, prompt them if its not a label in the list - if [ "$(echo "$labels" | grep "^$label\$")" = "" ]; then - PS3="Select the label of the backup to restore: " - select label in $labels; do - break - done - fi - echo "Restoring $mname with label $label... " - - # Backup targets - db_target="${mname}_${label}_dbfiles.tar" - - # Get database volume name, or, if it doesn't exist, make the name we expect it to be - db_vol_name=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname" | grep db) - if [ -z "$db_vol_name" ]; then - db_vol_name="${mname}_db" - else - container_tool volume rm "$db_vol_name" 2> /dev/null # If volume removal fails, its fine - fi - - # Recreate volume and extract to the database volume - container_tool volume create --label "com.docker.compose.project=$mname" "$db_vol_name" - container_tool run --rm --privileged -v "$db_vol_name":/db -v "$MDL_BACKUP_DIR":/backup "$MDL_SHELL_IMAGE" tar xf "/backup/$db_target" -C /db - - echo "Done restoring the fast backup of database for $mname with label $label." - - # Unset environment variables - unset_env "$mname" - -done diff --git a/libexec/mdl-restore.sh b/libexec/mdl-restore.sh index f74f9b9..3e09d48 100755 --- a/libexec/mdl-restore.sh +++ b/libexec/mdl-restore.sh @@ -14,6 +14,7 @@ Options: -h, --help Show this help message and exit. -b, --box Use backup sets in Box instead of the local backup folder. -x, --extract If compressed, leave the decompressed files when done extracting them. +-f, --fastdb Use an unsafe fast database backup instead of a SQL dump. -r, --rm Remove the local copy of the backup when done. EOF } @@ -22,6 +23,8 @@ EOF [[ $* =~ -b || $* =~ --box ]] && box=true || box=false [[ $* =~ -x || $* =~ --extract ]] && extract=true || extract=false [[ $* =~ -r || $* =~ --rm ]] && remove_when_done=true || remove_when_done=false +modules='src data db' && [[ $* =~ -f || $* =~ --fastdb ]] && modules='src data dbfiles' +[[ $modules =~ dbfiles ]] && echo "${yellow}Warning: Fast database restores are unsafe for production use.$norm" >&2 # Check necessary utilities requires "${MDL_CONTAINER_TOOL[0]}" tar bzip2 gzip xz find sed grep uniq @@ -56,23 +59,23 @@ for mname in $mnames; do fi # Backup targets - declare data_target src_target db_target # Explicitly declared to make shellcheck happy - for t in data src db; do + declare data_target src_target db_target dbfiles_target # Explicitly declared to make shellcheck happy + for t in $modules; do target="${t}_target"; local_target="local_${t}_target" - declare $target= $local_target= + declare "$target"= "$local_target"= # Find filenames of target files while IFS= read -r file; do - [[ -z ${!target} && $file =~ ^${mname}_${label}_${t}\. ]] && declare $target="$file" + [[ -z ${!target} && $file =~ ^${mname}_${label}_${t}\. ]] && declare "$target"="$file" done <<< "$files" # Find filenames of local files, in case we're looking in Box while IFS= read -r file; do - [[ $file =~ ^${mname}_${label}_${t}\. ]] && declare $local_target="$file" + [[ $file =~ ^${mname}_${label}_${t}\. ]] && declare "$local_target"="$file" done <<< "$local_files" done # List (and if requested, download from Box) each target. Abort if a target can't be found. echo "Using $backup_source_desc backup set with label $ul$label$rmul:" - for t in data src db; do + for t in $modules; do target="${t}_target"; local_target="local_${t}_target" echo " - $bold$t:$norm ${!target:-${red}Not found, so we will abort$norm}" # If target is not found, abort. Otherwise, download if Box.com is the source. @@ -87,7 +90,7 @@ for mname in $mnames; do fi # If `extract` is requested, decompress the files if $extract && new_target=$(decompress "$MDL_BACKUP_DIR/${!target}"); then - declare $target="$(basename "$new_target")" + declare "$target"="$(basename "$new_target")" echo " - Extracted $ul${!target}$rmul." fi done @@ -140,47 +143,59 @@ for mname in $mnames; do " & pid_src=$! - # Start a MariaDB container to restore the database - ( + # Either fast database backup or a full SQL dump restore + if [[ -n $dbfiles_target ]]; then + # Fast database restore echo "Restoring $ul$db_vol_name$norm volume..." - sql_path="$(mktemp -d)/${mname}_backup.sql" - if ! decompress "$MDL_BACKUP_DIR/$db_target" "$sql_path" -k > /dev/null; then - # If decompression fails, it probably isn't compressed. Point at original file instead. - sql_path="$MDL_BACKUP_DIR/$db_target" - fi - db_runner="${mname}_worker_db_restore" - container_tool run -d --rm --name "$db_runner" \ - --privileged \ - -e MARIADB_ROOT_PASSWORD="${ROOT_PASSWORD:-password}" \ - -e MARIADB_USER="${DB_USERNAME:-moodleuser}" \ - -e MARIADB_PASSWORD="${DB_PASSWORD:-password}" \ - -e MARIADB_DATABASE="${DB_NAME:-moodle}" \ - -e MARIADB_COLLATE=utf8mb4_unicode_ci \ - -e MARIADB_SKIP_TEST_DB=yes \ - -v "$db_vol_name":/bitnami/mariadb \ - -v "$sql_path":/docker-entrypoint-initdb.d/restore.sql:Z,ro \ - "$MARIADB_IMAGE" > /dev/null - # MariaDB doesn't have a "run task and exit" mode, so we just wait until - # the logs indicate it has finished, then we stop it. - last_check=0 - until container_tool logs --since "$last_check" "$db_runner" 2>&1 | grep -q 'MariaDB setup finished'; do - last_check=$(($(date +%s)-1)) - sleep 5 - done - container_tool stop "$db_runner" > /dev/null - ) & - db_pid=$! + container_tool run --rm --name "${mname}_worker_tar_dbfiles" -v "$db_vol_name":/data -v "$MDL_BACKUP_DIR":/backup:Z,ro "$MDL_SHELL_IMAGE" \ + sh -c "tar xf '/backup/$dbfiles_target' -C /data" & + pid_db=$! + elif [[ -n $db_target ]]; then + # Start a MariaDB container to restore the database + ( + echo "Restoring $ul$db_vol_name$norm volume..." + sql_path="$(mktemp -d)/${mname}_backup.sql" + if ! decompress "$MDL_BACKUP_DIR/$db_target" "$sql_path" -k > /dev/null; then + # If decompression fails, it probably isn't compressed. Point at original file instead. + sql_path="$MDL_BACKUP_DIR/$db_target" + fi + db_runner="${mname}_worker_db_restore" + container_tool run -d --rm --name "$db_runner" \ + --privileged \ + -e MARIADB_ROOT_PASSWORD="${ROOT_PASSWORD:-password}" \ + -e MARIADB_USER="${DB_USERNAME:-moodleuser}" \ + -e MARIADB_PASSWORD="${DB_PASSWORD:-password}" \ + -e MARIADB_DATABASE="${DB_NAME:-moodle}" \ + -e MARIADB_COLLATE=utf8mb4_unicode_ci \ + -e MARIADB_SKIP_TEST_DB=yes \ + -v "$db_vol_name":/bitnami/mariadb \ + -v "$sql_path":/docker-entrypoint-initdb.d/restore.sql:Z,ro \ + "$MARIADB_IMAGE" > /dev/null + # MariaDB doesn't have a "run task and exit" mode, so we just wait until + # the logs indicate it has finished, then we stop it. + last_check=0 + until container_tool logs --since "$last_check" "$db_runner" 2>&1 | grep -q 'MariaDB setup finished'; do + last_check=$(($(date +%s)-1)) + sleep 5 + done + container_tool stop "$db_runner" > /dev/null + ) & + pid_db=$! + fi # When done, clean up. Down the stack and remove the temp volume. wait $pid_src container_tool volume rm -f "$temp_vol_name" > /dev/null - wait $pid_data $db_pid + wait $pid_data + [[ -n $pid_db ]] && wait "$pid_db" branchver="$branchver" "$scr_dir/mdl-stop.sh" "$mname" -q # Remove the local backup files when done, if they specified that option if $remove_when_done; then echo Removing local backup files... - rm -fv "$MDL_BACKUP_DIR/$data_target" "$MDL_BACKUP_DIR/$src_target" "$MDL_BACKUP_DIR/$db_target" + rm -fv "$MDL_BACKUP_DIR/$data_target" "$MDL_BACKUP_DIR/$src_target" + [[ -n $db_target ]] && rm -fv "$MDL_BACKUP_DIR/$db_target" + [[ -n $dbfiles_target ]] && rm -fv "$MDL_BACKUP_DIR/$dbfiles_target" fi # Update Moodle config From 217163cd558c251f9953142ea701ef6a226cd976 Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Fri, 31 Oct 2025 11:43:00 -0500 Subject: [PATCH 02/12] fix: Improve `mdl install-plugin` to use version.php for component name Whereas it is good practice to use your component name as the output directory of a zipped plugin, so plugin authors do not use the exact frankenstyle component name. However, they do use the correct frankenstyle name in version.php because it is required. To make `mdl install-plugin` more compatible, we update it to inspect version.php and get the component name (and thus type and install path) from there. --- libexec/mdl-install-plugin.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/libexec/mdl-install-plugin.sh b/libexec/mdl-install-plugin.sh index c8a0b4e..7520e4c 100755 --- a/libexec/mdl-install-plugin.sh +++ b/libexec/mdl-install-plugin.sh @@ -122,6 +122,8 @@ for mname in $mnames; do # Process each zip file for zip_file in "${zip_files[@]}"; do zip_filename=$(basename "$zip_file") + # Clean up temp dirs from previous iteration, if applicable + rm -Rf "$temp_unzipped" "$temp_downloaded" # If the zip file is a URL, download it first if [[ $zip_file =~ ^https?:// ]]; then mkdir -p "$temp_downloaded" @@ -133,18 +135,19 @@ for mname in $mnames; do # Unzip the plugin, and copy its contents to its final destination mkdir -p "$temp_unzipped" unzip -o -qq -d "$temp_unzipped" "$zip_file" || { echo "Failed to unzip $zip_file" >&2; exit 1; } - # We use the directory inside the unzipped archive to determine plugin type and name. There should - # only be one directory, but we put safeguards in place to make sure. - # Example: block_my_great_plugin -> type=block, name=my_great_plugin - pkg_name=$(find "$temp_unzipped" -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 basename | head -n1) - type=${pkg_name%%_*} - name=${pkg_name#*_} + # We inspect version.php to get the component name, and from there we can determine type and name. + # Example: `$plugin->component = 'local_my_plugin';` -> `local_my_plugin` -> type=local, name=my_plugin + plugin_dir=$(find "$temp_unzipped" -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 basename | head -n1) + version_file="$temp_unzipped/$plugin_dir/version.php" + [ ! -f "$version_file" ] && echo "${red}Could not find version.php in plugin $zip_filename." >&2 && continue + component_full_name=$(grep -E "\\\$plugin->component[[:space:]]*=" "$version_file" | sed -E "s/.*\\\$plugin->component[[:space:]]*=[[:space:]]*'([^']+)'.*/\1/") + [ -z "$component_full_name" ] && echo "${red}Could not determine component name from version file for $zip_filename." >&2 && continue + type=${component_full_name%%_*} + name=${component_full_name#*_} dest="$(plugin_type_path "$type")/$name" echo "Installing $type plugin $name to $ul$dest$norm." mkdir -p "$temp_moodle/$dest" - ( shopt -s dotglob && cp -R "$temp_unzipped/$pkg_name"/* "$temp_moodle/$dest" ) - # Clean up - rm -Rf "$temp_unzipped" "$temp_downloaded" + ( shopt -s dotglob && cp -R "$temp_unzipped/$plugin_dir"/* "$temp_moodle/$dest" ) done # Copy all final work into the container while IFS= read -r -d '' dir; do From bdb1506ead2cdf412c5625e20b5fb11c01676bbc Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Fri, 19 Dec 2025 09:46:00 -0600 Subject: [PATCH 03/12] docs: Remove separate Fast DB scripts from script reference --- docs/scripts.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/scripts.md b/docs/scripts.md index 1d5b91b..a201a0e 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -30,14 +30,6 @@ are listed at the bottom to make this reference easier to read. 
{{ man['mdl-exec-sql'] }}
-### `mdl fast-db-backup` - -
{{ man['mdl-fast-db-backup'] }}
- -### `mdl fast-db-restore` - -
{{ man['mdl-fast-db-restore'] }}
- ### `mdl info` 
{{ man['mdl-info'] }}
From 5eadd2a633a1e876cb7a81fd82a808e18208e3a5 Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Fri, 31 Oct 2025 11:58:51 -0500 Subject: [PATCH 04/12] fix: Adjust version list in `mdl init` to just list x.y version We want it to just list versions as x.y (i.e. 4.5), but when we changed versions.txt to use the entire image page, we broke this. By piping `cut -d':' -f2`, we only look at the tag of the image, which is our desired effect. Fixes #15. --- libexec/mdl-init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/mdl-init.sh b/libexec/mdl-init.sh index cd0ac3f..927ed6d 100755 --- a/libexec/mdl-init.sh +++ b/libexec/mdl-init.sh @@ -299,7 +299,7 @@ if [[ -n $mname ]]; then for ver_string in "${versions[@]}"; do IFS=' ' read -ra var_array <<< "$ver_string" branch_array+=("${var_array[0]}") - moodle_array+=("$(echo "${var_array[1]}" | cut -d'.' -f1-2)") + moodle_array+=("$(echo "${var_array[1]}" | cut -d':' -f2 | cut -d'.' -f1-2)") done PS3="Select the version to install: " select moodle_ver in "${moodle_array[@]}"; do From 16b52b6c97a8afb92444403de448f97a627c85fd Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Tue, 9 Dec 2025 17:04:05 -0600 Subject: [PATCH 05/12] fix: Improve environment and label name edge cases The rule for Moodle and Docker names: Must consist only of lowercase alphanumeric characters, hyphens, and underscores as well as start with a letter or number. We discovered two issues: - If a name had invalid characters, the project will fail to start up. - If a name has an underscore (which is valid), backup operations are buggy. To fix this, we add name validation in `mdl init`, and we improve spots where labels are calculated for backup handling. Instead of referencing the second or third item when separating the name into a list by underscores, we remove the start and end parts of the name, leaving just the full label. We can also combine some of the logic into a single awk command to make it cleaner. Fixes #9. --- lib/mdl-common.sh | 15 +++++++++++++++ libexec/mdl-init.sh | 7 +++++++ libexec/mdl-list.sh | 6 +++--- libexec/mdl-restore.sh | 3 +-- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/mdl-common.sh b/lib/mdl-common.sh index 34a1dce..50fc228 100644 --- a/lib/mdl-common.sh +++ b/lib/mdl-common.sh @@ -122,6 +122,21 @@ function decompress() { return 2 } +# Function that uses awk to safely extract label from filename. Expects to receive +# filename from pipe, such as: `echo $mname | extract_label ` +function extract_label() { + cat | awk -v type="$2" -v mname="$1" ' + # Filter essentially looks for: /_\./ for example type "src" matches "_src." + /_'"$2"'\./ { + # Remove leading "_", so mname "moodle" matches "moodle_" at start of name. + sub("^" mname "_", ""); + # Remove ending "_.*", so type "src" matches "_src.tar", "_src.tar.bz2", etc at end of name. + sub("_" type "\\..*$", ""); + print $0 + } + ' +} + # Used to clear all known vars to proactively avoid data leaks. # Usage: unset_env function unset_env() { diff --git a/libexec/mdl-init.sh b/libexec/mdl-init.sh index 927ed6d..3071e4d 100755 --- a/libexec/mdl-init.sh +++ b/libexec/mdl-init.sh @@ -169,6 +169,13 @@ elif [[ -z $mname ]]; then fi if [[ -n $mname ]]; then + # Validate mname. Abort if we received an invalid name. + if ! [[ $mname =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then + echo "${red}The environment name $ul$mname$rmul is not valid.$norm" >&2 + echo "${yellow}Names can only contain lowercase alphanumeric characters, hyphens, and underscores,$norm" >&2 + echo "${yellow}and must start with a letter or number.$norm" >&2 + exit 1 + fi if [[ ! -d "$MDL_ENVS_DIR/$mname" ]] || $force; then echo "Creating environment: $ul$mname$rmul" mkdir -p "$MDL_ENVS_DIR/$mname" diff --git a/libexec/mdl-list.sh b/libexec/mdl-list.sh index 203ca7c..3cabe50 100755 --- a/libexec/mdl-list.sh +++ b/libexec/mdl-list.sh @@ -71,9 +71,9 @@ for mname in $mnames; do fi # Collect the list of backups - $type_backup && labels="$(find "$MDL_BACKUP_DIR" -name "${mname}_*_src.*" | cut -d"_" -f2- | sed -e "s/_src\..*//" | uniq | sort)" - $type_box && box_labels="$("$scr_dir/mdl-box.sh" "$mname" ls | awk -F'_' '$3 ~ /src/' | cut -d"_" -f2- | sed -e "s/_src\..*//" | uniq | sort)" - $type_fastdb && fast_labels=$(find "$MDL_BACKUP_DIR" -name "${mname}_*_dbfiles.tar" | cut -d"_" -f2- | sed -e "s/_dbfiles.tar//" | sort) + $type_backup && labels="$(find "$MDL_BACKUP_DIR" -name "${mname}_*_src.*" -print0 | xargs -0 -r -n1 basename | extract_label "$mname" src | sort | uniq)" + $type_box && box_labels="$("$scr_dir/mdl-box.sh" "$mname" ls | extract_label "$mname" src | sort | uniq)" + $type_fastdb && fast_labels=$(find "$MDL_BACKUP_DIR" -name "${mname}_*_dbfiles.*" -print0 | xargs -0 -r -n1 basename | extract_label "$mname" dbfiles | sort | uniq) # Output: Normal Backups if $type_backup && ! $quiet; then diff --git a/libexec/mdl-restore.sh b/libexec/mdl-restore.sh index 3e09d48..7303537 100755 --- a/libexec/mdl-restore.sh +++ b/libexec/mdl-restore.sh @@ -43,8 +43,7 @@ for mname in $mnames; do $box && files=$("$scr_dir/mdl-box.sh" "$mname" ls) || files=$local_files local_files=$(echo "$local_files" | xargs -r -n1 basename) files=$(echo "$files" | xargs -r -n1 basename) - src_files=$(echo "$files" | awk -F'_' '$3 ~ /src/') - labels="$(echo "$src_files" | cut -d"_" -f2- | sed -e "s/_src\..*//" | uniq | sort)" + labels="$(echo "$files" | extract_label "$mname" src | sort | uniq)" # What timestamp of backup do they want? (Select from the list if they did not provide) $box && backup_source_desc='Box.com' || backup_source_desc='local' From cf6a321d70b38781e43e5de1b6f206311a0cdf50 Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Wed, 10 Dec 2025 14:26:21 -0600 Subject: [PATCH 06/12] fix: Use `command -v` instead of `which` for command existence checks Recently, macOS changed its implementation of `which`, causing it to return its error message on stdout instead of stderr when a command is not found. This breaks conventional usage of `which` in scripts to check for command existence. To address this, we replace all instances of `which ` with `command -v ` which is more reliable and consistent across different Unix-like systems. --- installers/install.sh | 4 ++-- installers/uninstall.sh | 8 ++++---- lib/mdl-common.sh | 2 +- libexec/mdl-box.sh | 2 +- libexec/mdl-info.sh | 2 +- libexec/mdl-init.sh | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/installers/install.sh b/installers/install.sh index e26febc..d7174d8 100755 --- a/installers/install.sh +++ b/installers/install.sh @@ -20,13 +20,13 @@ branch=main [[ $1 != -* && -n $1 ]] && branch=$1 # Requires curl -if [[ -z $(which curl 2>/dev/null) ]]; then +if ! command -v curl &>/dev/null; then echo "$(tput bold)$(tput setaf 1)This command requires $(tput smul)curl$(tput rmul) to work.$(tput sgr0)" >&2 exit 1 fi # If `mdl` is already installed, it must not be a symlink -mdl_path=$(which mdl 2>/dev/null) +mdl_path=$(command -v mdl) if [[ -n $mdl_path && -L $mdl_path ]]; then echo 'The mdl CLI is already installed in developer mode. You do not have to reinstall' >&2 echo 'it. If you want to reinstall it in normal mode, please uninstall it first.' >&2 diff --git a/installers/uninstall.sh b/installers/uninstall.sh index 4a5ce3e..b38cfb2 100755 --- a/installers/uninstall.sh +++ b/installers/uninstall.sh @@ -1,7 +1,7 @@ #!/bin/bash # Check that `mdl` is installed -if [[ -z $(which mdl) ]]; then +if ! command -v mdl &>/dev/null; then echo "Could not find mdl. Are you sure it is installed?" >&2 exit 1 fi @@ -12,10 +12,10 @@ if [[ $EUID -ne 0 ]]; then fi # Determine paths and load common functions -base=$(realpath "$(dirname "$(realpath "$(which mdl)")")/..") +base=$(realpath "$(dirname "$(realpath "$(command -v mdl)")")/..") # Explicitly set scr_dir in case we're running from a curl pipe scr_dir="$base/libexec" -[[ -L $(which mdl) ]] && linked=true || linked=false +[[ -L $(command -v mdl) ]] && linked=true || linked=false # shellcheck source=../lib/mdl-common.sh [[ -f $base/lib/mdl-common.sh ]] && . "$base/lib/mdl-common.sh" # shellcheck source=../lib/mdl-ui.sh @@ -31,7 +31,7 @@ if $linked; then echo 'It appears you installed mdl in developer mode, which just installs a symlink to' echo 'the project in your path.' echo - yorn "Do you want to remove the symlink?" 'y' && sudo rm "$(which mdl)" + yorn "Do you want to remove the symlink?" 'y' && sudo rm "$(command -v mdl)" else yorn "Remove the mdl executable and its associated files?" 'y' && \ sudo rm -fv "$base"/bin/mdl "$base"/lib/mdl-*.sh "$base"/libexec/mdl-*.sh \ diff --git a/lib/mdl-common.sh b/lib/mdl-common.sh index 50fc228..f004d04 100644 --- a/lib/mdl-common.sh +++ b/lib/mdl-common.sh @@ -55,7 +55,7 @@ function container_tool() { "${MDL_CONTAINER_TOOL[@]}" "$@"; } function requires() { local ok=true for cmd in "$@"; do - if [[ -z $(which "$cmd" 2>/dev/null) ]]; then + if ! command -v "$cmd" &>/dev/null; then echo "${red}${bold}This command requires $ul$cmd$rmul to work.$norm" >&2 ok=false elif [[ $cmd =~ docker || $cmd =~ podman ]]; then diff --git a/libexec/mdl-box.sh b/libexec/mdl-box.sh index 8dbdea3..df2ab7a 100755 --- a/libexec/mdl-box.sh +++ b/libexec/mdl-box.sh @@ -145,7 +145,7 @@ for mname in $mnames; do url="https://account.box.com/api/oauth2/authorize?response_type=code&client_id=$BOX_CLIENT_ID&redirect_uri=$BOX_REDIRECT_URI" echo "Go to the following link to get the authorization code:" echo "$ul$url$norm" - [[ -n $(which open 2>/dev/null) ]] && open "$url" + command -v open &>/dev/null && open "$url" read -r -p "Authorization code: " auth_code if [[ -z $auth_code ]]; then echo "You entered a blank authorization code. Aborting." diff --git a/libexec/mdl-info.sh b/libexec/mdl-info.sh index a6941ab..4120d79 100755 --- a/libexec/mdl-info.sh +++ b/libexec/mdl-info.sh @@ -92,7 +92,7 @@ backup_status=true && [[ ! -d "$MDL_BACKUP_DIR" ]] && ok=false && backup_status= compose_status=true && [[ ! -d "$MDL_COMPOSE_DIR" ]] && ok=false && compose_status=false container_tool_status=true && [[ -z "$("${MDL_CONTAINER_TOOL[0]}" --version 2> /dev/null)" ]] && ok=false && container_tool_status=false compose_tool_status=true && [[ -z "$("${MDL_COMPOSE_TOOL[0]}" --version 2> /dev/null)" ]] && ok=false && compose_tool_status=false -mdl_path=$(which mdl) +mdl_path=$(command -v mdl) mdl_realpath=$(realpath "$mdl_path") [[ -L $mdl_path ]] && mdl_status="in dev mode at $ul$mdl_realpath$rmul" || mdl_status="at $ul$mdl_path$rmul" diff --git a/libexec/mdl-init.sh b/libexec/mdl-init.sh index 3071e4d..ef75f38 100755 --- a/libexec/mdl-init.sh +++ b/libexec/mdl-init.sh @@ -139,7 +139,7 @@ if $should_init_system; then install -d "$MDL_ENVS_DIR" install -d "$MDL_BACKUP_DIR" install -d "$MDL_COMPOSE_DIR" - if [[ -L $(which mdl) ]]; then + if [[ -L $(command -v mdl) ]]; then # If in dev mode, link to the project compose file. compose_file=$(realpath "$scr_dir/../compose/compose.yml") echo 'Since mdl is in developer mode, installing symlink to compose file at:' From ddb88959cd64bb3a8463da3ac137d350464a3a28 Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Wed, 10 Dec 2025 14:36:03 -0600 Subject: [PATCH 07/12] config: Add words to dictionary --- custom-words.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/custom-words.txt b/custom-words.txt index 3020e5d..18678de 100644 --- a/custom-words.txt +++ b/custom-words.txt @@ -8,6 +8,7 @@ behaviour bitnamilegacy booktool branchver +bsdtar cachelock Caddyfile calendartype @@ -52,7 +53,10 @@ moodledata moodleuser mygr mymoodle +pbzip pids +pigz +pixz plib profilefield purgecaches From ddab5d5590d0e1597b940c2ac953e1fb59d8d361 Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Wed, 10 Dec 2025 16:40:20 -0600 Subject: [PATCH 08/12] feat: Speed up and streamline backup/restore process Use standard `docker cp` tooling to transfer tar archives of the volumes, instead of running tar within the container. Also, if pbzip2, pigz, or pixz exist on the system, they will be used, instead of their standard compression tool equivalents that are only single-threaded (bzip2, gzip, xz). If bsdtar is available on the system, it will use that to filter the tar stream of the paths that don't need to be included in the backup. However, if bsdtar is not available, we also handle removing these paths during the restore process as well. So, we always ensure the caches and sessions are eliminated during restore. Closes #6. --- lib/mdl-common.sh | 17 ++-- libexec/mdl-backup.sh | 182 +++++++++++++++++++++-------------------- libexec/mdl-restore.sh | 51 ++++++++---- 3 files changed, 140 insertions(+), 110 deletions(-) diff --git a/lib/mdl-common.sh b/lib/mdl-common.sh index f004d04..e2daf3c 100644 --- a/lib/mdl-common.sh +++ b/lib/mdl-common.sh @@ -87,8 +87,18 @@ function support_long_options() { fi } +function calc_compression_tool() { + local ext=${1##*.} + local cmd + [[ $ext == bz2 ]] && cmd=bzip2 && command -v pbzip2 &>/dev/null && cmd=pbzip2 + [[ $ext == gz ]] && cmd=gzip && command -v pigz &>/dev/null && cmd=pigz + [[ $ext == xz ]] && cmd=xz && command -v pixz &>/dev/null && cmd=pixz + echo "$cmd" +} + # Receives file path and decompresses it. Can detect bzip2, gzip and xz files. If none of -# those extensions match the filename, it throws an error. After successful decompression, +# those extensions match the filename, it throws an error. It will always try to use the +# parallel processing version of the command if available. After successful decompression, # the original file is deleted unless you specify `--keep`. # # Parameters: @@ -99,14 +109,11 @@ function decompress() { local file_path=$1 local out=$2 local ext=${file_path##*.} - local cmd + local cmd=$(calc_compression_tool "$ext") # Check options [[ $* =~ -k || $* =~ --keep ]] && keep=true || keep=false # File path is required [[ -z $file_path ]] && return 1 - [[ $ext == bz2 ]] && cmd=bzip2 - [[ $ext == gz ]] && cmd=gzip - [[ $ext == xz ]] && cmd=xz if [[ -n $cmd ]]; then # If they didn't provide an explicit output path, use file path sans extension [[ -z $out ]] && out=${file_path%".$ext"} diff --git a/libexec/mdl-backup.sh b/libexec/mdl-backup.sh index 1fc052a..ecfff95 100755 --- a/libexec/mdl-backup.sh +++ b/libexec/mdl-backup.sh @@ -19,12 +19,19 @@ label=$default_label # Declarations source_host= source_type= -compress_flag= ssh_args= container_tool=${MDL_CONTAINER_TOOL[*]} verbose=false dry_run=false +# Functions +function ssh_wrap() { + echo "${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}$1${source_host:+"\\\"\""}" +} +function eval_ssh_wrap() { + eval "$(ssh_wrap "$1")" +} + # Help display_help() { cat < /dev/null && source_host_reachable=true || source_host_reachable=false fi - # If "container", the container must be active in order to dump the database for the "db" module. + # If "container", the containers must be active in order to backup data for their module. source_db_container='' + source_moodle_container='' if $is_container; then - if [[ " $modules " == *" db "* ]]; then - source_db_container_cmd=" - ${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} - $container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep mariadb | head -1 - ${source_host:+"\\\"\""} - " - source_db_container=$(eval "$source_db_container_cmd") + if [[ " $modules " == *" db "* || $modules =~ fastdb ]]; then + # Find the database container name + source_db_container=$(eval_ssh_wrap "$container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep mariadb | head -1") [[ -z $source_db_container ]] && echo "${red}The database container cannot be found. Start the environment to perform a backup." >&2 && exit 1 + # Determine db path if not provided (usually won't be for container mode) + if [[ -z $source_db_path && $modules =~ fastdb ]]; then + source_db_path=$(eval_ssh_wrap "$container_tool inspect '$source_db_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"db\"))) | .Destination'") + [[ -z $source_db_path ]] && echo "${red}Could not determine ${ul}db$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1 + fi fi - if [[ $modules =~ data || $modules =~ src || $modules =~ fastdb ]]; then - vols_cmd=" - ${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} - $container_tool volume ls -q --filter 'label=com.docker.compose.project=$mname' - ${source_host:+"\\\"\""} - " - vols=$(eval "$vols_cmd") - [[ $modules =~ data ]] && source_data_volume=$(grep data <<< "$vols") - [[ $modules =~ src ]] && source_src_volume=$(grep src <<< "$vols") - [[ $modules =~ fastdb ]] && source_db_volume=$(grep db <<< "$vols") + if [[ $modules =~ data || $modules =~ src ]]; then + # Find the moodle container name + source_moodle_container=$(eval_ssh_wrap "$container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep moodle | head -1") + [[ -z $source_moodle_container ]] && echo "${red}The Moodle container cannot be found. Start the environment to perform a backup." >&2 && exit 1 + # Determine src/data paths if not provided (usually won't be for container mode) + if [[ -z $source_src_path && $modules =~ src ]]; then + source_src_path=$(eval_ssh_wrap "$container_tool inspect '$source_moodle_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"src\"))) | .Destination'") + [[ -z $source_src_path ]] && echo "${red}Could not determine ${ul}src$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1 + fi + if [[ -z $source_data_path && $modules =~ data ]]; then + source_data_path=$(eval_ssh_wrap "$container_tool inspect '$source_moodle_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"data\"))) | .Destination'") + [[ -z $source_data_path ]] && echo "${red}Could not determine ${ul}data$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1 + fi fi fi # Password mask source_db_password_mask=${source_db_password//?/*} - # Compression extension - compress_ext='' - case "$compress_arg" in - bzip2) compress_ext='.bz2' ;; - gzip) compress_ext='.gz' ;; - xz) compress_ext='.xz' ;; - esac - # Targets [[ $modules =~ data ]] && data_target="$MDL_BACKUP_DIR/${mname}_${label}_data.tar$compress_ext" [[ $modules =~ src ]] && src_target="$MDL_BACKUP_DIR/${mname}_${label}_src.tar$compress_ext" @@ -287,25 +298,24 @@ for mname in $mnames; do echo -e "$ul$bold$action_word $mname environment$norm" if $verbose; then echo - echo "$bold Type:$norm $source_type" - [[ -n $source_host ]] && echo "$bold Host:$norm $source_host ($($source_host_reachable && echo "${green}reachable" || echo "${red}unreachable!")$norm)" - [[ -n $source_src_path ]] && echo "$bold 'src' Path:$norm $source_src_path" - [[ -n $source_data_path ]] && echo "$bold 'data' Path:$norm $source_data_path" - [[ -n $source_src_volume ]] && echo "$bold 'src' Volume:$norm $source_src_volume" - [[ -n $source_data_volume ]] && echo "$bold 'data' Volume:$norm $source_data_volume" - [[ -n $source_db_volume ]] && echo "$bold 'db' Volume:$norm $source_db_volume" - [[ -n $source_db_container ]] && echo "$bold DB Container:$norm $source_db_container" - [[ -n $source_db_name ]] && echo "$bold DB Name:$norm $source_db_name" - [[ -n $source_db_username ]] && echo "$bold DB Username:$norm $source_db_username" - [[ -n $source_db_password ]] && echo "$bold DB Password:$norm $source_db_password_mask" + echo "$bold Type:$norm $source_type" + [[ -n $source_host ]] && echo "$bold Host:$norm $source_host ($($source_host_reachable && echo "${green}reachable" || echo "${red}unreachable!")$norm)" + [[ -n $source_moodle_container ]] && echo "$bold Mdl Container:$norm $source_moodle_container" + [[ -n $source_src_path ]] && echo "$bold 'src' Path:$norm $source_src_path" + [[ -n $source_data_path ]] && echo "$bold 'data' Path:$norm $source_data_path" + [[ -n $source_db_container ]] && echo "$bold DB Container:$norm $source_db_container" + [[ -n $source_db_path ]] && echo "$bold 'db' Path:$norm $source_db_path" + [[ -n $source_db_name ]] && echo "$bold DB Name:$norm $source_db_name" + [[ -n $source_db_username ]] && echo "$bold DB Username:$norm $source_db_username" + [[ -n $source_db_password ]] && echo "$bold DB Password:$norm $source_db_password_mask" echo - [[ -n $label ]] && echo "$bold Label:$norm $label" - echo "$bold Modules:$norm $modules" - echo "$bold Compress:$norm $compress_arg" - [[ -n $src_target ]] && echo "$bold 'src' Path:$norm $src_target" - [[ -n $data_target ]] && echo "$bold 'data' Path:$norm $data_target" - [[ -n $db_target ]] && echo "$bold DB Path:$norm $db_target" - [[ -n $fastdb_target ]] && echo "$bold Fast DB Path:$norm $fastdb_target" + [[ -n $label ]] && echo "$bold Label:$norm $label" + echo "$bold Modules:$norm $modules" + echo "$bold Compress:$norm $compress_arg" + [[ -n $src_target ]] && echo "$bold 'src' Path:$norm $src_target" + [[ -n $data_target ]] && echo "$bold 'data' Path:$norm $data_target" + [[ -n $db_target ]] && echo "$bold DB Path:$norm $db_target" + [[ -n $fastdb_target ]] && echo "$bold Fast DB Path:$norm $fastdb_target" echo fi @@ -315,43 +325,37 @@ for mname in $mnames; do fi # MODULE: data - # shellcheck disable=SC2034 - data_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} - $is_container && data_cmd="$data_cmd $container_tool run --rm --name '${mname}_worker_bk_data' -v '$source_data_volume':/data $MDL_SHELL_IMAGE" - data_cmd="$data_cmd \ - tar c $compress_flag \ - --exclude='./trashdir' \ - --exclude='./temp' \ - --exclude='./sessions' \ - --exclude='./localcache' \ - --exclude='./cache' \ - --exclude='./moodle-cron.log' \ - -C $($is_container && echo /data || echo "$source_data_path") . \ - " - data_cmd=$data_cmd${source_host:+"\\\"\""} + data_cmd='' + $is_container && data_cmd=$(ssh_wrap "$container_tool cp $source_moodle_container:$source_data_path/. -") + $is_container || data_cmd=$(ssh_wrap "tar c -C '$source_data_path' .") + if command -v bsdtar &>/dev/null; then + data_cmd="$data_cmd | bsdtar cf - \ + --exclude localcache \ + --exclude cache \ + --exclude sessions \ + --exclude temp \ + --exclude trashdir \ + --exclude moodle-cron.log \ + @- \ + " + else + echo "${yellow}Warning: Since ${ul}bsdtar$rmul isn't found, caches/logs will not be excluded from backup.$norm" >&2 + fi # MODULE: src + src_cmd='' + $is_container && src_cmd=$(ssh_wrap "$container_tool cp $source_moodle_container:$source_src_path/. -") # shellcheck disable=SC2034 - src_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} - $is_container && src_cmd="$src_cmd $container_tool run --rm --name '${mname}_worker_bk_src' -v '$source_src_volume':/src $MDL_SHELL_IMAGE" - src_cmd="$src_cmd \ - tar c $compress_flag \ - -C $($is_container && echo /src || echo "$source_src_path") . \ - " - src_cmd=$src_cmd${source_host:+"\\\"\""} + $is_container || src_cmd=$(ssh_wrap "tar c -C '$source_src_path' .") # MODULE: fastdb + fastdb_cmd='' # shellcheck disable=SC2034 - fastdb_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} - $is_container && fastdb_cmd="$fastdb_cmd $container_tool run --rm --name '${mname}_worker_bk_fastdb' -v '$source_db_volume':/db $MDL_SHELL_IMAGE" - $is_container && fastdb_cmd="$fastdb_cmd tar c $compress_flag -C /db ." - fastdb_cmd=$fastdb_cmd${source_host:+"\\\"\""} + $is_container && fastdb_cmd=$(ssh_wrap "$container_tool cp $source_db_container:$source_db_path/. -") # MODULE: db - # TODO: When piping to compression program, a failed status of mysqldump will be lost. - # shellcheck disable=SC2034 - db_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""} - $is_container && db_cmd="$db_cmd $container_tool exec '$source_db_container'" + db_cmd='' + $is_container && db_cmd="$container_tool exec '$source_db_container'" db_cmd="$db_cmd \ mysqldump \ --user='$source_db_username' \ @@ -360,22 +364,21 @@ for mname in $mnames; do -C -Q -e --create-options \ $source_db_name \ " - db_cmd=$db_cmd${source_host:+"\\\"\""} - [[ $compress_arg != none ]] && db_cmd="$db_cmd | $compress_arg -cq9" + db_cmd=$(ssh_wrap "$db_cmd") # Actually EXECUTE the commands pids=() + $verbose && echo for t in $valid_modules; do if [[ " $modules " == *" $t "* ]]; then - targ_var=${t}_target; targ=${!targ_var} - cmd_var=${t}_cmd; cmd=${!cmd_var} + targ_var="${t}_target"; targ="${!targ_var}" + cmd_var="${t}_cmd"; cmd="${!cmd_var}" + [[ -n $compression_tool ]] && cmd="$cmd | $compression_tool -cq9" echo "$mname $t: $targ" - # In verbose mode, output the command, but mask the password and eliminate whitespace by echoing with word splitting. - # shellcheck disable=2086 - $verbose && echo ${cmd//password=\'$source_db_password\'/password=\'$source_db_password_mask\'} - if $dry_run; then - echo -e "${red}Not executed. This is a dry run.$norm\n" - else + # In verbose mode, output the command, but mask password. Eliminate whitespace with `xargs`. + # Ref: https://stackoverflow.com/questions/369758/how-to-trim-whitespace-from-a-bash-variable + $verbose && echo "${bold}Command:$norm ${cmd//password=\'$source_db_password\'/password=\'$source_db_password_mask\'}" | xargs + if ! $dry_run; then cmd="$cmd > '$targ'" # Execute the command, and handle success/fail scenarios eval "$cmd" && success=true || success=false @@ -384,13 +387,14 @@ for mname in $mnames; do rm "$targ" echo "Removed $(basename "$targ") because the $t backup failed." >&2 fi - fi - fi & - pids+=($!) + fi & + pids+=($!) + fi done # TODO: It'd be nice if we check if any of the steps failed, and exit non-zero if so. wait "${pids[@]}" + $dry_run && echo -e "${red}Commands not executed. This is a dry run.$norm\n" echo "$action_word of $mname environment is complete!" # Unset environment variables diff --git a/libexec/mdl-restore.sh b/libexec/mdl-restore.sh index 7303537..505deca 100755 --- a/libexec/mdl-restore.sh +++ b/libexec/mdl-restore.sh @@ -104,9 +104,14 @@ for mname in $mnames; do # Restore src to a temp volume first, to retrieve the git branch version temp_vol_name="${mname}_temp" + worker_name="${mname}_worker_src_restore" + compression_tool=$(calc_compression_tool "$src_target") + pipe_cmd=(cat) && [[ -n $compression_tool ]] && pipe_cmd=("$compression_tool" -d -c) container_tool volume rm -f "$temp_vol_name" > /dev/null - container_tool run --rm --name "${mname}_worker_tar_src" -v "$temp_vol_name":/src -v "$MDL_BACKUP_DIR":/backup:Z,ro "$MDL_SHELL_IMAGE" \ - tar xf "/backup/$src_target" -C /src + # We create this container and never start it, because we need an owner of the volume for `docker cp` + container_tool create --name "$worker_name" -v "$temp_vol_name":/src busybox > /dev/null + container_tool cp - "$worker_name":/src < <("${pipe_cmd[@]}" "$MDL_BACKUP_DIR/$src_target") > /dev/null + container_tool rm -f "$worker_name" > /dev/null branchver=$(src_vol_name="$temp_vol_name" "$scr_dir/mdl-moodle-version.sh" "$mname") . "$scr_dir/mdl-calc-images.sh" "$mname" @@ -115,23 +120,34 @@ for mname in $mnames; do # Create the stack, so we have the volumes that are auto-attached to the stack branchver="$branchver" "$scr_dir/mdl-start.sh" "$mname" -q -n - # Find all the volume names + # Find all the volume and container names, and their volume mount points. vols=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname") db_vol_name=$(grep db <<< "$vols") data_vol_name=$(grep data <<< "$vols") src_vol_name=$(grep src <<< "$vols") + containers=$(container_tool ps -a -f "label=com.docker.compose.project=$mname" --format '{{.Names}}') + db_container=$(grep mariadb <<< "$containers" | head -1) + db_path=$(container_tool inspect "$db_container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("db"))) | .Destination') + moodle_container=$(grep moodle <<< "$containers" | head -1) + data_path=$(container_tool inspect "$moodle_container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("data"))) | .Destination') + src_path=$(container_tool inspect "$moodle_container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("src"))) | .Destination') # Extract src and data to their volumes, set permissions appropriately. # Ref: https://docs.moodle.org/4x/sv/Security_recommendations#Running_Moodle_on_a_dedicated_server - echo "Restoring $ul$src_vol_name$norm and $ul$data_vol_name$norm volumes..." - container_tool run --rm --name "${mname}_worker_tar_data" -v "$data_vol_name":/data -v "$MDL_BACKUP_DIR":/backup:Z,ro "$MDL_SHELL_IMAGE" \ - sh -c "\ - mkdir -p /data/sessions /data/trashdir /data/temp /data/localcache /data/cache - tar xf '/backup/$data_target' -C /data - chown -R daemon:daemon /data - find /data -type d -print0 | xargs -0 chmod 700 - find /data -type f -print0 | xargs -0 chmod 600 - " & + echo "Restoring $ul$src_vol_name:$src_path$norm and $ul$data_vol_name:$data_path$norm volumes..." + ( + compression_tool=$(calc_compression_tool "$data_target") + pipe_cmd=(cat) && [[ -n $compression_tool ]] && pipe_cmd=("$compression_tool" -d -c) + container_tool cp - "$moodle_container":"$data_path" < <("${pipe_cmd[@]}" "$MDL_BACKUP_DIR/$data_target") > /dev/null + container_tool run --rm --name "${mname}_worker_fix_data_perms" -v "$data_vol_name":/data "$MDL_SHELL_IMAGE" \ + sh -c "\ + (cd /data && rm -rf sessions trashdir temp localcache cache moodle-cron.log) + (cd /data && mkdir -p sessions trashdir temp localcache cache) + chown -R daemon:daemon /data + find /data -type d -print0 | xargs -0 chmod 700 + find /data -type f -print0 | xargs -0 chmod 600 + " + ) & pid_data=$! container_tool run --rm --name "${mname}_worker_cp_src" -v "$src_vol_name":/src -v "$temp_vol_name":/temp:ro "$MDL_SHELL_IMAGE" \ sh -c "\ @@ -145,14 +161,17 @@ for mname in $mnames; do # Either fast database backup or a full SQL dump restore if [[ -n $dbfiles_target ]]; then # Fast database restore - echo "Restoring $ul$db_vol_name$norm volume..." - container_tool run --rm --name "${mname}_worker_tar_dbfiles" -v "$db_vol_name":/data -v "$MDL_BACKUP_DIR":/backup:Z,ro "$MDL_SHELL_IMAGE" \ - sh -c "tar xf '/backup/$dbfiles_target' -C /data" & + echo "Restoring $ul$db_vol_name:$db_path$norm volume via fast database restore..." + ( + compression_tool=$(calc_compression_tool "$dbfiles_target") + pipe_cmd=(cat) && [[ -n $compression_tool ]] && pipe_cmd=("$compression_tool" -d -c) + container_tool cp - "$db_container":"$db_path" < <("${pipe_cmd[@]}" "$MDL_BACKUP_DIR/$dbfiles_target") > /dev/null + ) & pid_db=$! elif [[ -n $db_target ]]; then # Start a MariaDB container to restore the database ( - echo "Restoring $ul$db_vol_name$norm volume..." + echo "Restoring $ul$db_vol_name$norm volume via database dump..." sql_path="$(mktemp -d)/${mname}_backup.sql" if ! decompress "$MDL_BACKUP_DIR/$db_target" "$sql_path" -k > /dev/null; then # If decompression fails, it probably isn't compressed. Point at original file instead. From 6907da0b51b875a86182b5593c4a7a0753c185b5 Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Mon, 15 Dec 2025 15:58:02 -0600 Subject: [PATCH 09/12] fix: Remove hard-coded bitnami references Future Moodle images may not be based on Bitnami and may have different src/data paths. We try to dynamically calculate these paths by querying the container of its mount points. This approach helped us realize that the DB volume really is pointed at the file system incorrectly for the Bitnami implementation of MariaDB. Should be /bitnami/mariadb/data, not /bitnami/mariadb. This change is more accurate and also will make it compatible with other non-Bitnami images for MySQL/MariaDB databases. Closes #13. --- compose/compose.yml | 2 +- lib/mdl-common.sh | 25 +++++++++++++++++++------ libexec/mdl-cli.sh | 8 +++++--- libexec/mdl-init.sh | 36 ++++++++++++++++++++---------------- libexec/mdl-restore.sh | 7 +++---- 5 files changed, 48 insertions(+), 30 deletions(-) diff --git a/compose/compose.yml b/compose/compose.yml index 61f6aad..1d43151 100644 --- a/compose/compose.yml +++ b/compose/compose.yml @@ -14,7 +14,7 @@ services: - MARIADB_SKIP_TEST_DB=yes volumes: - /etc/localtime:/etc/localtime:ro - - db:/bitnami/mariadb + - db:/bitnami/mariadb/data moodle: image: $MOODLE_IMAGE privileged: true diff --git a/lib/mdl-common.sh b/lib/mdl-common.sh index e2daf3c..291c963 100644 --- a/lib/mdl-common.sh +++ b/lib/mdl-common.sh @@ -171,22 +171,32 @@ function export_env() { export mname=$1 } -# Updates config.php with the environment variables. +# Updates config.php with the environment variables. Assumes that export_env has already been run. # Usage: update_config function update_config() { local -r env_dir="$MDL_ENVS_DIR/$1" - local -r env_config_file="$env_dir/src/config.php" + + # Get the Moodle container and its mount paths for this environment + local -r container="$(container_tool ps -a -f "label=com.docker.compose.project=$1" --format '{{.Names}}' | grep moodle | head -1)" + local -r data_path=${container:+$(container_tool inspect "$container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("data"))) | .Destination')} # Get desired wwwroot value if [ -z "$WWWROOT" ]; then - local -r defaultwwwroot="$(grep -o -E "CFG->wwwroot\s*=\s*'(.*)';" "$env_config_file" | cut -d"'" -f2)" + local defaultwwwroot='' + # Determine default wwwroot from existing config if possible (container must be running) + if [[ -n $container ]]; then + local -r src_path=${container:+$(container_tool inspect "$container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("src"))) | .Destination')} + if [[ -n $src_path ]]; then + defaultwwwroot=$(container_tool exec -t "$container" php -r "define('CLI_SCRIPT', true); require '$src_path/config.php'; echo \$CFG->wwwroot;") + fi + fi local altwwwroot1="http://$HOSTNAME" [ "$defaultwwwroot" = "$altwwwroot1" ] && altwwwroot1='' local altwwwroot2="http://$MOODLE_HOST" [ "$defaultwwwroot" = "$altwwwroot2" ] && altwwwroot2='' echo 'You can avoid this prompt by setting WWWROOT in your .env file.' PS3="Select a desired wwwroot value or type your own: " - select WWWROOT in "$defaultwwwroot" $altwwwroot1 $altwwwroot2; do + select WWWROOT in $defaultwwwroot $altwwwroot1 $altwwwroot2; do WWWROOT="${WWWROOT:-$REPLY}" break done @@ -217,9 +227,11 @@ function update_config() { replace_config_value dbname "$DB_NAME" | replace_config_value dbuser "$DB_USERNAME" | replace_config_value dbpass "$DB_PASSWORD" | - replace_config_value wwwroot "$WWWROOT" | - replace_config_value dataroot "${DATA_ROOT:-/bitnami/moodledata}" + replace_config_value wwwroot "$WWWROOT" ) + if [[ -n "$DATA_ROOT" || -n "$data_path" ]]; then + config_content=$(replace_config_value "$config_content" dataroot "${DATA_ROOT:-$data_path}") + fi # Run custom updates if provided custom_script="$env_dir/custom-config.sh" [[ -f "$custom_script" ]] && config_content=$(. "$custom_script" "$config_content") @@ -233,6 +245,7 @@ EOF container_tool run --rm -t --name "${1}_worker_update_config" \ -v "$env_dir":/env:Z,ro \ -v "${1}_src":/src \ + -e data_path="$data_path" \ "$MDL_SHELL_IMAGE" sh -c "$cmd" } diff --git a/libexec/mdl-cli.sh b/libexec/mdl-cli.sh index 4397330..7ebf0ec 100755 --- a/libexec/mdl-cli.sh +++ b/libexec/mdl-cli.sh @@ -53,19 +53,21 @@ mnames=$("$scr_dir"/mdl-select-env.sh "$1") for mname in $mnames; do - # Get the Moodle container for this environment + # Get the Moodle container and base directory for this environment container="$(container_tool ps -f "label=com.docker.compose.project=$mname" --format '{{.Names}}' | grep moodle | head -1)" [[ -z $container ]] && echo "${red}Could not find a container running Moodle for $ul$mname$rmul!$norm" >&2 && exit 1 + base_dir=$(container_tool inspect "$container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("src"))) | .Destination') + [[ -z $base_dir ]] && echo "${red}Could not determine Moodle base directory for $ul$mname$rmul!$norm" >&2 && exit 1 cmd="$2" # If they did not provide a cmd, list the available commands if [[ -z $cmd ]]; then echo "${bold}${ul}Available Commands$norm" - container_tool exec -t "$container" find /bitnami/moodle/admin/cli -maxdepth 1 -type f -exec basename {} .php \; | sort | sed 's/^/ - /' + container_tool exec -t "$container" find "$base_dir/admin/cli" -maxdepth 1 -type f -exec basename {} .php \; | sort | sed 's/^/ - /' exit fi # Run the command, passing any additional arguments they passed on to this script - container_tool exec $paramI -t "$container" php "/bitnami/moodle/admin/cli/$cmd.php" "${@:3}" + container_tool exec $paramI -t "$container" php "$base_dir/admin/cli/$cmd.php" "${@:3}" done diff --git a/libexec/mdl-init.sh b/libexec/mdl-init.sh index ef75f38..907a416 100755 --- a/libexec/mdl-init.sh +++ b/libexec/mdl-init.sh @@ -318,10 +318,12 @@ if [[ -n $mname ]]; then fi done echo - # Start the environment. Bitnami image will automatically bootstrap install. Wait to finish. + # Start the environment, automatically bootstrapping install. Wait to finish. branchver="$branchver" "$scr_dir/mdl-start.sh" "$mname" -q moodle_svc=$(container_tool ps --filter "label=com.docker.compose.project=$mname" --format '{{.Names}}' | grep moodle) src_vol_name=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname" | grep src) + src_path=$(container_tool inspect "$moodle_svc" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("src"))) | .Destination') + data_path=$(container_tool inspect "$moodle_svc" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("data"))) | .Destination') # Do git install once standard install completes. function git_cmd() { container_tool run --rm -t --name "$mname-git-$(uuidgen)" -v "$src_vol_name":/git "$MDL_GIT_IMAGE" -c safe.directory=/git "$@" @@ -359,17 +361,19 @@ if [[ -n $mname ]]; then } !skip { print } ' - config_file=/bitnami/moodle/config.php + config_file=$src_path/config.php revised_config_file=$(mktemp) container_tool exec -it "$moodle_svc" awk "$awk_cmd" "$config_file" > "$revised_config_file" container_tool cp "$revised_config_file" "$moodle_svc":"$config_file" - # Bitnami image unfortunately does not install the git repo. So, we add git after the fact. - # This works fine since the Moodle repo branch will always be even with or slightly ahead of the Bitnami image. - targetbranch="MOODLE_${branchver}_STABLE" - git_cmd init -b main - git_cmd remote add origin https://github.com/moodle/moodle.git - git_cmd fetch -np origin "$targetbranch" - git_cmd checkout -f "$targetbranch" + # If image does not install the git repo, we add [g]it after the fact. This works fine since + # the Moodle repo branch will always be even with or slightly ahead of the image. + if ! git_cmd status &>/dev/null; then + targetbranch="MOODLE_${branchver}_STABLE" + git_cmd init -b main + git_cmd remote add origin https://github.com/moodle/moodle.git + git_cmd fetch -np origin "$targetbranch" + git_cmd checkout -f "$targetbranch" + fi ) > /dev/null & git_pid=$! yorn 'Do you want to optimize the git repository? It will save space but take more time.' 'n' && do_gc=true || do_gc=false @@ -382,13 +386,13 @@ if [[ -n $mname ]]; then "$scr_dir/mdl-cli.sh" "$mname" upgrade --non-interactive # After upgrades, we need to fix permissions. # Ref: https://docs.moodle.org/4x/sv/Security_recommendations#Running_Moodle_on_a_dedicated_server - container_tool exec -it "${moodle_svc}" bash -c ' - chown -R daemon:daemon /bitnami/moodle /bitnami/moodledata - find /bitnami/moodle -type d -print0 | xargs -0 chmod 755 - find /bitnami/moodle -type f -print0 | xargs -0 chmod 644 - find /bitnami/moodledata -type d -print0 | xargs -0 chmod 700 - find /bitnami/moodledata -type f -print0 | xargs -0 chmod 600 - ' + container_tool exec -it "${moodle_svc}" bash -c " + chown -R daemon:daemon '$src_path' '$data_path' + find '$src_path' -type d -print0 | xargs -0 chmod 755 + find '$src_path' -type f -print0 | xargs -0 chmod 644 + find '$data_path' -type d -print0 | xargs -0 chmod 700 + find '$data_path' -type f -print0 | xargs -0 chmod 600 + " fi echo 🎉 Done! else diff --git a/libexec/mdl-restore.sh b/libexec/mdl-restore.sh index 505deca..db213d8 100755 --- a/libexec/mdl-restore.sh +++ b/libexec/mdl-restore.sh @@ -186,7 +186,7 @@ for mname in $mnames; do -e MARIADB_DATABASE="${DB_NAME:-moodle}" \ -e MARIADB_COLLATE=utf8mb4_unicode_ci \ -e MARIADB_SKIP_TEST_DB=yes \ - -v "$db_vol_name":/bitnami/mariadb \ + -v "$db_vol_name":"$db_path" \ -v "$sql_path":/docker-entrypoint-initdb.d/restore.sql:Z,ro \ "$MARIADB_IMAGE" > /dev/null # MariaDB doesn't have a "run task and exit" mode, so we just wait until @@ -206,6 +206,8 @@ for mname in $mnames; do container_tool volume rm -f "$temp_vol_name" > /dev/null wait $pid_data [[ -n $pid_db ]] && wait "$pid_db" + # Update config before destroying the stack so we know the data mount point + export_env_and_update_config "$mname" branchver="$branchver" "$scr_dir/mdl-stop.sh" "$mname" -q # Remove the local backup files when done, if they specified that option @@ -216,9 +218,6 @@ for mname in $mnames; do [[ -n $dbfiles_target ]] && rm -fv "$MDL_BACKUP_DIR/$dbfiles_target" fi - # Update Moodle config - export_env_and_update_config "$mname" - echo "Done restoring $ul$mname$rmul from $backup_source_desc backup set with label $ul$label$norm." # Unset environment variables From 69c46cf83c49d23466a11a53038b94f67c77b84c Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Wed, 17 Dec 2025 12:47:12 -0600 Subject: [PATCH 10/12] refactor: Change compose file name to `default.yml` Changing the compose file name from `compose.yml` to `default.yml` paves the way for the provided compose file to be the default configuration but to support customize configs as well. See #14. --- compose/{compose.yml => default.yml} | 0 installers/install.sh | 2 +- libexec/mdl-calc-compose-path.sh | 11 ++++++----- libexec/mdl-init.sh | 10 +++++----- 4 files changed, 12 insertions(+), 11 deletions(-) rename compose/{compose.yml => default.yml} (100%) diff --git a/compose/compose.yml b/compose/default.yml similarity index 100% rename from compose/compose.yml rename to compose/default.yml diff --git a/installers/install.sh b/installers/install.sh index d7174d8..ed8e457 100755 --- a/installers/install.sh +++ b/installers/install.sh @@ -85,4 +85,4 @@ echo 🎉 The mdl CLI is installed! echo # Initialize the system -mdl init --no-title -c "${MDL_BASE_URL/main/$branch}/compose/compose.yml" +mdl init --no-title -c "${MDL_BASE_URL/main/$branch}/compose/default.yml" diff --git a/libexec/mdl-calc-compose-path.sh b/libexec/mdl-calc-compose-path.sh index 267eeac..a1279f3 100755 --- a/libexec/mdl-calc-compose-path.sh +++ b/libexec/mdl-calc-compose-path.sh @@ -6,8 +6,8 @@ display_help() { cat < -Looks at the version of a Moodle environment, based on its Git branch, and returns the -full path of the ${ul}compose.yml${rmul} file that should be used. +Looks at the version of a Moodle environment, based on its Git branch and custom configs, +and returns the full path of the compose file that should be used. Options: -h, --help Show this help message and exit. @@ -18,6 +18,7 @@ EOF requires realpath -# Right now, all configs can use the same `compose.yml` file, but if that changes, -# this script will inform scripts which file to use based on Moodle version. -realpath "$MDL_COMPOSE_DIR/compose.yml" +# The default config for all versions is `default.yml` file. But if the environment config +# provides a specific compose file, use that one instead. Try first relative to the +# compose directory, then the absolute path. +realpath "$MDL_COMPOSE_DIR/default.yml" \ No newline at end of file diff --git a/libexec/mdl-init.sh b/libexec/mdl-init.sh index 907a416..6da0627 100755 --- a/libexec/mdl-init.sh +++ b/libexec/mdl-init.sh @@ -4,7 +4,7 @@ . "${0%/*}/../lib/mdl-ui.sh" # Defaults -compose_file_url=$MDL_BASE_URL/compose/compose.yml +compose_file_url=$MDL_BASE_URL/compose/default.yml display_title=true force=false install_moodle=true @@ -141,19 +141,19 @@ if $should_init_system; then install -d "$MDL_COMPOSE_DIR" if [[ -L $(command -v mdl) ]]; then # If in dev mode, link to the project compose file. - compose_file=$(realpath "$scr_dir/../compose/compose.yml") + compose_file=$(realpath "$scr_dir/../compose/default.yml") echo 'Since mdl is in developer mode, installing symlink to compose file at:' echo "$ul$compose_file$rmul" - ln -s -F "$compose_file" "$MDL_COMPOSE_DIR/compose.yml" + ln -s -F "$compose_file" "$MDL_COMPOSE_DIR/$(basename "$compose_file")" elif [[ -n $compose_file_url ]]; then # Download the provided compose file URL. echo 'Downloading compose file from:' echo "$ul$compose_file_url$rmul" - if ! curl -fsL "$compose_file_url" -o "$MDL_COMPOSE_DIR/compose.yml"; then + if ! curl -fsL "$compose_file_url" -o "$MDL_COMPOSE_DIR/$(basename "$compose_file_url")"; then echo "Failed to download compose file. Please check your internet connection or the URL." >&2 exit 1 fi - elif [[ -e $MDL_COMPOSE_DIR/compose.yml ]]; then + elif [[ -e $MDL_COMPOSE_DIR/default.yml ]]; then # A blank compose file URL was provided, but it's ok, because a file is present. echo 'Skipping compose file download. That is ok because one is already installed.' else From 95f4caf40d759fb907ec80e059aa0b3d47916b4e Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Wed, 17 Dec 2025 15:30:05 -0600 Subject: [PATCH 11/12] feat: Add support for custom compose files Whereas the default.yml compose file will be used if an environment does not specify a custom configuration with the `COMPOSE_FILE` variable, this feature improves the `mdl calc-compose-path` script to actually do some work: It will check the existence of the compose file path (first relative to the `MDL_COMPOSE_DIR` directory, then as an absolute path), and it will output the path of the compose file, and throw an error if the compose file does not exist. This will enable users to customize their compose files. Closes #14. --- libexec/mdl-calc-compose-path.sh | 21 ++++++++++++++++++--- libexec/mdl-info.sh | 9 ++++++--- libexec/mdl-init.sh | 5 ++++- libexec/mdl-logs.sh | 1 + libexec/mdl-start.sh | 1 + libexec/mdl-status.sh | 1 + libexec/mdl-stop.sh | 1 + 7 files changed, 32 insertions(+), 7 deletions(-) diff --git a/libexec/mdl-calc-compose-path.sh b/libexec/mdl-calc-compose-path.sh index a1279f3..a097b5f 100755 --- a/libexec/mdl-calc-compose-path.sh +++ b/libexec/mdl-calc-compose-path.sh @@ -17,8 +17,23 @@ EOF [[ $* =~ -h || $* =~ --help ]] && display_help && exit requires realpath +mname=$("$scr_dir/mdl-select-env.sh" "$1" --no-all) +export_env "$mname" # The default config for all versions is `default.yml` file. But if the environment config -# provides a specific compose file, use that one instead. Try first relative to the -# compose directory, then the absolute path. -realpath "$MDL_COMPOSE_DIR/default.yml" \ No newline at end of file +# provides a specific compose file, use that one instead. First try setting the path +# relative to the compose directory, and if that doesn't exist (throws an error), then use +# the absolute path. +compose_file=${COMPOSE_FILE:-default.yml} +if [[ -f "$MDL_COMPOSE_DIR/$compose_file" ]]; then + compose_path="$MDL_COMPOSE_DIR/$compose_file" +else + abs="$(realpath "$compose_file" 2>/dev/null)" + [[ -f $abs ]] && compose_path=$abs +fi +if [[ -n $compose_path ]]; then + echo "$compose_path" +else + echo "${red}Could not find compose file $ul$compose_file$rmul for $ul$mname$rmul.$norm" >&2 + exit 1 +fi diff --git a/libexec/mdl-info.sh b/libexec/mdl-info.sh index 4120d79..ca9d217 100755 --- a/libexec/mdl-info.sh +++ b/libexec/mdl-info.sh @@ -166,12 +166,12 @@ for mname in $mnames; do # ENVIRONMENT INFORMATION fields=( MOODLE_HOST WWWROOT MOODLE_PORT \ - MOODLE_IMAGE MARIADB_IMAGE \ + MOODLE_IMAGE MARIADB_IMAGE COMPOSE_FILE \ DB_TYPE DB_HOST DATA_ROOT DB_NAME ROOT_PASSWORD DB_USERNAME DB_PASSWORD \ SOURCE_HOST SOURCE_DATA_PATH SOURCE_SRC_PATH SOURCE_DB_NAME SOURCE_DB_USERNAME SOURCE_DB_PASSWORD \ BOX_CLIENT_ID BOX_CLIENT_SECRET BOX_REDIRECT_URI BOX_FOLDER_ID \ - mname running env_path custom_path db_vol_name data_vol_name src_vol_name \ - env_status custom_status db_status data_status src_status + mname running env_path compose_path custom_path db_vol_name data_vol_name src_vol_name \ + env_status compose_path_status custom_status db_status data_status src_status ) # Standard configs export_env "$mname" @@ -186,6 +186,7 @@ for mname in $mnames; do # Additional calculated fields . "$scr_dir/mdl-calc-images.sh" "$mname" env_path="$MDL_ENVS_DIR/$mname/.env" + compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname" 2>/dev/null) custom_path="$MDL_ENVS_DIR/$mname/custom-config.sh" vols=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname") db_vol_name=$(grep db <<< "$vols") @@ -194,6 +195,7 @@ for mname in $mnames; do running=false && [[ -n $(container_tool ps -q -f name="$mname") ]] && running=true running_string="${red}not running$norm" && $running && running_string="${green}running$norm" env_status=false && [ -f "$env_path" ] && env_status=true + compose_path_status=false && [ -n "$compose_path" ] && compose_path_status=true custom_status=false && [ -f "$custom_path" ] && custom_status=true db_status=false && [ -n "$db_vol_name" ] && db_status=true data_status=false && [ -n "$data_vol_name" ] && data_status=true @@ -219,6 +221,7 @@ for mname in $mnames; do echo pretty_line 'Paths and Volumes' pretty_line 'Environment file' "$env_path" "$env_status" + pretty_line 'Compose file' "${compose_path:-$COMPOSE_FILE}" "$compose_path_status" pretty_line 'Custom config file' "$custom_path" "$custom_status" pretty_line 'Database volume' "${db_vol_name:-${red}missing$norm}" "$db_status" pretty_line 'Data volume' "${data_vol_name:-${red}missing$norm}" "$data_status" diff --git a/libexec/mdl-init.sh b/libexec/mdl-init.sh index 6da0627..9de95eb 100755 --- a/libexec/mdl-init.sh +++ b/libexec/mdl-init.sh @@ -182,6 +182,9 @@ if [[ -n $mname ]]; then export_env "$mname" echo "Environment created at: $ul$MDL_ENVS_DIR/$mname$rmul" echo + echo "${ul}Compose Configuration$rmul" + COMPOSE_FILE=$(ask "Compose file" "${COMPOSE_FILE:-default.yml}") + echo echo "${ul}Database Configuration$rmul" DB_NAME=$(ask "Database name" "$DB_NAME") ROOT_PASSWORD=$(ask "Root password" "$ROOT_PASSWORD") @@ -255,7 +258,7 @@ if [[ -n $mname ]]; then env_file="$MDL_ENVS_DIR/$mname/.env" # Find any custom variables in the .env file that are not in the default list. variables=( - ROOT_PASSWORD DB_NAME DB_USERNAME DB_PASSWORD MOODLE_HOST WWWROOT MOODLE_PORT + COMPOSE_FILE ROOT_PASSWORD DB_NAME DB_USERNAME DB_PASSWORD MOODLE_HOST WWWROOT MOODLE_PORT SOURCE_HOST SOURCE_DATA_PATH SOURCE_SRC_PATH SOURCE_DB_NAME SOURCE_DB_USERNAME SOURCE_DB_PASSWORD BOX_CLIENT_ID BOX_CLIENT_SECRET BOX_REDIRECT_URI BOX_FOLDER_ID ) diff --git a/libexec/mdl-logs.sh b/libexec/mdl-logs.sh index 3aa4c99..70adfb4 100755 --- a/libexec/mdl-logs.sh +++ b/libexec/mdl-logs.sh @@ -24,6 +24,7 @@ containers="$(container_tool ps -q -f name="$mname" 2> /dev/null)" [ -z "$containers" ] && echo "The $mname stack is not running." && exit 1 compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname") +[[ -z $compose_path ]] && exit 1 . "$scr_dir/mdl-calc-images.sh" "$mname" export_env_and_update_config "$mname" compose_tool -p "$mname" -f "$compose_path" logs "${@:2}" diff --git a/libexec/mdl-start.sh b/libexec/mdl-start.sh index 4a93750..01ef9d6 100755 --- a/libexec/mdl-start.sh +++ b/libexec/mdl-start.sh @@ -43,6 +43,7 @@ done for mname in $mnames; do compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname") + [[ -z $compose_path ]] && continue $quiet || echo "Starting $mname..." . "$scr_dir/mdl-calc-images.sh" "$mname" export_env_and_update_config "$mname" diff --git a/libexec/mdl-status.sh b/libexec/mdl-status.sh index 9c5e0f8..cf1bfa0 100755 --- a/libexec/mdl-status.sh +++ b/libexec/mdl-status.sh @@ -35,6 +35,7 @@ for mname in $mnames; do data_vol_name=$(grep data <<< "$vols") src_vol_name=$(grep src <<< "$vols") compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname") + [[ -z $compose_path ]] && continue $quiet || echo "${ul}Environment: $bold$mname$norm" # Status diff --git a/libexec/mdl-stop.sh b/libexec/mdl-stop.sh index 53e6613..f28794f 100755 --- a/libexec/mdl-stop.sh +++ b/libexec/mdl-stop.sh @@ -32,6 +32,7 @@ for mname in $mnames; do fi compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname") + [[ -z $compose_path ]] && continue $quiet || echo "Stopping $mname..." . "$scr_dir/mdl-calc-images.sh" "$mname" From 95dbec91537e025cc7b030dc755e3c1202e17b36 Mon Sep 17 00:00:00 2001 From: Josh Curtiss Date: Thu, 18 Dec 2025 11:57:12 -0600 Subject: [PATCH 12/12] config: Eliminate privileged escalation in MariaDB container MariaDB can safely run without using privileged mode. Moodle itself may need privileged mode is some scenarios, especially when using podman, so for now we leave privileged mode enabled for the Moodle container. --- compose/default.yml | 1 - libexec/mdl-restore.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/compose/default.yml b/compose/default.yml index 1d43151..cf87981 100644 --- a/compose/default.yml +++ b/compose/default.yml @@ -2,7 +2,6 @@ name: $mname services: mariadb: image: $MARIADB_IMAGE - privileged: true restart: unless-stopped networks: - backend diff --git a/libexec/mdl-restore.sh b/libexec/mdl-restore.sh index db213d8..85ce16a 100755 --- a/libexec/mdl-restore.sh +++ b/libexec/mdl-restore.sh @@ -179,7 +179,6 @@ for mname in $mnames; do fi db_runner="${mname}_worker_db_restore" container_tool run -d --rm --name "$db_runner" \ - --privileged \ -e MARIADB_ROOT_PASSWORD="${ROOT_PASSWORD:-password}" \ -e MARIADB_USER="${DB_USERNAME:-moodleuser}" \ -e MARIADB_PASSWORD="${DB_PASSWORD:-password}" \