diff --git a/compose/compose.yml b/compose/default.yml
similarity index 95%
rename from compose/compose.yml
rename to compose/default.yml
index 61f6aad..cf87981 100644
--- a/compose/compose.yml
+++ b/compose/default.yml
@@ -2,7 +2,6 @@ name: $mname
services:
mariadb:
image: $MARIADB_IMAGE
- privileged: true
restart: unless-stopped
networks:
- backend
@@ -14,7 +13,7 @@ services:
- MARIADB_SKIP_TEST_DB=yes
volumes:
- /etc/localtime:/etc/localtime:ro
- - db:/bitnami/mariadb
+ - db:/bitnami/mariadb/data
moodle:
image: $MOODLE_IMAGE
privileged: true
diff --git a/custom-words.txt b/custom-words.txt
index 3020e5d..18678de 100644
--- a/custom-words.txt
+++ b/custom-words.txt
@@ -8,6 +8,7 @@ behaviour
bitnamilegacy
booktool
branchver
+bsdtar
cachelock
Caddyfile
calendartype
@@ -52,7 +53,10 @@ moodledata
moodleuser
mygr
mymoodle
+pbzip
pids
+pigz
+pixz
plib
profilefield
purgecaches
diff --git a/docs/scripts.md b/docs/scripts.md
index 1d5b91b..a201a0e 100644
--- a/docs/scripts.md
+++ b/docs/scripts.md
@@ -30,14 +30,6 @@ are listed at the bottom to make this reference easier to read.
{{ man['mdl-exec-sql'] }}
-### `mdl fast-db-backup`
-
-{{ man['mdl-fast-db-backup'] }}
-
-### `mdl fast-db-restore`
-
-{{ man['mdl-fast-db-restore'] }}
-
### `mdl info`
{{ man['mdl-info'] }}
diff --git a/installers/install.sh b/installers/install.sh
index e26febc..ed8e457 100755
--- a/installers/install.sh
+++ b/installers/install.sh
@@ -20,13 +20,13 @@ branch=main
[[ $1 != -* && -n $1 ]] && branch=$1
# Requires curl
-if [[ -z $(which curl 2>/dev/null) ]]; then
+if ! command -v curl &>/dev/null; then
echo "$(tput bold)$(tput setaf 1)This command requires $(tput smul)curl$(tput rmul) to work.$(tput sgr0)" >&2
exit 1
fi
# If `mdl` is already installed, it must not be a symlink
-mdl_path=$(which mdl 2>/dev/null)
+mdl_path=$(command -v mdl)
if [[ -n $mdl_path && -L $mdl_path ]]; then
echo 'The mdl CLI is already installed in developer mode. You do not have to reinstall' >&2
echo 'it. If you want to reinstall it in normal mode, please uninstall it first.' >&2
@@ -85,4 +85,4 @@ echo 🎉 The mdl CLI is installed!
echo
# Initialize the system
-mdl init --no-title -c "${MDL_BASE_URL/main/$branch}/compose/compose.yml"
+mdl init --no-title -c "${MDL_BASE_URL/main/$branch}/compose/default.yml"
diff --git a/installers/uninstall.sh b/installers/uninstall.sh
index 4a5ce3e..b38cfb2 100755
--- a/installers/uninstall.sh
+++ b/installers/uninstall.sh
@@ -1,7 +1,7 @@
#!/bin/bash
# Check that `mdl` is installed
-if [[ -z $(which mdl) ]]; then
+if ! command -v mdl &>/dev/null; then
echo "Could not find mdl. Are you sure it is installed?" >&2
exit 1
fi
@@ -12,10 +12,10 @@ if [[ $EUID -ne 0 ]]; then
fi
# Determine paths and load common functions
-base=$(realpath "$(dirname "$(realpath "$(which mdl)")")/..")
+base=$(realpath "$(dirname "$(realpath "$(command -v mdl)")")/..")
# Explicitly set scr_dir in case we're running from a curl pipe
scr_dir="$base/libexec"
-[[ -L $(which mdl) ]] && linked=true || linked=false
+[[ -L $(command -v mdl) ]] && linked=true || linked=false
# shellcheck source=../lib/mdl-common.sh
[[ -f $base/lib/mdl-common.sh ]] && . "$base/lib/mdl-common.sh"
# shellcheck source=../lib/mdl-ui.sh
@@ -31,7 +31,7 @@ if $linked; then
echo 'It appears you installed mdl in developer mode, which just installs a symlink to'
echo 'the project in your path.'
echo
- yorn "Do you want to remove the symlink?" 'y' && sudo rm "$(which mdl)"
+ yorn "Do you want to remove the symlink?" 'y' && sudo rm "$(command -v mdl)"
else
yorn "Remove the mdl executable and its associated files?" 'y' && \
sudo rm -fv "$base"/bin/mdl "$base"/lib/mdl-*.sh "$base"/libexec/mdl-*.sh \
diff --git a/lib/mdl-common.sh b/lib/mdl-common.sh
index 34a1dce..291c963 100644
--- a/lib/mdl-common.sh
+++ b/lib/mdl-common.sh
@@ -55,7 +55,7 @@ function container_tool() { "${MDL_CONTAINER_TOOL[@]}" "$@"; }
function requires() {
local ok=true
for cmd in "$@"; do
- if [[ -z $(which "$cmd" 2>/dev/null) ]]; then
+ if ! command -v "$cmd" &>/dev/null; then
echo "${red}${bold}This command requires $ul$cmd$rmul to work.$norm" >&2
ok=false
elif [[ $cmd =~ docker || $cmd =~ podman ]]; then
@@ -87,8 +87,18 @@ function support_long_options() {
fi
}
+function calc_compression_tool() {
+ local ext=${1##*.}
+ local cmd
+ [[ $ext == bz2 ]] && cmd=bzip2 && command -v pbzip2 &>/dev/null && cmd=pbzip2
+ [[ $ext == gz ]] && cmd=gzip && command -v pigz &>/dev/null && cmd=pigz
+ [[ $ext == xz ]] && cmd=xz && command -v pixz &>/dev/null && cmd=pixz
+ echo "$cmd"
+}
+
# Receives file path and decompresses it. Can detect bzip2, gzip and xz files. If none of
-# those extensions match the filename, it throws an error. After successful decompression,
+# those extensions match the filename, it throws an error. It will always try to use the
+# parallel processing version of the command if available. After successful decompression,
# the original file is deleted unless you specify `--keep`.
#
# Parameters:
@@ -99,14 +109,11 @@ function decompress() {
local file_path=$1
local out=$2
local ext=${file_path##*.}
- local cmd
+ local cmd=$(calc_compression_tool "$ext")
# Check options
[[ $* =~ -k || $* =~ --keep ]] && keep=true || keep=false
# File path is required
[[ -z $file_path ]] && return 1
- [[ $ext == bz2 ]] && cmd=bzip2
- [[ $ext == gz ]] && cmd=gzip
- [[ $ext == xz ]] && cmd=xz
if [[ -n $cmd ]]; then
# If they didn't provide an explicit output path, use file path sans extension
[[ -z $out ]] && out=${file_path%".$ext"}
@@ -122,6 +129,21 @@ function decompress() {
return 2
}
+# Function that uses awk to safely extract label from filename. Expects to receive
+# filename from pipe, such as: `echo $mname | extract_label `
+function extract_label() {
+ cat | awk -v type="$2" -v mname="$1" '
+ # Filter essentially looks for: /_\./ for example type "src" matches "_src."
+ /_'"$2"'\./ {
+ # Remove leading "_", so mname "moodle" matches "moodle_" at start of name.
+ sub("^" mname "_", "");
+ # Remove ending "_.*", so type "src" matches "_src.tar", "_src.tar.bz2", etc at end of name.
+ sub("_" type "\\..*$", "");
+ print $0
+ }
+ '
+}
+
# Used to clear all known vars to proactively avoid data leaks.
# Usage: unset_env
function unset_env() {
@@ -149,22 +171,32 @@ function export_env() {
export mname=$1
}
-# Updates config.php with the environment variables.
+# Updates config.php with the environment variables. Assumes that export_env has already been run.
# Usage: update_config
function update_config() {
local -r env_dir="$MDL_ENVS_DIR/$1"
- local -r env_config_file="$env_dir/src/config.php"
+
+ # Get the Moodle container and its mount paths for this environment
+ local -r container="$(container_tool ps -a -f "label=com.docker.compose.project=$1" --format '{{.Names}}' | grep moodle | head -1)"
+ local -r data_path=${container:+$(container_tool inspect "$container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("data"))) | .Destination')}
# Get desired wwwroot value
if [ -z "$WWWROOT" ]; then
- local -r defaultwwwroot="$(grep -o -E "CFG->wwwroot\s*=\s*'(.*)';" "$env_config_file" | cut -d"'" -f2)"
+ local defaultwwwroot=''
+ # Determine default wwwroot from existing config if possible (container must be running)
+ if [[ -n $container ]]; then
+ local -r src_path=${container:+$(container_tool inspect "$container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("src"))) | .Destination')}
+ if [[ -n $src_path ]]; then
+ defaultwwwroot=$(container_tool exec -t "$container" php -r "define('CLI_SCRIPT', true); require '$src_path/config.php'; echo \$CFG->wwwroot;")
+ fi
+ fi
local altwwwroot1="http://$HOSTNAME"
[ "$defaultwwwroot" = "$altwwwroot1" ] && altwwwroot1=''
local altwwwroot2="http://$MOODLE_HOST"
[ "$defaultwwwroot" = "$altwwwroot2" ] && altwwwroot2=''
echo 'You can avoid this prompt by setting WWWROOT in your .env file.'
PS3="Select a desired wwwroot value or type your own: "
- select WWWROOT in "$defaultwwwroot" $altwwwroot1 $altwwwroot2; do
+ select WWWROOT in $defaultwwwroot $altwwwroot1 $altwwwroot2; do
WWWROOT="${WWWROOT:-$REPLY}"
break
done
@@ -195,9 +227,11 @@ function update_config() {
replace_config_value dbname "$DB_NAME" |
replace_config_value dbuser "$DB_USERNAME" |
replace_config_value dbpass "$DB_PASSWORD" |
- replace_config_value wwwroot "$WWWROOT" |
- replace_config_value dataroot "${DATA_ROOT:-/bitnami/moodledata}"
+ replace_config_value wwwroot "$WWWROOT"
)
+ if [[ -n "$DATA_ROOT" || -n "$data_path" ]]; then
+ config_content=$(replace_config_value "$config_content" dataroot "${DATA_ROOT:-$data_path}")
+ fi
# Run custom updates if provided
custom_script="$env_dir/custom-config.sh"
[[ -f "$custom_script" ]] && config_content=$(. "$custom_script" "$config_content")
@@ -211,6 +245,7 @@ EOF
container_tool run --rm -t --name "${1}_worker_update_config" \
-v "$env_dir":/env:Z,ro \
-v "${1}_src":/src \
+ -e data_path="$data_path" \
"$MDL_SHELL_IMAGE" sh -c "$cmd"
}
diff --git a/libexec/mdl-backup.sh b/libexec/mdl-backup.sh
index 5300eb2..ecfff95 100755
--- a/libexec/mdl-backup.sh
+++ b/libexec/mdl-backup.sh
@@ -3,7 +3,7 @@
. "${0%/*}/../lib/mdl-common.sh"
# Valid Options
-valid_modules='src data db'
+valid_modules='src data db fastdb'
valid_compress='bzip2 gzip xz none'
# Defaults
@@ -19,12 +19,19 @@ label=$default_label
# Declarations
source_host=
source_type=
-compress_flag=
ssh_args=
container_tool=${MDL_CONTAINER_TOOL[*]}
verbose=false
dry_run=false
+# Functions
+function ssh_wrap() {
+ echo "${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}$1${source_host:+"\\\"\""}"
+}
+function eval_ssh_wrap() {
+ eval "$(ssh_wrap "$1")"
+}
+
# Help
display_help() {
cat <&2
exit 1
fi
@@ -158,6 +182,16 @@ if [[ ! "$valid_compress" =~ $compress_arg ]]; then
exit 1
fi
+# Fast DB backup is only for container sources
+if [[ $modules =~ fastdb ]]; then
+ if $is_container; then
+ echo -e "${yellow}Warning: Fast database backups are unsafe for production use.$norm\n" >&2
+ else
+ echo -e "${red}Error: Fast database backups are only supported for container sources.$norm\n" >&2
+ exit 1
+ fi
+fi
+
# Check necessary utilities
cmds=(tar "${MDL_CONTAINER_TOOL[0]}")
[[ $compress_arg != none ]] && cmds+=("$compress_arg")
@@ -221,67 +255,67 @@ for mname in $mnames; do
ping -c 1 "$source_host_server" &> /dev/null && source_host_reachable=true || source_host_reachable=false
fi
- # If "container", the container must be active in order to dump the database for the "db" module.
+ # If "container", the containers must be active in order to backup data for their module.
source_db_container=''
+ source_moodle_container=''
if $is_container; then
- if [[ $modules =~ db ]]; then
- source_db_container_cmd="
- ${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
- $container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep mariadb | head -1
- ${source_host:+"\\\"\""}
- "
- source_db_container=$(eval "$source_db_container_cmd")
+ if [[ " $modules " == *" db "* || $modules =~ fastdb ]]; then
+ # Find the database container name
+ source_db_container=$(eval_ssh_wrap "$container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep mariadb | head -1")
[[ -z $source_db_container ]] && echo "${red}The database container cannot be found. Start the environment to perform a backup." >&2 && exit 1
+ # Determine db path if not provided (usually won't be for container mode)
+ if [[ -z $source_db_path && $modules =~ fastdb ]]; then
+ source_db_path=$(eval_ssh_wrap "$container_tool inspect '$source_db_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"db\"))) | .Destination'")
+ [[ -z $source_db_path ]] && echo "${red}Could not determine ${ul}db$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1
+ fi
fi
if [[ $modules =~ data || $modules =~ src ]]; then
- vols_cmd="
- ${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
- $container_tool volume ls -q --filter 'label=com.docker.compose.project=$mname'
- ${source_host:+"\\\"\""}
- "
- vols=$(eval "$vols_cmd")
- [[ $modules =~ data ]] && source_data_volume=$(grep data <<< "$vols")
- [[ $modules =~ src ]] && source_src_volume=$(grep src <<< "$vols")
+ # Find the moodle container name
+ source_moodle_container=$(eval_ssh_wrap "$container_tool ps -f 'label=com.docker.compose.project=$mname' --format '{{.Names}}' | grep moodle | head -1")
+ [[ -z $source_moodle_container ]] && echo "${red}The Moodle container cannot be found. Start the environment to perform a backup." >&2 && exit 1
+ # Determine src/data paths if not provided (usually won't be for container mode)
+ if [[ -z $source_src_path && $modules =~ src ]]; then
+ source_src_path=$(eval_ssh_wrap "$container_tool inspect '$source_moodle_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"src\"))) | .Destination'")
+ [[ -z $source_src_path ]] && echo "${red}Could not determine ${ul}src$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1
+ fi
+ if [[ -z $source_data_path && $modules =~ data ]]; then
+ source_data_path=$(eval_ssh_wrap "$container_tool inspect '$source_moodle_container' | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains(\"data\"))) | .Destination'")
+ [[ -z $source_data_path ]] && echo "${red}Could not determine ${ul}data$rmul directory for $ul$mname$rmul!$norm" >&2 && exit 1
+ fi
fi
fi
# Password mask
source_db_password_mask=${source_db_password//?/*}
- # Compression extension
- compress_ext=''
- case "$compress_arg" in
- bzip2) compress_ext='.bz2' ;;
- gzip) compress_ext='.gz' ;;
- xz) compress_ext='.xz' ;;
- esac
-
# Targets
[[ $modules =~ data ]] && data_target="$MDL_BACKUP_DIR/${mname}_${label}_data.tar$compress_ext"
[[ $modules =~ src ]] && src_target="$MDL_BACKUP_DIR/${mname}_${label}_src.tar$compress_ext"
- [[ $modules =~ db ]] && db_target="$MDL_BACKUP_DIR/${mname}_${label}_db.sql$compress_ext"
+ [[ $modules =~ fastdb ]] && fastdb_target="$MDL_BACKUP_DIR/${mname}_${label}_dbfiles.tar$compress_ext"
+ [[ " $modules " == *" db "* ]] && db_target="$MDL_BACKUP_DIR/${mname}_${label}_db.sql$compress_ext"
action_word='Backing up'
echo -e "$ul$bold$action_word $mname environment$norm"
if $verbose; then
echo
- echo "$bold Type:$norm $source_type"
- [[ -n $source_host ]] && echo "$bold Host:$norm $source_host ($($source_host_reachable && echo "${green}reachable" || echo "${red}unreachable!")$norm)"
- [[ -n $source_src_path ]] && echo "$bold 'src' Path:$norm $source_src_path"
- [[ -n $source_data_path ]] && echo "$bold 'data' Path:$norm $source_data_path"
- [[ -n $source_src_volume ]] && echo "$bold 'src' Volume:$norm $source_src_volume"
- [[ -n $source_data_volume ]] && echo "$bold 'data' Volume:$norm $source_data_volume"
- [[ -n $source_db_container ]] && echo "$bold DB Container:$norm $source_db_container"
- [[ -n $source_db_name ]] && echo "$bold DB Name:$norm $source_db_name"
- [[ -n $source_db_username ]] && echo "$bold DB Username:$norm $source_db_username"
- [[ -n $source_db_password ]] && echo "$bold DB Password:$norm $source_db_password_mask"
+ echo "$bold Type:$norm $source_type"
+ [[ -n $source_host ]] && echo "$bold Host:$norm $source_host ($($source_host_reachable && echo "${green}reachable" || echo "${red}unreachable!")$norm)"
+ [[ -n $source_moodle_container ]] && echo "$bold Mdl Container:$norm $source_moodle_container"
+ [[ -n $source_src_path ]] && echo "$bold 'src' Path:$norm $source_src_path"
+ [[ -n $source_data_path ]] && echo "$bold 'data' Path:$norm $source_data_path"
+ [[ -n $source_db_container ]] && echo "$bold DB Container:$norm $source_db_container"
+ [[ -n $source_db_path ]] && echo "$bold 'db' Path:$norm $source_db_path"
+ [[ -n $source_db_name ]] && echo "$bold DB Name:$norm $source_db_name"
+ [[ -n $source_db_username ]] && echo "$bold DB Username:$norm $source_db_username"
+ [[ -n $source_db_password ]] && echo "$bold DB Password:$norm $source_db_password_mask"
echo
- [[ -n $label ]] && echo "$bold Label:$norm $label"
- echo "$bold Modules:$norm $modules"
- echo "$bold Compress:$norm $compress_arg"
- [[ -n $src_target ]] && echo "$bold 'src' Path:$norm $src_target"
- [[ -n $data_target ]] && echo "$bold 'data' Path:$norm $data_target"
- [[ -n $db_target ]] && echo "$bold DB Path:$norm $db_target"
+ [[ -n $label ]] && echo "$bold Label:$norm $label"
+ echo "$bold Modules:$norm $modules"
+ echo "$bold Compress:$norm $compress_arg"
+ [[ -n $src_target ]] && echo "$bold 'src' Path:$norm $src_target"
+ [[ -n $data_target ]] && echo "$bold 'data' Path:$norm $data_target"
+ [[ -n $db_target ]] && echo "$bold DB Path:$norm $db_target"
+ [[ -n $fastdb_target ]] && echo "$bold Fast DB Path:$norm $fastdb_target"
echo
fi
@@ -290,32 +324,38 @@ for mname in $mnames; do
exit 1
fi
+ # MODULE: data
+ data_cmd=''
+ $is_container && data_cmd=$(ssh_wrap "$container_tool cp $source_moodle_container:$source_data_path/. -")
+ $is_container || data_cmd=$(ssh_wrap "tar c -C '$source_data_path' .")
+ if command -v bsdtar &>/dev/null; then
+ data_cmd="$data_cmd | bsdtar cf - \
+ --exclude localcache \
+ --exclude cache \
+ --exclude sessions \
+ --exclude temp \
+ --exclude trashdir \
+ --exclude moodle-cron.log \
+ @- \
+ "
+ else
+ echo "${yellow}Warning: Since ${ul}bsdtar$rmul isn't found, caches/logs will not be excluded from backup.$norm" >&2
+ fi
+
+ # MODULE: src
+ src_cmd=''
+ $is_container && src_cmd=$(ssh_wrap "$container_tool cp $source_moodle_container:$source_src_path/. -")
# shellcheck disable=SC2034
- data_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
- $is_container && data_cmd="$data_cmd $container_tool run --rm --name '${mname}_worker_bk_data' -v '$source_data_volume':/data $MDL_SHELL_IMAGE"
- data_cmd="$data_cmd \
- tar c $compress_flag \
- --exclude='./trashdir' \
- --exclude='./temp' \
- --exclude='./sessions' \
- --exclude='./localcache' \
- --exclude='./cache' \
- --exclude='./moodle-cron.log' \
- -C $($is_container && echo /data || echo "$source_data_path") . \
- "
- data_cmd=$data_cmd${source_host:+"\\\"\""}
- # shellcheck disable=SC2034
- src_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
- $is_container && src_cmd="$src_cmd $container_tool run --rm --name '${mname}_worker_bk_src' -v '$source_src_volume':/src $MDL_SHELL_IMAGE"
- src_cmd="$src_cmd \
- tar c $compress_flag \
- -C $($is_container && echo /src || echo "$source_src_path") . \
- "
- src_cmd=$src_cmd${source_host:+"\\\"\""}
- # TODO: When piping to compression program, a failed status of mysqldump will be lost.
+ $is_container || src_cmd=$(ssh_wrap "tar c -C '$source_src_path' .")
+
+ # MODULE: fastdb
+ fastdb_cmd=''
# shellcheck disable=SC2034
- db_cmd=${source_host:+"ssh $ssh_args $source_host $source_sudo \"sh -l -c \\\""}
- $is_container && db_cmd="$db_cmd $container_tool exec '$source_db_container'"
+ $is_container && fastdb_cmd=$(ssh_wrap "$container_tool cp $source_db_container:$source_db_path/. -")
+
+ # MODULE: db
+ db_cmd=''
+ $is_container && db_cmd="$container_tool exec '$source_db_container'"
db_cmd="$db_cmd \
mysqldump \
--user='$source_db_username' \
@@ -324,20 +364,21 @@ for mname in $mnames; do
-C -Q -e --create-options \
$source_db_name \
"
- db_cmd=$db_cmd${source_host:+"\\\"\""}
- [[ $compress_arg != none ]] && db_cmd="$db_cmd | $compress_arg -cq9"
+ db_cmd=$(ssh_wrap "$db_cmd")
+
+ # Actually EXECUTE the commands
pids=()
- for t in data src db; do
- if [[ $modules =~ $t ]]; then
- targ_var=${t}_target; targ=${!targ_var}
- cmd_var=${t}_cmd; cmd=${!cmd_var}
+ $verbose && echo
+ for t in $valid_modules; do
+ if [[ " $modules " == *" $t "* ]]; then
+ targ_var="${t}_target"; targ="${!targ_var}"
+ cmd_var="${t}_cmd"; cmd="${!cmd_var}"
+ [[ -n $compression_tool ]] && cmd="$cmd | $compression_tool -cq9"
echo "$mname $t: $targ"
- # In verbose mode, output the command, but mask the password and eliminate whitespace by echoing with word splitting.
- # shellcheck disable=2086
- $verbose && echo ${cmd//password=\'$source_db_password\'/password=\'$source_db_password_mask\'}
- if $dry_run; then
- echo -e "${red}Not executed. This is a dry run.$norm\n"
- else
+ # In verbose mode, output the command, but mask password. Eliminate whitespace with `xargs`.
+ # Ref: https://stackoverflow.com/questions/369758/how-to-trim-whitespace-from-a-bash-variable
+ $verbose && echo "${bold}Command:$norm ${cmd//password=\'$source_db_password\'/password=\'$source_db_password_mask\'}" | xargs
+ if ! $dry_run; then
cmd="$cmd > '$targ'"
# Execute the command, and handle success/fail scenarios
eval "$cmd" && success=true || success=false
@@ -346,13 +387,14 @@ for mname in $mnames; do
rm "$targ"
echo "Removed $(basename "$targ") because the $t backup failed." >&2
fi
- fi
- fi &
- pids+=($!)
+ fi &
+ pids+=($!)
+ fi
done
# TODO: It'd be nice if we check if any of the steps failed, and exit non-zero if so.
wait "${pids[@]}"
+ $dry_run && echo -e "${red}Commands not executed. This is a dry run.$norm\n"
echo "$action_word of $mname environment is complete!"
# Unset environment variables
diff --git a/libexec/mdl-box.sh b/libexec/mdl-box.sh
index 8dbdea3..df2ab7a 100755
--- a/libexec/mdl-box.sh
+++ b/libexec/mdl-box.sh
@@ -145,7 +145,7 @@ for mname in $mnames; do
url="https://account.box.com/api/oauth2/authorize?response_type=code&client_id=$BOX_CLIENT_ID&redirect_uri=$BOX_REDIRECT_URI"
echo "Go to the following link to get the authorization code:"
echo "$ul$url$norm"
- [[ -n $(which open 2>/dev/null) ]] && open "$url"
+ command -v open &>/dev/null && open "$url"
read -r -p "Authorization code: " auth_code
if [[ -z $auth_code ]]; then
echo "You entered a blank authorization code. Aborting."
diff --git a/libexec/mdl-calc-compose-path.sh b/libexec/mdl-calc-compose-path.sh
index 267eeac..a097b5f 100755
--- a/libexec/mdl-calc-compose-path.sh
+++ b/libexec/mdl-calc-compose-path.sh
@@ -6,8 +6,8 @@ display_help() {
cat <
-Looks at the version of a Moodle environment, based on its Git branch, and returns the
-full path of the ${ul}compose.yml${rmul} file that should be used.
+Looks at the version of a Moodle environment, based on its Git branch and custom configs,
+and returns the full path of the compose file that should be used.
Options:
-h, --help Show this help message and exit.
@@ -17,7 +17,23 @@ EOF
[[ $* =~ -h || $* =~ --help ]] && display_help && exit
requires realpath
+mname=$("$scr_dir/mdl-select-env.sh" "$1" --no-all)
+export_env "$mname"
-# Right now, all configs can use the same `compose.yml` file, but if that changes,
-# this script will inform scripts which file to use based on Moodle version.
-realpath "$MDL_COMPOSE_DIR/compose.yml"
+# The default config for all versions is `default.yml` file. But if the environment config
+# provides a specific compose file, use that one instead. First try setting the path
+# relative to the compose directory, and if that doesn't exist (throws an error), then use
+# the absolute path.
+compose_file=${COMPOSE_FILE:-default.yml}
+if [[ -f "$MDL_COMPOSE_DIR/$compose_file" ]]; then
+ compose_path="$MDL_COMPOSE_DIR/$compose_file"
+else
+ abs="$(realpath "$compose_file" 2>/dev/null)"
+ [[ -f $abs ]] && compose_path=$abs
+fi
+if [[ -n $compose_path ]]; then
+ echo "$compose_path"
+else
+ echo "${red}Could not find compose file $ul$compose_file$rmul for $ul$mname$rmul.$norm" >&2
+ exit 1
+fi
diff --git a/libexec/mdl-cli.sh b/libexec/mdl-cli.sh
index 4397330..7ebf0ec 100755
--- a/libexec/mdl-cli.sh
+++ b/libexec/mdl-cli.sh
@@ -53,19 +53,21 @@ mnames=$("$scr_dir"/mdl-select-env.sh "$1")
for mname in $mnames; do
- # Get the Moodle container for this environment
+ # Get the Moodle container and base directory for this environment
container="$(container_tool ps -f "label=com.docker.compose.project=$mname" --format '{{.Names}}' | grep moodle | head -1)"
[[ -z $container ]] && echo "${red}Could not find a container running Moodle for $ul$mname$rmul!$norm" >&2 && exit 1
+ base_dir=$(container_tool inspect "$container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("src"))) | .Destination')
+ [[ -z $base_dir ]] && echo "${red}Could not determine Moodle base directory for $ul$mname$rmul!$norm" >&2 && exit 1
cmd="$2"
# If they did not provide a cmd, list the available commands
if [[ -z $cmd ]]; then
echo "${bold}${ul}Available Commands$norm"
- container_tool exec -t "$container" find /bitnami/moodle/admin/cli -maxdepth 1 -type f -exec basename {} .php \; | sort | sed 's/^/ - /'
+ container_tool exec -t "$container" find "$base_dir/admin/cli" -maxdepth 1 -type f -exec basename {} .php \; | sort | sed 's/^/ - /'
exit
fi
# Run the command, passing any additional arguments they passed on to this script
- container_tool exec $paramI -t "$container" php "/bitnami/moodle/admin/cli/$cmd.php" "${@:3}"
+ container_tool exec $paramI -t "$container" php "$base_dir/admin/cli/$cmd.php" "${@:3}"
done
diff --git a/libexec/mdl-fast-db-backup.sh b/libexec/mdl-fast-db-backup.sh
deleted file mode 100755
index 6764441..0000000
--- a/libexec/mdl-fast-db-backup.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/bash
-
-. "${0%/*}/../lib/mdl-common.sh"
-
-display_help() {
- cat < [LABEL]
-
-Makes a fast database backup, which is just a tar archive of the raw database files. The
-reason this is fast is because the archive process is faster than a database dump, and
-because the restore process directly restores the database files, as opposed to the
-traditional restore which saves the dump in the environment and requires the dump to be
-processed by the database container on startup.
-$bold$red
-This is unsafe for production but often works fine for development purposes.
-$norm
-Options:
--h, --help Show this help message and exit.
-EOF
-}
-
-[[ $* =~ -h || $* =~ --help ]] && display_help && exit
-
-requires "${MDL_CONTAINER_TOOL[0]}"
-
-mnames=$("$scr_dir"/mdl-select-env.sh "$1")
-
-echo '
-WARNING: This makes a fast database backup, which is just a tar archive of the
-filesystem. This should not be used for production purposes.
-'
-
-for mname in $mnames; do
-
- echo "Fast backup of the $mname database."
-
- # Abort if the volume can't be found
- db_vol_name=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname" | grep db)
- if [ -z "$db_vol_name" ]; then
- echo "Database volume for $mname could not be found."
- exit 1
- fi
-
- # What label on the backup do they want? (Defaults to "local_branchver_yyyymmdd")
- branchver=$("$scr_dir"/mdl-moodle-version.sh "$mname")
- defaultlabel="local_${branchver}_$(date +"%Y%m%d")"
- label="$2"
- if [ "$label" = "" ]; then
- echo -n "Enter the label to put on the backup [$defaultlabel]: "
- read -r label
- label="${label:-$defaultlabel}"
- fi
-
- "$scr_dir/mdl-stop.sh" "$mname"
-
- db_target="${mname}_${label}_dbfiles.tar"
- container_tool run --rm --privileged -v "$db_vol_name":/db -v "$MDL_BACKUP_DIR":/backup "$MDL_SHELL_IMAGE" tar cf "/backup/$db_target" -C /db .
-
- echo "Fast backup of $mname is done!"
-
-done
diff --git a/libexec/mdl-fast-db-restore.sh b/libexec/mdl-fast-db-restore.sh
deleted file mode 100755
index 1040d46..0000000
--- a/libexec/mdl-fast-db-restore.sh
+++ /dev/null
@@ -1,73 +0,0 @@
-#!/bin/bash
-
-. "${0%/*}/../lib/mdl-common.sh"
-
-display_help() {
- cat < [LABEL]
-
-Restores a fast database backup, just overwriting the raw database files. $bold${red}This is unsafe
-for production.$norm However, it can make the development cycle of restoring to a previous
-state much faster; whereas you do need to restart the environment, the database will
-start up instantaneously since it doesn't have to import a database dump file.
-
-Options:
--h, --help Show this help message and exit.
-EOF
-}
-
-[[ $* =~ -h || $* =~ --help ]] && display_help && exit
-
-requires "${MDL_CONTAINER_TOOL[0]}"
-
-mnames=$("$scr_dir/mdl-select-env.sh" "$1")
-
-echo '
-WARNING: This is restoring a fast database backup, which is just restoring the
-database filesystem. Whereas this often works in a dev environment, it should
-never be used for production purposes.
-'
-
-for mname in $mnames; do
-
- export_env "$mname"
- echo "Preparing to restore a fast database backup of $mname..."
-
- # Stop the services if they're running
- "$scr_dir/mdl-stop.sh" "$mname"
- echo
-
- # What timestamp of backup do they want? (Select from the list if they did not provide)
- labels=$(find "$MDL_BACKUP_DIR" -name "${mname}_*_dbfiles.tar" | cut -d"_" -f2- | sed -e "s/_dbfiles.tar//")
- [ -z "$labels" ] && echo "There are no fast backup files for $mname." && exit 1
- label="$2"
- # Even if they provided a label, prompt them if its not a label in the list
- if [ "$(echo "$labels" | grep "^$label\$")" = "" ]; then
- PS3="Select the label of the backup to restore: "
- select label in $labels; do
- break
- done
- fi
- echo "Restoring $mname with label $label... "
-
- # Backup targets
- db_target="${mname}_${label}_dbfiles.tar"
-
- # Get database volume name, or, if it doesn't exist, make the name we expect it to be
- db_vol_name=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname" | grep db)
- if [ -z "$db_vol_name" ]; then
- db_vol_name="${mname}_db"
- else
- container_tool volume rm "$db_vol_name" 2> /dev/null # If volume removal fails, its fine
- fi
-
- # Recreate volume and extract to the database volume
- container_tool volume create --label "com.docker.compose.project=$mname" "$db_vol_name"
- container_tool run --rm --privileged -v "$db_vol_name":/db -v "$MDL_BACKUP_DIR":/backup "$MDL_SHELL_IMAGE" tar xf "/backup/$db_target" -C /db
-
- echo "Done restoring the fast backup of database for $mname with label $label."
-
- # Unset environment variables
- unset_env "$mname"
-
-done
diff --git a/libexec/mdl-info.sh b/libexec/mdl-info.sh
index a6941ab..ca9d217 100755
--- a/libexec/mdl-info.sh
+++ b/libexec/mdl-info.sh
@@ -92,7 +92,7 @@ backup_status=true && [[ ! -d "$MDL_BACKUP_DIR" ]] && ok=false && backup_status=
compose_status=true && [[ ! -d "$MDL_COMPOSE_DIR" ]] && ok=false && compose_status=false
container_tool_status=true && [[ -z "$("${MDL_CONTAINER_TOOL[0]}" --version 2> /dev/null)" ]] && ok=false && container_tool_status=false
compose_tool_status=true && [[ -z "$("${MDL_COMPOSE_TOOL[0]}" --version 2> /dev/null)" ]] && ok=false && compose_tool_status=false
-mdl_path=$(which mdl)
+mdl_path=$(command -v mdl)
mdl_realpath=$(realpath "$mdl_path")
[[ -L $mdl_path ]] && mdl_status="in dev mode at $ul$mdl_realpath$rmul" || mdl_status="at $ul$mdl_path$rmul"
@@ -166,12 +166,12 @@ for mname in $mnames; do
# ENVIRONMENT INFORMATION
fields=(
MOODLE_HOST WWWROOT MOODLE_PORT \
- MOODLE_IMAGE MARIADB_IMAGE \
+ MOODLE_IMAGE MARIADB_IMAGE COMPOSE_FILE \
DB_TYPE DB_HOST DATA_ROOT DB_NAME ROOT_PASSWORD DB_USERNAME DB_PASSWORD \
SOURCE_HOST SOURCE_DATA_PATH SOURCE_SRC_PATH SOURCE_DB_NAME SOURCE_DB_USERNAME SOURCE_DB_PASSWORD \
BOX_CLIENT_ID BOX_CLIENT_SECRET BOX_REDIRECT_URI BOX_FOLDER_ID \
- mname running env_path custom_path db_vol_name data_vol_name src_vol_name \
- env_status custom_status db_status data_status src_status
+ mname running env_path compose_path custom_path db_vol_name data_vol_name src_vol_name \
+ env_status compose_path_status custom_status db_status data_status src_status
)
# Standard configs
export_env "$mname"
@@ -186,6 +186,7 @@ for mname in $mnames; do
# Additional calculated fields
. "$scr_dir/mdl-calc-images.sh" "$mname"
env_path="$MDL_ENVS_DIR/$mname/.env"
+ compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname" 2>/dev/null)
custom_path="$MDL_ENVS_DIR/$mname/custom-config.sh"
vols=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname")
db_vol_name=$(grep db <<< "$vols")
@@ -194,6 +195,7 @@ for mname in $mnames; do
running=false && [[ -n $(container_tool ps -q -f name="$mname") ]] && running=true
running_string="${red}not running$norm" && $running && running_string="${green}running$norm"
env_status=false && [ -f "$env_path" ] && env_status=true
+ compose_path_status=false && [ -n "$compose_path" ] && compose_path_status=true
custom_status=false && [ -f "$custom_path" ] && custom_status=true
db_status=false && [ -n "$db_vol_name" ] && db_status=true
data_status=false && [ -n "$data_vol_name" ] && data_status=true
@@ -219,6 +221,7 @@ for mname in $mnames; do
echo
pretty_line 'Paths and Volumes'
pretty_line 'Environment file' "$env_path" "$env_status"
+ pretty_line 'Compose file' "${compose_path:-$COMPOSE_FILE}" "$compose_path_status"
pretty_line 'Custom config file' "$custom_path" "$custom_status"
pretty_line 'Database volume' "${db_vol_name:-${red}missing$norm}" "$db_status"
pretty_line 'Data volume' "${data_vol_name:-${red}missing$norm}" "$data_status"
diff --git a/libexec/mdl-init.sh b/libexec/mdl-init.sh
index cd0ac3f..9de95eb 100755
--- a/libexec/mdl-init.sh
+++ b/libexec/mdl-init.sh
@@ -4,7 +4,7 @@
. "${0%/*}/../lib/mdl-ui.sh"
# Defaults
-compose_file_url=$MDL_BASE_URL/compose/compose.yml
+compose_file_url=$MDL_BASE_URL/compose/default.yml
display_title=true
force=false
install_moodle=true
@@ -139,21 +139,21 @@ if $should_init_system; then
install -d "$MDL_ENVS_DIR"
install -d "$MDL_BACKUP_DIR"
install -d "$MDL_COMPOSE_DIR"
- if [[ -L $(which mdl) ]]; then
+ if [[ -L $(command -v mdl) ]]; then
# If in dev mode, link to the project compose file.
- compose_file=$(realpath "$scr_dir/../compose/compose.yml")
+ compose_file=$(realpath "$scr_dir/../compose/default.yml")
echo 'Since mdl is in developer mode, installing symlink to compose file at:'
echo "$ul$compose_file$rmul"
- ln -s -F "$compose_file" "$MDL_COMPOSE_DIR/compose.yml"
+ ln -s -F "$compose_file" "$MDL_COMPOSE_DIR/$(basename "$compose_file")"
elif [[ -n $compose_file_url ]]; then
# Download the provided compose file URL.
echo 'Downloading compose file from:'
echo "$ul$compose_file_url$rmul"
- if ! curl -fsL "$compose_file_url" -o "$MDL_COMPOSE_DIR/compose.yml"; then
+ if ! curl -fsL "$compose_file_url" -o "$MDL_COMPOSE_DIR/$(basename "$compose_file_url")"; then
echo "Failed to download compose file. Please check your internet connection or the URL." >&2
exit 1
fi
- elif [[ -e $MDL_COMPOSE_DIR/compose.yml ]]; then
+ elif [[ -e $MDL_COMPOSE_DIR/default.yml ]]; then
# A blank compose file URL was provided, but it's ok, because a file is present.
echo 'Skipping compose file download. That is ok because one is already installed.'
else
@@ -169,12 +169,22 @@ elif [[ -z $mname ]]; then
fi
if [[ -n $mname ]]; then
+ # Validate mname. Abort if we received an invalid name.
+ if ! [[ $mname =~ ^[a-z0-9][a-z0-9_-]*$ ]]; then
+ echo "${red}The environment name $ul$mname$rmul is not valid.$norm" >&2
+ echo "${yellow}Names can only contain lowercase alphanumeric characters, hyphens, and underscores,$norm" >&2
+ echo "${yellow}and must start with a letter or number.$norm" >&2
+ exit 1
+ fi
if [[ ! -d "$MDL_ENVS_DIR/$mname" ]] || $force; then
echo "Creating environment: $ul$mname$rmul"
mkdir -p "$MDL_ENVS_DIR/$mname"
export_env "$mname"
echo "Environment created at: $ul$MDL_ENVS_DIR/$mname$rmul"
echo
+ echo "${ul}Compose Configuration$rmul"
+ COMPOSE_FILE=$(ask "Compose file" "${COMPOSE_FILE:-default.yml}")
+ echo
echo "${ul}Database Configuration$rmul"
DB_NAME=$(ask "Database name" "$DB_NAME")
ROOT_PASSWORD=$(ask "Root password" "$ROOT_PASSWORD")
@@ -248,7 +258,7 @@ if [[ -n $mname ]]; then
env_file="$MDL_ENVS_DIR/$mname/.env"
# Find any custom variables in the .env file that are not in the default list.
variables=(
- ROOT_PASSWORD DB_NAME DB_USERNAME DB_PASSWORD MOODLE_HOST WWWROOT MOODLE_PORT
+ COMPOSE_FILE ROOT_PASSWORD DB_NAME DB_USERNAME DB_PASSWORD MOODLE_HOST WWWROOT MOODLE_PORT
SOURCE_HOST SOURCE_DATA_PATH SOURCE_SRC_PATH SOURCE_DB_NAME SOURCE_DB_USERNAME SOURCE_DB_PASSWORD
BOX_CLIENT_ID BOX_CLIENT_SECRET BOX_REDIRECT_URI BOX_FOLDER_ID
)
@@ -299,7 +309,7 @@ if [[ -n $mname ]]; then
for ver_string in "${versions[@]}"; do
IFS=' ' read -ra var_array <<< "$ver_string"
branch_array+=("${var_array[0]}")
- moodle_array+=("$(echo "${var_array[1]}" | cut -d'.' -f1-2)")
+ moodle_array+=("$(echo "${var_array[1]}" | cut -d':' -f2 | cut -d'.' -f1-2)")
done
PS3="Select the version to install: "
select moodle_ver in "${moodle_array[@]}"; do
@@ -311,10 +321,12 @@ if [[ -n $mname ]]; then
fi
done
echo
- # Start the environment. Bitnami image will automatically bootstrap install. Wait to finish.
+ # Start the environment, automatically bootstrapping install. Wait to finish.
branchver="$branchver" "$scr_dir/mdl-start.sh" "$mname" -q
moodle_svc=$(container_tool ps --filter "label=com.docker.compose.project=$mname" --format '{{.Names}}' | grep moodle)
src_vol_name=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname" | grep src)
+ src_path=$(container_tool inspect "$moodle_svc" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("src"))) | .Destination')
+ data_path=$(container_tool inspect "$moodle_svc" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("data"))) | .Destination')
# Do git install once standard install completes.
function git_cmd() {
container_tool run --rm -t --name "$mname-git-$(uuidgen)" -v "$src_vol_name":/git "$MDL_GIT_IMAGE" -c safe.directory=/git "$@"
@@ -352,17 +364,19 @@ if [[ -n $mname ]]; then
}
!skip { print }
'
- config_file=/bitnami/moodle/config.php
+ config_file=$src_path/config.php
revised_config_file=$(mktemp)
container_tool exec -it "$moodle_svc" awk "$awk_cmd" "$config_file" > "$revised_config_file"
container_tool cp "$revised_config_file" "$moodle_svc":"$config_file"
- # Bitnami image unfortunately does not install the git repo. So, we add git after the fact.
- # This works fine since the Moodle repo branch will always be even with or slightly ahead of the Bitnami image.
- targetbranch="MOODLE_${branchver}_STABLE"
- git_cmd init -b main
- git_cmd remote add origin https://github.com/moodle/moodle.git
- git_cmd fetch -np origin "$targetbranch"
- git_cmd checkout -f "$targetbranch"
+ # If image does not install the git repo, we add [g]it after the fact. This works fine since
+ # the Moodle repo branch will always be even with or slightly ahead of the image.
+ if ! git_cmd status &>/dev/null; then
+ targetbranch="MOODLE_${branchver}_STABLE"
+ git_cmd init -b main
+ git_cmd remote add origin https://github.com/moodle/moodle.git
+ git_cmd fetch -np origin "$targetbranch"
+ git_cmd checkout -f "$targetbranch"
+ fi
) > /dev/null &
git_pid=$!
yorn 'Do you want to optimize the git repository? It will save space but take more time.' 'n' && do_gc=true || do_gc=false
@@ -375,13 +389,13 @@ if [[ -n $mname ]]; then
"$scr_dir/mdl-cli.sh" "$mname" upgrade --non-interactive
# After upgrades, we need to fix permissions.
# Ref: https://docs.moodle.org/4x/sv/Security_recommendations#Running_Moodle_on_a_dedicated_server
- container_tool exec -it "${moodle_svc}" bash -c '
- chown -R daemon:daemon /bitnami/moodle /bitnami/moodledata
- find /bitnami/moodle -type d -print0 | xargs -0 chmod 755
- find /bitnami/moodle -type f -print0 | xargs -0 chmod 644
- find /bitnami/moodledata -type d -print0 | xargs -0 chmod 700
- find /bitnami/moodledata -type f -print0 | xargs -0 chmod 600
- '
+ container_tool exec -it "${moodle_svc}" bash -c "
+ chown -R daemon:daemon '$src_path' '$data_path'
+ find '$src_path' -type d -print0 | xargs -0 chmod 755
+ find '$src_path' -type f -print0 | xargs -0 chmod 644
+ find '$data_path' -type d -print0 | xargs -0 chmod 700
+ find '$data_path' -type f -print0 | xargs -0 chmod 600
+ "
fi
echo 🎉 Done!
else
diff --git a/libexec/mdl-install-plugin.sh b/libexec/mdl-install-plugin.sh
index c8a0b4e..7520e4c 100755
--- a/libexec/mdl-install-plugin.sh
+++ b/libexec/mdl-install-plugin.sh
@@ -122,6 +122,8 @@ for mname in $mnames; do
# Process each zip file
for zip_file in "${zip_files[@]}"; do
zip_filename=$(basename "$zip_file")
+ # Clean up temp dirs from previous iteration, if applicable
+ rm -Rf "$temp_unzipped" "$temp_downloaded"
# If the zip file is a URL, download it first
if [[ $zip_file =~ ^https?:// ]]; then
mkdir -p "$temp_downloaded"
@@ -133,18 +135,19 @@ for mname in $mnames; do
# Unzip the plugin, and copy its contents to its final destination
mkdir -p "$temp_unzipped"
unzip -o -qq -d "$temp_unzipped" "$zip_file" || { echo "Failed to unzip $zip_file" >&2; exit 1; }
- # We use the directory inside the unzipped archive to determine plugin type and name. There should
- # only be one directory, but we put safeguards in place to make sure.
- # Example: block_my_great_plugin -> type=block, name=my_great_plugin
- pkg_name=$(find "$temp_unzipped" -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 basename | head -n1)
- type=${pkg_name%%_*}
- name=${pkg_name#*_}
+ # We inspect version.php to get the component name, and from there we can determine type and name.
+ # Example: `$plugin->component = 'local_my_plugin';` -> `local_my_plugin` -> type=local, name=my_plugin
+ plugin_dir=$(find "$temp_unzipped" -maxdepth 1 -mindepth 1 -type d -print0 | xargs -0 basename | head -n1)
+ version_file="$temp_unzipped/$plugin_dir/version.php"
+ [ ! -f "$version_file" ] && echo "${red}Could not find version.php in plugin $zip_filename." >&2 && continue
+ component_full_name=$(grep -E "\\\$plugin->component[[:space:]]*=" "$version_file" | sed -E "s/.*\\\$plugin->component[[:space:]]*=[[:space:]]*'([^']+)'.*/\1/")
+ [ -z "$component_full_name" ] && echo "${red}Could not determine component name from version file for $zip_filename." >&2 && continue
+ type=${component_full_name%%_*}
+ name=${component_full_name#*_}
dest="$(plugin_type_path "$type")/$name"
echo "Installing $type plugin $name to $ul$dest$norm."
mkdir -p "$temp_moodle/$dest"
- ( shopt -s dotglob && cp -R "$temp_unzipped/$pkg_name"/* "$temp_moodle/$dest" )
- # Clean up
- rm -Rf "$temp_unzipped" "$temp_downloaded"
+ ( shopt -s dotglob && cp -R "$temp_unzipped/$plugin_dir"/* "$temp_moodle/$dest" )
done
# Copy all final work into the container
while IFS= read -r -d '' dir; do
diff --git a/libexec/mdl-list.sh b/libexec/mdl-list.sh
index 203ca7c..3cabe50 100755
--- a/libexec/mdl-list.sh
+++ b/libexec/mdl-list.sh
@@ -71,9 +71,9 @@ for mname in $mnames; do
fi
# Collect the list of backups
- $type_backup && labels="$(find "$MDL_BACKUP_DIR" -name "${mname}_*_src.*" | cut -d"_" -f2- | sed -e "s/_src\..*//" | uniq | sort)"
- $type_box && box_labels="$("$scr_dir/mdl-box.sh" "$mname" ls | awk -F'_' '$3 ~ /src/' | cut -d"_" -f2- | sed -e "s/_src\..*//" | uniq | sort)"
- $type_fastdb && fast_labels=$(find "$MDL_BACKUP_DIR" -name "${mname}_*_dbfiles.tar" | cut -d"_" -f2- | sed -e "s/_dbfiles.tar//" | sort)
+ $type_backup && labels="$(find "$MDL_BACKUP_DIR" -name "${mname}_*_src.*" -print0 | xargs -0 -r -n1 basename | extract_label "$mname" src | sort | uniq)"
+ $type_box && box_labels="$("$scr_dir/mdl-box.sh" "$mname" ls | extract_label "$mname" src | sort | uniq)"
+ $type_fastdb && fast_labels=$(find "$MDL_BACKUP_DIR" -name "${mname}_*_dbfiles.*" -print0 | xargs -0 -r -n1 basename | extract_label "$mname" dbfiles | sort | uniq)
# Output: Normal Backups
if $type_backup && ! $quiet; then
diff --git a/libexec/mdl-logs.sh b/libexec/mdl-logs.sh
index 3aa4c99..70adfb4 100755
--- a/libexec/mdl-logs.sh
+++ b/libexec/mdl-logs.sh
@@ -24,6 +24,7 @@ containers="$(container_tool ps -q -f name="$mname" 2> /dev/null)"
[ -z "$containers" ] && echo "The $mname stack is not running." && exit 1
compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname")
+[[ -z $compose_path ]] && exit 1
. "$scr_dir/mdl-calc-images.sh" "$mname"
export_env_and_update_config "$mname"
compose_tool -p "$mname" -f "$compose_path" logs "${@:2}"
diff --git a/libexec/mdl-restore.sh b/libexec/mdl-restore.sh
index f74f9b9..85ce16a 100755
--- a/libexec/mdl-restore.sh
+++ b/libexec/mdl-restore.sh
@@ -14,6 +14,7 @@ Options:
-h, --help Show this help message and exit.
-b, --box Use backup sets in Box instead of the local backup folder.
-x, --extract If compressed, leave the decompressed files when done extracting them.
+-f, --fastdb Use an unsafe fast database backup instead of a SQL dump.
-r, --rm Remove the local copy of the backup when done.
EOF
}
@@ -22,6 +23,8 @@ EOF
[[ $* =~ -b || $* =~ --box ]] && box=true || box=false
[[ $* =~ -x || $* =~ --extract ]] && extract=true || extract=false
[[ $* =~ -r || $* =~ --rm ]] && remove_when_done=true || remove_when_done=false
+modules='src data db' && [[ $* =~ -f || $* =~ --fastdb ]] && modules='src data dbfiles'
+[[ $modules =~ dbfiles ]] && echo "${yellow}Warning: Fast database restores are unsafe for production use.$norm" >&2
# Check necessary utilities
requires "${MDL_CONTAINER_TOOL[0]}" tar bzip2 gzip xz find sed grep uniq
@@ -40,8 +43,7 @@ for mname in $mnames; do
$box && files=$("$scr_dir/mdl-box.sh" "$mname" ls) || files=$local_files
local_files=$(echo "$local_files" | xargs -r -n1 basename)
files=$(echo "$files" | xargs -r -n1 basename)
- src_files=$(echo "$files" | awk -F'_' '$3 ~ /src/')
- labels="$(echo "$src_files" | cut -d"_" -f2- | sed -e "s/_src\..*//" | uniq | sort)"
+ labels="$(echo "$files" | extract_label "$mname" src | sort | uniq)"
# What timestamp of backup do they want? (Select from the list if they did not provide)
$box && backup_source_desc='Box.com' || backup_source_desc='local'
@@ -56,23 +58,23 @@ for mname in $mnames; do
fi
# Backup targets
- declare data_target src_target db_target # Explicitly declared to make shellcheck happy
- for t in data src db; do
+ declare data_target src_target db_target dbfiles_target # Explicitly declared to make shellcheck happy
+ for t in $modules; do
target="${t}_target"; local_target="local_${t}_target"
- declare $target= $local_target=
+ declare "$target"= "$local_target"=
# Find filenames of target files
while IFS= read -r file; do
- [[ -z ${!target} && $file =~ ^${mname}_${label}_${t}\. ]] && declare $target="$file"
+ [[ -z ${!target} && $file =~ ^${mname}_${label}_${t}\. ]] && declare "$target"="$file"
done <<< "$files"
# Find filenames of local files, in case we're looking in Box
while IFS= read -r file; do
- [[ $file =~ ^${mname}_${label}_${t}\. ]] && declare $local_target="$file"
+ [[ $file =~ ^${mname}_${label}_${t}\. ]] && declare "$local_target"="$file"
done <<< "$local_files"
done
# List (and if requested, download from Box) each target. Abort if a target can't be found.
echo "Using $backup_source_desc backup set with label $ul$label$rmul:"
- for t in data src db; do
+ for t in $modules; do
target="${t}_target"; local_target="local_${t}_target"
echo " - $bold$t:$norm ${!target:-${red}Not found, so we will abort$norm}"
# If target is not found, abort. Otherwise, download if Box.com is the source.
@@ -87,7 +89,7 @@ for mname in $mnames; do
fi
# If `extract` is requested, decompress the files
if $extract && new_target=$(decompress "$MDL_BACKUP_DIR/${!target}"); then
- declare $target="$(basename "$new_target")"
+ declare "$target"="$(basename "$new_target")"
echo " - Extracted $ul${!target}$rmul."
fi
done
@@ -102,9 +104,14 @@ for mname in $mnames; do
# Restore src to a temp volume first, to retrieve the git branch version
temp_vol_name="${mname}_temp"
+ worker_name="${mname}_worker_src_restore"
+ compression_tool=$(calc_compression_tool "$src_target")
+ pipe_cmd=(cat) && [[ -n $compression_tool ]] && pipe_cmd=("$compression_tool" -d -c)
container_tool volume rm -f "$temp_vol_name" > /dev/null
- container_tool run --rm --name "${mname}_worker_tar_src" -v "$temp_vol_name":/src -v "$MDL_BACKUP_DIR":/backup:Z,ro "$MDL_SHELL_IMAGE" \
- tar xf "/backup/$src_target" -C /src
+ # We create this container and never start it, because we need an owner of the volume for `docker cp`
+ container_tool create --name "$worker_name" -v "$temp_vol_name":/src busybox > /dev/null
+ container_tool cp - "$worker_name":/src < <("${pipe_cmd[@]}" "$MDL_BACKUP_DIR/$src_target") > /dev/null
+ container_tool rm -f "$worker_name" > /dev/null
branchver=$(src_vol_name="$temp_vol_name" "$scr_dir/mdl-moodle-version.sh" "$mname")
. "$scr_dir/mdl-calc-images.sh" "$mname"
@@ -113,23 +120,34 @@ for mname in $mnames; do
# Create the stack, so we have the volumes that are auto-attached to the stack
branchver="$branchver" "$scr_dir/mdl-start.sh" "$mname" -q -n
- # Find all the volume names
+ # Find all the volume and container names, and their volume mount points.
vols=$(container_tool volume ls -q --filter "label=com.docker.compose.project=$mname")
db_vol_name=$(grep db <<< "$vols")
data_vol_name=$(grep data <<< "$vols")
src_vol_name=$(grep src <<< "$vols")
+ containers=$(container_tool ps -a -f "label=com.docker.compose.project=$mname" --format '{{.Names}}')
+ db_container=$(grep mariadb <<< "$containers" | head -1)
+ db_path=$(container_tool inspect "$db_container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("db"))) | .Destination')
+ moodle_container=$(grep moodle <<< "$containers" | head -1)
+ data_path=$(container_tool inspect "$moodle_container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("data"))) | .Destination')
+ src_path=$(container_tool inspect "$moodle_container" | jq -r '.[] .Mounts[] | select(.Name != null and (.Name | contains("src"))) | .Destination')
# Extract src and data to their volumes, set permissions appropriately.
# Ref: https://docs.moodle.org/4x/sv/Security_recommendations#Running_Moodle_on_a_dedicated_server
- echo "Restoring $ul$src_vol_name$norm and $ul$data_vol_name$norm volumes..."
- container_tool run --rm --name "${mname}_worker_tar_data" -v "$data_vol_name":/data -v "$MDL_BACKUP_DIR":/backup:Z,ro "$MDL_SHELL_IMAGE" \
- sh -c "\
- mkdir -p /data/sessions /data/trashdir /data/temp /data/localcache /data/cache
- tar xf '/backup/$data_target' -C /data
- chown -R daemon:daemon /data
- find /data -type d -print0 | xargs -0 chmod 700
- find /data -type f -print0 | xargs -0 chmod 600
- " &
+ echo "Restoring $ul$src_vol_name:$src_path$norm and $ul$data_vol_name:$data_path$norm volumes..."
+ (
+ compression_tool=$(calc_compression_tool "$data_target")
+ pipe_cmd=(cat) && [[ -n $compression_tool ]] && pipe_cmd=("$compression_tool" -d -c)
+ container_tool cp - "$moodle_container":"$data_path" < <("${pipe_cmd[@]}" "$MDL_BACKUP_DIR/$data_target") > /dev/null
+ container_tool run --rm --name "${mname}_worker_fix_data_perms" -v "$data_vol_name":/data "$MDL_SHELL_IMAGE" \
+ sh -c "\
+ (cd /data && rm -rf sessions trashdir temp localcache cache moodle-cron.log)
+ (cd /data && mkdir -p sessions trashdir temp localcache cache)
+ chown -R daemon:daemon /data
+ find /data -type d -print0 | xargs -0 chmod 700
+ find /data -type f -print0 | xargs -0 chmod 600
+ "
+ ) &
pid_data=$!
container_tool run --rm --name "${mname}_worker_cp_src" -v "$src_vol_name":/src -v "$temp_vol_name":/temp:ro "$MDL_SHELL_IMAGE" \
sh -c "\
@@ -140,52 +158,65 @@ for mname in $mnames; do
" &
pid_src=$!
- # Start a MariaDB container to restore the database
- (
- echo "Restoring $ul$db_vol_name$norm volume..."
- sql_path="$(mktemp -d)/${mname}_backup.sql"
- if ! decompress "$MDL_BACKUP_DIR/$db_target" "$sql_path" -k > /dev/null; then
- # If decompression fails, it probably isn't compressed. Point at original file instead.
- sql_path="$MDL_BACKUP_DIR/$db_target"
- fi
- db_runner="${mname}_worker_db_restore"
- container_tool run -d --rm --name "$db_runner" \
- --privileged \
- -e MARIADB_ROOT_PASSWORD="${ROOT_PASSWORD:-password}" \
- -e MARIADB_USER="${DB_USERNAME:-moodleuser}" \
- -e MARIADB_PASSWORD="${DB_PASSWORD:-password}" \
- -e MARIADB_DATABASE="${DB_NAME:-moodle}" \
- -e MARIADB_COLLATE=utf8mb4_unicode_ci \
- -e MARIADB_SKIP_TEST_DB=yes \
- -v "$db_vol_name":/bitnami/mariadb \
- -v "$sql_path":/docker-entrypoint-initdb.d/restore.sql:Z,ro \
- "$MARIADB_IMAGE" > /dev/null
- # MariaDB doesn't have a "run task and exit" mode, so we just wait until
- # the logs indicate it has finished, then we stop it.
- last_check=0
- until container_tool logs --since "$last_check" "$db_runner" 2>&1 | grep -q 'MariaDB setup finished'; do
- last_check=$(($(date +%s)-1))
- sleep 5
- done
- container_tool stop "$db_runner" > /dev/null
- ) &
- db_pid=$!
+ # Either fast database backup or a full SQL dump restore
+ if [[ -n $dbfiles_target ]]; then
+ # Fast database restore
+ echo "Restoring $ul$db_vol_name:$db_path$norm volume via fast database restore..."
+ (
+ compression_tool=$(calc_compression_tool "$dbfiles_target")
+ pipe_cmd=(cat) && [[ -n $compression_tool ]] && pipe_cmd=("$compression_tool" -d -c)
+ container_tool cp - "$db_container":"$db_path" < <("${pipe_cmd[@]}" "$MDL_BACKUP_DIR/$dbfiles_target") > /dev/null
+ ) &
+ pid_db=$!
+ elif [[ -n $db_target ]]; then
+ # Start a MariaDB container to restore the database
+ (
+ echo "Restoring $ul$db_vol_name$norm volume via database dump..."
+ sql_path="$(mktemp -d)/${mname}_backup.sql"
+ if ! decompress "$MDL_BACKUP_DIR/$db_target" "$sql_path" -k > /dev/null; then
+ # If decompression fails, it probably isn't compressed. Point at original file instead.
+ sql_path="$MDL_BACKUP_DIR/$db_target"
+ fi
+ db_runner="${mname}_worker_db_restore"
+ container_tool run -d --rm --name "$db_runner" \
+ -e MARIADB_ROOT_PASSWORD="${ROOT_PASSWORD:-password}" \
+ -e MARIADB_USER="${DB_USERNAME:-moodleuser}" \
+ -e MARIADB_PASSWORD="${DB_PASSWORD:-password}" \
+ -e MARIADB_DATABASE="${DB_NAME:-moodle}" \
+ -e MARIADB_COLLATE=utf8mb4_unicode_ci \
+ -e MARIADB_SKIP_TEST_DB=yes \
+ -v "$db_vol_name":"$db_path" \
+ -v "$sql_path":/docker-entrypoint-initdb.d/restore.sql:Z,ro \
+ "$MARIADB_IMAGE" > /dev/null
+ # MariaDB doesn't have a "run task and exit" mode, so we just wait until
+ # the logs indicate it has finished, then we stop it.
+ last_check=0
+ until container_tool logs --since "$last_check" "$db_runner" 2>&1 | grep -q 'MariaDB setup finished'; do
+ last_check=$(($(date +%s)-1))
+ sleep 5
+ done
+ container_tool stop "$db_runner" > /dev/null
+ ) &
+ pid_db=$!
+ fi
# When done, clean up. Down the stack and remove the temp volume.
wait $pid_src
container_tool volume rm -f "$temp_vol_name" > /dev/null
- wait $pid_data $db_pid
+ wait $pid_data
+ [[ -n $pid_db ]] && wait "$pid_db"
+ # Update config before destroying the stack so we know the data mount point
+ export_env_and_update_config "$mname"
branchver="$branchver" "$scr_dir/mdl-stop.sh" "$mname" -q
# Remove the local backup files when done, if they specified that option
if $remove_when_done; then
echo Removing local backup files...
- rm -fv "$MDL_BACKUP_DIR/$data_target" "$MDL_BACKUP_DIR/$src_target" "$MDL_BACKUP_DIR/$db_target"
+ rm -fv "$MDL_BACKUP_DIR/$data_target" "$MDL_BACKUP_DIR/$src_target"
+ [[ -n $db_target ]] && rm -fv "$MDL_BACKUP_DIR/$db_target"
+ [[ -n $dbfiles_target ]] && rm -fv "$MDL_BACKUP_DIR/$dbfiles_target"
fi
- # Update Moodle config
- export_env_and_update_config "$mname"
-
echo "Done restoring $ul$mname$rmul from $backup_source_desc backup set with label $ul$label$norm."
# Unset environment variables
diff --git a/libexec/mdl-start.sh b/libexec/mdl-start.sh
index 4a93750..01ef9d6 100755
--- a/libexec/mdl-start.sh
+++ b/libexec/mdl-start.sh
@@ -43,6 +43,7 @@ done
for mname in $mnames; do
compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname")
+ [[ -z $compose_path ]] && continue
$quiet || echo "Starting $mname..."
. "$scr_dir/mdl-calc-images.sh" "$mname"
export_env_and_update_config "$mname"
diff --git a/libexec/mdl-status.sh b/libexec/mdl-status.sh
index 9c5e0f8..cf1bfa0 100755
--- a/libexec/mdl-status.sh
+++ b/libexec/mdl-status.sh
@@ -35,6 +35,7 @@ for mname in $mnames; do
data_vol_name=$(grep data <<< "$vols")
src_vol_name=$(grep src <<< "$vols")
compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname")
+ [[ -z $compose_path ]] && continue
$quiet || echo "${ul}Environment: $bold$mname$norm"
# Status
diff --git a/libexec/mdl-stop.sh b/libexec/mdl-stop.sh
index 53e6613..f28794f 100755
--- a/libexec/mdl-stop.sh
+++ b/libexec/mdl-stop.sh
@@ -32,6 +32,7 @@ for mname in $mnames; do
fi
compose_path=$("$scr_dir/mdl-calc-compose-path.sh" "$mname")
+ [[ -z $compose_path ]] && continue
$quiet || echo "Stopping $mname..."
. "$scr_dir/mdl-calc-images.sh" "$mname"