diff --git a/emhttp/languages/en_US/helptext.txt b/emhttp/languages/en_US/helptext.txt index e0267fa77f..02f6d2ebe4 100644 --- a/emhttp/languages/en_US/helptext.txt +++ b/emhttp/languages/en_US/helptext.txt @@ -2011,6 +2011,10 @@ For the other schedules choose here the time of the day the mover should start. Write mover messages to the syslog file. :end +:mover_progress_help: +Enable progress display and allow a user override. +:end + :notifications_display_help: In *Detailed* view all notifications will be displayed on screen as soon as they arrive.
Notifications can be acknowledged individually or all at once. diff --git a/emhttp/plugins/dynamix/ArrayOperation.page b/emhttp/plugins/dynamix/ArrayOperation.page index 5386755d87..2b031f2de6 100644 --- a/emhttp/plugins/dynamix/ArrayOperation.page +++ b/emhttp/plugins/dynamix/ArrayOperation.page @@ -251,6 +251,7 @@ toggle_diskio(true); var mymonitor = new NchanSubscriber('/sub/mymonitor',{subscriber:'websocket'}); +var maxmoverlines = 5 ; mymonitor.on('message', function(state) { switch (state) { case '0': // normal operation @@ -263,6 +264,9 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',false); $('#mover-text').html("_(Move)_ _(will immediately invoke the Mover)_.  onclick=\"$.cookie('one','tab2')\">(_(Schedule)_)"); + for (let i = 0; i < maxmoverlines; i++) { + $('#moverrow'+i ).hide(); + } break; case '1': // parity running @@ -275,6 +279,9 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ -- _(Parity operation is running)_"); + for (let i = 0; i < maxmoverlines; i++) { + $('#moverrow'+i ).hide(); + } break; case '2': // mover running @@ -283,6 +290,8 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ - _(Mover is running)_."); + + break; case '3': // btrfs running @@ -291,6 +300,9 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ -- _(BTRFS operation is running)_"); + for (let i = 0; i < maxmoverlines; i++) { + $('#moverrow'+i ).hide(); + } break; } @@ -303,6 +315,33 @@ arraymonitor.on('message', function(state) { if (state==1 && !timers.arraymonitor) timers.arraymonitor = setTimeout(refresh,1250); }); +var moverStatus = new NchanSubscriber('/sub/mover',{subscriber:'websocket'}); +moverStatus.on('message', function(moverdata) { + var moverlines = 0; + var selectlines = [] ; + $.each(moverdata.split(';'),function(k,v) { + var keydata = v.split('#'); + if(keydata[0]=="Showlines") {moverlines=keydata[1];} + if(keydata[0]=="Selectlines") {selectlines=keydata[1].split(',');} + $('#'+keydata[0]).html(keydata[1]); + }); + + if(moverlines > maxmoverlines) moverlines = maxmoverlines; + if(moverlines == 0) { + for (let i = 0; i < 5; i++) { + $('#moverrow'+i ).hide(); + }; + }; + for (let i = 0; i < moverlines; i++) { + var selectline = i + 1; + var selectlinestr = selectline.toString(); + var checkselectline = selectlines.includes(selectlinestr); + if (checkselectline == true) $('#moverrow'+i ).show(); else $('#moverrow'+i ).hide(); + }; +}); + +moverStatus.start(); + var devices = new NchanSubscriber('/sub/devices',{subscriber:'websocket'}); devices.on('message', function(msg,meta) { switch (meta.id.channel()0) { @@ -670,6 +709,11 @@ window.onunload = function(){
+ + + + +
diff --git a/emhttp/plugins/dynamix/MoverSettings.page b/emhttp/plugins/dynamix/MoverSettings.page index d7587dc98e..10b452ab71 100644 --- a/emhttp/plugins/dynamix/MoverSettings.page +++ b/emhttp/plugins/dynamix/MoverSettings.page @@ -117,6 +117,15 @@ _(Mover logging)_: :mover_logging_help: +_(Mover Progress)_: +: + +:mover_progress_help: + diff --git a/emhttp/plugins/dynamix/nchan/parity_list b/emhttp/plugins/dynamix/nchan/parity_list old mode 100755 new mode 100644 index e66c5893f6..1f8fe20165 --- a/emhttp/plugins/dynamix/nchan/parity_list +++ b/emhttp/plugins/dynamix/nchan/parity_list @@ -21,6 +21,7 @@ $md5_old = -1; $spot_old = -1; $fs_old = -1; $proc_old = -1; +$mover_old = -1; require_once "$docroot/webGui/include/Helpers.php"; require_once "$docroot/webGui/include/publish.php"; @@ -117,6 +118,51 @@ while (true) { elseif (exec("zpool status|grep -c 'scrub in progress'")>0) $process = 4; else $process = 0; + + if ($process == 2) { + $moverdata = []; + if ( file_exists("$varroot/mover.ini")) { + $moverstat = parse_ini_file("$varroot/mover.ini"); + $mover = $moverstat; + $perc = $moverstat["Filepercent"]; + $value1 = "
$perc%
"; + if ($moverstat["TotalToArray"]>0) { + $toperc = ($moverstat["TotalToArray"]-$moverstat["RemainToArray"])/$moverstat["TotalToArray"] * 100; + if ($moverstat["RemainToArray"]>0) { + $perc= round($toperc,2) ; + $value2 = "
"._("Percentage Complete ").$perc."%
"; + } else {$value2 = _("Transfer complete") ;} + } else { + $value2 = _("Nothing to transfer") ;} + if ($moverstat["TotalFromArray"]>0) { + $fromperc = ($moverstat["TotalFromArray"] - $moverstat["RemainFromArray"])/$moverstat["TotalFromArray"] * 100; + if ($moverstat["RemainFromArray"]>0) { + $perc= round($fromperc,2) ; + $value3 = "
"._("Percentage Complete ").$perc."%
"; + } else {$value3 = _("Transfer complete") ;} + } else { + $value3 = _("Nothing to transfer") ;} + $moverdata = []; + $moverdata[] = "Showlines#{$moverstat['Showlines']}"; + $moverdata[] = "Selectlines#{$moverstat['Selectlines']}"; + $moverdata[] = "movertitle0#"._($moverstat['movertitle1']); + $moverdata[] = "movertitle1#"._($moverstat['movertitle2']); + $moverdata[] = "movertitle2#"._($moverstat['movertitle3']); + $moverdata[] = "movertitle3#"._($moverstat['movertitle4']); + $moverdata[] = "movertitle4#"._($moverstat['movertitle5']); + $moverdata[] = "moverline0#{$moverstat['File']}"; + $moverdata[] = "moverline1#{$moverstat['Action']}"; + $moverdata[] = "moverline2#$value1"; + $moverdata[] = "moverline3#$value2"; + $moverdata[] = "moverline4#$value3"; + } else { + $moverdata[] = "Showlines#0"; $mover = -1 ; + } + } else { + $moverdata = []; + $moverdata[] = "Showlines#0"; $mover = -1 ; + } + $echo = implode(';',$data); $md5_new = md5($echo,true); if ($md5_new !== $md5_old) { @@ -135,6 +181,11 @@ while (true) { publish('mymonitor', $process); $proc_old = $process; } + if ($mover !== $mover_old){ + publish('mover', implode(';',$moverdata)); + $mover_old = $mover; + } + sleep(1); } ?> diff --git a/sbin/moverprogress b/sbin/moverprogress new file mode 100755 index 0000000000..a3e284bc33 --- /dev/null +++ b/sbin/moverprogress @@ -0,0 +1,303 @@ +#!/usr/bin/php + $val) { + if(is_array($val)) { + $res[] = PHP_EOL."[$key]"; + foreach($val as $skey => $sval) $res[] = "$skey = ".(is_numeric($sval) ? $sval : '"'.$sval.'"'); + } else { + $res[] = "$key = ".(is_numeric($val) ? $val : '"'.$val.'"'); + } + } + + file_put_contents($file, implode(PHP_EOL, $res)); +} + + +function save_moverstate($var, $val) { + $config_file = "/usr/local/emhttp/state/mover.ini"; + $config = @parse_ini_file($config_file, true); + $config[$var] = $val; + save_ini_file($config_file, $config); + return (isset($config[$var][$val])) ? $config[$var][$val] : FALSE; +} + +function save_moverstatefull($state) { + $config_file = "/usr/local/emhttp/state/mover.ini"; + $config = @parse_ini_file($config_file, true); + $config = $state; + save_ini_file($config_file, $config); + return (isset($config)) ? $config : FALSE; +} + +function startMover($options="") { + global $vars, $cfg, $cron, $runcmd ; + + + + $pid = @file_get_contents("/var/run/mover.pid"); + if ($pid) { + echo "mover: mover already running\n" ; + exit(); + } + + + if (isset($cfg)) { + # Only start if shfs includes pools + if ($cfg["shareCacheEnabled"] != "yes") { + echo "mover: cache not enabled\n" ; + exit(2) ; + } + } + + $LOGLEVEL = 0 ; + + if ($cfg["shareMoverLogging"] == "yes") $LOGLEVEL=1 ; + exec("mountpoint -q /mnt/user0",$output, $rtnvar) ; + if ($rtnvar > 0) { + echo "mover: array devices not mounted" ; + exit(3) ; + } + + if ($LOGLEVEL >0) echo "mover: started\n" ; + file_put_contents("/var/run/mover.pid", getmypid()) ; + + $path= '/boot/config/pools'; + $directory = '/boot/config/pools/'; + $scanned_directory = array_diff(scandir($directory), array('..', '.')); + $pools = glob($path."/*.cfg"); + $DUARGSI = $DUARGSO = "" ; + writestatus() ; + $config_file = "/usr/local/emhttp/state/mover.ini"; + $config = @parse_ini_file($config_file, true); + + foreach ($pools as $pool) { + $sfiles = glob("/mnt/".basename($pool , ".cfg")."/*", GLOB_ONLYDIR ); + + foreach ($sfiles as $sfile) { + + if (is_file("/boot/config/shares/".basename($sfile).".cfg")) { + $ini = parse_ini_file("/boot/config/shares/".basename($sfile).".cfg", true) ; + + if ($ini["shareUseCache"] == "yes" ) { + $DUARGSI .= " ".$sfile ; + } + } + } + + } + + $sfiles=array() ; + exec('ls -dv /mnt/disk[0-9]*/*/',$sfiles) ; + + $total0 = 0 ; + foreach ($sfiles as $sfile) { + + if (is_file("/boot/config/shares/".basename($sfile).".cfg")) { + $ini = parse_ini_file("/boot/config/shares/".basename($sfile).".cfg", true) ; + + if ($ini["shareUseCache"] == "prefer" ) { + if (!isset($ini["shareCachePool"])) $ini["shareCachePool"]="cache" ; + if (is_dir($sfile)) { + + $DUARGSO .= " ".$sfile ; + } + } + } + } + + $total=0; + if ($DUARGSI != "") $total = shell_exec("du -sc ".$DUARGSI." | grep total | awk '{print $1}'" ) ; else $total = 0; + if ($DUARGSO != "") $totalo = shell_exec("du -sc ".$DUARGSO." | grep total | awk '{print $1}'" ) ; else $totalo = 0; + + + printf("mover: shares to array %s To array bytes %d\n",$DUARGSI,$total) ; + printf("mover: shares from array %s From array bytes %d\n",$DUARGSO,$totalo) ; + $config_file = "/usr/local/emhttp/state/mover.ini"; + $config = @parse_ini_file($config_file, true); + $pools = glob($path."/*.cfg"); + + $config["TotalToArray"] = $total ; + $config["RemainToArray"] = $total; + $config["TotalFromArray"] = $totalo; + $config["RemainFromArray"] = $totalo ; + save_moverstatefull($config) ; + + sleep(10); + + foreach ($pools as $pool) { + $sfiles = glob("/mnt/".basename($pool , ".cfg")."/*", GLOB_ONLYDIR ); + + foreach ($sfiles as $sfile) { + + if (is_file("/boot/config/shares/".basename($sfile).".cfg")) { + $ini = parse_ini_file("/boot/config/shares/".basename($sfile).".cfg", true) ; + + if ($ini["shareUseCache"] == "yes" ) { + $find=array() ; + exec("find ".$sfile." -depth ", $find) ; + + #$LOGLEVEL = 1 ; + + foreach ($find as $movefile) { + if (!is_dir($movefile)) { $tomove = shell_exec("du -sc \"".$movefile."\" | grep total | awk '{print $1}'" ) ; } else {$tomove = 0 ;} + + $config["File"] = $movefile ; + $config["Action"] = "Move share:".basename($sfile)." to array" ; + $config["Filepercent"] = 0 ; + save_moverstatefull($config) ; + + $movecmd="echo \"".$movefile."\" | /usr/local/bin/move -d ".$LOGLEVEL ; + if ($runcmd) shell_exec($movecmd) ; else printf("Command running %s\n File size %d\n " ,$movecmd, $tomove) ; + + $config["RemainToArray"] = $config["RemainToArray"] - $tomove ; + + save_moverstatefull($config) ; + + } + + if ($LOGLEVEL > 0) printf("mover: remaining to transfer to array %d %s\n",$config["RemainToArray"],$sfile) ; + } + } + } + + } + + + $sfiles = glob("/mnt/".basename($pool , ".cfg")."/*", GLOB_ONLYDIR ); + $sfiles=array() ; + exec('ls -dv /mnt/disk[0-9]*/*/',$sfiles) ; + + + foreach ($sfiles as $sfile) { + + if (is_file("/boot/config/shares/".basename($sfile).".cfg")) { + $ini = parse_ini_file("/boot/config/shares/".basename($sfile).".cfg", true) ; + + if ($ini["shareUseCache"] == "prefer" ) { + if (!isset($ini["shareCachePool"])) $ini["shareCachePool"]="cache" ; + if (is_dir($sfile)) { + + + $find=array() ; + exec("find ".$sfile." -depth ", $find) ; + + #$LOGLEVEL = 1 ; + + foreach ($find as $movefile) { + if (!is_dir($movefile)) { $frommove = shell_exec("du -sc \"".$movefile."\" | grep total | awk '{print $1}'" ) ; } else {$frommove = 0 ;} + + $config["File"] = $movefile ; + $config["Action"] = "Move share:".basename($sfile)." from array" ; + $config["Filepercent"] = 0 ; + save_moverstatefull($config) ; + + $movecmd="echo \"".$movefile."\" | /usr/local/bin/move -d ".$LOGLEVEL ; + if ($runcmd) shell_exec($movecmd) ; else printf("Command running %s\n File size %d\n " ,$movecmd, $frommove) ; + + $config["RemainFromArray"] = $config["RemainFromArray"] - $frommove ; + + save_moverstatefull($config) ; + + } + + if ($LOGLEVEL > 0) printf("mover: remaining to transfer from array %d %s\n",$config["RemainFromArray"],$sfile) ; + } + } + + } + } + if ($LOGLEVEL > 0) echo "mover: finished\n"; + + unlink("/var/run/mover.pid"); + +} + + + +/*function killtree() { + local pid=$1 child + $mypid=getmypid() + for child in $(pgrep -P $pid); do + killtree $child + done + [ $pid -ne $$ ] && kill -TERM $pid +} +*/ +# Caution: stopping mover like this can lead to partial files on the destination +# and possible incomplete hard link transfer. Not recommended to do this. +function stopMover() { + $pid = @file_get_contents("/var/run/mover.pid"); + + if (!$pid) { + echo "mover: not running\n" ; + exit(1); + } + + #killtree $(cat $PIDFILE) + sleep(2) ; + unlink("/var/run/mover.pid"); + echo "mover: stopped" ; + +} + +$COMMAND = ""; +if (isset($argv[1])) $COMMAND = $argv[1]; +switch ($COMMAND) { +case 'start': + startMover() ; + break; + +case 'stop': + stopMover() ; + break; + +case 'status': + if (is_file("/var/run/mover.pid")) { + $pid=file_get_contents("/var/run/mover.pid") ; + printf("%d\n",$pid) ; + } + break; + + + +default: + + startMover(); + break; +} +?> \ No newline at end of file diff --git a/sbin/moverprogressB b/sbin/moverprogressB new file mode 100755 index 0000000000..ff8b53e79f --- /dev/null +++ b/sbin/moverprogressB @@ -0,0 +1,218 @@ +#!/bin/bash +#Copyright 2005-2023, Lime Technology +#License: GPLv2 only + +# This is the 'mover' script used for moving files between a pool and the main unRAID array. +# It is typically invoked via cron. + +# First we check if it's valid for this script run: pool use in shfs must be enabled and +# an instance of the script must not already be running. + +# Next, check each of the top-level directories (shares) on each pool. +# If, and only if, the 'Use Cache' setting for the share is set to "yes", we use 'find' to +# list the objects (files and directories) of that share directory, moving them to the array. +# Next, we check each of the top-level directories (shares) on each array disk (in sorted order). +# If, and only if, the 'Use Cache' setting for the share is set to "prefer", we use 'find' to +# list the objects (files and directories) of that share directory, moving them to the pool +# associted with the share. + +# The script is set up so that hidden directories (i.e., directory names beginning with a '.' +# character) at the topmost level of a pool or an array disk are not moved. This behavior +# can be turned off by uncommenting the following line: +# shopt -s dotglob + +# Files at the top level of a pool or an array disk are never moved. + +# The 'find' command generates a list of all files and directories of a share. +# For each file, if the file is not "in use" by any process (as detected by 'in_use' command), +# then the file is moved, and upon success, deleted from the source disk. If the file already +# exists on the target, it is not moved and the source is not deleted. All meta-data of moved +# files/directories is preserved: permissions, ownership, extended attributes, and access/modified +# timestamps. + +# If an error occurs in copying a file, the partial file, if present, is deleted and the +# operation continues on to the next file. + +PIDFILE="/var/run/mover.pid" +CFGFILE="/boot/config/share.cfg" +LOGLEVEL=0 +MOVERSTATUS="/usr/local/emhttp/state/mover.ini" + +writestatus() { + echo "TotalToArray=${TOTALTO}" > $MOVERSTATUS + echo "RemainToArray=${TO}" >> $MOVERSTATUS + echo "TotalFromArray=${TOTALFROM}" >> $MOVERSTATUS + echo "RemainFromArray=${FROM}" >> $MOVERSTATUS + echo "File=" >> $MOVERSTATUS + echo "Action=Moving share ${SHARE}" >> $MOVERSTATUS + echo "Filepercent=" >> $MOVERSTATUS + echo "Showlines=5" >> $MOVERSTATUS + echo "Selectlines=2,3,4,5" >> $MOVERSTATUS + echo "movertitle1=File being processed:" >> $MOVERSTATUS + echo "movertitle2=Action:" >> $MOVERSTATUS + echo "movertitle3=Progress:" >> $MOVERSTATUS + echo "movertitle4=Transfering To Array:" >> $MOVERSTATUS + echo "movertitle5=Transfering From Array:" >> $MOVERSTATUS +} + +start() { + if [ -f $PIDFILE ]; then + if ps h $(cat $PIDFILE) | grep mover ; then + echo "mover: already running" + exit 1 + fi + fi + + if [ -f $CFGFILE ]; then + # Only start if shfs includes pools + if ! grep -qs 'shareCacheEnabled="yes"' $CFGFILE ; then + echo "mover: cache not enabled" + exit 2 + fi + if grep -qs 'shareMoverLogging="yes"' $CFGFILE ; then + LOGLEVEL=1 + fi + fi + if ! mountpoint -q /mnt/user0 ; then + echo "mover: array devices not mounted" + exit 3 + fi + + TOTALFROM=0 + TOTALTO=0 + writestatus + + echo $$ >/var/run/mover.pid + [[ $LOGLEVEL -gt 0 ]] && echo "mover: started" + + TOSHARES=0 + FROMSHARES=0 + DUARGSI="" + + shopt -s nullglob + + # Check for objects to move from pools to array + for POOL in /boot/config/pools/*.cfg ; do + for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do + SHARE=$(basename "$SHAREPATH") + if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then + DUARGSI="${DUARGSI} ${SHAREPATH}" + ((TOSHARES=TOSHARES+1)) + fi + done + done + + + if [[ "$TOSHARES" > 0 ]] + then + TO=$(du -sc ${DUARGSI} | grep total | awk '{print $1}' ) + else + TO=0 + fi + + DUARGSO="" + + # Check for objects to move from array to pools + for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do + SHARE=$(basename "$SHAREPATH") + if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then + eval "$(grep -s shareCachePool '/boot/config/shares/${SHARE}.cfg' | tr -d '\r')" + if [[ -z "$shareCachePool" ]]; then + shareCachePool="cache" + fi + if [[ -d "/mnt/$shareCachePool" ]]; then + DUARGSO="${DUARGSO} ${SHAREPATH}" + ((FROMSHARES=FROMSHARES+1)) + fi + fi + done + + if [[ "$FROMSHARES" > 0 ]] + then + FROM=$(du -sc ${DUARGSO} | grep total | awk '{print $1}') + else + FROM=0 + fi + TOTALFROM=$FROM + TOTALTO=$TO + writestatus + + # Check for objects to move from pools to array + for POOL in /boot/config/pools/*.cfg ; do + for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do + SHARE=$(basename "$SHAREPATH") + writestatus + if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then + TOMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') + [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${TOMOVE}" + find "${SHAREPATH%/}" -depth | /usr/local/bin/move -d $LOGLEVEL + ((TO=TO-TOMOVE)) + [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${TO}" + writestatus + fi + done + done + + # Check for objects to move from array to pools + for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do + SHARE=$(basename "$SHAREPATH") + writestatus + if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then + eval "$(grep -s shareCachePool '/boot/config/shares/${SHARE}.cfg' | tr -d '\r')" + if [[ -z "$shareCachePool" ]]; then + shareCachePool="cache" + fi + if [[ -d "/mnt/$shareCachePool" ]]; then + FROMMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') + [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${FROMMOVE}" + find "${SHAREPATH%/}" -depth | /usr/local/bin/move -d $LOGLEVEL + ((FROM=FROM-FROMMOVE)) + [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${FROM}" + writestatus + fi + fi + done + sleep 5 + rm -f $PIDFILE + rm -f $MOVERSTATUS + [[ $LOGLEVEL -gt 0 ]] && echo "mover: finished" +} + +killtree() { + local pid=$1 child + + for child in $(pgrep -P $pid); do + killtree $child + done + [ $pid -ne $$ ] && kill -TERM $pid +} + +# Caution: stopping mover like this can lead to partial files on the destination +# and possible incomplete hard link transfer. Not recommended to do this. +stop() { + if [ ! -f $PIDFILE ]; then + echo "mover: not running" + exit 0 + fi + killtree $(cat $PIDFILE) + sleep 2 + rm -f $PIDFILE + echo "mover: stopped" +} + +case $1 in +start) + start + ;; +stop) + stop + ;; +status) + [ -f $PIDFILE ] + ;; +*) + # Default is "start" + # echo "Usage: $0 (start|stop|status)" + start + ;; +esac