From 07ebe4f8c8b89af190b737746bd7b5fd4cc6b693 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Tue, 8 Mar 2022 20:21:35 +0000 Subject: [PATCH 01/17] Initial Commit for Mover Status rc3+ --- plugins/dynamix/ArrayOperation.page | 20 ++++++++++++++++++++ plugins/dynamix/nchan/parity_list | 28 ++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/plugins/dynamix/ArrayOperation.page b/plugins/dynamix/ArrayOperation.page index 9af44b10fd..198bc421c3 100644 --- a/plugins/dynamix/ArrayOperation.page +++ b/plugins/dynamix/ArrayOperation.page @@ -261,6 +261,8 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',false); $('#mover-text').html("_(Move)_ _(will immediately invoke the Mover)_.  onclick=\"$.cookie('one','tab2',{path:'/'})\">(_(Schedule)_)"); + $('#moverrow1' ).hide(); + $('#moverrow2' ).hide(); break; case '1': // parity running @@ -273,6 +275,8 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ -- _(Parity operation is running)_"); + $('#moverrow1' ).hide(); + $('#moverrow2' ).hide(); break; case '2': // mover running @@ -281,6 +285,10 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ - _(Mover is running)_."); + // $('#moverline1').html("Test Line 1") + // $('#moverline2').html("Test Line 2") + $('#moverrow1' ).show(); + $('#moverrow2' ).show(); break; case '3': // btrfs running @@ -289,6 +297,8 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ -- _(BTRFS operation is running)_"); + $('#moverrow1' ).hide(); + $('#moverrow2' ).hide(); break; } @@ -301,6 +311,13 @@ arraymonitor.on('message', function(state) { if (state==1 && !timers.arraymonitor) timers.arraymonitor = setTimeout(refresh,1250); }); +var moverStatus = new NchanSubscriber('/sub/mover',{subscriber:'websocket'}); +moverStatus.on('message', function(moverdata) { +$.each(moverdata.split(';'),function(k,v) {if ($('#moverline'+k).length>0) $('#moverline'+k).html(v);}); +}); + +moverStatus.start(); + var devices = new NchanSubscriber('/sub/devices',{subscriber:'websocket'}); devices.on('message', function(msg,meta) { switch (meta.id.channel()0) { @@ -648,6 +665,9 @@ function formatWarning(val) {
+ + +
diff --git a/plugins/dynamix/nchan/parity_list b/plugins/dynamix/nchan/parity_list index f98ec576be..ba1d21b343 100755 --- a/plugins/dynamix/nchan/parity_list +++ b/plugins/dynamix/nchan/parity_list @@ -116,10 +116,38 @@ while (true) { elseif (exec('pgrep -cf /sbin/btrfs')>0) $process = 3; else $process = 0; + + if ($process == 2) { + $moverdata = []; + $moverdata[] = "true" ; + if ( file_exists("$varroot/mover.ini")) { + $moverstat = parse_ini_file("$varroot/mover.ini") ; + if ($moverstat["TotalToArray"]>0) { + $toperc = ($moverstat["TotalToArray"]-$moverstat["RemainToArray"])/$moverstat["TotalToArray"] * 100; + if ($moverstat["RemainToArray"]>0) { $moverdata[] = _("Percentage Complete ").round($toperc,2)."%" ; } else {$moverdata[] = _("Transfer complete") ;} + } else { + $moverdata[] = _("Nothing to transfer") ;} + if ($moverstat["TotalFromArray"]>0) { + $fromperc = ($moverstat["TotalFromArray"] - $moverstat["RemainFromArray"])/$moverstat["TotalFromArray"] * 100; + if ($moverstat["RemainFromArray"]>0) { $moverdata[] = _("Percentage Complete ").round($fromperc,2)."%" ; } else {$moverdata[] = _("Transfer complete") ;} + } else { + $moverdata[] = _("Nothing to transfer") ;} + } else { + $moverdata[] = _('Error'); + $moverdata[] = _('Error'); + } + } else { + $moverdata = []; + $moverdata[] = "false" ; + $moverdata[] = _('Not Running'); + $moverdata[] = _('Not Running'); + } + publish('parity', implode(';',$data)); publish('paritymonitor', $spot>0 ? 1 : 0); publish('fsState', $fsState); publish('mymonitor', $process); + publish('mover', implode(';',$moverdata)); sleep(1); } ?> From 70a3707f4bf846bef3099910decbe1b0e22e0299 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Tue, 8 Mar 2022 20:30:08 +0000 Subject: [PATCH 02/17] Code clean up --- plugins/dynamix/ArrayOperation.page | 2 -- 1 file changed, 2 deletions(-) diff --git a/plugins/dynamix/ArrayOperation.page b/plugins/dynamix/ArrayOperation.page index 198bc421c3..2a258fd607 100644 --- a/plugins/dynamix/ArrayOperation.page +++ b/plugins/dynamix/ArrayOperation.page @@ -285,8 +285,6 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ - _(Mover is running)_."); - // $('#moverline1').html("Test Line 1") - // $('#moverline2').html("Test Line 2") $('#moverrow1' ).show(); $('#moverrow2' ).show(); From 0b933a9db57f1677c123bacac6c4c6c3cbd6e274 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Wed, 9 Mar 2022 20:49:41 +0000 Subject: [PATCH 03/17] Add additional lines and progress barss --- plugins/dynamix/ArrayOperation.page | 31 +++++++++++++++++++---------- plugins/dynamix/nchan/parity_list | 23 +++++++++++++++++---- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/plugins/dynamix/ArrayOperation.page b/plugins/dynamix/ArrayOperation.page index 2a258fd607..8fa2957e2e 100644 --- a/plugins/dynamix/ArrayOperation.page +++ b/plugins/dynamix/ArrayOperation.page @@ -249,6 +249,7 @@ toggle_diskio(true); var mymonitor = new NchanSubscriber('/sub/mymonitor',{subscriber:'websocket'}); +var maxmoverlines = 5 ; mymonitor.on('message', function(state) { switch (state) { case '0': // normal operation @@ -261,8 +262,9 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',false); $('#mover-text').html("_(Move)_ _(will immediately invoke the Mover)_.  onclick=\"$.cookie('one','tab2',{path:'/'})\">(_(Schedule)_)"); - $('#moverrow1' ).hide(); - $('#moverrow2' ).hide(); + for (let i = 0; i < maxmoverlines; i++) { + $('#moverrow'+i ).hide(); + } break; case '1': // parity running @@ -275,8 +277,9 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ -- _(Parity operation is running)_"); - $('#moverrow1' ).hide(); - $('#moverrow2' ).hide(); + for (let i = 0; i < maxmoverlines; i++) { + $('#moverrow'+i ).hide(); + } break; case '2': // mover running @@ -285,8 +288,10 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ - _(Mover is running)_."); - $('#moverrow1' ).show(); - $('#moverrow2' ).show(); + for (let i = 0; i < maxmoverlines; i++) { + $('#moverrow'+i ).show(); + } + break; case '3': // btrfs running @@ -295,8 +300,9 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ -- _(BTRFS operation is running)_"); - $('#moverrow1' ).hide(); - $('#moverrow2' ).hide(); + for (let i = 0; i < maxmoverlines; i++) { + $('#moverrow'+i ).hide(); + } break; } @@ -312,6 +318,7 @@ arraymonitor.on('message', function(state) { var moverStatus = new NchanSubscriber('/sub/mover',{subscriber:'websocket'}); moverStatus.on('message', function(moverdata) { $.each(moverdata.split(';'),function(k,v) {if ($('#moverline'+k).length>0) $('#moverline'+k).html(v);}); + }); moverStatus.start(); @@ -663,9 +670,11 @@ function formatWarning(val) {
- - - + + + + +
diff --git a/plugins/dynamix/nchan/parity_list b/plugins/dynamix/nchan/parity_list index ba1d21b343..2ec8fae91d 100755 --- a/plugins/dynamix/nchan/parity_list +++ b/plugins/dynamix/nchan/parity_list @@ -119,26 +119,41 @@ while (true) { if ($process == 2) { $moverdata = []; - $moverdata[] = "true" ; + if ( file_exists("$varroot/mover.ini")) { $moverstat = parse_ini_file("$varroot/mover.ini") ; + $moverdata[] = $moverstat["File"] ; + $moverdata[] = $moverstat["Action"] ; + $perc = $moverstat["Filepercent"] ; + $moverdata[] = "
$perc%
" ; if ($moverstat["TotalToArray"]>0) { $toperc = ($moverstat["TotalToArray"]-$moverstat["RemainToArray"])/$moverstat["TotalToArray"] * 100; - if ($moverstat["RemainToArray"]>0) { $moverdata[] = _("Percentage Complete ").round($toperc,2)."%" ; } else {$moverdata[] = _("Transfer complete") ;} + if ($moverstat["RemainToArray"]>0) { + $perc= round($toperc,2) ; + $moverdata[] = "
"._("Percentage Complete ").$perc."%
" ; + } else {$moverdata[] = _("Transfer complete") ;} } else { $moverdata[] = _("Nothing to transfer") ;} if ($moverstat["TotalFromArray"]>0) { $fromperc = ($moverstat["TotalFromArray"] - $moverstat["RemainFromArray"])/$moverstat["TotalFromArray"] * 100; - if ($moverstat["RemainFromArray"]>0) { $moverdata[] = _("Percentage Complete ").round($fromperc,2)."%" ; } else {$moverdata[] = _("Transfer complete") ;} + if ($moverstat["RemainFromArray"]>0) { + $perc= round($fromperc,2) ; + $moverdata[] = "
"._("Percentage Complete ").$perc."%
"; + } else {$moverdata[] = _("Transfer complete") ;} } else { $moverdata[] = _("Nothing to transfer") ;} } else { + $moverdata[] = _('Error'); + $moverdata[] = _('Error'); + $moverdata[] = _('Error'); $moverdata[] = _('Error'); $moverdata[] = _('Error'); } } else { $moverdata = []; - $moverdata[] = "false" ; + $moverdata[] = _('Not Running'); + $moverdata[] = _('Not Running'); + $moverdata[] = _('Not Running'); $moverdata[] = _('Not Running'); $moverdata[] = _('Not Running'); } From 942ce08b3bc0212e7def89fefe30a98e1b8a3196 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Sat, 2 Apr 2022 13:58:30 +0100 Subject: [PATCH 04/17] Update ArrayOperation.page --- plugins/dynamix/ArrayOperation.page | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/dynamix/ArrayOperation.page b/plugins/dynamix/ArrayOperation.page index 8fa2957e2e..a39b1b7297 100644 --- a/plugins/dynamix/ArrayOperation.page +++ b/plugins/dynamix/ArrayOperation.page @@ -261,7 +261,7 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',false); - $('#mover-text').html("_(Move)_ _(will immediately invoke the Mover)_.  onclick=\"$.cookie('one','tab2',{path:'/'})\">(_(Schedule)_)"); + $('#mover-text').html("_(Move)_ _(will immediately invoke the Mover)_.  onclick=\"$.cookie('one','tab2)\">(_(Schedule)_)"); for (let i = 0; i < maxmoverlines; i++) { $('#moverrow'+i ).hide(); } @@ -670,9 +670,9 @@ function formatWarning(val) {
- - - + + +
From 2ef7202e0a3520992753cead6d72b0aab4f0d1a3 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Sat, 2 Apr 2022 13:59:51 +0100 Subject: [PATCH 05/17] Revert "Update ArrayOperation.page" This reverts commit 942ce08b3bc0212e7def89fefe30a98e1b8a3196. --- plugins/dynamix/ArrayOperation.page | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/dynamix/ArrayOperation.page b/plugins/dynamix/ArrayOperation.page index a39b1b7297..8fa2957e2e 100644 --- a/plugins/dynamix/ArrayOperation.page +++ b/plugins/dynamix/ArrayOperation.page @@ -261,7 +261,7 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',false); - $('#mover-text').html("_(Move)_ _(will immediately invoke the Mover)_.  onclick=\"$.cookie('one','tab2)\">(_(Schedule)_)"); + $('#mover-text').html("_(Move)_ _(will immediately invoke the Mover)_.  onclick=\"$.cookie('one','tab2',{path:'/'})\">(_(Schedule)_)"); for (let i = 0; i < maxmoverlines; i++) { $('#moverrow'+i ).hide(); } @@ -670,9 +670,9 @@ function formatWarning(val) { - - - + + +
From 298a997a47b27e24884b2802ba2731d0ef440a26 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Sun, 26 Nov 2023 15:03:54 +0000 Subject: [PATCH 06/17] Add mover file. --- sbin/mover | 97 +++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 89 insertions(+), 8 deletions(-) diff --git a/sbin/mover b/sbin/mover index ab25bfd1bc..0e5cabd535 100755 --- a/sbin/mover +++ b/sbin/mover @@ -1,5 +1,5 @@ #!/bin/bash -#Copyright 2005-2020, Lime Technology +#Copyright 2005-2023, Lime Technology #License: GPLv2 only # This is the 'mover' script used for moving files between a pool and the main unRAID array. @@ -35,7 +35,18 @@ PIDFILE="/var/run/mover.pid" CFGFILE="/boot/config/share.cfg" -DEBUGGING="" +LOGLEVEL=0 +MOVERSTATUS="/usr/local/emhttp/state/mover.ini" + +writestatus() { + echo "TotalToArray=${TOTALTO}" > $MOVERSTATUS + echo "RemainToArray=${TO}" >> $MOVERSTATUS + echo "TotalFromArray=${TOTALFROM}" >> $MOVERSTATUS + echo "RemainFromArray=${FROM}" >> $MOVERSTATUS + echo "File=Test" >> $MOVERSTATUS + echo "Action=Moving share ${SHARE}" >> $MOVERSTATUS + echo "Filepercent=50" >> $MOVERSTATUS +} start() { if [ -f $PIDFILE ]; then @@ -51,14 +62,25 @@ start() { echo "mover: cache not enabled" exit 2 fi + if grep -qs 'shareMoverLogging="yes"' $CFGFILE ; then + LOGLEVEL=1 + fi fi if ! mountpoint -q /mnt/user0 ; then echo "mover: array devices not mounted" exit 3 fi + TOTALFROM=0 + TOTALTO=0 + writestatus + echo $$ >/var/run/mover.pid - echo "mover: started" + [[ $LOGLEVEL -gt 0 ]] && echo "mover: started" + + TOSHARES=0 + FROMSHARES=0 + DUARGSI="" shopt -s nullglob @@ -67,27 +89,86 @@ start() { for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do SHARE=$(basename "$SHAREPATH") if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then - find "${SHAREPATH%/}" -depth | /usr/local/bin/move $DEBUGGING + DUARGSI="${DUARGSI} ${SHAREPATH}" + ((TOSHARES=TOSHARES+1)) fi done done + + if [[ "$TOSHARES" > 0 ]] + then + TO=$(du -sc ${DUARGSI} | grep total | awk '{print $1}' ) + else + TO=0 + fi + + DUARGSO="" + # Check for objects to move from array to pools - ls -dvc1 /mnt/disk[0-9]*/*/ | while read SHAREPATH ; do + for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do SHARE=$(basename "$SHAREPATH") if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then - eval $(grep -s shareCachePool "/boot/config/shares/${SHARE}.cfg" | tr -d '\r') + eval "$(grep -s shareCachePool /boot/config/shares/${SHARE}.cfg | tr -d '\r')" if [[ -z "$shareCachePool" ]]; then shareCachePool="cache" fi if [[ -d "/mnt/$shareCachePool" ]]; then - find "${SHAREPATH%/}" -depth | /usr/local/bin/move $DEBUGGING + DUARGSO="${DUARGSO} ${SHAREPATH}" + ((FROMSHARES=FROMSHARES+1)) fi fi done + if [[ "$FROMSHARES" > 0 ]] + then + FROM=$(du -sc ${DUARGSO} | grep total | awk '{print $1}') + else + FROM=0 + fi + TOTALFROM=$FROM + TOTALTO=$TO + writestatus + + # Check for objects to move from pools to array + for POOL in /boot/config/pools/*.cfg ; do + for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do + SHARE=$(basename "$SHAREPATH") + writestatus + if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then + TOMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') + [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${TOMOVE}" + find "${SHAREPATH%/}" -depth | /usr/local/sbin/move -d $LOGLEVEL + ((TO=TO-TOMOVE)) + [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${TO}" + writestatus + fi + done + done + + # Check for objects to move from array to pools + for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do + SHARE=$(basename "$SHAREPATH") + writestatus + if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then + eval "$(grep -s shareCachePool /boot/config/shares/${SHARE}.cfg | tr -d '\r')" + if [[ -z "$shareCachePool" ]]; then + shareCachePool="cache" + fi + if [[ -d "/mnt/$shareCachePool" ]]; then + FROMMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') + [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${FROMMOVE}" + find "${SHAREPATH%/}" -depth | /usr/local/sbin/move -d $LOGLEVEL + ((FROM=FROM-FROMMOVE)) + [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${FROM}" + writestatus + fi + fi + done + sleep 5 rm -f $PIDFILE - echo "mover: finished" + rm -f $MOVERSTATUS + [[ $LOGLEVEL -gt 0 ]] && echo "mover: finished" } killtree() { From ce37cb70a243a999f09acc0ebd5554f38ae3fbd6 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Sun, 26 Nov 2023 15:31:05 +0000 Subject: [PATCH 07/17] Update mover --- sbin/mover | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sbin/mover b/sbin/mover index 0e5cabd535..f5477c31bc 100755 --- a/sbin/mover +++ b/sbin/mover @@ -109,7 +109,7 @@ start() { for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do SHARE=$(basename "$SHAREPATH") if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then - eval "$(grep -s shareCachePool /boot/config/shares/${SHARE}.cfg | tr -d '\r')" + eval "$(grep -s shareCachePool '/boot/config/shares/${SHARE}.cfg' | tr -d '\r')" if [[ -z "$shareCachePool" ]]; then shareCachePool="cache" fi @@ -151,7 +151,7 @@ start() { SHARE=$(basename "$SHAREPATH") writestatus if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then - eval "$(grep -s shareCachePool /boot/config/shares/${SHARE}.cfg | tr -d '\r')" + eval "$(grep -s shareCachePool '/boot/config/shares/${SHARE}.cfg' | tr -d '\r')" if [[ -z "$shareCachePool" ]]; then shareCachePool="cache" fi From d8dd7ed315f3b4efcdd0c7ec7f776c4ac1b68bf2 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Sun, 26 Nov 2023 17:29:33 +0000 Subject: [PATCH 08/17] Update mover --- sbin/mover | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sbin/mover b/sbin/mover index f5477c31bc..51559643d6 100755 --- a/sbin/mover +++ b/sbin/mover @@ -43,9 +43,9 @@ writestatus() { echo "RemainToArray=${TO}" >> $MOVERSTATUS echo "TotalFromArray=${TOTALFROM}" >> $MOVERSTATUS echo "RemainFromArray=${FROM}" >> $MOVERSTATUS - echo "File=Test" >> $MOVERSTATUS - echo "Action=Moving share ${SHARE}" >> $MOVERSTATUS - echo "Filepercent=50" >> $MOVERSTATUS + echo "File=Processing" >> $MOVERSTATUS + echo "Action=Processing" >> $MOVERSTATUS + echo "Filepercent=Processing" >> $MOVERSTATUS } start() { @@ -138,7 +138,7 @@ start() { if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then TOMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${TOMOVE}" - find "${SHAREPATH%/}" -depth | /usr/local/sbin/move -d $LOGLEVEL + find "${SHAREPATH%/}" -depth | /usr/local/bin/move -d $LOGLEVEL ((TO=TO-TOMOVE)) [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${TO}" writestatus @@ -158,7 +158,7 @@ start() { if [[ -d "/mnt/$shareCachePool" ]]; then FROMMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${FROMMOVE}" - find "${SHAREPATH%/}" -depth | /usr/local/sbin/move -d $LOGLEVEL + find "${SHAREPATH%/}" -depth | /usr/local/bin/move -d $LOGLEVEL ((FROM=FROM-FROMMOVE)) [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${FROM}" writestatus From 6e697432d2df4b58ad87fe8a9aa96aa41255f17c Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:40:16 +0000 Subject: [PATCH 09/17] Change processing. --- emhttp/plugins/dynamix/ArrayOperation.page | 35 +++++++++++++----- emhttp/plugins/dynamix/nchan/parity_list | 41 ++++++++++++---------- sbin/mover | 13 +++++-- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/emhttp/plugins/dynamix/ArrayOperation.page b/emhttp/plugins/dynamix/ArrayOperation.page index 07997612db..f2352121bd 100644 --- a/emhttp/plugins/dynamix/ArrayOperation.page +++ b/emhttp/plugins/dynamix/ArrayOperation.page @@ -290,9 +290,7 @@ mymonitor.on('message', function(state) { $('#mover-button').prop('disabled',true); $('#mover-text').html("_(Disabled)_ - _(Mover is running)_."); - for (let i = 0; i < maxmoverlines; i++) { - $('#moverrow'+i ).show(); - } + break; @@ -319,8 +317,27 @@ arraymonitor.on('message', function(state) { var moverStatus = new NchanSubscriber('/sub/mover',{subscriber:'websocket'}); moverStatus.on('message', function(moverdata) { -$.each(moverdata.split(';'),function(k,v) {if ($('#moverline'+k).length>0) $('#moverline'+k).html(v);}); + var moverlines = 0; + var selectlines = [] ; + $.each(moverdata.split(';'),function(k,v) { + var keydata = v.split('#'); + if(keydata[0]=="Showlines") {moverlines=keydata[1];} + if(keydata[0]=="Selectlines") {selectlines=keydata[1].split(',');} + $('#'+keydata[0]).html(keydata[1]); + }); + if(moverlines > maxmoverlines) moverlines = maxmoverlines; + if(moverlines == 0) { + for (let i = 0; i < 5; i++) { + $('#moverrow'+i ).hide(); + }; + }; + for (let i = 0; i < moverlines; i++) { + var selectline = i + 1; + var selectlinestr = selectline.toString(); + var checkselectline = selectlines.includes(selectlinestr); + if (checkselectline == true) $('#moverrow'+i ).show(); else $('#moverrow'+i ).hide(); + }; }); moverStatus.start(); @@ -682,11 +699,11 @@ function formatWarning(val) { - - - - - + + + + +
diff --git a/emhttp/plugins/dynamix/nchan/parity_list b/emhttp/plugins/dynamix/nchan/parity_list index 72787504f1..f9136286fd 100644 --- a/emhttp/plugins/dynamix/nchan/parity_list +++ b/emhttp/plugins/dynamix/nchan/parity_list @@ -120,40 +120,43 @@ while (true) { if ( file_exists("$varroot/mover.ini")) { $moverstat = parse_ini_file("$varroot/mover.ini") ; - $moverdata[] = $moverstat["File"] ; - $moverdata[] = $moverstat["Action"] ; $perc = $moverstat["Filepercent"] ; - $moverdata[] = "
$perc%
" ; + $value1 = "
$perc%
" ; if ($moverstat["TotalToArray"]>0) { $toperc = ($moverstat["TotalToArray"]-$moverstat["RemainToArray"])/$moverstat["TotalToArray"] * 100; if ($moverstat["RemainToArray"]>0) { $perc= round($toperc,2) ; - $moverdata[] = "
"._("Percentage Complete ").$perc."%
" ; - } else {$moverdata[] = _("Transfer complete") ;} + $value2 = "
"._("Percentage Complete ").$perc."%
" ; + } else {$value2 = _("Transfer complete") ;} } else { - $moverdata[] = _("Nothing to transfer") ;} + $value2 = _("Nothing to transfer") ;} if ($moverstat["TotalFromArray"]>0) { $fromperc = ($moverstat["TotalFromArray"] - $moverstat["RemainFromArray"])/$moverstat["TotalFromArray"] * 100; if ($moverstat["RemainFromArray"]>0) { $perc= round($fromperc,2) ; - $moverdata[] = "
"._("Percentage Complete ").$perc."%
"; - } else {$moverdata[] = _("Transfer complete") ;} + $value3 = "
"._("Percentage Complete ").$perc."%
"; + } else {$value3 = _("Transfer complete") ;} } else { - $moverdata[] = _("Nothing to transfer") ;} + $value3 = _("Nothing to transfer") ;} + $moverdata = []; + $moverdata[] = "Showlines#{$moverstat['Showlines']}"; + $moverdata[] = "Selectlines#{$moverstat['Selectlines']}"; + $moverdata[] = "movertitle0#"._($moverstat['movertitle1']); + $moverdata[] = "movertitle1#"._($moverstat['movertitle2']); + $moverdata[] = "movertitle2#"._($moverstat['movertitle3']); + $moverdata[] = "movertitle3#"._($moverstat['movertitle4']); + $moverdata[] = "movertitle4#"._($moverstat['movertitle5']); + $moverdata[] = "moverline0#{$moverstat['File']}"; + $moverdata[] = "moverline1#{$moverstat['Action']}"; + $moverdata[] = "moverline2#$value1"; + $moverdata[] = "moverline3#$value2"; + $moverdata[] = "moverline4#$value3"; } else { - $moverdata[] = _('Error'); - $moverdata[] = _('Error'); - $moverdata[] = _('Error'); - $moverdata[] = _('Error'); - $moverdata[] = _('Error'); + $moverdata[] = "Showlines#0"; } } else { $moverdata = []; - $moverdata[] = _('Not Running'); - $moverdata[] = _('Not Running'); - $moverdata[] = _('Not Running'); - $moverdata[] = _('Not Running'); - $moverdata[] = _('Not Running'); + $moverdata[] = "Showlines#0"; } publish('parity', implode(';',$data)); diff --git a/sbin/mover b/sbin/mover index 51559643d6..ff8b53e79f 100755 --- a/sbin/mover +++ b/sbin/mover @@ -43,9 +43,16 @@ writestatus() { echo "RemainToArray=${TO}" >> $MOVERSTATUS echo "TotalFromArray=${TOTALFROM}" >> $MOVERSTATUS echo "RemainFromArray=${FROM}" >> $MOVERSTATUS - echo "File=Processing" >> $MOVERSTATUS - echo "Action=Processing" >> $MOVERSTATUS - echo "Filepercent=Processing" >> $MOVERSTATUS + echo "File=" >> $MOVERSTATUS + echo "Action=Moving share ${SHARE}" >> $MOVERSTATUS + echo "Filepercent=" >> $MOVERSTATUS + echo "Showlines=5" >> $MOVERSTATUS + echo "Selectlines=2,3,4,5" >> $MOVERSTATUS + echo "movertitle1=File being processed:" >> $MOVERSTATUS + echo "movertitle2=Action:" >> $MOVERSTATUS + echo "movertitle3=Progress:" >> $MOVERSTATUS + echo "movertitle4=Transfering To Array:" >> $MOVERSTATUS + echo "movertitle5=Transfering From Array:" >> $MOVERSTATUS } start() { From 2dc0d787dcb0d9a3da10f8e87f611c6cb8e62e6e Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:33:43 +0000 Subject: [PATCH 10/17] Update mover files --- sbin/mover | 104 ++-------------- sbin/moverprogress | 303 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 311 insertions(+), 96 deletions(-) create mode 100644 sbin/moverprogress diff --git a/sbin/mover b/sbin/mover index ff8b53e79f..ab25bfd1bc 100755 --- a/sbin/mover +++ b/sbin/mover @@ -1,5 +1,5 @@ #!/bin/bash -#Copyright 2005-2023, Lime Technology +#Copyright 2005-2020, Lime Technology #License: GPLv2 only # This is the 'mover' script used for moving files between a pool and the main unRAID array. @@ -35,25 +35,7 @@ PIDFILE="/var/run/mover.pid" CFGFILE="/boot/config/share.cfg" -LOGLEVEL=0 -MOVERSTATUS="/usr/local/emhttp/state/mover.ini" - -writestatus() { - echo "TotalToArray=${TOTALTO}" > $MOVERSTATUS - echo "RemainToArray=${TO}" >> $MOVERSTATUS - echo "TotalFromArray=${TOTALFROM}" >> $MOVERSTATUS - echo "RemainFromArray=${FROM}" >> $MOVERSTATUS - echo "File=" >> $MOVERSTATUS - echo "Action=Moving share ${SHARE}" >> $MOVERSTATUS - echo "Filepercent=" >> $MOVERSTATUS - echo "Showlines=5" >> $MOVERSTATUS - echo "Selectlines=2,3,4,5" >> $MOVERSTATUS - echo "movertitle1=File being processed:" >> $MOVERSTATUS - echo "movertitle2=Action:" >> $MOVERSTATUS - echo "movertitle3=Progress:" >> $MOVERSTATUS - echo "movertitle4=Transfering To Array:" >> $MOVERSTATUS - echo "movertitle5=Transfering From Array:" >> $MOVERSTATUS -} +DEBUGGING="" start() { if [ -f $PIDFILE ]; then @@ -69,25 +51,14 @@ start() { echo "mover: cache not enabled" exit 2 fi - if grep -qs 'shareMoverLogging="yes"' $CFGFILE ; then - LOGLEVEL=1 - fi fi if ! mountpoint -q /mnt/user0 ; then echo "mover: array devices not mounted" exit 3 fi - TOTALFROM=0 - TOTALTO=0 - writestatus - echo $$ >/var/run/mover.pid - [[ $LOGLEVEL -gt 0 ]] && echo "mover: started" - - TOSHARES=0 - FROMSHARES=0 - DUARGSI="" + echo "mover: started" shopt -s nullglob @@ -96,86 +67,27 @@ start() { for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do SHARE=$(basename "$SHAREPATH") if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then - DUARGSI="${DUARGSI} ${SHAREPATH}" - ((TOSHARES=TOSHARES+1)) + find "${SHAREPATH%/}" -depth | /usr/local/bin/move $DEBUGGING fi done done - - if [[ "$TOSHARES" > 0 ]] - then - TO=$(du -sc ${DUARGSI} | grep total | awk '{print $1}' ) - else - TO=0 - fi - - DUARGSO="" - # Check for objects to move from array to pools - for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do + ls -dvc1 /mnt/disk[0-9]*/*/ | while read SHAREPATH ; do SHARE=$(basename "$SHAREPATH") if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then - eval "$(grep -s shareCachePool '/boot/config/shares/${SHARE}.cfg' | tr -d '\r')" + eval $(grep -s shareCachePool "/boot/config/shares/${SHARE}.cfg" | tr -d '\r') if [[ -z "$shareCachePool" ]]; then shareCachePool="cache" fi if [[ -d "/mnt/$shareCachePool" ]]; then - DUARGSO="${DUARGSO} ${SHAREPATH}" - ((FROMSHARES=FROMSHARES+1)) + find "${SHAREPATH%/}" -depth | /usr/local/bin/move $DEBUGGING fi fi done - if [[ "$FROMSHARES" > 0 ]] - then - FROM=$(du -sc ${DUARGSO} | grep total | awk '{print $1}') - else - FROM=0 - fi - TOTALFROM=$FROM - TOTALTO=$TO - writestatus - - # Check for objects to move from pools to array - for POOL in /boot/config/pools/*.cfg ; do - for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do - SHARE=$(basename "$SHAREPATH") - writestatus - if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then - TOMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') - [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${TOMOVE}" - find "${SHAREPATH%/}" -depth | /usr/local/bin/move -d $LOGLEVEL - ((TO=TO-TOMOVE)) - [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${TO}" - writestatus - fi - done - done - - # Check for objects to move from array to pools - for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do - SHARE=$(basename "$SHAREPATH") - writestatus - if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then - eval "$(grep -s shareCachePool '/boot/config/shares/${SHARE}.cfg' | tr -d '\r')" - if [[ -z "$shareCachePool" ]]; then - shareCachePool="cache" - fi - if [[ -d "/mnt/$shareCachePool" ]]; then - FROMMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') - [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${FROMMOVE}" - find "${SHAREPATH%/}" -depth | /usr/local/bin/move -d $LOGLEVEL - ((FROM=FROM-FROMMOVE)) - [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${FROM}" - writestatus - fi - fi - done - sleep 5 rm -f $PIDFILE - rm -f $MOVERSTATUS - [[ $LOGLEVEL -gt 0 ]] && echo "mover: finished" + echo "mover: finished" } killtree() { diff --git a/sbin/moverprogress b/sbin/moverprogress new file mode 100644 index 0000000000..a3e284bc33 --- /dev/null +++ b/sbin/moverprogress @@ -0,0 +1,303 @@ +#!/usr/bin/php + $val) { + if(is_array($val)) { + $res[] = PHP_EOL."[$key]"; + foreach($val as $skey => $sval) $res[] = "$skey = ".(is_numeric($sval) ? $sval : '"'.$sval.'"'); + } else { + $res[] = "$key = ".(is_numeric($val) ? $val : '"'.$val.'"'); + } + } + + file_put_contents($file, implode(PHP_EOL, $res)); +} + + +function save_moverstate($var, $val) { + $config_file = "/usr/local/emhttp/state/mover.ini"; + $config = @parse_ini_file($config_file, true); + $config[$var] = $val; + save_ini_file($config_file, $config); + return (isset($config[$var][$val])) ? $config[$var][$val] : FALSE; +} + +function save_moverstatefull($state) { + $config_file = "/usr/local/emhttp/state/mover.ini"; + $config = @parse_ini_file($config_file, true); + $config = $state; + save_ini_file($config_file, $config); + return (isset($config)) ? $config : FALSE; +} + +function startMover($options="") { + global $vars, $cfg, $cron, $runcmd ; + + + + $pid = @file_get_contents("/var/run/mover.pid"); + if ($pid) { + echo "mover: mover already running\n" ; + exit(); + } + + + if (isset($cfg)) { + # Only start if shfs includes pools + if ($cfg["shareCacheEnabled"] != "yes") { + echo "mover: cache not enabled\n" ; + exit(2) ; + } + } + + $LOGLEVEL = 0 ; + + if ($cfg["shareMoverLogging"] == "yes") $LOGLEVEL=1 ; + exec("mountpoint -q /mnt/user0",$output, $rtnvar) ; + if ($rtnvar > 0) { + echo "mover: array devices not mounted" ; + exit(3) ; + } + + if ($LOGLEVEL >0) echo "mover: started\n" ; + file_put_contents("/var/run/mover.pid", getmypid()) ; + + $path= '/boot/config/pools'; + $directory = '/boot/config/pools/'; + $scanned_directory = array_diff(scandir($directory), array('..', '.')); + $pools = glob($path."/*.cfg"); + $DUARGSI = $DUARGSO = "" ; + writestatus() ; + $config_file = "/usr/local/emhttp/state/mover.ini"; + $config = @parse_ini_file($config_file, true); + + foreach ($pools as $pool) { + $sfiles = glob("/mnt/".basename($pool , ".cfg")."/*", GLOB_ONLYDIR ); + + foreach ($sfiles as $sfile) { + + if (is_file("/boot/config/shares/".basename($sfile).".cfg")) { + $ini = parse_ini_file("/boot/config/shares/".basename($sfile).".cfg", true) ; + + if ($ini["shareUseCache"] == "yes" ) { + $DUARGSI .= " ".$sfile ; + } + } + } + + } + + $sfiles=array() ; + exec('ls -dv /mnt/disk[0-9]*/*/',$sfiles) ; + + $total0 = 0 ; + foreach ($sfiles as $sfile) { + + if (is_file("/boot/config/shares/".basename($sfile).".cfg")) { + $ini = parse_ini_file("/boot/config/shares/".basename($sfile).".cfg", true) ; + + if ($ini["shareUseCache"] == "prefer" ) { + if (!isset($ini["shareCachePool"])) $ini["shareCachePool"]="cache" ; + if (is_dir($sfile)) { + + $DUARGSO .= " ".$sfile ; + } + } + } + } + + $total=0; + if ($DUARGSI != "") $total = shell_exec("du -sc ".$DUARGSI." | grep total | awk '{print $1}'" ) ; else $total = 0; + if ($DUARGSO != "") $totalo = shell_exec("du -sc ".$DUARGSO." | grep total | awk '{print $1}'" ) ; else $totalo = 0; + + + printf("mover: shares to array %s To array bytes %d\n",$DUARGSI,$total) ; + printf("mover: shares from array %s From array bytes %d\n",$DUARGSO,$totalo) ; + $config_file = "/usr/local/emhttp/state/mover.ini"; + $config = @parse_ini_file($config_file, true); + $pools = glob($path."/*.cfg"); + + $config["TotalToArray"] = $total ; + $config["RemainToArray"] = $total; + $config["TotalFromArray"] = $totalo; + $config["RemainFromArray"] = $totalo ; + save_moverstatefull($config) ; + + sleep(10); + + foreach ($pools as $pool) { + $sfiles = glob("/mnt/".basename($pool , ".cfg")."/*", GLOB_ONLYDIR ); + + foreach ($sfiles as $sfile) { + + if (is_file("/boot/config/shares/".basename($sfile).".cfg")) { + $ini = parse_ini_file("/boot/config/shares/".basename($sfile).".cfg", true) ; + + if ($ini["shareUseCache"] == "yes" ) { + $find=array() ; + exec("find ".$sfile." -depth ", $find) ; + + #$LOGLEVEL = 1 ; + + foreach ($find as $movefile) { + if (!is_dir($movefile)) { $tomove = shell_exec("du -sc \"".$movefile."\" | grep total | awk '{print $1}'" ) ; } else {$tomove = 0 ;} + + $config["File"] = $movefile ; + $config["Action"] = "Move share:".basename($sfile)." to array" ; + $config["Filepercent"] = 0 ; + save_moverstatefull($config) ; + + $movecmd="echo \"".$movefile."\" | /usr/local/bin/move -d ".$LOGLEVEL ; + if ($runcmd) shell_exec($movecmd) ; else printf("Command running %s\n File size %d\n " ,$movecmd, $tomove) ; + + $config["RemainToArray"] = $config["RemainToArray"] - $tomove ; + + save_moverstatefull($config) ; + + } + + if ($LOGLEVEL > 0) printf("mover: remaining to transfer to array %d %s\n",$config["RemainToArray"],$sfile) ; + } + } + } + + } + + + $sfiles = glob("/mnt/".basename($pool , ".cfg")."/*", GLOB_ONLYDIR ); + $sfiles=array() ; + exec('ls -dv /mnt/disk[0-9]*/*/',$sfiles) ; + + + foreach ($sfiles as $sfile) { + + if (is_file("/boot/config/shares/".basename($sfile).".cfg")) { + $ini = parse_ini_file("/boot/config/shares/".basename($sfile).".cfg", true) ; + + if ($ini["shareUseCache"] == "prefer" ) { + if (!isset($ini["shareCachePool"])) $ini["shareCachePool"]="cache" ; + if (is_dir($sfile)) { + + + $find=array() ; + exec("find ".$sfile." -depth ", $find) ; + + #$LOGLEVEL = 1 ; + + foreach ($find as $movefile) { + if (!is_dir($movefile)) { $frommove = shell_exec("du -sc \"".$movefile."\" | grep total | awk '{print $1}'" ) ; } else {$frommove = 0 ;} + + $config["File"] = $movefile ; + $config["Action"] = "Move share:".basename($sfile)." from array" ; + $config["Filepercent"] = 0 ; + save_moverstatefull($config) ; + + $movecmd="echo \"".$movefile."\" | /usr/local/bin/move -d ".$LOGLEVEL ; + if ($runcmd) shell_exec($movecmd) ; else printf("Command running %s\n File size %d\n " ,$movecmd, $frommove) ; + + $config["RemainFromArray"] = $config["RemainFromArray"] - $frommove ; + + save_moverstatefull($config) ; + + } + + if ($LOGLEVEL > 0) printf("mover: remaining to transfer from array %d %s\n",$config["RemainFromArray"],$sfile) ; + } + } + + } + } + if ($LOGLEVEL > 0) echo "mover: finished\n"; + + unlink("/var/run/mover.pid"); + +} + + + +/*function killtree() { + local pid=$1 child + $mypid=getmypid() + for child in $(pgrep -P $pid); do + killtree $child + done + [ $pid -ne $$ ] && kill -TERM $pid +} +*/ +# Caution: stopping mover like this can lead to partial files on the destination +# and possible incomplete hard link transfer. Not recommended to do this. +function stopMover() { + $pid = @file_get_contents("/var/run/mover.pid"); + + if (!$pid) { + echo "mover: not running\n" ; + exit(1); + } + + #killtree $(cat $PIDFILE) + sleep(2) ; + unlink("/var/run/mover.pid"); + echo "mover: stopped" ; + +} + +$COMMAND = ""; +if (isset($argv[1])) $COMMAND = $argv[1]; +switch ($COMMAND) { +case 'start': + startMover() ; + break; + +case 'stop': + stopMover() ; + break; + +case 'status': + if (is_file("/var/run/mover.pid")) { + $pid=file_get_contents("/var/run/mover.pid") ; + printf("%d\n",$pid) ; + } + break; + + + +default: + + startMover(); + break; +} +?> \ No newline at end of file From 534b0a25761c1f2f5265cf8dfb2ac807339ae5c9 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:41:11 +0000 Subject: [PATCH 11/17] Add bash version --- sbin/moverprogressB | 218 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 sbin/moverprogressB diff --git a/sbin/moverprogressB b/sbin/moverprogressB new file mode 100644 index 0000000000..ff8b53e79f --- /dev/null +++ b/sbin/moverprogressB @@ -0,0 +1,218 @@ +#!/bin/bash +#Copyright 2005-2023, Lime Technology +#License: GPLv2 only + +# This is the 'mover' script used for moving files between a pool and the main unRAID array. +# It is typically invoked via cron. + +# First we check if it's valid for this script run: pool use in shfs must be enabled and +# an instance of the script must not already be running. + +# Next, check each of the top-level directories (shares) on each pool. +# If, and only if, the 'Use Cache' setting for the share is set to "yes", we use 'find' to +# list the objects (files and directories) of that share directory, moving them to the array. +# Next, we check each of the top-level directories (shares) on each array disk (in sorted order). +# If, and only if, the 'Use Cache' setting for the share is set to "prefer", we use 'find' to +# list the objects (files and directories) of that share directory, moving them to the pool +# associted with the share. + +# The script is set up so that hidden directories (i.e., directory names beginning with a '.' +# character) at the topmost level of a pool or an array disk are not moved. This behavior +# can be turned off by uncommenting the following line: +# shopt -s dotglob + +# Files at the top level of a pool or an array disk are never moved. + +# The 'find' command generates a list of all files and directories of a share. +# For each file, if the file is not "in use" by any process (as detected by 'in_use' command), +# then the file is moved, and upon success, deleted from the source disk. If the file already +# exists on the target, it is not moved and the source is not deleted. All meta-data of moved +# files/directories is preserved: permissions, ownership, extended attributes, and access/modified +# timestamps. + +# If an error occurs in copying a file, the partial file, if present, is deleted and the +# operation continues on to the next file. + +PIDFILE="/var/run/mover.pid" +CFGFILE="/boot/config/share.cfg" +LOGLEVEL=0 +MOVERSTATUS="/usr/local/emhttp/state/mover.ini" + +writestatus() { + echo "TotalToArray=${TOTALTO}" > $MOVERSTATUS + echo "RemainToArray=${TO}" >> $MOVERSTATUS + echo "TotalFromArray=${TOTALFROM}" >> $MOVERSTATUS + echo "RemainFromArray=${FROM}" >> $MOVERSTATUS + echo "File=" >> $MOVERSTATUS + echo "Action=Moving share ${SHARE}" >> $MOVERSTATUS + echo "Filepercent=" >> $MOVERSTATUS + echo "Showlines=5" >> $MOVERSTATUS + echo "Selectlines=2,3,4,5" >> $MOVERSTATUS + echo "movertitle1=File being processed:" >> $MOVERSTATUS + echo "movertitle2=Action:" >> $MOVERSTATUS + echo "movertitle3=Progress:" >> $MOVERSTATUS + echo "movertitle4=Transfering To Array:" >> $MOVERSTATUS + echo "movertitle5=Transfering From Array:" >> $MOVERSTATUS +} + +start() { + if [ -f $PIDFILE ]; then + if ps h $(cat $PIDFILE) | grep mover ; then + echo "mover: already running" + exit 1 + fi + fi + + if [ -f $CFGFILE ]; then + # Only start if shfs includes pools + if ! grep -qs 'shareCacheEnabled="yes"' $CFGFILE ; then + echo "mover: cache not enabled" + exit 2 + fi + if grep -qs 'shareMoverLogging="yes"' $CFGFILE ; then + LOGLEVEL=1 + fi + fi + if ! mountpoint -q /mnt/user0 ; then + echo "mover: array devices not mounted" + exit 3 + fi + + TOTALFROM=0 + TOTALTO=0 + writestatus + + echo $$ >/var/run/mover.pid + [[ $LOGLEVEL -gt 0 ]] && echo "mover: started" + + TOSHARES=0 + FROMSHARES=0 + DUARGSI="" + + shopt -s nullglob + + # Check for objects to move from pools to array + for POOL in /boot/config/pools/*.cfg ; do + for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do + SHARE=$(basename "$SHAREPATH") + if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then + DUARGSI="${DUARGSI} ${SHAREPATH}" + ((TOSHARES=TOSHARES+1)) + fi + done + done + + + if [[ "$TOSHARES" > 0 ]] + then + TO=$(du -sc ${DUARGSI} | grep total | awk '{print $1}' ) + else + TO=0 + fi + + DUARGSO="" + + # Check for objects to move from array to pools + for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do + SHARE=$(basename "$SHAREPATH") + if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then + eval "$(grep -s shareCachePool '/boot/config/shares/${SHARE}.cfg' | tr -d '\r')" + if [[ -z "$shareCachePool" ]]; then + shareCachePool="cache" + fi + if [[ -d "/mnt/$shareCachePool" ]]; then + DUARGSO="${DUARGSO} ${SHAREPATH}" + ((FROMSHARES=FROMSHARES+1)) + fi + fi + done + + if [[ "$FROMSHARES" > 0 ]] + then + FROM=$(du -sc ${DUARGSO} | grep total | awk '{print $1}') + else + FROM=0 + fi + TOTALFROM=$FROM + TOTALTO=$TO + writestatus + + # Check for objects to move from pools to array + for POOL in /boot/config/pools/*.cfg ; do + for SHAREPATH in /mnt/$(basename "$POOL" .cfg)/*/ ; do + SHARE=$(basename "$SHAREPATH") + writestatus + if grep -qs 'shareUseCache="yes"' "/boot/config/shares/${SHARE}.cfg" ; then + TOMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') + [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${TOMOVE}" + find "${SHAREPATH%/}" -depth | /usr/local/bin/move -d $LOGLEVEL + ((TO=TO-TOMOVE)) + [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${TO}" + writestatus + fi + done + done + + # Check for objects to move from array to pools + for SHAREPATH in $(ls -dv /mnt/disk[0-9]*/*/) ; do + SHARE=$(basename "$SHAREPATH") + writestatus + if grep -qs 'shareUseCache="prefer"' "/boot/config/shares/${SHARE}.cfg" ; then + eval "$(grep -s shareCachePool '/boot/config/shares/${SHARE}.cfg' | tr -d '\r')" + if [[ -z "$shareCachePool" ]]; then + shareCachePool="cache" + fi + if [[ -d "/mnt/$shareCachePool" ]]; then + FROMMOVE=$(du -sc ${SHAREPATH%/} | grep total | awk '{print $1}') + [[ $LOGLEVEL -gt 0 ]] && echo "Moving ${SHAREPATH%/} Bytes ${FROMMOVE}" + find "${SHAREPATH%/}" -depth | /usr/local/bin/move -d $LOGLEVEL + ((FROM=FROM-FROMMOVE)) + [[ $LOGLEVEL -gt 0 ]] && echo "RemainFromArray=${FROM}" + writestatus + fi + fi + done + sleep 5 + rm -f $PIDFILE + rm -f $MOVERSTATUS + [[ $LOGLEVEL -gt 0 ]] && echo "mover: finished" +} + +killtree() { + local pid=$1 child + + for child in $(pgrep -P $pid); do + killtree $child + done + [ $pid -ne $$ ] && kill -TERM $pid +} + +# Caution: stopping mover like this can lead to partial files on the destination +# and possible incomplete hard link transfer. Not recommended to do this. +stop() { + if [ ! -f $PIDFILE ]; then + echo "mover: not running" + exit 0 + fi + killtree $(cat $PIDFILE) + sleep 2 + rm -f $PIDFILE + echo "mover: stopped" +} + +case $1 in +start) + start + ;; +stop) + stop + ;; +status) + [ -f $PIDFILE ] + ;; +*) + # Default is "start" + # echo "Usage: $0 (start|stop|status)" + start + ;; +esac From ac6aa90f72520c661bd58c104191f142a299f6a9 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:45:23 +0000 Subject: [PATCH 12/17] Make script executable --- sbin/moverprogress | 0 sbin/moverprogressB | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 sbin/moverprogress mode change 100644 => 100755 sbin/moverprogressB diff --git a/sbin/moverprogress b/sbin/moverprogress old mode 100644 new mode 100755 diff --git a/sbin/moverprogressB b/sbin/moverprogressB old mode 100644 new mode 100755 From a91f1d52a61fb35eae3fc0d5829b8d9f7b906b7f Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:55:34 +0000 Subject: [PATCH 13/17] Add new options to mover. --- emhttp/plugins/dynamix/MoverSettings.page | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/emhttp/plugins/dynamix/MoverSettings.page b/emhttp/plugins/dynamix/MoverSettings.page index d7587dc98e..abf1c10220 100644 --- a/emhttp/plugins/dynamix/MoverSettings.page +++ b/emhttp/plugins/dynamix/MoverSettings.page @@ -117,6 +117,15 @@ _(Mover logging)_: :mover_logging_help: +_(Mover Progress)_: +: + +:mover_logging_help: + From d0b67d4cca86b2b10c0dac59e18a74d3cae4e37b Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Tue, 28 Nov 2023 08:58:32 +0000 Subject: [PATCH 14/17] Add help text --- emhttp/languages/en_US/helptext.txt | 4 ++++ emhttp/plugins/dynamix/MoverSettings.page | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/emhttp/languages/en_US/helptext.txt b/emhttp/languages/en_US/helptext.txt index e0267fa77f..02f6d2ebe4 100644 --- a/emhttp/languages/en_US/helptext.txt +++ b/emhttp/languages/en_US/helptext.txt @@ -2011,6 +2011,10 @@ For the other schedules choose here the time of the day the mover should start. Write mover messages to the syslog file. :end +:mover_progress_help: +Enable progress display and allow a user override. +:end + :notifications_display_help: In *Detailed* view all notifications will be displayed on screen as soon as they arrive.
Notifications can be acknowledged individually or all at once. diff --git a/emhttp/plugins/dynamix/MoverSettings.page b/emhttp/plugins/dynamix/MoverSettings.page index abf1c10220..10b452ab71 100644 --- a/emhttp/plugins/dynamix/MoverSettings.page +++ b/emhttp/plugins/dynamix/MoverSettings.page @@ -124,7 +124,7 @@ _(Mover Progress)_: -:mover_logging_help: +:mover_progress_help: From 8713e5b653978301057181297a1a3b96e6c1a296 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Sat, 2 Dec 2023 12:22:42 +0000 Subject: [PATCH 15/17] Update parity_list --- emhttp/plugins/dynamix/nchan/parity_list | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/parity_list b/emhttp/plugins/dynamix/nchan/parity_list index afdbdd89a7..a94f3192bd 100644 --- a/emhttp/plugins/dynamix/nchan/parity_list +++ b/emhttp/plugins/dynamix/nchan/parity_list @@ -120,7 +120,6 @@ while (true) { if ($process == 2) { $moverdata = []; - if ( file_exists("$varroot/mover.ini")) { $moverstat = parse_ini_file("$varroot/mover.ini") ; $perc = $moverstat["Filepercent"] ; @@ -162,9 +161,6 @@ while (true) { $moverdata[] = "Showlines#0"; } - publish('parity', implode(';',$data)); - if (time()-$timer >= 2) { - // update every 2 seconds $echo = implode(';',$data); $md5_new = md5($echo,true); if ($md5_new !== $md5_old) { @@ -181,10 +177,14 @@ while (true) { } if ($process !== $proc_old) { publish('mymonitor', $process); - publish('mover', implode(';',$moverdata)); $timer = time(); $proc_old = $process; } + if ($mover !== $mover_old){ + publish('mover', implode(';',$moverdata)); + $mover_old = $mover; + } + sleep(1); } ?> From 50f5930eb81c3a22cf356816769f173202a201c7 Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Sat, 2 Dec 2023 12:25:05 +0000 Subject: [PATCH 16/17] Update parity_list --- emhttp/plugins/dynamix/nchan/parity_list | 1 - 1 file changed, 1 deletion(-) diff --git a/emhttp/plugins/dynamix/nchan/parity_list b/emhttp/plugins/dynamix/nchan/parity_list index a94f3192bd..82c7d658e8 100644 --- a/emhttp/plugins/dynamix/nchan/parity_list +++ b/emhttp/plugins/dynamix/nchan/parity_list @@ -177,7 +177,6 @@ while (true) { } if ($process !== $proc_old) { publish('mymonitor', $process); - $timer = time(); $proc_old = $process; } if ($mover !== $mover_old){ From 7d08940dd1c2c8bc6565a36461eb491c2718e29c Mon Sep 17 00:00:00 2001 From: SimonFair <39065407+SimonFair@users.noreply.github.com> Date: Sat, 2 Dec 2023 12:38:39 +0000 Subject: [PATCH 17/17] Update parity_list --- emhttp/plugins/dynamix/nchan/parity_list | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/emhttp/plugins/dynamix/nchan/parity_list b/emhttp/plugins/dynamix/nchan/parity_list index 82c7d658e8..1f8fe20165 100644 --- a/emhttp/plugins/dynamix/nchan/parity_list +++ b/emhttp/plugins/dynamix/nchan/parity_list @@ -21,6 +21,7 @@ $md5_old = -1; $spot_old = -1; $fs_old = -1; $proc_old = -1; +$mover_old = -1; require_once "$docroot/webGui/include/Helpers.php"; require_once "$docroot/webGui/include/publish.php"; @@ -121,14 +122,15 @@ while (true) { if ($process == 2) { $moverdata = []; if ( file_exists("$varroot/mover.ini")) { - $moverstat = parse_ini_file("$varroot/mover.ini") ; - $perc = $moverstat["Filepercent"] ; - $value1 = "
$perc%
" ; + $moverstat = parse_ini_file("$varroot/mover.ini"); + $mover = $moverstat; + $perc = $moverstat["Filepercent"]; + $value1 = "
$perc%
"; if ($moverstat["TotalToArray"]>0) { $toperc = ($moverstat["TotalToArray"]-$moverstat["RemainToArray"])/$moverstat["TotalToArray"] * 100; if ($moverstat["RemainToArray"]>0) { $perc= round($toperc,2) ; - $value2 = "
"._("Percentage Complete ").$perc."%
" ; + $value2 = "
"._("Percentage Complete ").$perc."%
"; } else {$value2 = _("Transfer complete") ;} } else { $value2 = _("Nothing to transfer") ;} @@ -154,11 +156,11 @@ while (true) { $moverdata[] = "moverline3#$value2"; $moverdata[] = "moverline4#$value3"; } else { - $moverdata[] = "Showlines#0"; + $moverdata[] = "Showlines#0"; $mover = -1 ; } } else { $moverdata = []; - $moverdata[] = "Showlines#0"; + $moverdata[] = "Showlines#0"; $mover = -1 ; } $echo = implode(';',$data);