grab_one

#!/bin/bash

# Logical flow:
#
# If there is a PID file
#  If the PID is valid
#   Exit
# (Over)Write a PID file.
# If we were supplied a channel name:
#  get its url file
# Otherwise
#  Find the oldest url file
# Go to the folder containing that url file.
# touch the url file
# Download everything from the url specified in the url file.
# Update the file's timestamp to the stream's timestamp if possible
# Sort the files by date
# Create an sha256sums.part file
# Rename sha256sums.part to sha256sums
# Remove the PID file
# Exit

BASEDIR=/srv/hackgrab
HOSTNAME=`hostname`
AGEOUT='90'
MAXDL='20'
MAXLIST='100'
MAXDELAY='60'

cd ${BASEDIR}/content

if [ -e /dev/shm/Hackgrab_PID ]
then
    OLDPID=`cat /dev/shm/Hackgrab_PID`
    if ps -p ${OLDPID}
    then
	exit 2
    fi
fi

echo $$ > /dev/shm/Hackgrab_PID

if [ "${1}" ]
then
    TORUN="${1}/url"
else
    TORUN=`ls -tr */url | head -n1`
fi

FOLDER=`dirname "${TORUN}"`

echo -n "${HOSTNAME} started ${FOLDER} at "
date +"%Y-%m-%d %H:%M:%S"

cd "${BASEDIR}/content/${FOLDER}"

if [ -e .ag_do ]
then
    rm .ag_do
fi

touch url

if [ -e .lock ]
then
    exit 3
fi

touch .lock

if [ "${1}" ]
then
    :
else
    DELAY=$(($RANDOM % ${MAXDELAY}))
    UNTIL=`date --date="${DELAY} minutes" +"%H:%M"`
    echo "Wait ${UNTIL}  ${FOLDER}" > ${BASEDIR}/metadata/status_${HOSTNAME}
    echo "Countermeasure delay of ${DELAY} minutes, until approximately ${UNTIL}."
    sleep $(($DELAY * 60))
fi

touch url 

if [ -x dothis ]
then
    echo "Running pre-download script."
    echo "Prescript   ${FOLDER}" > ${BASEDIR}/metadata/status_${HOSTNAME}
    
    if ./dothis
    then
	rm dothis
    else
	chmod -x dothis
    fi
fi

URL="`cat url`"

if [ -e .maxdl ]
then
    MAXDL=`cat .maxdl`
fi

if [ -e .maxlist ]
then
    MAXLIST=`cat .maxlist`
fi

echo "Downloading ${FOLDER}" > ${BASEDIR}/metadata/status_${HOSTNAME} && \
    #yt-dlp -S vcodec:h264 --remote-components ejs:github --playlist-end ${MAXLIST} --ignore-errors --write-description --download-archive archive --max-downloads ${MAXDL} "${URL}"
    yt-dlp -S vcodec:h264 --remote-components ejs:github --playlist-end ${MAXLIST} --write-info-json --ignore-errors --write-description --download-archive archive --max-downloads ${MAXDL} "${URL}"

echo "Sorting     ${FOLDER}" > ${BASEDIR}/metadata/status_${HOSTNAME}

shopt -s nullglob 

for i in *.mp4 *.webm
do
    #Extract the date
    echo Timestamping ${i}
    DATE=`/usr/bin/mediainfo "${i}" | grep "Tagged date" | head -n1 | cut -f2 -d":"`

    #If there is one, use it to set the file's timestamp.
    if [ "${DATE}" ]
    then
	touch --date="${DATE}" "${i}"
    fi
done

${BASEDIR}/bin/datesort.py

echo "Hashing     ${FOLDER}" > ${BASEDIR}/metadata/status_${HOSTNAME} && \
    #sha256sum */*/*/*.mp4 | tee sha256sums.part && \
    #mv sha256sums.part sha256sums
    ${BASEDIR}/bin/update_hash.py

rm -f .lock

INACTIVE="TRUE"

TODAY=`date +"%Y/%m/%d"`
i=`date +"%Y/%m/%d" --date="${AGEOUT} days ago"`

while [ ${INACTIVE} = "TRUE" ] && [ ${i} != ${TODAY} ]
do
    [ -e ${i} ] && INACTIVE="FALSE" && echo "Content found within the last ${AGEOUT} days."
    i=`date +"%Y/%m/%d" --date="tomorrow ${i}"`
done

[ ${INACTIVE} = "FALSE" ]  || (mv -f url url_hold && echo "Deactivated ${FOLDER} due to inactivity.")

[ -e .lowpri ] && mv -f url url_hold && echo "Deactivated ${FOLDER} due to low priority."

cd ${BASEDIR} && rm /dev/shm/Hackgrab_PID && echo "Idle" > ${BASEDIR}/metadata/status_${HOSTNAME}

echo -n "${HOSTNAME} completed ${FOLDER} at "
date +"%Y-%m-%d %H:%M:%S"
echo