From 5233dd6d26b25f25f0e7a84f0da57b306639f661 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 23 May 2018 13:55:18 -0400 Subject: [PATCH 1/5] add workaround for when a package includes a restoration --- mmfunctions | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mmfunctions b/mmfunctions index cb9d3cf6..4a95822a 100755 --- a/mmfunctions +++ b/mmfunctions @@ -6,6 +6,7 @@ OBJECTS_FIND_EXCLUSIONS=(! -name ".*") OBJECTS_FIND_EXCLUSIONS+=(! -path "*/access/*") OBJECTS_FIND_EXCLUSIONS+=(! -path "*/service/*") OBJECTS_FIND_EXCLUSIONS+=(! -path "*/trimmed_materials/*") +OBJECTS_FIND_EXCLUSIONS+=(! -path "*/restoration/*") # load configuration file if [ -f "${TEMP_MMCONFIG}" ] ; then @@ -997,8 +998,14 @@ _find_input (){ fi ISOBJECT="Y" INPUTFILES=$(_maketemp) + if [[ -d "${1}/objects/restoration" ]] ; then + LOOKHERE="${1}/objects/restoration" + _report -dt "This package contains a restoration, using that for processing." + else + LOOKHERE="${1}/objects" + fi # find av files in a directory and output to a temp list - find "${1}/objects" -type f -size +0 "${OBJECTS_FIND_EXCLUSIONS[@]}" | while read file ; do + find "${LOOKHERE}" -type f -size +0 "${OBJECTS_FIND_EXCLUSIONS[@]}" | while read file ; do streamcount=$(ffprobe -loglevel quiet "$file" -show_entries format=nb_streams -of default=nw=1:nk=1) duration_ts=$(ffprobe -loglevel quiet "$file" -show_entries stream=duration_ts -of default=nw=1:nk=1) if [[ "$streamcount" > 0 && "${duration_ts}" != 1 ]] ; then @@ -1033,7 +1040,7 @@ _find_input (){ fi fi else - _report -wt "A valid source file isn't found in ${1}/objects." + _report -wt "A valid source file isn't found in ${LOOKHERE}." _writeerrorlog "_find_input" "A valid source file was not found, so the script could not proceed." exit 1 fi From c83ff4b61e4c70958643192fe7263c09bf6d8277 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 23 May 2018 14:10:04 -0400 Subject: [PATCH 2/5] initial support for multifile inputs in ingest file --- ingestfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ingestfile b/ingestfile index 4994952c..5c317cc7 100755 --- a/ingestfile +++ b/ingestfile @@ -616,14 +616,14 @@ FROM=$(_get_filesystem "${INPUT}") TO=$(_get_filesystem "${OBJECTSDIR}/") if [ "${FROM}" == "${TO}" -a "${CLEANUPDECISION}" == "Remove source file after successful ingest" ] ; then _report -dt "Because ${INPUT} and ${OBJECTSDIR} are on the same filesystem and preservation mode is enabled we will mv rather than rsync." - _run_critical mv -v -n "${INPUT}" "${OBJECTSDIR}/" + _run_critical mv -v -n "${INPUT[@]}" "${OBJECTSDIR}/" if [ ! "${#FILES[@]}" = 0 ] ; then _run_critical mv -v -n "${FILES[@]}" "${LOGDIR}/" fi else _report -dt "STATUS Copying the original file to library storage at ${OBJECTSDIR}." LOGFILE="${LOGDIR}/rsync_$(_get_iso8601_c)_$(basename ${0})_${VERSION}.txt" - "${SCRIPTDIR}/migratefiles" -l "${LOGFILE}" -o "${OBJECTSDIR}" "${INPUT}" + "${SCRIPTDIR}/migratefiles" -l "${LOGFILE}" -o "${OBJECTSDIR}" "${INPUT[@]}" if [ ! "${#FILES[@]}" = 0 ] ; then _run_critical rsync -rt --progress --log-file="${LOGDIR}/rsync_$(_get_iso8601_c)_$(basename $0)_${VERSION}.txt" "${FILES[@]}" "${LOGDIR}/" fi From 369f9fbbe71223cc7a1812056c0d7ac6285c3cc9 Mon Sep 17 00:00:00 2001 From: Dave Rice Date: Wed, 23 May 2018 14:14:45 -0400 Subject: [PATCH 3/5] add preservation option to ask for a restoration file --- ingestfile | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ingestfile b/ingestfile index 5c317cc7..2fa72c61 100755 --- a/ingestfile +++ b/ingestfile @@ -79,6 +79,13 @@ _ask_digitization_logs(){ [[ ! -z "${FILES}" ]] && _writeingestlog "digitization_logs: ${FILES}\n" } +_ask_restoration(){ + _report -qn "Drag in any restoration master: " + read -e -a RESTORATION_FILE + [[ "${RESTORATION_FILE[0]}" = "q" ]] && exit 0 + [[ ! -z "${RESTORATION_FILE}" ]] && _writeingestlog "restoration: ${RESTORATION_FILE}\n" +} + # spreadsheet fields are currently CUNY-specific _parse_spreadsheet(){ DIGITIZATIONLOG=$(_maketemp) @@ -492,6 +499,7 @@ fi _setup_package if [[ "${MODE}" == "audio" ]] ; then _ask_digitization_logs + _ask_restoration _ask_photos "${MEDIAID}" if [[ -f ${FILES} ]] ; then _parse_spreadsheet @@ -561,6 +569,7 @@ fi # MICROSERVICES BEFORE PROCESSING if [[ "${MODE}" == "preservation" ]] ; then _ask_digitization_logs + _ask_restoration if [[ "${INPUT#*.}" = "mkv" ]] ; then "${SCRIPTDIR}/makemkvchapters" "${INPUT}" else @@ -620,6 +629,10 @@ if [ "${FROM}" == "${TO}" -a "${CLEANUPDECISION}" == "Remove source file after s if [ ! "${#FILES[@]}" = 0 ] ; then _run_critical mv -v -n "${FILES[@]}" "${LOGDIR}/" fi + if [ ! "${#RESTORATION_FILE[@]}" = 0 ] ; then + _mkdir2 "${OBJECTSDIR}/restoration" + _run_critical mv -v -n "${RESTORATION_FILE[@]}" "${OBJECTSDIR}/restoration/" + fi else _report -dt "STATUS Copying the original file to library storage at ${OBJECTSDIR}." LOGFILE="${LOGDIR}/rsync_$(_get_iso8601_c)_$(basename ${0})_${VERSION}.txt" @@ -627,6 +640,10 @@ else if [ ! "${#FILES[@]}" = 0 ] ; then _run_critical rsync -rt --progress --log-file="${LOGDIR}/rsync_$(_get_iso8601_c)_$(basename $0)_${VERSION}.txt" "${FILES[@]}" "${LOGDIR}/" fi + if [ ! "${#RESTORATION_FILE[@]}" = 0 ] ; then + _mkdir2 "${OBJECTSDIR}/restoration" + _run_critical rsync -rt --progress --log-file="${LOGDIR}/rsync_$(_get_iso8601_c)_$(basename $0)_${VERSION}.txt" "${RESTORATION_FILE[@]}" "${OBJECTSDIR}/restoration/" + fi fi # set makederivopts From 69c9047612c02cc8603e4f801c3386e59a5eda67 Mon Sep 17 00:00:00 2001 From: CSchloss385 Date: Thu, 24 May 2018 11:06:08 -0400 Subject: [PATCH 4/5] removed ask restoration function from preservation mode --- ingestfile | 1 - 1 file changed, 1 deletion(-) diff --git a/ingestfile b/ingestfile index 2fa72c61..1e33accf 100755 --- a/ingestfile +++ b/ingestfile @@ -569,7 +569,6 @@ fi # MICROSERVICES BEFORE PROCESSING if [[ "${MODE}" == "preservation" ]] ; then _ask_digitization_logs - _ask_restoration if [[ "${INPUT#*.}" = "mkv" ]] ; then "${SCRIPTDIR}/makemkvchapters" "${INPUT}" else From c56838661e94cce3f21c06617bb7a1fb44d9c5d4 Mon Sep 17 00:00:00 2001 From: CSchloss385 Date: Thu, 24 May 2018 11:06:31 -0400 Subject: [PATCH 5/5] Removed restoration exclusion --- mmfunctions | 1 - 1 file changed, 1 deletion(-) diff --git a/mmfunctions b/mmfunctions index 4a95822a..81801305 100755 --- a/mmfunctions +++ b/mmfunctions @@ -6,7 +6,6 @@ OBJECTS_FIND_EXCLUSIONS=(! -name ".*") OBJECTS_FIND_EXCLUSIONS+=(! -path "*/access/*") OBJECTS_FIND_EXCLUSIONS+=(! -path "*/service/*") OBJECTS_FIND_EXCLUSIONS+=(! -path "*/trimmed_materials/*") -OBJECTS_FIND_EXCLUSIONS+=(! -path "*/restoration/*") # load configuration file if [ -f "${TEMP_MMCONFIG}" ] ; then