date_loop

#!/bin/bash

__usage__="usage: ${0} START END INCREMENT COMMAND [-v]

Loop over a range of dates (UTC) in steps increments of multiples of hours.

Positional arguments:

    START END
        Start and end timesteps, inclusive (YYYYMMDDHH[NN], i.e., %Y%m%d%H[%M]).

    INCREMENT
        Timestep increment in the smallest timestep unit, e.g., minutes for *%M.

    COMMAND
        The command executed for each timestep. To use the datetime components,
        add them as escaped variables (\${DATE}, \${YYYY}, \${MM}, \${DD}, \${HH}, \${NN}).

Optional arguments:

    -v [N=1]
        Print the current timestep for Nth iteration (N in [1..999]).

    -p N
        Parallelize over N processes at a time (N in [1..99]).
"

main()
{
    parse_cli "${@}" || return ${?}

    # Set timezone to UTC to avoid winter-/summertime change issues
    export TZ=UTC

    # Determine resolution based on start date length
    case ${#__date_start__} in
        10) __res__=hour;;
        12) __res__=minute;;
        *)  echo "invalid start datetime ${__date_start__} string of length ${#__date_start__}" >&2
            return 1
        ;;
    esac

    check_datetimes || return ${?}

    # Determine start date
    case "${__res__}" in
        "hour"  ) __date_curr__="$(TZ=UTC date +%Y%m%d%H   -d "${__date_start__:0:8} ${__date_start__:8:2} - ${__increment__} hours")" ;;
        "minute") __date_curr__="$(TZ=UTC date +%Y%m%d%H%M -d "${__date_start__:0:8} ${__date_start__:8:4} - ${__increment__} minutes")" ;;
    esac

    # Loop over dates (dont use while loop to avoid infinite loop)
    __pids_active__=()
    __dates_active__=()
    __iter_max__=$((__date_end__ - __date_start__)) # bigger than actual number of iterations
    for iter_i in $(seq 0 ${__iter_max__})
    do
        # Increment date
        case "${__res__}" in
            "hour"  ) __date_curr__="$(TZ=UTC date +%Y%m%d%H   -d "${__date_curr__:0:8} ${__date_curr__:8:2} + ${__increment__} hours")";;
            "minute") __date_curr__="$(TZ=UTC date +%Y%m%d%H%M -d "${__date_curr__:0:8} ${__date_curr__:8:4} + ${__increment__} minutes")";;
        esac

        # Are we done yet?
        [ ${__date_curr__} -gt ${__date_end__} ] && break

        # Define variables to access the datetime components
        DATE="${__date_curr__}"
        YYYY="${__date_curr__:0:4}"
        MM="${__date_curr__:4:2}"
        DD="${__date_curr__:6:2}"
        HH="${__date_curr__:8:2}"
        NN="${__date_curr__:10:2}"

        # Print current timestep if requested
        ${__verbose__} && [ $((iter_i%__vb_n__)) -eq 0 ] && echo "${__date_curr__}"

        # Execute command
        eval ${__cmd__} &
        __pids_active__[iter_i%__num_procs__]=${!}
        __dates_active__[iter_i%__num_procs__]=${__date_curr__}

        # Once N processes have been started in the background,
        # wait for them to finish before starting new ones
        # For sequential execution, this is simply done every timestep
        if [ $(((iter_i+1)%__num_procs__)) -eq 0 ]
        then
            wait_for_processes || return ${?}
        fi
    done

    # Just in case
    wait

    return 0
}

check_datetimes()
{
    # Check dates and increment
    if [ "${__res__}" == "hour" ]
    then
        echo "${__date_start__}" | grep -q '[12][0-9]\{3\}[01][0-9][0-3][0-9][012][0-9]'
        err="invalid start date ${__date_start__} for length 10, expect format YYYYMMDDHH, i.e., %Y%m%d%H"
        [ ${?} -ne 0 ] && { echo "${err}" >&2; return 1; }
        echo "${__date_end__}" | grep -q '[12][0-9]\{3\}[01][0-9][0-3][0-9][012][0-9]'
        err="invalid end date ${__date_end__} for length 10, expect format YYYYMMDDHH, i.e., %Y%m%d%H"
        [ ${?} -ne 0 ] && { echo "${err}" >&2; return 1; }
    elif [ "${__res__}" == "minute" ]
    then
        echo "${__date_start__}" | grep -q '[12][0-9]\{3\}[01][0-9][0-3][0-9][012][0-9][0-5][0-9]'
        err="invalid start date ${__date_start__} for length 12, expect format YYYYMMDDHHNN, i.e., %Y%m%d%H%M"
        [ ${?} -ne 0 ] && { echo "${err}" >&2; return 1; }
        echo "${__date_end__}" | grep -q '[12][0-9]\{3\}[01][0-9][0-3][0-9][012][0-9][0-5][0-9]'
        err="invalid end date ${__date_end__} for length 12, expect format YYYYMMDDHHNN, i.e., %Y%m%d%H%M"
        [ ${?} -ne 0 ] && { echo "${err}" >&2; return 1; }
    else
        echo "invalid start date ${__date_start__} of length ${#__date_start__}" >&2
        return 1
    fi
    err="invalid non-zero increment ${__increment__}"
    [ ${__increment__} -eq 0 ] && { echo "${err}" >&2; return 1; }
    return 0
}

wait_for_processes()
{
    # Wait in turn for each active process
    # Count the number of failed processes
    # Even if one fails, the others can finish
    local nerr=0
    local stats=()
    local pid date stat ierr i
    for (( i=0; i<${__num_procs__}; i++ ))
    do
        pid=${__pids_active__[i]}
        date=${__dates_active__[i]}
        wait ${pid}
        stat=${?}
        stats[i]=${stat}
        [ ${stat} -ne 0 ] && nerr=$((nerr + 1))
    done

    # Just in case
    wait

    # In case of failed processes, abort execution
    if [ ${nerr} -gt 0 ]
    then
        ierr=0
        for (( i=0; i<${__num_procs__}; i++ ))
        do
            pid=${__pids_active__[i]}
            date=${__dates_active__[i]}
            stat=${stats[i]}
            echo -n " $((i+1))/${__num_procs__} [${pid}] "
            if [ ${stat} -eq 0 ]
            then
                echo "OK ${date}" >&2
            else
                echo "ERROR $((ierr+1))/${nerr} (${stat}) ${date}" >&2
                ierr=$((ierr+1))
            fi
        done
        return ${nerr}
    fi

    return 0
}

parse_cli()
{
    # Set defaults
    __debug__=false
    __verbose__=false
    __num_procs__=1

    # Parse command line arguments
    local n=0
    local args=("${@}")
    local i arg
    for ((i=0; i < ${#args[@]}; i++))
    do
        arg=${args[i]}
        ${__debug__} && echo "${i} ${arg}" >&2
        case "${arg}" in
         "-v")
             __verbose__=true
             echo "${args[i+1]}" | grep -q '^[1-9][0-9]\{,2\}$'
             if [ ${?} -eq 0 ]
             then
                 i=$((i+1))
                 __vb_n__=${args[i]}
             else
                 __vb_n__=1
             fi
         ;;
         "-p")
             echo "${args[i+1]}" | grep -q '^[1-9][0-9]\?$'
             if [ ${?} -ne 0 ]
             then
                 echo "'-p' not followed by number of processes" >&2
                 return 4
             fi
             i=$((i+1))
             __num_procs__=${args[i]}
         ;;
         *)
             n=$((n+1))
             case "${n}" in
                 1) __date_start__="${arg}" ;;
                 2) __date_end__="${arg}" ;;
                 3) __increment__="${arg}" ;;
                 4) __cmd__="${arg}" ;;
             esac
         ;;
        esac
    done
    if [ ${n} -ne 4 ]
    then
        echo -e "error: wrong number of positional arguments: ${n}\n\n${__usage__}" >&2
        return 2
    fi
    return 0
}

main "${@}"