mincer/mincer-worker at master · EnvisionX/mincer · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
#!/bin/sh -e
# -*- shell-script -*-

##
## Mincer worker
##

if [ "$1" = "-h" -o "$1" = "--help" -o -z "$1" -o -z "$2" ]; then
    echo "Usage: $0 <config-file> <file-to-process>" 1>&2
    exit 1
fi

MINCER_CONFIG=`readlink --canonicalize-missing "$1"`
. "$MINCER_CONFIG"

[ -z "$CALLBACK" ] && CALLBACK="$ROOT_DIR"/callback.sh
[ -z "$INCOMING_DIR" ] && INCOMING_DIR="$ROOT_DIR"/incoming
[ -z "$FAILED_DIR" ] && FAILED_DIR="$ROOT_DIR"/failed
[ -z "$HISTORY_DIR" ] && HISTORY_DIR="$ROOT_DIR"/history
[ -z "$WORK_DIR" ] && WORK_DIR="$ROOT_DIR"/tmp

mkdir -p -- "$WORK_DIR"
cd "$WORK_DIR"
for i in *; do test -f "$i" && rm -f -- "$i"; done

log(){ echo `date '+%Y-%m-%d %H:%M:%S %Z'` "<worker-$$>" "$1"; }

INPUT_FILE_BASENAME="$2"
INPUT_FILE="$INCOMING_DIR"/"$INPUT_FILE_BASENAME"

if [ ! -s "$INPUT_FILE" ]; then
    log "skipping empty file $INPUT_FILE"
    rm -f -- "$INPUT_FILE"
    exit 0
fi

if [ ! -r "$INPUT_FILE" ]; then
    log "file is not readable"
    mkdir -p -- "$FAILED_DIR"
    mv -f -- "$INPUT_FILE" "$FAILED_DIR"
    exit 1
fi

log "STARTING: $CALLBACK $INPUT_FILE"

export MINCER_CONFIG
if "$CALLBACK" "$INPUT_FILE"; then
    log "processed"
    # Copy result files as .tmp files in all output dirs
    for DIR in "$OUTPUT_DIR" "$OUTPUT_DIR1" \
               "$OUTPUT_DIR2" "$OUTPUT_DIR3"; do
        [ -z "$DIR" ] && continue
        log "copying to $DIR:"
        for i in *; do
            [ -f "$i" ] || continue
            log "  $i"
            mkdir -p -- "$DIR" && cp -f -- "$i" "$DIR"/"$i".tmp || \
                {
                    # Remove all created tmp files
                    for DIR in "$OUTPUT_DIR" "$OUTPUT_DIR1" \
                               "$OUTPUT_DIR2" "$OUTPUT_DIR3"; do
                        [ -z "$DIR" ] && continue
                        for i in *; do
                            [ -f "$i" ] || continue
                            rm -f -- "$DIR"/"$i".tmp || :
                        done
                    done
                    mkdir -p -- "$FAILED_DIR"
                    mv -f -- "$INPUT_FILE" "$FAILED_DIR"
                    exit 1
                }
        done
    done
    # Rename all copied tmp files to origin names
    for DIR in "$OUTPUT_DIR" "$OUTPUT_DIR1" \
               "$OUTPUT_DIR2" "$OUTPUT_DIR3"; do
        [ -z "$DIR" ] && continue
        for i in *; do
            [ -f "$i" ] || continue
            mv -f -- "$DIR"/"$i".tmp "$DIR"/"$i"
        done
    done
    ## post process
    set +e
    if [ "$HISTORY" = "plain" ]; then
        mkdir -p -- "$HISTORY_DIR"
        mv -f -- "$INPUT_FILE" "$HISTORY_DIR"
        log "moved to history"
    elif [ "$HISTORY" = "gzip" ]; then
        mkdir -p -- "$HISTORY_DIR"
        if echo "$INPUT_FILE_BASENAME" | grep -qE '\.gz$'; then
            mv -f -- "$INPUT_FILE" "$HISTORY_DIR"
            log "already gzipped. moved to history"
        else
            gzip -f > "$HISTORY_DIR"/"$INPUT_FILE_BASENAME".gz < "$INPUT_FILE"
            log "gzipped to history"
        fi
    fi
    rm -f -- "$INPUT_FILE"
else
    log "failed"
    mkdir -p -- "$FAILED_DIR"
    mv -f -- "$INPUT_FILE" "$FAILED_DIR"
    exit 1
fi

exit 0