setlX/test_performance at master · sdro/setlX · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
#!/bin/bash

tmpFile=$(mktemp -t setlx-XXXXXXXXXX)

# set overrides for setlX launcher script
export OVERRIDE_setlXJarDirectory="$PWD/interpreter/"
export OVERRIDE_setlXjavaParameters="-d64 -Xmx6144m"

# other locations for tests
setlXlaunchScript="$PWD/interpreter/setlX"

$setlXlaunchScript --version | head -n 1 > "$tmpFile"
if [ $? -ne 0 ]
  then
    cat "$tmpFile"
    echo "Please make sure the interpreter is already compiled and executable."

    rm -f "$tmpFile"
    exit 1
fi

version="v$(cat "$tmpFile" | tr '.' '-')"

resultFileName="example_SetlX_code/performance_test_code/results/$(hostname | tr '.' '-')_$version"

resultFileNameReal="$resultFileName.real.csv"
resultFileNameUser="$resultFileName.user.csv"
resultFileNameSys="$resultFileName.sys.csv"

rm -f "$resultFileNameReal" "$resultFileNameUser" "$resultFileNameSys"

nSkipped=0
sumRealTime=0.0
sumUserTime=0.0
sumSysTime=0.0

# discover all files
examples=()
while read -r -d $'\0' path
  do
    examples=("${examples[@]}" "$path")
done < <( find -L "example_SetlX_code/performance_test_code" -name "*.stlx" -print0 )

# count number of files
count="${#examples[@]}"

# loop through all SetlX files, execute them and compare to reference result
started=0
for path in "${examples[@]}"
  do
    shortname="${path#example_SetlX_code/performance_test_code/}"
    directoryname="$(dirname $path)"
    filename="$(basename $path)"

    started=$(bc <<< "$started + 1")
    echo "($started/$count) executing file '$shortname' ..."

    cd "$directoryname"
    execTime=$(
        time -p (
            # execute with setlX
            $setlXlaunchScript --predictableRandom "$filename" > "$tmpFile" 2>&1
        ) 2>&1
    )
    cd - >/dev/null

    diff "$path.reference" "$tmpFile" > /dev/null
    if [ $? -eq 1 ]
      then

        cat $tmpFile
        echo "executing file '$filename' failed, skipping this test."

        nSkipped=$(bc <<< "$nSkipped + 1")

        echo

    else
        read -a timeArray <<<$execTime
        realTime=${timeArray[1]}
        userTime=${timeArray[3]}
        sysTime=${timeArray[5]}

        echo "$filename,$realTime" >> "$resultFileNameReal"
        echo "$filename,$userTime" >> "$resultFileNameUser"
        echo "$filename,$sysTime"  >> "$resultFileNameSys"

        echo "real: $realTime, user: $userTime, sys: $sysTime"

        sumRealTime=$(bc <<< "$sumRealTime + $realTime")
        sumUserTime=$(bc <<< "$sumUserTime + $userTime")
        sumSysTime=$(bc <<< "$sumSysTime + $sysTime")

        echo
    fi

done

cat "$resultFileNameReal" | sort > "$tmpFile"
echo "code example,$version - time in seconds (real)" > "$resultFileNameReal"
cat "$tmpFile" >> "$resultFileNameReal"
echo "overall,$sumRealTime" >> "$resultFileNameReal"

cat "$resultFileNameUser" | sort > "$tmpFile"
echo "code example,$version - time in seconds (user)" > "$resultFileNameUser"
cat "$tmpFile" >> "$resultFileNameUser"
echo "overall,$sumUserTime" >> "$resultFileNameUser"

cat "$resultFileNameSys" | sort > "$tmpFile"
echo "code example,$version - time in seconds (sys)" > "$resultFileNameSys"
cat "$tmpFile" >> "$resultFileNameSys"
echo "overall,$sumSysTime" >> "$resultFileNameSys"

rm -f "$tmpFile"

echo "overall execution time (excluding $nSkipped tests, which failed):"
echo "real: $sumRealTime, user: $sumUserTime, sys: $sumSysTime"
echo

exit 0