Parallel execution of unit tests (#2435)
Running unit tests is quite a bottleneck while developing or while waiting for GitHub CI results... Try to run the tests in parallel, using the `parallel` tool. By default, tests still run one after the other, as usual; to enable parallel execution you need `NDPI_FORCE_PARALLEL_UTESTS=1 ./tests/do.sh` Please note that the output is quite different in parallel mode! A big part of the script has been rewritten to avoid code dupication between "serial" and "parallel" path On my notebook: ``` ivan@ivan-Latitude-E6540:~/svnrepos/nDPI(parallel)$ time ./tests/do.sh [...] real 3m12,684s [...] ivan@ivan-Latitude-E6540:~/svnrepos/nDPI(parallel)$ time NDPI_FORCE_PARALLEL_UTESTS=1 ./tests/do.sh [...] real 0m58,463s ```
This commit is contained in:
+127
-70
@@ -5,6 +5,9 @@
|
||||
#
|
||||
# NDPI_FORCE_UPDATING_UTESTS_RESULTS=1 ./tests/do.sh
|
||||
#
|
||||
# To run tests in parallel: (you need **GNU** `parallel` program)
|
||||
#
|
||||
# NDPI_FORCE_PARALLEL_UTESTS=1 ./tests/do.sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
@@ -18,6 +21,11 @@ if [ "${NDPI_FORCE_UPDATING_UTESTS_RESULTS}" = "1" ]; then
|
||||
FORCE_UPDATING_UTESTS_RESULTS=1
|
||||
fi
|
||||
|
||||
FORCE_PARALLEL_UTESTS=0
|
||||
if [ "${NDPI_FORCE_PARALLEL_UTESTS}" = "1" ]; then
|
||||
FORCE_PARALLEL_UTESTS=1
|
||||
fi
|
||||
|
||||
#Remember: valgrind and *SAN are incompatible!
|
||||
CMD_PREFIX="${CMD_PREFIX}"
|
||||
if [ "${NDPI_TESTS_WINE}" = "1" ]; then
|
||||
@@ -40,6 +48,20 @@ GLOBAL_CONTEXT_ENABLED=@GLOBAL_CONTEXT_ENABLED@
|
||||
GLOBAL_CONTEXT_CFGS="caches_global"
|
||||
READER="${CMD_PREFIX} ../../../example/ndpiReader${EXE_SUFFIX} --cfg=filename.config,../../../example/config.txt -A -p ../../../example/protos.txt -c ../../../example/categories.txt -r ../../../example/risky_domains.txt -j ../../../example/ja3_fingerprints.csv -S ../../../example/sha1_fingerprints.csv -G ../../../lists -q -K JSON -k /dev/null -t -v 2"
|
||||
|
||||
|
||||
#These exports are used in parallel mode
|
||||
export CMD_DIFF
|
||||
export CMD_WDIFF
|
||||
export CMD_COLORDIFF
|
||||
export PCRE2_ENABLED
|
||||
export PCRE_PCAPS
|
||||
export NBPF_ENABLED
|
||||
export NBPF_PCAPS
|
||||
export READER
|
||||
export FORCE_UPDATING_UTESTS_RESULTS
|
||||
export FORCE_PARALLEL_UTESTS
|
||||
|
||||
|
||||
RC=0
|
||||
|
||||
if [ ! -x "../example/ndpiReader${EXE_SUFFIX}" ]; then
|
||||
@@ -48,6 +70,16 @@ if [ ! -x "../example/ndpiReader${EXE_SUFFIX}" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#For paralell tests you need `parallel` from GNU, not from `moreutils` package!
|
||||
#On Ubuntu, for example, you might need to explicitly run `apt install parallel`
|
||||
if [ $FORCE_PARALLEL_UTESTS -eq 1 ]; then
|
||||
if ! parallel -V | grep -qoE 'GNU parallel'; then
|
||||
echo "$0: To run the test in parallel mode you need **GNU** `parallel`"
|
||||
echo "$0: Try something like `apt install parallel`"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ${GPROF_ENABLED} -eq 1 ]; then
|
||||
GPROF="${GPROF:-$(which pprof)}"
|
||||
if [ ! -x "${GPROF}" ]; then
|
||||
@@ -73,78 +105,99 @@ fuzzy_testing() {
|
||||
fi
|
||||
}
|
||||
|
||||
build_results() {
|
||||
for f in $PCAPS; do
|
||||
#echo $f
|
||||
run_single_pcap()
|
||||
{
|
||||
f=$1
|
||||
|
||||
if [ ! -f ./pcap/$f ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
SKIP_PCAP=0;
|
||||
if [ $PCRE2_ENABLED -eq 0 ]; then
|
||||
for p in $PCRE_PCAPS; do
|
||||
if [ $f = $p ]; then
|
||||
SKIP_PCAP=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ $NBPF_ENABLED -eq 0 ]; then
|
||||
for p in $NBPF_PCAPS; do
|
||||
if [ $f = $p ]; then
|
||||
SKIP_PCAP=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ $SKIP_PCAP -eq 1 ]; then
|
||||
if [ $FORCE_PARALLEL_UTESTS -eq 1 ]; then
|
||||
printf "SKIPPED\n"
|
||||
else
|
||||
printf "%-48s\tSKIPPED\n" "$f"
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
|
||||
CMD="$READER -i pcap/$f -w /tmp/reader.$$.out $READER_EXTRA_PARAM"
|
||||
CPUPROFILE=./result/$f.cprof HEAPPROFILE=./result/$f $CMD
|
||||
CMD_RET=$?
|
||||
if [ $CMD_RET -eq 0 ] && [ -f /tmp/reader.$$.out ]; then
|
||||
# create result files if not present
|
||||
if [ ! -f result/$f.out ]; then
|
||||
CMD="$READER -i pcap/$f -w result/$f.out $READER_EXTRA_PARAM"
|
||||
$CMD
|
||||
if [ ! -f result/$f.out ]; then
|
||||
cp /tmp/reader.$$.out result/$f.out
|
||||
fi
|
||||
done
|
||||
NUM_DIFF=`${CMD_DIFF} result/$f.out /tmp/reader.$$.out | wc -l`
|
||||
else
|
||||
if [ $FORCE_PARALLEL_UTESTS -eq 1 ]; then
|
||||
printf "ERROR (ndpiReader${EXE_SUFFIX} exit code: ${CMD_RET})\n"
|
||||
else
|
||||
printf "%-48s\tERROR (ndpiReader${EXE_SUFFIX} exit code: ${CMD_RET})\n" "$f"
|
||||
FAILURES+=("$f") #TODO: find a way to update this variable also in parallel mode
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ $NUM_DIFF -eq 0 ]; then
|
||||
if [ $FORCE_PARALLEL_UTESTS -eq 0 ]; then
|
||||
printf "%-48s\tOK\n" "$f"
|
||||
fi
|
||||
return 0
|
||||
else
|
||||
if [ $FORCE_PARALLEL_UTESTS -eq 1 ]; then
|
||||
printf "ERROR\n"
|
||||
else
|
||||
printf "%-48s\tERROR\n" "$f"
|
||||
FAILURES+=("$f") #TODO: find a way to update this variable also in parallel mode
|
||||
fi
|
||||
echo "$CMD [old vs new]"
|
||||
${CMD_DIFF} result/$f.out /tmp/reader.$$.out
|
||||
if [ ! -z "${CMD_COLORDIFF}" -a ! -z "${CMD_WDIFF}" ]; then
|
||||
${CMD_WDIFF} -n -3 result/$f.out /tmp/reader.$$.out | sort | uniq | ${CMD_COLORDIFF}
|
||||
fi
|
||||
if [ $FORCE_UPDATING_UTESTS_RESULTS -eq 1 ]; then
|
||||
cp /tmp/reader.$$.out result/$f.out
|
||||
fi
|
||||
fi
|
||||
|
||||
/bin/rm -f /tmp/reader.$$.out
|
||||
|
||||
return 1
|
||||
}
|
||||
export -f run_single_pcap
|
||||
|
||||
check_results() {
|
||||
for f in $PCAPS; do
|
||||
if [ -n "$*" ]; then
|
||||
SKIP_PCAP=1
|
||||
for i in $* ; do [ "$f" = "$i" ] && SKIP_PCAP=0 && break ; done
|
||||
[ $SKIP_PCAP = 1 ] && continue
|
||||
fi
|
||||
SKIP_PCAP=0
|
||||
if [ $PCRE2_ENABLED -eq 0 ]; then
|
||||
for p in $PCRE_PCAPS; do
|
||||
if [ $f = $p ]; then
|
||||
SKIP_PCAP=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ $NBPF_ENABLED -eq 0 ]; then
|
||||
for p in $NBPF_PCAPS; do
|
||||
if [ $f = $p ]; then
|
||||
SKIP_PCAP=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ $SKIP_PCAP -eq 1 ]; then
|
||||
printf "%-48s\tSKIPPED\n" "$f"
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ -f result/$f.out ]; then
|
||||
CMD="$READER -i pcap/$f -w /tmp/reader.$$.out $READER_EXTRA_PARAM"
|
||||
CPUPROFILE=./result/$f.cprof HEAPPROFILE=./result/$f $CMD
|
||||
CMD_RET=$?
|
||||
if [ $CMD_RET -eq 0 ]; then
|
||||
NUM_DIFF=`${CMD_DIFF} result/$f.out /tmp/reader.$$.out | wc -l`
|
||||
else
|
||||
printf "%-48s\tERROR (ndpiReader${EXE_SUFFIX} exit code: ${CMD_RET})\n" "$f"
|
||||
RC=$(( RC + 1 ))
|
||||
FAILURES+=("$f.out")
|
||||
continue
|
||||
fi
|
||||
|
||||
if [ $NUM_DIFF -eq 0 ]; then
|
||||
printf "%-48s\tOK\n" "$f"
|
||||
else
|
||||
printf "%-48s\tERROR\n" "$f"
|
||||
echo "$CMD [old vs new]"
|
||||
${CMD_DIFF} result/$f.out /tmp/reader.$$.out
|
||||
if [ ! -z "${CMD_COLORDIFF}" -a ! -z "${CMD_WDIFF}" ]; then
|
||||
${CMD_WDIFF} -n -3 result/$f.out /tmp/reader.$$.out | sort | uniq | ${CMD_COLORDIFF}
|
||||
fi
|
||||
RC=$(( RC + 1 ))
|
||||
FAILURES+=("$f.out")
|
||||
if [ $FORCE_UPDATING_UTESTS_RESULTS -eq 1 ]; then
|
||||
cp /tmp/reader.$$.out result/$f.out
|
||||
fi
|
||||
fi
|
||||
|
||||
/bin/rm -f /tmp/reader.$$.out
|
||||
fi
|
||||
done
|
||||
if [ $FORCE_PARALLEL_UTESTS -eq 1 ]; then
|
||||
parallel --bar --tag "run_single_pcap" ::: $PCAPS
|
||||
RET=$? #Number of failed job up to 100
|
||||
RC=$(( RC + $RET ))
|
||||
else
|
||||
for f in $PCAPS; do
|
||||
run_single_pcap $f
|
||||
RET=$?
|
||||
RC=$(( RC + $RET ))
|
||||
done
|
||||
fi
|
||||
|
||||
if [ ${GPROF_ENABLED} -eq 1 ]; then
|
||||
GPROF_ARGS='-nodecount 100 -nodefraction 0 -symbolize=fastlocal'
|
||||
@@ -177,15 +230,19 @@ for d in $(find ./cfgs/* -type d -maxdepth 0 2>/dev/null) ; do
|
||||
|
||||
cd ./cfgs/"$(basename $d)"
|
||||
|
||||
PCAPS=`cd pcap; /bin/ls *.*cap*`
|
||||
if [ "$#" -ne 0 ]; then
|
||||
PCAPS=$*
|
||||
else
|
||||
PCAPS=`cd pcap; /bin/ls *.*cap*`
|
||||
fi
|
||||
FAILURES=()
|
||||
READER_EXTRA_PARAM=""
|
||||
[ -f config.txt ] && READER_EXTRA_PARAM=$(< config.txt)
|
||||
export READER_EXTRA_PARAM
|
||||
|
||||
echo "Run configuration \""$(basename $d)"\" [$READER_EXTRA_PARAM]"
|
||||
|
||||
build_results
|
||||
check_results $*
|
||||
check_results
|
||||
|
||||
test ${#FAILURES} -ne 0 && printf '%s: %s\n' "${0}" "${RC} pcap(s) failed"
|
||||
test ${#FAILURES} -ne 0 && echo "Failed: ${FAILURES[@]}"
|
||||
|
||||
Reference in New Issue
Block a user