Simplified the process of adding lines to AC_AUTOMATA_t.
Use the ndpi_string_to_automa() function to add patterns with domain names.
For other cases can use ndpi_add_string_value_to_automa().
ac_automata_feature(ac_automa, AC_FEATURE_LC) allows adding
and compare data in a case insensitive manner. For mandatory pattern comparison
from the end of the line, the "ac_pattern.rep.at_end=1" flag is used.
This eliminated unnecessary conversions to lowercase and adding "$" for
end-of-line matching in domain name patterns.
ac_match_handler() has been renamed ac_domain_match_handler() and has been greatly simplified.
ac_domain_match_handler() looks for the template with the highest domain level.
For special cases it is possible to manually specify the domain level.
Added test for checking ambiguous domain names like:
- short.weixin.qq.com is QQ, not Wechat
- instagram.faae1-1.fna.fbcdn.net is Instagram, not Facebook
If you specify a NULL handler when creating the AC_AUTOMATA_t structure,
then a pattern with the maximum length that satisfies the search conditions will be found
(exact match, from the beginning of the string, from the end of the string, or a substring).
Added debugging for ac_automata_search.
To do this, you need to enable debugging globally using ac_automata_enable_debug(1) and
enable debugging in the AC_AUTOMATA_t structure using ac_automata_name("name", AC_FEATURE_DEBUG).
The search will display "name" and a list of matching patterns.
Running "AHO_DEBUG=1 ndpiReader ..." will show the lines that were searched for templates
and which templates were found.
The ac_automata_dump() prototype has been changed. Now it outputs data to a file.
If it is specified as NULL, then the output will be directed to stdout.
If you need to get data as a string, then use open_memstream().
Added the ability to run individual tests via the do.sh script
84 lines
2.2 KiB
Bash
Executable File
84 lines
2.2 KiB
Bash
Executable File
#!/bin/sh
|
|
|
|
cd "$(dirname "${0}")"
|
|
|
|
FUZZY_TESTING_ENABLED=@BUILD_FUZZTARGETS@
|
|
|
|
GCRYPT_ENABLED=@GCRYPT_ENABLED@
|
|
GCRYPT_PCAPS="gquic.pcap quic-23.pcap quic-24.pcap quic-27.pcap quic-28.pcap quic-29.pcap quic-mvfst-22.pcap quic-mvfst-27.pcapng quic-mvfst-exp.pcap quic_q50.pcap quic_t50.pcap quic_t51.pcap quic_0RTT.pcap quic_interop_V.pcapng quic-33.pcapng doq.pcapng doq_adguard.pcapng dlt_ppp.pcap os_detected.pcapng"
|
|
READER="../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt -r ../example/risky_domains.txt -j ../example/ja3_fingerprints.csv -S ../example/sha1_fingerprints.csv"
|
|
|
|
RC=0
|
|
PCAPS=`cd pcap; /bin/ls *.pcap *.pcapng`
|
|
|
|
if [ ! -x "../example/ndpiReader" ]; then
|
|
echo "$0: Missing $(realpath ../example/ndpiReader)"
|
|
echo "$0: Run ./configure and make first"
|
|
exit 1
|
|
fi
|
|
|
|
fuzzy_testing() {
|
|
if [ -f ../fuzz/fuzz_ndpi_reader ]; then
|
|
../fuzz/fuzz_ndpi_reader -max_total_time="${MAX_TOTAL_TIME:-592}" -print_pcs=1 -workers="${FUZZY_WORKERS:-0}" -jobs="${FUZZY_JOBS:-0}" pcap/
|
|
fi
|
|
}
|
|
|
|
build_results() {
|
|
for f in $PCAPS; do
|
|
#echo $f
|
|
# create result files if not present
|
|
if [ ! -f result/$f.out ]; then
|
|
CMD="$READER -q -t -i pcap/$f -w result/$f.out -v 2"
|
|
$CMD
|
|
fi
|
|
done
|
|
}
|
|
|
|
check_results() {
|
|
for f in $PCAPS; do
|
|
if [ -n "$*" ]; then
|
|
SKIP_PCAP=1
|
|
for i in $* ; do [ "$f" = "$i" ] && SKIP_PCAP=0 && break ; done
|
|
[ $SKIP_PCAP = 1 ] && continue
|
|
fi
|
|
SKIP_PCAP=0
|
|
if [ $GCRYPT_ENABLED -eq 0 ]; then
|
|
for g in $GCRYPT_PCAPS; do
|
|
if [ $f = $g ]; then
|
|
SKIP_PCAP=1
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
if [ $SKIP_PCAP -eq 1 ]; then
|
|
printf "%-32s\tSKIPPED\n" "$f"
|
|
continue
|
|
fi
|
|
|
|
if [ -f result/$f.out ]; then
|
|
CMD="$READER -q -t -i pcap/$f -w /tmp/reader.out -v 2"
|
|
$CMD
|
|
NUM_DIFF=`diff result/$f.out /tmp/reader.out | wc -l`
|
|
|
|
if [ $NUM_DIFF -eq 0 ]; then
|
|
printf "%-32s\tOK\n" "$f"
|
|
else
|
|
printf "%-32s\tERROR\n" "$f"
|
|
echo "$CMD [old vs new]"
|
|
diff result/$f.out /tmp/reader.out
|
|
RC=1
|
|
fi
|
|
|
|
/bin/rm /tmp/reader.out
|
|
fi
|
|
done
|
|
}
|
|
|
|
if [ $FUZZY_TESTING_ENABLED -eq 1 ]; then
|
|
fuzzy_testing
|
|
fi
|
|
build_results
|
|
check_results $*
|
|
|
|
exit $RC
|