Add unit tests to travis. Move ndpi serializer tests to unit tests.

This commit is contained in:
Alfredo Cardigliano
2020-09-21 17:24:06 +02:00
parent a263ac9024
commit e6d206fd15
6 changed files with 303 additions and 152 deletions
+13
View File
@@ -13,6 +13,7 @@ matrix:
packages:
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
- os: linux
@@ -23,6 +24,7 @@ matrix:
packages:
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
- os: linux
@@ -34,6 +36,7 @@ matrix:
packages:
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
- os: linux
@@ -45,6 +48,7 @@ matrix:
packages:
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
# Targets below have been disabled as we have no way
@@ -57,6 +61,7 @@ matrix:
# apt:
# packages:
# - libpcap-dev
# - libjson-c-dev
# - autogen
#
# - os: linux
@@ -67,6 +72,7 @@ matrix:
# apt:
# packages:
# - libpcap-dev
# - libjson-c-dev
# - autogen
- os: linux
@@ -79,6 +85,7 @@ matrix:
- g++-8
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
env:
- MATRIX_EVAL="CC=gcc-8"
@@ -93,6 +100,7 @@ matrix:
- g++-9
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
env:
- MATRIX_EVAL="CC=gcc-9"
@@ -108,6 +116,7 @@ matrix:
- clang-8
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
env:
- MATRIX_EVAL="CC=clang-8"
@@ -123,6 +132,7 @@ matrix:
- clang-7
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
env:
- MATRIX_EVAL="CC=clang-7"
@@ -140,6 +150,7 @@ matrix:
- clang-7
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
- name: fuzzm
env: CXXFLAGS="-g3 -O0 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" CFLAGS="-g3 -O0 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=memory -fsanitize=fuzzer-no-link" LDFLAGS="-g3 -O0 -fsanitize=memory" QA_FUZZ=msan CC=clang && CXX=clang++
@@ -151,6 +162,7 @@ matrix:
packages:
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
- name: fuzzu
env: CXXFLAGS="-g3 -O0 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fsanitize=fuzzer-no-link" CFLAGS="-g3 -O0 -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=undefined -fno-sanitize-recover=undefined,integer -fsanitize=fuzzer-no-link" LDFLAGS="-g3 -O0 -fsanitize=undefined" QA_FUZZ=ubsan CC=clang-7 && CXX=clang++-7
@@ -165,6 +177,7 @@ matrix:
- clang-7
- libpcap-dev
- libgcrypt20-dev
- libjson-c-dev
- autogen
+2 -2
View File
@@ -5,7 +5,7 @@ AC_CONFIG_MACRO_DIR([m4])
AM_INIT_AUTOMAKE([foreign subdir-objects])
EXTRA_TARGETS="example tests"
EXTRA_TARGETS="example tests tests/unit"
AC_ARG_WITH(only-libndpi, AS_HELP_STRING([--with-only-libndpi], [Build only libndpi (no examples, tests etc)]))
AS_IF([test "${with_only_libndpi+set}" = set],[
EXTRA_TARGETS=""
@@ -189,7 +189,7 @@ if test "${with_pcre+set}" = set; then :
fi
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
AC_CONFIG_HEADERS(src/include/ndpi_config.h)
AC_SUBST(GIT_RELEASE)
AC_SUBST(NDPI_MAJOR)
-150
View File
@@ -3386,155 +3386,6 @@ void automataUnitTest() {
/* *********************************************** */
void serializerUnitTest() {
ndpi_serializer serializer, deserializer;
int i, loop_id;
u_int8_t trace = 0;
ndpi_serialization_format fmt;
for(loop_id=0; loop_id<3; loop_id++) {
switch(loop_id) {
case 0:
if (trace) printf("--- TLV test ---\n");
fmt = ndpi_serialization_format_tlv;
break;
case 1:
if (trace) printf("--- JSON test ---\n");
fmt = ndpi_serialization_format_json;
break;
case 2:
if (trace) printf("--- CSV test ---\n");
fmt = ndpi_serialization_format_csv;
break;
}
assert(ndpi_init_serializer(&serializer, fmt) != -1);
for(i=0; i<16; i++) {
char kbuf[32], vbuf[32];
snprintf(kbuf, sizeof(kbuf), "Key %d", i);
snprintf(vbuf, sizeof(vbuf), "Value %d", i);
assert(ndpi_serialize_uint32_uint32(&serializer, i, i*i) != -1);
assert(ndpi_serialize_uint32_string(&serializer, i, "Data") != -1);
assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
assert(ndpi_serialize_string_uint32(&serializer, kbuf, i*i) != -1);
assert(ndpi_serialize_string_float(&serializer, kbuf, (float)(i*i), "%f") != -1);
if ((i&0x3) == 0x3) ndpi_serialize_end_of_record(&serializer);
}
if (fmt == ndpi_serialization_format_json) {
assert(ndpi_serialize_start_of_list(&serializer, "List") != -1);
for(i=0; i<4; i++) {
char kbuf[32], vbuf[32];
snprintf(kbuf, sizeof(kbuf), "Ignored");
snprintf(vbuf, sizeof(vbuf), "Item %d", i);
assert(ndpi_serialize_uint32_uint32(&serializer, i, i*i) != -1);
assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
assert(ndpi_serialize_string_float(&serializer, kbuf, (float)(i*i), "%f") != -1);
}
assert(ndpi_serialize_end_of_list(&serializer) != -1);
assert(ndpi_serialize_string_string(&serializer, "Last", "Ok") != -1);
if(trace) {
u_int32_t buffer_len = 0;
char *buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
printf("%s\n", buffer);
}
} else if (fmt == ndpi_serialization_format_csv) {
if(trace) {
u_int32_t buffer_len = 0;
char *buffer;
buffer = ndpi_serializer_get_header(&serializer, &buffer_len);
printf("%s\n", buffer);
buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
printf("%s\n", buffer);
}
} else {
if(trace)
printf("Serialization size: %u\n", ndpi_serializer_get_buffer_len(&serializer));
assert(ndpi_init_deserializer(&deserializer, &serializer) != -1);
while(1) {
ndpi_serialization_type kt, et;
et = ndpi_deserialize_get_item_type(&deserializer, &kt);
if(et == ndpi_serialization_unknown) {
break;
} else if(et == ndpi_serialization_end_of_record) {
if (trace) printf("EOR\n");
} else {
u_int32_t k32, v32;
ndpi_string ks, vs;
float vf;
switch(kt) {
case ndpi_serialization_uint32:
ndpi_deserialize_key_uint32(&deserializer, &k32);
if(trace) printf("%u=", k32);
break;
case ndpi_serialization_string:
ndpi_deserialize_key_string(&deserializer, &ks);
if (trace) {
u_int8_t bkp = ks.str[ks.str_len];
ks.str[ks.str_len] = '\0';
printf("%s=", ks.str);
ks.str[ks.str_len] = bkp;
}
break;
default:
printf("ERROR: Unsupported TLV key type %u\n", kt);
exit(0);
return;
}
switch(et) {
case ndpi_serialization_uint32:
assert(ndpi_deserialize_value_uint32(&deserializer, &v32) != -1);
if(trace) printf("%u\n", v32);
break;
case ndpi_serialization_string:
assert(ndpi_deserialize_value_string(&deserializer, &vs) != -1);
if(trace) {
u_int8_t bkp = vs.str[vs.str_len];
vs.str[vs.str_len] = '\0';
printf("%s\n", vs.str);
vs.str[vs.str_len] = bkp;
}
break;
case ndpi_serialization_float:
assert(ndpi_deserialize_value_float(&deserializer, &vf) != -1);
if(trace) printf("%f\n", vf);
break;
default:
if (trace) printf("\n");
printf("serializerUnitTest: unsupported type %u detected!\n", et);
return;
}
}
ndpi_deserialize_next(&deserializer);
}
}
ndpi_term_serializer(&serializer);
}
if (trace)
exit(0);
}
/* *********************************************** */
// #define RUN_DATA_ANALYSIS_THEN_QUIT 1
void analyzeUnitTest() {
@@ -3676,7 +3527,6 @@ int orginal_main(int argc, char **argv) {
hllUnitTest();
bitmapUnitTest();
automataUnitTest();
serializerUnitTest();
analyzeUnitTest();
ndpi_self_check_host_match();
analysisUnitTest();
+10
View File
@@ -3,6 +3,7 @@
cd "$(dirname "${0}")"
READER="../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt"
UNIT="./unit/unit"
RC=0
PCAPS=`cd pcap; /bin/ls *.pcap`
@@ -45,8 +46,17 @@ check_results() {
done
}
check_unit() {
$UNIT
UNIT_RC=$?
if [ $UNIT_RC -ne 0 ]; then
RC=1
fi
}
fuzzy_testing
build_results
check_results
check_unit
exit $RC
+33
View File
@@ -0,0 +1,33 @@
CC=@CC@
CXX=@CXX@
SRCHOME=../../src
JSON_INC = $(shell pkg-config --cflags json-c)
JSON_LIB = $(shell pkg-config --libs json-c)
CFLAGS=-g -fPIC -DPIC -I$(SRCHOME)/include $(JSON_INC) @CFLAGS@
LIBNDPI=$(SRCHOME)/lib/libndpi.a
LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @ADDITIONAL_LIBS@ $(JSON_LIB) -lpthread -lm @LDFLAGS@
HEADERS=$(SRCHOME)/include/ndpi_api.h $(SRCHOME)/include/ndpi_typedefs.h $(SRCHOME)/include/ndpi_protocol_ids.h
OBJS=unit
PREFIX?=@prefix@
all: unit
EXECUTABLE_SOURCES := unit.c
COMMON_SOURCES := $(filter-out $(EXECUTABLE_SOURCES),$(wildcard *.c ))
unit: $(LIBNDPI) unit.o
$(CC) $(CFLAGS) unit.o -o $@ $(LDFLAGS)
%.o: %.c $(HEADERS) Makefile
$(CC) $(CFLAGS) -c $< -o $@
clean:
/bin/rm -f *.o unit
/bin/rm -f .*.o.cmd .*.o.d
/bin/rm -rf build
distclean: clean
/bin/rm -f Makefile
+245
View File
@@ -0,0 +1,245 @@
/*
* unit.c
*
* Copyright (C) 2019-20 - ntop.org
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/
#ifdef linux
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <sched.h>
#endif /* linux */
#ifdef WIN32
#include <winsock2.h>
#include <process.h>
#include <io.h>
#define getopt getopt____
#else
#include <unistd.h>
#include <netinet/in.h>
#endif
#include <stdio.h>
#include <stdlib.h>
#include <getopt.h>
#include <string.h>
#include <stdarg.h>
#include <search.h>
#include <pcap.h>
#include <signal.h>
#include <pthread.h>
#include <sys/socket.h>
#include <assert.h>
#include <math.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <libgen.h>
#include "json.h"
#include "ndpi_config.h"
#include "ndpi_api.h"
static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;
static int verbose = 0;
/* *********************************************** */
int serializerUnitTest() {
ndpi_serializer serializer, deserializer;
int i, loop_id;
u_int8_t verbose = 0;
ndpi_serialization_format fmt;
u_int32_t buffer_len;
char *buffer;
enum json_tokener_error jerr;
json_object *j;
for(loop_id=0; loop_id<3; loop_id++) {
switch(loop_id) {
case 0:
if (verbose) printf("--- TLV test ---\n");
fmt = ndpi_serialization_format_tlv;
break;
case 1:
if (verbose) printf("--- JSON test ---\n");
fmt = ndpi_serialization_format_json;
break;
case 2:
if (verbose) printf("--- CSV test ---\n");
fmt = ndpi_serialization_format_csv;
break;
}
assert(ndpi_init_serializer(&serializer, fmt) != -1);
for(i=0; i<16; i++) {
char kbuf[32], vbuf[32];
snprintf(kbuf, sizeof(kbuf), "Key %d", i);
snprintf(vbuf, sizeof(vbuf), "Value %d", i);
assert(ndpi_serialize_uint32_uint32(&serializer, i, i*i) != -1);
assert(ndpi_serialize_uint32_string(&serializer, i, "Data") != -1);
assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
assert(ndpi_serialize_string_uint32(&serializer, kbuf, i*i) != -1);
assert(ndpi_serialize_string_float(&serializer, kbuf, (float)(i*i), "%f") != -1);
if ((i&0x3) == 0x3) ndpi_serialize_end_of_record(&serializer);
}
if (fmt == ndpi_serialization_format_json) {
assert(ndpi_serialize_start_of_list(&serializer, "List") != -1);
for(i=0; i<4; i++) {
char kbuf[32], vbuf[32];
snprintf(kbuf, sizeof(kbuf), "Ignored");
snprintf(vbuf, sizeof(vbuf), "Item %d", i);
assert(ndpi_serialize_uint32_uint32(&serializer, i, i*i) != -1);
assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
assert(ndpi_serialize_string_float(&serializer, kbuf, (float)(i*i), "%f") != -1);
}
assert(ndpi_serialize_end_of_list(&serializer) != -1);
assert(ndpi_serialize_string_string(&serializer, "Last", "Ok") != -1);
buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
if(verbose)
printf("%s\n", buffer);
/* Decoding JSON to validate syntax */
jerr = json_tokener_success;
j = json_tokener_parse_verbose(buffer, &jerr);
if (j == NULL) {
printf("%s: ERROR (json validation failed)\n", __FUNCTION__);
return -1;
} else {
/* Validation ok */
json_object_put(j);
}
} else if (fmt == ndpi_serialization_format_csv) {
if(verbose) {
u_int32_t buffer_len = 0;
char *buffer;
buffer = ndpi_serializer_get_header(&serializer, &buffer_len);
printf("%s\n", buffer);
buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
printf("%s\n", buffer);
}
} else {
if(verbose)
printf("Serialization size: %u\n", ndpi_serializer_get_buffer_len(&serializer));
assert(ndpi_init_deserializer(&deserializer, &serializer) != -1);
while(1) {
ndpi_serialization_type kt, et;
et = ndpi_deserialize_get_item_type(&deserializer, &kt);
if(et == ndpi_serialization_unknown) {
break;
} else if(et == ndpi_serialization_end_of_record) {
if (verbose) printf("EOR\n");
} else {
u_int32_t k32, v32;
ndpi_string ks, vs;
float vf;
switch(kt) {
case ndpi_serialization_uint32:
ndpi_deserialize_key_uint32(&deserializer, &k32);
if(verbose) printf("%u=", k32);
break;
case ndpi_serialization_string:
ndpi_deserialize_key_string(&deserializer, &ks);
if (verbose) {
u_int8_t bkp = ks.str[ks.str_len];
ks.str[ks.str_len] = '\0';
printf("%s=", ks.str);
ks.str[ks.str_len] = bkp;
}
break;
default:
printf("%s: ERROR (unsupported TLV key type %u)\n", __FUNCTION__, kt);
return -1;
}
switch(et) {
case ndpi_serialization_uint32:
assert(ndpi_deserialize_value_uint32(&deserializer, &v32) != -1);
if(verbose) printf("%u\n", v32);
break;
case ndpi_serialization_string:
assert(ndpi_deserialize_value_string(&deserializer, &vs) != -1);
if(verbose) {
u_int8_t bkp = vs.str[vs.str_len];
vs.str[vs.str_len] = '\0';
printf("%s\n", vs.str);
vs.str[vs.str_len] = bkp;
}
break;
case ndpi_serialization_float:
assert(ndpi_deserialize_value_float(&deserializer, &vf) != -1);
if(verbose) printf("%f\n", vf);
break;
default:
if (verbose) printf("\n");
printf("%s: ERROR (unsupported type %u detected)\n", __FUNCTION__, et);
return -1;
}
}
ndpi_deserialize_next(&deserializer);
}
}
ndpi_term_serializer(&serializer);
}
printf("%s: OK\n", __FUNCTION__);
return 0;
}
/* *********************************************** */
int main(int argc, char **argv) {
if (ndpi_get_api_version() != NDPI_API_VERSION) {
printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
return -1;
}
ndpi_info_mod = ndpi_init_detection_module(ndpi_no_prefs);
if (ndpi_info_mod == NULL)
return -1;
/* Tests */
if (serializerUnitTest() != 0) return -1;
return 0;
}