Added support for roaring bitmap v3 (#2355)

* Integrated RoaringBitmap v3

* Renamed ndpi_bitmap64 ro ndpi_bitmap64_fuse

* Fixes to ndpi_bitmap for new roaring library

* Fixes for bitmap serialization

* Fixed format

* Warning fix

* Conversion fix

* Warning fix

* Added check for roaring v3 support

* Updated file name

* Updated path

* Uses clang-9 (instead of clang-7) for builds

* Fixed fuzz_ds_bitmap64_fuse

* Fixes nDPI printf handling

* Disabled printf

* Yet another printf fix

* Cleaup

* Fx for compiling on older platforms

* Fixes for old compilers

* Initialization changes

* Added compiler check

* Fixes for old compilers

* Inline function is not static inline

* Added missing include
This commit is contained in:
Luca Deri
2024-03-25 08:15:19 +01:00
committed by GitHub
parent be0fd7444f
commit 51f5fc7140
86 changed files with 33332 additions and 4377 deletions
+1 -1
View File
@@ -107,7 +107,7 @@ jobs:
msan: "--with-sanitizer"
nBPF: ""
lto_gold_linker: "--with-lto-and-gold-linker"
- compiler: "clang-7" # "Oldest" clang easily available
- compiler: "clang-9" # "Oldest" clang easily available
os: ubuntu-20.04
arch: "x86_64"
gcrypt: ""
+19
View File
@@ -120,6 +120,25 @@ AC_CHECK_TOOL(AR, ar, [false])
AC_LANG_WERROR
AC_MSG_CHECKING([whether roaring v3 works])
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
#if __has_include (<stdio.h>)
#endif
]])],
[
AC_MSG_RESULT([yes])
],
[
AC_MSG_RESULT([no])
AC_DEFINE_UNQUOTED(USE_ROARING_V2, "1", [Use CRoaring 2.1.x])
])
GCC_VERSION=`gcc --version | cut -d ' ' -f 3 | head -1|cut -d '.' -f 1`
if [ "${GCC_VERSION}" -lt "7" ]; then
AC_DEFINE_UNQUOTED(USE_ROARING_V2, "1", [Use CRoaring 2.1.x])
fi
NDPI_MAJOR=`echo "${PACKAGE_VERSION}" | cut -d . -f 1`
NDPI_MINOR=`echo "${PACKAGE_VERSION}" | cut -d . -f 2`
NDPI_PATCH=`echo "${PACKAGE_VERSION}" | cut -d . -f 3`
+3 -3
View File
@@ -5605,7 +5605,7 @@ void compressedBitmapUnitTest() {
size_t ser;
char *buf;
ndpi_bitmap_iterator *it;
u_int32_t value;
u_int64_t value;
for(i=0; i<1000; i++) {
u_int32_t v = rand();
@@ -5621,12 +5621,12 @@ void compressedBitmapUnitTest() {
assert(ser > 0);
if(trace) printf("len: %u\n", (unsigned int)ser);
b1 = ndpi_bitmap_deserialize(buf);
b1 = ndpi_bitmap_deserialize(buf, ser);
assert(b1);
assert((it = ndpi_bitmap_iterator_alloc(b)));
while(ndpi_bitmap_iterator_next(it, &value)) {
if(trace) printf("%u ", value);
if(trace) printf("%lu ", (unsigned long)value);
}
if(trace) printf("\n");
+16 -16
View File
@@ -2,7 +2,7 @@ bin_PROGRAMS = fuzz_process_packet fuzz_ndpi_reader fuzz_ndpi_reader_alloc_fail
#Alghoritms
bin_PROGRAMS += fuzz_alg_bins fuzz_alg_hll fuzz_alg_hw_rsi_outliers_da fuzz_alg_jitter fuzz_alg_ses_des fuzz_alg_crc32_md5 fuzz_alg_bytestream
#Data structures
bin_PROGRAMS += fuzz_ds_patricia fuzz_ds_ahocorasick fuzz_ds_libcache fuzz_ds_tree fuzz_ds_ptree fuzz_ds_hash fuzz_ds_cmsketch fuzz_ds_bitmap64 fuzz_ds_domain_classify
bin_PROGRAMS += fuzz_ds_patricia fuzz_ds_ahocorasick fuzz_ds_libcache fuzz_ds_tree fuzz_ds_ptree fuzz_ds_hash fuzz_ds_cmsketch fuzz_ds_bitmap64_fuse fuzz_ds_domain_classify
#Third party
bin_PROGRAMS += fuzz_libinjection fuzz_binaryfusefilter
#Internal crypto
@@ -328,20 +328,20 @@ fuzz_ds_cmsketch_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
$(fuzz_ds_cmsketch_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
fuzz_ds_bitmap64_SOURCES = fuzz_ds_bitmap64.cpp fuzz_common_code.c
fuzz_ds_bitmap64_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
fuzz_ds_bitmap64_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
fuzz_ds_bitmap64_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS)
fuzz_ds_bitmap64_LDFLAGS = $(LIBS)
fuzz_ds_bitmap64_fuse_SOURCES = fuzz_ds_bitmap64_fuse.cpp fuzz_common_code.c
fuzz_ds_bitmap64_fuse_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
fuzz_ds_bitmap64_fuse_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
fuzz_ds_bitmap64_fuse_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS)
fuzz_ds_bitmap64_fuse_LDFLAGS = $(LIBS)
if HAS_FUZZLDFLAGS
fuzz_ds_bitmap64_CXXFLAGS += $(LIB_FUZZING_ENGINE)
fuzz_ds_bitmap64_CFLAGS += $(LIB_FUZZING_ENGINE)
fuzz_ds_bitmap64_LDFLAGS += $(LIB_FUZZING_ENGINE)
fuzz_ds_bitmap64_fuse_CXXFLAGS += $(LIB_FUZZING_ENGINE)
fuzz_ds_bitmap64_fuse_CFLAGS += $(LIB_FUZZING_ENGINE)
fuzz_ds_bitmap64_fuse_LDFLAGS += $(LIB_FUZZING_ENGINE)
endif
# force usage of CXX for linker
fuzz_ds_bitmap64_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
fuzz_ds_bitmap64_fuse_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
$(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
$(fuzz_ds_bitmap64_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
$(fuzz_ds_bitmap64_fuse_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
fuzz_ds_domain_classify_SOURCES = fuzz_ds_domain_classify.cpp fuzz_common_code.c
fuzz_ds_domain_classify_CXXFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
@@ -729,15 +729,15 @@ files_corpus_fuzz_readerutils_parseprotolist := $(wildcard corpus/fuzz_readerut
fuzz_readerutils_parseprotolist_seed_corpus.zip: $(files_corpus_fuzz_readerutils_parseprotolist)
zip -j fuzz_readerutils_parseprotolist_seed_corpus.zip $(files_corpus_fuzz_readerutils_parseprotolist)
files_corpus_fuzz_ds_bitmap64 := $(wildcard corpus/fuzz_ds_bitmap64/*)
fuzz_ds_bitmap64_seed_corpus.zip: $(files_corpus_fuzz_ds_bitmap64)
zip -j fuzz_ds_bitmap64_seed_corpus.zip $(files_corpus_fuzz_ds_bitmap64)
files_corpus_fuzz_ds_bitmap64_fuse := $(wildcard corpus/fuzz_ds_bitmap64_fuse/*)
fuzz_ds_bitmap64_fuse_seed_corpus.zip: $(files_corpus_fuzz_ds_bitmap64_fuse)
zip -j fuzz_ds_bitmap64_fuse_seed_corpus.zip $(files_corpus_fuzz_ds_bitmap64_fuse)
files_corpus_fuzz_ds_domain_classify := $(wildcard corpus/fuzz_ds_domain_classify/*)
fuzz_ds_domain_classify_seed_corpus.zip: $(files_corpus_fuzz_ds_domain_classify)
zip -j fuzz_ds_domain_classify_seed_corpus.zip $(files_corpus_fuzz_ds_domain_classify)
corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_filecfg_categories_seed_corpus.zip fuzz_filecfg_config_seed_corpus.zip fuzz_filecfg_category_seed_corpus.zip fuzz_is_stun_udp_seed_corpus.zip fuzz_is_stun_tcp_seed_corpus.zip fuzz_filecfg_malicious_sha1_seed_corpus.zip fuzz_filecfg_malicious_ja3_seed_corpus.zip fuzz_filecfg_risk_domains_seed_corpus.zip
corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_fuse_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_filecfg_categories_seed_corpus.zip fuzz_filecfg_config_seed_corpus.zip fuzz_filecfg_category_seed_corpus.zip fuzz_is_stun_udp_seed_corpus.zip fuzz_is_stun_tcp_seed_corpus.zip fuzz_filecfg_malicious_sha1_seed_corpus.zip fuzz_filecfg_malicious_ja3_seed_corpus.zip fuzz_filecfg_risk_domains_seed_corpus.zip
cp corpus/fuzz_*seed_corpus.zip .
#Create dictionaries exactly as expected by oss-fuzz.
@@ -787,7 +787,7 @@ distdir:
-o -path './corpus/fuzz_alg_crc32_md5/*' \
-o -path './corpus/fuzz_alg_bytestream/*' \
-o -path './corpus/fuzz_ds_libcache/*' \
-o -path './corpus/fuzz_ds_bitmap64/*' \
-o -path './corpus/fuzz_ds_bitmap64_fuse/*' \
-o -path './corpus/fuzz_ds_domain_classify/*' \
-o -path './corpus/fuzz_ds_ptree/*' | xargs -I'{}' cp -r '{}' '$(distdir)/{}'
@@ -7,23 +7,23 @@
extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
FuzzedDataProvider fuzzed_data(data, size);
u_int16_t i, num_iteration, is_added = 0;
ndpi_bitmap64 *b;
ndpi_bitmap64_fuse *b;
bool rc;
u_int64_t value, value_added;
/* To allow memory allocation failures */
fuzz_set_alloc_callbacks_and_seed(size);
b = ndpi_bitmap64_alloc();
b = ndpi_bitmap64_fuse_alloc();
if(fuzzed_data.ConsumeBool())
ndpi_bitmap64_compress(b);
ndpi_bitmap64_fuse_compress(b);
num_iteration = fuzzed_data.ConsumeIntegral<u_int16_t>();
for (i = 0; i < num_iteration; i++) {
value = fuzzed_data.ConsumeIntegral<u_int64_t>();
rc = ndpi_bitmap64_set(b, value);
rc = ndpi_bitmap64_fuse_set(b, value);
/* Keep one random entry really added */
if (rc == true && is_added == 0 && fuzzed_data.ConsumeBool()) {
value_added = value;
@@ -32,23 +32,23 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
}
if(fuzzed_data.ConsumeBool())
ndpi_bitmap64_compress(b);
ndpi_bitmap64_fuse_compress(b);
/* "Random" search */
num_iteration = fuzzed_data.ConsumeIntegral<u_int8_t>();
for (i = 0; i < num_iteration; i++) {
value = fuzzed_data.ConsumeIntegral<u_int64_t>();
ndpi_bitmap64_isset(b, value);
ndpi_bitmap64_fuse_isset(b, value);
}
/* Search of an added entry */
if (is_added) {
ndpi_bitmap64_isset(b, value_added);
ndpi_bitmap64_fuse_isset(b, value_added);
}
ndpi_bitmap64_size(b);
ndpi_bitmap64_fuse_size(b);
ndpi_bitmap64_free(b);
ndpi_bitmap64_fuse_free(b);
return 0;
}
+12 -14
View File
@@ -2012,18 +2012,16 @@ extern "C" {
*/
ndpi_bitmap* ndpi_bitmap_alloc(void);
ndpi_bitmap* ndpi_bitmap_alloc_size(u_int32_t size);
void ndpi_bitmap_free(ndpi_bitmap* b);
ndpi_bitmap* ndpi_bitmap_copy(ndpi_bitmap* b);
u_int64_t ndpi_bitmap_cardinality(ndpi_bitmap* b);
bool ndpi_bitmap_is_empty(ndpi_bitmap* b);
void ndpi_bitmap_set(ndpi_bitmap* b, u_int32_t value);
void ndpi_bitmap_unset(ndpi_bitmap* b, u_int32_t value);
bool ndpi_bitmap_isset(ndpi_bitmap* b, u_int32_t value);
void ndpi_bitmap_clear(ndpi_bitmap* b);
void ndpi_bitmap_set(ndpi_bitmap* b, u_int64_t value);
void ndpi_bitmap_unset(ndpi_bitmap* b, u_int64_t value);
bool ndpi_bitmap_isset(ndpi_bitmap* b, u_int64_t value);
size_t ndpi_bitmap_serialize(ndpi_bitmap* b, char **buf);
ndpi_bitmap* ndpi_bitmap_deserialize(char *buf);
ndpi_bitmap* ndpi_bitmap_deserialize(char *buf, size_t buf_len);
void ndpi_bitmap_and(ndpi_bitmap* a, ndpi_bitmap* b_and);
ndpi_bitmap* ndpi_bitmap_and_alloc(ndpi_bitmap* a, ndpi_bitmap* b_and);
@@ -2035,7 +2033,7 @@ extern "C" {
ndpi_bitmap_iterator* ndpi_bitmap_iterator_alloc(ndpi_bitmap* b);
void ndpi_bitmap_iterator_free(ndpi_bitmap* b);
bool ndpi_bitmap_iterator_next(ndpi_bitmap_iterator* i, u_int32_t *value);
bool ndpi_bitmap_iterator_next(ndpi_bitmap_iterator* i, u_int64_t *value);
/* ******************************* */
@@ -2045,16 +2043,16 @@ extern "C" {
This is
- a probabilistic datastructure !!! (i.e. be prepared to false positives)
- immutable (i.e. adding keys after a search (i.e. ndpi_bitmap64_isset)
- immutable (i.e. adding keys after a search (i.e. ndpi_bitmap64_fuse_isset)
is not allowed
*/
ndpi_bitmap64* ndpi_bitmap64_alloc(void);
bool ndpi_bitmap64_set(ndpi_bitmap64 *b, u_int64_t value);
bool ndpi_bitmap64_compress(ndpi_bitmap64 *b);
bool ndpi_bitmap64_isset(ndpi_bitmap64 *b, u_int64_t value);
void ndpi_bitmap64_free(ndpi_bitmap64 *b);
u_int32_t ndpi_bitmap64_size(ndpi_bitmap64 *b);
ndpi_bitmap64_fuse* ndpi_bitmap64_fuse_alloc(void);
bool ndpi_bitmap64_fuse_set(ndpi_bitmap64_fuse *b, u_int64_t value);
bool ndpi_bitmap64_fuse_compress(ndpi_bitmap64_fuse *b);
bool ndpi_bitmap64_fuse_isset(ndpi_bitmap64_fuse *b, u_int64_t value);
void ndpi_bitmap64_fuse_free(ndpi_bitmap64_fuse *b);
u_int32_t ndpi_bitmap64_fuse_size(ndpi_bitmap64_fuse *b);
/* ******************************* */
+1
View File
@@ -176,6 +176,7 @@
#define get_u_int32_t(X,O) (*(u_int32_t *)((&(((u_int8_t *)X)[O]))))
#if defined(__arm__)
#include <stdint.h>
#include <string.h>
static inline uint64_t get_u_int64_t(const uint8_t* X, int O)
{
uint64_t tmp;
+2 -1
View File
@@ -1141,6 +1141,7 @@ typedef struct ndpi_proto {
typedef void ndpi_bitmap;
typedef void ndpi_bitmap64;
typedef void ndpi_bitmap64_fuse; /* probabilistic */
typedef void ndpi_bitmap_iterator;
typedef void ndpi_filter;
@@ -1155,7 +1156,7 @@ typedef struct {
typedef struct {
struct {
u_int16_t class_id;
ndpi_bitmap64 *domains;
ndpi_bitmap64_fuse *domains;
} classes[MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS];
} ndpi_domain_classify;
+112 -24
View File
@@ -1,7 +1,7 @@
/*
* ndpi_bitmap.c
*
* Copyright (C) 2011-23 - ntop.org and contributors
* Copyright (C) 2011-24 - ntop.org and contributors
*
* This file is part of nDPI, an open source deep packet inspection
* library based on the OpenDPI and PACE technology by ipoque GmbH
@@ -35,145 +35,226 @@
#include "ndpi_includes.h"
#include "ndpi_encryption.h"
#ifdef USE_ROARING_V2
#include "third_party/include/roaring_v2.h"
#else
#include "third_party/include/roaring.h"
#endif
/* ******************************************* */
ndpi_bitmap* ndpi_bitmap_alloc() {
#ifdef USE_ROARING_V2
return((ndpi_bitmap*)roaring_bitmap_create());
}
/* ******************************************* */
ndpi_bitmap* ndpi_bitmap_alloc_size(u_int32_t size) {
return((ndpi_bitmap*)roaring_bitmap_create_with_capacity(size));
#else
return((ndpi_bitmap*)roaring64_bitmap_create());
#endif
}
/* ******************************************* */
void ndpi_bitmap_free(ndpi_bitmap* b) {
#ifdef USE_ROARING_V2
roaring_bitmap_free((const roaring_bitmap_t *)b);
#else
roaring64_bitmap_free((roaring64_bitmap_t *)b);
#endif
}
/* ******************************************* */
ndpi_bitmap* ndpi_bitmap_copy(ndpi_bitmap* b) {
#ifdef USE_ROARING_V2
return(roaring_bitmap_copy(b));
#else
return(roaring64_bitmap_copy(b));
#endif
}
/* ******************************************* */
u_int64_t ndpi_bitmap_cardinality(ndpi_bitmap* b) {
#ifdef USE_ROARING_V2
return(roaring_bitmap_get_cardinality((const roaring_bitmap_t *)b));
#else
return(roaring64_bitmap_get_cardinality((roaring64_bitmap_t *)b));
#endif
}
/* ******************************************* */
void ndpi_bitmap_set(ndpi_bitmap* b, u_int32_t value) {
void ndpi_bitmap_set(ndpi_bitmap* b, u_int64_t value) {
#ifdef USE_ROARING_V2
roaring_bitmap_add((roaring_bitmap_t *)b, value);
#else
roaring64_bitmap_add((roaring64_bitmap_t *)b, value);
#endif
}
/* ******************************************* */
void ndpi_bitmap_unset(ndpi_bitmap* b, u_int32_t value) {
void ndpi_bitmap_unset(ndpi_bitmap* b, u_int64_t value) {
#ifdef USE_ROARING_V2
roaring_bitmap_remove((roaring_bitmap_t *)b, value);
#else
roaring64_bitmap_remove((roaring64_bitmap_t *)b, value);
#endif
}
/* ******************************************* */
bool ndpi_bitmap_isset(ndpi_bitmap* b, u_int32_t value) {
return(roaring_bitmap_contains((const roaring_bitmap_t *)b, value));
}
bool ndpi_bitmap_isset(ndpi_bitmap* b, u_int64_t value) {
bool ret;
#ifdef USE_ROARING_V2
ret = roaring_bitmap_contains((const roaring_bitmap_t *)b, value);
#else
ret = roaring64_bitmap_contains((const roaring64_bitmap_t *)b, value);
#endif
/* ******************************************* */
void ndpi_bitmap_clear(ndpi_bitmap* b) {
roaring_bitmap_clear((roaring_bitmap_t *)b);
return(ret);
}
/* ******************************************* */
size_t ndpi_bitmap_serialize(ndpi_bitmap* b, char **buf) {
const roaring_bitmap_t *r = (const roaring_bitmap_t *)b;
size_t s = roaring_bitmap_size_in_bytes(r);
size_t s;
#ifdef USE_ROARING_V2
const roaring_bitmap_t *r = (const roaring_bitmap_t *)b;
s = roaring_bitmap_portable_size_in_bytes(r);
#else
const roaring64_bitmap_t *r = (const roaring64_bitmap_t *)b;
s = roaring64_bitmap_portable_size_in_bytes(r);
#endif
*buf = (char*)ndpi_malloc(s);
if((*buf) == NULL) return(0);
return(roaring_bitmap_serialize(r, *buf));
#ifdef USE_ROARING_V2
return(roaring_bitmap_portable_serialize(r, *buf));
#else
return(roaring64_bitmap_portable_serialize(r, *buf));
#endif
}
/* ******************************************* */
ndpi_bitmap* ndpi_bitmap_deserialize(char *buf) {
return((ndpi_bitmap*)roaring_bitmap_deserialize(buf));
ndpi_bitmap* ndpi_bitmap_deserialize(char *buf, size_t buf_len) {
#ifdef USE_ROARING_V2
return((ndpi_bitmap*)roaring_bitmap_portable_deserialize_safe(buf, buf_len));
#else
return((ndpi_bitmap*)roaring64_bitmap_portable_deserialize_safe(buf, buf_len));
#endif
}
/* ******************************************* */
/* b = b & b_and */
void ndpi_bitmap_and(ndpi_bitmap* a, ndpi_bitmap* b_and) {
#ifdef USE_ROARING_V2
roaring_bitmap_and_inplace((roaring_bitmap_t*)a, (roaring_bitmap_t*)b_and);
#else
roaring64_bitmap_and_inplace((roaring64_bitmap_t*)a, (roaring64_bitmap_t*)b_and);
#endif
}
/* ******************************************* */
/* b = b & b_and */
ndpi_bitmap* ndpi_bitmap_and_alloc(ndpi_bitmap* a, ndpi_bitmap* b_and) {
#ifdef USE_ROARING_V2
return((ndpi_bitmap*)roaring_bitmap_and((roaring_bitmap_t*)a, (roaring_bitmap_t*)b_and));
#else
return((ndpi_bitmap*)roaring64_bitmap_and((roaring64_bitmap_t*)a, (roaring64_bitmap_t*)b_and));
#endif
}
/* ******************************************* */
/* b = b & !b_and */
void ndpi_bitmap_andnot(ndpi_bitmap* a, ndpi_bitmap* b_and) {
#ifdef USE_ROARING_V2
roaring_bitmap_andnot_inplace((roaring_bitmap_t*)a, (roaring_bitmap_t*)b_and);
#else
roaring64_bitmap_andnot_inplace((roaring64_bitmap_t*)a, (roaring64_bitmap_t*)b_and);
#endif
}
/* ******************************************* */
/* b = b | b_or */
void ndpi_bitmap_or(ndpi_bitmap* a, ndpi_bitmap* b_or) {
#ifdef USE_ROARING_V2
roaring_bitmap_or_inplace((roaring_bitmap_t*)a, (roaring_bitmap_t*)b_or);
#else
roaring64_bitmap_or_inplace((roaring64_bitmap_t*)a, (roaring64_bitmap_t*)b_or);
#endif
}
/* ******************************************* */
/* b = b | b_or */
ndpi_bitmap* ndpi_bitmap_or_alloc(ndpi_bitmap* a, ndpi_bitmap* b_or) {
#ifdef USE_ROARING_V2
return((ndpi_bitmap*)roaring_bitmap_or((roaring_bitmap_t*)a, (roaring_bitmap_t*)b_or));
#else
return((ndpi_bitmap*)roaring64_bitmap_or((roaring64_bitmap_t*)a, (roaring64_bitmap_t*)b_or));
#endif
}
/* ******************************************* */
/* b = b ^ b_xor */
void ndpi_bitmap_xor(ndpi_bitmap* a, ndpi_bitmap* b_xor) {
#ifdef USE_ROARING_V2
roaring_bitmap_xor_inplace((roaring_bitmap_t*)a, (roaring_bitmap_t*)b_xor);
#else
roaring64_bitmap_xor_inplace((roaring64_bitmap_t*)a, (roaring64_bitmap_t*)b_xor);
#endif
}
/* ******************************************* */
void ndpi_bitmap_optimize(ndpi_bitmap* a) {
#ifdef USE_ROARING_V2
roaring_bitmap_run_optimize(a);
#else
roaring64_bitmap_run_optimize(a);
#endif
}
/* ******************************************* */
ndpi_bitmap_iterator* ndpi_bitmap_iterator_alloc(ndpi_bitmap* b) {
return(roaring_create_iterator((roaring_bitmap_t*)b));
#ifdef USE_ROARING_V2
return((ndpi_bitmap_iterator*)roaring_create_iterator((roaring_bitmap_t*)b));
#else
return((ndpi_bitmap_iterator*)roaring64_iterator_create((const roaring64_bitmap_t*)b));
#endif
}
/* ******************************************* */
void ndpi_bitmap_iterator_free(ndpi_bitmap* b) {
#ifdef USE_ROARING_V2
roaring_free_uint32_iterator((roaring_uint32_iterator_t*)b);
#else
roaring64_iterator_free((roaring64_iterator_t*)b);
#endif
}
/* ******************************************* */
bool ndpi_bitmap_is_empty(ndpi_bitmap* b) {
#ifdef USE_ROARING_V2
return(roaring_bitmap_is_empty((roaring_bitmap_t*)b));
#else
return(roaring64_bitmap_is_empty((roaring64_bitmap_t*)b));
#endif
}
/* ******************************************* */
@@ -182,8 +263,15 @@ bool ndpi_bitmap_is_empty(ndpi_bitmap* b) {
true is returned when a value is present, false when we reached the end
*/
bool ndpi_bitmap_iterator_next(ndpi_bitmap_iterator* i, uint32_t *value) {
uint32_t num = roaring_read_uint32_iterator((roaring_uint32_iterator_t*)i, value, 1);
bool ndpi_bitmap_iterator_next(ndpi_bitmap_iterator* i, u_int64_t *value) {
#ifdef USE_ROARING_V2
uint32_t ret;
uint32_t num = roaring_read_uint32_iterator((roaring_uint32_iterator_t*)i, &ret, 1);
*value = (uint32_t)ret;
#else
uint64_t num = roaring64_iterator_read((roaring64_iterator_t*)i, value, 1);
#endif
return((num == 1) ? true /* found */ : false /* not found */);
}
@@ -1,7 +1,7 @@
/*
* ndpi_bitmap64.c
* ndpi_bitmap64_fuse.c
*
* Copyright (C) 2011-23 - ntop.org and contributors
* Copyright (C) 2011-24 - ntop.org and contributors
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
@@ -30,7 +30,7 @@
#include "ndpi_api.h"
#include "third_party/include/binaryfusefilter.h"
#define NDPI_BITMAP64_REALLOC_SIZE 4096
#define NDPI_BITMAP64_FUSE_REALLOC_SIZE 4096
// #define PRINT_DUPLICATED_HASHS
@@ -39,16 +39,16 @@ typedef struct {
u_int64_t *entries;
bool is_compressed;
binary_fuse16_t bitmap;
} ndpi_bitmap64_t;
} ndpi_bitmap64_fuse_t;
/* ********************************************************** */
ndpi_bitmap64* ndpi_bitmap64_alloc() {
ndpi_bitmap64_t *rc = (ndpi_bitmap64_t*)ndpi_malloc(sizeof(ndpi_bitmap64_t));
ndpi_bitmap64_fuse* ndpi_bitmap64_fuse_alloc() {
ndpi_bitmap64_fuse_t *rc = (ndpi_bitmap64_fuse_t*)ndpi_malloc(sizeof(ndpi_bitmap64_fuse_t));
if(!rc) return(rc);
rc->num_allocated_entries = NDPI_BITMAP64_REALLOC_SIZE, rc->num_used_entries = 0;
rc->num_allocated_entries = NDPI_BITMAP64_FUSE_REALLOC_SIZE, rc->num_used_entries = 0;
if((rc->entries = (u_int64_t*)ndpi_calloc(rc->num_allocated_entries, sizeof(u_int64_t))) == NULL) {
ndpi_free(rc);
return(NULL);
@@ -56,12 +56,12 @@ ndpi_bitmap64* ndpi_bitmap64_alloc() {
rc->is_compressed = false;
return((ndpi_bitmap64*)rc);
return((ndpi_bitmap64_fuse*)rc);
}
/* ********************************************************** */
static int ndpi_bitmap64_entry_compare(const void *_a, const void *_b) {
static int ndpi_bitmap64_fuse_entry_compare(const void *_a, const void *_b) {
u_int64_t *a = (u_int64_t*)_a, *b = (u_int64_t*)_b;
if(*a < *b) return -1;
@@ -72,8 +72,8 @@ static int ndpi_bitmap64_entry_compare(const void *_a, const void *_b) {
/* ********************************************************** */
/* Sort and compact memory before searching */
bool ndpi_bitmap64_compress(ndpi_bitmap64 *_b) {
ndpi_bitmap64_t *b = (ndpi_bitmap64_t*)_b;
bool ndpi_bitmap64_fuse_compress(ndpi_bitmap64_fuse *_b) {
ndpi_bitmap64_fuse_t *b = (ndpi_bitmap64_fuse_t*)_b;
u_int32_t i;
if(!b)
@@ -86,7 +86,7 @@ bool ndpi_bitmap64_compress(ndpi_bitmap64 *_b) {
if(b->num_used_entries > 1)
qsort(b->entries, b->num_used_entries,
sizeof(u_int64_t),
ndpi_bitmap64_entry_compare);
ndpi_bitmap64_fuse_entry_compare);
/* Now remove duplicates */
u_int64_t old_value = b->entries[0], new_len = 1;
@@ -128,8 +128,8 @@ bool ndpi_bitmap64_compress(ndpi_bitmap64 *_b) {
/* ********************************************************** */
bool ndpi_bitmap64_set(ndpi_bitmap64 *_b, u_int64_t value) {
ndpi_bitmap64_t *b = (ndpi_bitmap64_t*)_b;
bool ndpi_bitmap64_fuse_set(ndpi_bitmap64_fuse *_b, u_int64_t value) {
ndpi_bitmap64_fuse_t *b = (ndpi_bitmap64_fuse_t*)_b;
if(!b)
return(false);
@@ -146,7 +146,7 @@ bool ndpi_bitmap64_set(ndpi_bitmap64 *_b, u_int64_t value) {
if(b->num_used_entries >= b->num_allocated_entries) {
u_int64_t *rc;
u_int32_t new_len = b->num_allocated_entries + NDPI_BITMAP64_REALLOC_SIZE;
u_int32_t new_len = b->num_allocated_entries + NDPI_BITMAP64_FUSE_REALLOC_SIZE;
rc = (u_int64_t*)ndpi_realloc(b->entries,
sizeof(u_int64_t)*b->num_allocated_entries,
@@ -167,14 +167,14 @@ bool ndpi_bitmap64_set(ndpi_bitmap64 *_b, u_int64_t value) {
/* ********************************************************** */
bool ndpi_bitmap64_isset(ndpi_bitmap64 *_b, u_int64_t value) {
ndpi_bitmap64_t *b = (ndpi_bitmap64_t*)_b;
bool ndpi_bitmap64_fuse_isset(ndpi_bitmap64_fuse *_b, u_int64_t value) {
ndpi_bitmap64_fuse_t *b = (ndpi_bitmap64_fuse_t*)_b;
if(!b)
return(false);
if(!b->is_compressed) {
if(!ndpi_bitmap64_compress(b))
if(!ndpi_bitmap64_fuse_compress(b))
return(false); /* Compresssion failed */
}
@@ -183,8 +183,8 @@ bool ndpi_bitmap64_isset(ndpi_bitmap64 *_b, u_int64_t value) {
/* ********************************************************** */
void ndpi_bitmap64_free(ndpi_bitmap64 *_b) {
ndpi_bitmap64_t *b = (ndpi_bitmap64_t*)_b;
void ndpi_bitmap64_fuse_free(ndpi_bitmap64_fuse *_b) {
ndpi_bitmap64_fuse_t *b = (ndpi_bitmap64_fuse_t*)_b;
if(!b)
return;
@@ -199,15 +199,15 @@ void ndpi_bitmap64_free(ndpi_bitmap64 *_b) {
/* ********************************************************** */
u_int32_t ndpi_bitmap64_size(ndpi_bitmap64 *_b) {
ndpi_bitmap64_t *b = (ndpi_bitmap64_t*)_b;
u_int32_t ndpi_bitmap64_fuse_size(ndpi_bitmap64_fuse *_b) {
ndpi_bitmap64_fuse_t *b = (ndpi_bitmap64_fuse_t*)_b;
if(!b) return(0);
if(!b->is_compressed) {
if(!ndpi_bitmap64_compress(b))
if(!ndpi_bitmap64_fuse_compress(b))
return(0); /* Compresssion failed */
}
return(sizeof(ndpi_bitmap64) + binary_fuse16_size_in_bytes(&b->bitmap));
return(sizeof(ndpi_bitmap64_fuse) + binary_fuse16_size_in_bytes(&b->bitmap));
}
+8 -8
View File
@@ -57,7 +57,7 @@ void ndpi_domain_classify_free(ndpi_domain_classify *s) {
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
if(s->classes[i].domains != NULL) {
ndpi_bitmap64_free(s->classes[i].domains);
ndpi_bitmap64_fuse_free(s->classes[i].domains);
} else
break;
}
@@ -75,7 +75,7 @@ u_int32_t ndpi_domain_classify_size(ndpi_domain_classify *s) {
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
if(s->classes[i].domains != NULL) {
tot_len += ndpi_bitmap64_size(s->classes[i].domains);
tot_len += ndpi_bitmap64_fuse_size(s->classes[i].domains);
} else
break;
}
@@ -111,7 +111,7 @@ bool ndpi_domain_classify_add(ndpi_domain_classify *s,
break;
} else if(s->classes[i].class_id == 0) {
s->classes[i].class_id = class_id;
s->classes[i].domains = ndpi_bitmap64_alloc();
s->classes[i].domains = ndpi_bitmap64_fuse_alloc();
if(!s->classes[i].domains)
s->classes[i].class_id = 0;
@@ -125,7 +125,7 @@ bool ndpi_domain_classify_add(ndpi_domain_classify *s,
hash = ndpi_quick_hash64(domain, strlen(domain));
return(ndpi_bitmap64_set(s->classes[i].domains, hash));
return(ndpi_bitmap64_fuse_set(s->classes[i].domains, hash));
}
/* ********************************************************** */
@@ -146,7 +146,7 @@ u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
break;
} else if(s->classes[i].class_id == 0) {
s->classes[i].class_id = class_id;
s->classes[i].domains = ndpi_bitmap64_alloc();
s->classes[i].domains = ndpi_bitmap64_fuse_alloc();
if(!s->classes[i].domains)
s->classes[i].class_id = 0;
break;
@@ -179,7 +179,7 @@ u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
hash = ndpi_quick_hash64(line, strlen(line));
if(ndpi_bitmap64_set(s->classes[i].domains, hash))
if(ndpi_bitmap64_fuse_set(s->classes[i].domains, hash))
num_added++;
}
@@ -198,7 +198,7 @@ bool ndpi_domain_classify_finalize(ndpi_domain_classify *s) {
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
if(s->classes[i].class_id != 0) {
ndpi_bitmap64_compress(s->classes[i].domains);
ndpi_bitmap64_fuse_compress(s->classes[i].domains);
}
}
return(true);
@@ -258,7 +258,7 @@ const char* ndpi_domain_classify_longest_prefix(ndpi_domain_classify *s,
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
if(s->classes[i].class_id != 0) {
if(ndpi_bitmap64_isset(s->classes[i].domains, hash)) {
if(ndpi_bitmap64_fuse_isset(s->classes[i].domains, hash)) {
#ifdef DEBUG_CONTAINS
printf("[contains] %s = %d [%llu]\n",
hostname, s->classes[i].class_id, hash);
+1896 -135
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -43,7 +43,7 @@ typedef __kernel_size_t size_t;
#include "ndpi_api.h"
#include "ahocorasick.h"
#include "../../ndpi_replace_printf.h"
#include "../../../include/ndpi_replace_printf.h"
/* TODO: For different depth of node, number of outgoing edges differs
considerably, It is efficient to use different chunk size for
+10337 -4140
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -137,7 +137,7 @@
<ClCompile Include="..\src\lib\ndpi_binary_bitmap.c" />
<ClCompile Include="..\src\lib\ndpi_hash.c" />
<ClCompile Include="..\src\lib\ndpi_domain_classify.c" />
<ClCompile Include="..\src\lib\ndpi_bitmap64.c" />
<ClCompile Include="..\src\lib\ndpi_bitmap64_fuse.c" />
<ClCompile Include="..\src\lib\protocols\activision.c" />
<ClCompile Include="..\src\lib\protocols\afp.c" />
<ClCompile Include="..\src\lib\protocols\ajp.c" />