Swap from Aho-Corasick to an experimental/home-grown algorithm that uses a probabilistic
approach for handling Internet domain names. For switching back to Aho-Corasick it is necessary to edit ndpi-typedefs.h and uncomment the line // #define USE_LEGACY_AHO_CORASICK [1] With Aho-Corasick $ ./example/ndpiReader -G ./lists/ -i tests/pcap/ookla.pcap | grep Memory nDPI Memory statistics: nDPI Memory (once): 37.34 KB Flow Memory (per flow): 960 B Actual Memory: 33.09 MB Peak Memory: 33.09 MB [2] With the new algorithm $ ./example/ndpiReader -G ./lists/ -i tests/pcap/ookla.pcap | grep Memory nDPI Memory statistics: nDPI Memory (once): 37.31 KB Flow Memory (per flow): 960 B Actual Memory: 7.42 MB Peak Memory: 7.42 MB In essence from ~33 MB to ~7 MB This new algorithm will enable larger lists to be loaded (e.g. top 1M domans https://s3-us-west-1.amazonaws.com/umbrella-static/index.html) In ./lists there are file names that are named as <category>_<string>.list With -G ndpiReader can load all of them at startup
This commit is contained in:
+34
-21
@@ -80,6 +80,7 @@ static char *_customCategoryFilePath= NULL; /**< Custom categories file path */
|
||||
static char *_maliciousJA3Path = NULL; /**< Malicious JA3 signatures */
|
||||
static char *_maliciousSHA1Path = NULL; /**< Malicious SSL certificate SHA1 fingerprints */
|
||||
static char *_riskyDomainFilePath = NULL; /**< Risky domain files */
|
||||
static char *_categoriesDirPath = NULL; /**< Directory containing domain files */
|
||||
static u_int8_t live_capture = 0;
|
||||
static u_int8_t undetected_flows_deleted = 0;
|
||||
static FILE *csv_fp = NULL; /**< for CSV export */
|
||||
@@ -543,6 +544,7 @@ static void help(u_int long_help) {
|
||||
" -r <path> | Load risky domain file\n"
|
||||
" -j <path> | Load malicious JA3 fingeprints\n"
|
||||
" -S <path> | Load malicious SSL certificate SHA1 fingerprints\n"
|
||||
" -G <dir> | Bind domain names to categories loading files from <dir>\n"
|
||||
" -w <path> | Write test output on the specified file. This is useful for\n"
|
||||
" | testing purposes in order to compare results across runs\n"
|
||||
" -h | This help\n"
|
||||
@@ -647,6 +649,7 @@ static struct option longopts[] = {
|
||||
{ "filter", required_argument, NULL, 'f'},
|
||||
{ "flow-stats", required_argument, NULL, 'F'},
|
||||
{ "cpu-bind", required_argument, NULL, 'g'},
|
||||
{ "load-categories", required_argument, NULL, 'G'},
|
||||
{ "loops", required_argument, NULL, 'l'},
|
||||
{ "num-threads", required_argument, NULL, 'n'},
|
||||
{ "ignore-vlanid", no_argument, NULL, 'I'},
|
||||
@@ -965,7 +968,8 @@ static void parseOptions(int argc, char **argv) {
|
||||
lru_cache_ttls[i] = -1; /* Use the default value */
|
||||
}
|
||||
|
||||
while((opt = getopt_long(argc, argv, "a:Ab:B:e:Ec:C:dDFf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:X:w:zZ:q0123:456:7:89:m:MT:U:",
|
||||
while((opt = getopt_long(argc, argv,
|
||||
"a:Ab:B:e:Ec:C:dDFf:g:G:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:X:w:zZ:q0123:456:7:89:m:MT:U:",
|
||||
longopts, &option_idx)) != EOF) {
|
||||
#ifdef DEBUG_TRACE
|
||||
if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : "");
|
||||
@@ -1035,6 +1039,10 @@ static void parseOptions(int argc, char **argv) {
|
||||
#endif
|
||||
#endif
|
||||
|
||||
case 'G':
|
||||
_categoriesDirPath = optarg;
|
||||
break;
|
||||
|
||||
case 'l':
|
||||
num_loops = atoi(optarg);
|
||||
break;
|
||||
@@ -1107,6 +1115,7 @@ static void parseOptions(int argc, char **argv) {
|
||||
module_tmp = ndpi_init_detection_module(0);
|
||||
if(!module_tmp)
|
||||
break;
|
||||
|
||||
NDPI_BITMASK_SET_ALL(all);
|
||||
ndpi_set_protocol_detection_bitmask2(module_tmp, &all);
|
||||
ndpi_finalize_initialization(module_tmp);
|
||||
@@ -2646,6 +2655,30 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
if(_categoriesDirPath)
|
||||
ndpi_load_categories_dir(ndpi_thread_info[thread_id].workflow->ndpi_struct, _categoriesDirPath);
|
||||
|
||||
if(_riskyDomainFilePath)
|
||||
ndpi_load_risk_domain_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _riskyDomainFilePath);
|
||||
|
||||
if(_maliciousJA3Path)
|
||||
ndpi_load_malicious_ja3_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _maliciousJA3Path);
|
||||
|
||||
if(_maliciousSHA1Path)
|
||||
ndpi_load_malicious_sha1_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _maliciousSHA1Path);
|
||||
|
||||
if(_customCategoryFilePath) {
|
||||
char *label = strrchr(_customCategoryFilePath, '/');
|
||||
|
||||
if(label != NULL)
|
||||
label = &label[1];
|
||||
else
|
||||
label = _customCategoryFilePath;
|
||||
|
||||
ndpi_load_categories_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _customCategoryFilePath, label);
|
||||
}
|
||||
|
||||
/* Make sure to load lists before finalizing the initialization */
|
||||
ndpi_set_protocol_detection_bitmask2(ndpi_thread_info[thread_id].workflow->ndpi_struct, &enabled_bitmask);
|
||||
|
||||
// clear memory for results
|
||||
@@ -2661,26 +2694,6 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
|
||||
if(_protoFilePath != NULL)
|
||||
ndpi_load_protocols_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _protoFilePath);
|
||||
|
||||
if(_customCategoryFilePath) {
|
||||
char *label = strrchr(_customCategoryFilePath, '/');
|
||||
|
||||
if(label != NULL)
|
||||
label = &label[1];
|
||||
else
|
||||
label = _customCategoryFilePath;
|
||||
|
||||
ndpi_load_categories_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _customCategoryFilePath, label);
|
||||
}
|
||||
|
||||
if(_riskyDomainFilePath)
|
||||
ndpi_load_risk_domain_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _riskyDomainFilePath);
|
||||
|
||||
if(_maliciousJA3Path)
|
||||
ndpi_load_malicious_ja3_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _maliciousJA3Path);
|
||||
|
||||
if(_maliciousSHA1Path)
|
||||
ndpi_load_malicious_sha1_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _maliciousSHA1Path);
|
||||
|
||||
/* Enable/disable/configure LRU caches size here */
|
||||
for(i = 0; i < NDPI_LRUCACHE_MAX; i++) {
|
||||
if(lru_cache_sizes[i] != -1)
|
||||
|
||||
Reference in New Issue
Block a user