Swap from Aho-Corasick to an experimental/home-grown algorithm that uses a probabilistic
approach for handling Internet domain names. For switching back to Aho-Corasick it is necessary to edit ndpi-typedefs.h and uncomment the line // #define USE_LEGACY_AHO_CORASICK [1] With Aho-Corasick $ ./example/ndpiReader -G ./lists/ -i tests/pcap/ookla.pcap | grep Memory nDPI Memory statistics: nDPI Memory (once): 37.34 KB Flow Memory (per flow): 960 B Actual Memory: 33.09 MB Peak Memory: 33.09 MB [2] With the new algorithm $ ./example/ndpiReader -G ./lists/ -i tests/pcap/ookla.pcap | grep Memory nDPI Memory statistics: nDPI Memory (once): 37.31 KB Flow Memory (per flow): 960 B Actual Memory: 7.42 MB Peak Memory: 7.42 MB In essence from ~33 MB to ~7 MB This new algorithm will enable larger lists to be loaded (e.g. top 1M domans https://s3-us-west-1.amazonaws.com/umbrella-static/index.html) In ./lists there are file names that are named as <category>_<string>.list With -G ndpiReader can load all of them at startup
This commit is contained in:
@@ -807,6 +807,30 @@ extern "C" {
|
||||
*/
|
||||
int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str, const char* path, void *user_data);
|
||||
|
||||
/**
|
||||
* Loads a file (separated by <cr>) of domain names associated with the specified category
|
||||
*
|
||||
* @par ndpi_mod = the detection module
|
||||
* @par path = the path of the file
|
||||
* @par category_id = Id of the category to which domains will be associated
|
||||
* @return 0 if the file is loaded correctly;
|
||||
* -1 else
|
||||
*/
|
||||
int ndpi_load_category_file(struct ndpi_detection_module_struct *ndpi_str,
|
||||
char* path, ndpi_protocol_category_t category_id);
|
||||
|
||||
/**
|
||||
* Load files (whose name is <categoryid>_<label>.<extension>) stored
|
||||
* in a directory and bind each domain to the specified category.
|
||||
*
|
||||
* @par ndpi_mod = the detection module
|
||||
* @par path = the path of the file
|
||||
* @return 0 if the file is loaded correctly;
|
||||
* -1 else
|
||||
*/
|
||||
int ndpi_load_categories_dir(struct ndpi_detection_module_struct *ndpi_str,
|
||||
char* path);
|
||||
|
||||
/**
|
||||
* Read a file and load the list of risky domains
|
||||
*
|
||||
|
||||
@@ -367,7 +367,7 @@ typedef enum {
|
||||
NDPI_PROTOCOL_HOTS = 336, /* Heroes of the Storm */
|
||||
NDPI_PROTOCOL_FACEBOOK_REEL_STORY = 337,
|
||||
NDPI_PROTOCOL_SRTP = 338,
|
||||
NDPI_PROTOCOL_GAMBLING = 339,
|
||||
NDPI_PROTOCOL_FREE = 339, /* Formerly used by gambling now a category. It can be reused in the future */
|
||||
NDPI_PROTOCOL_EPICGAMES = 340,
|
||||
NDPI_PROTOCOL_GEFORCENOW = 341,
|
||||
NDPI_PROTOCOL_NVIDIA = 342,
|
||||
|
||||
@@ -31,7 +31,7 @@
|
||||
/* Used by both nDPI core and patricia code under third-party */
|
||||
#include "ndpi_patricia_typedefs.h"
|
||||
|
||||
#define USE_LEGACY_AHO_CORASICK
|
||||
// #define USE_LEGACY_AHO_CORASICK
|
||||
|
||||
#ifndef NDPI_CFFI_PREPROCESSING
|
||||
#ifndef u_char
|
||||
@@ -1101,6 +1101,9 @@ typedef enum {
|
||||
Crypto Currency e.g Bitcoin, Litecoin, Etherum ..et.
|
||||
*/
|
||||
NDPI_PROTOCOL_CATEGORY_CRYPTO_CURRENCY = 106,
|
||||
|
||||
/* Gambling websites */
|
||||
NDPI_PROTOCOL_CATEGORY_GAMBLING = 107,
|
||||
|
||||
/*
|
||||
IMPORTANT
|
||||
|
||||
Reference in New Issue
Block a user