Implementation of flow risk eception (work in progress)

This commit is contained in:
Luca Deri
2021-07-22 01:35:57 +02:00
parent fc9901292d
commit b01b60a2b5
7 changed files with 370 additions and 130 deletions
+12
View File
@@ -39,3 +39,15 @@ ip:213.75.170.11/32:443@CustomProtocol
ip:8.248.73.247:443@AmazonPrime
ip:54.80.47.130@AmazonPrime
#
# Risk Exceptions
#
# ip_risk_mask: used to mask flow risks for IP addresses
# host_risk_mask: used to mask exceptions for domain names and hosts
#
# Syntax: <name>=<64 bit mask to be put in AND with the risk
#
# For IPs, the flow risk is put in AND (source IP mask OR destination IP mask)
# For Flows with a hostname (e.g. TLS) the risk is also put in AND with the host_risk_mask
ip_risk_mask:192.168.1.6=0
host_risk_mask:"api-global.netflix.com"=0
+5 -2
View File
@@ -1297,6 +1297,7 @@ void update_tcp_flags_count(struct ndpi_flow_info* flow, struct ndpi_tcphdr* tcp
}
/* ****************************************************** */
/**
Function to process the packet:
determine the flow of a packet and try to decode it
@@ -1567,11 +1568,13 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
#endif
*flow_risk = flow->risk;
return(flow->detected_protocol);
}
int ndpi_is_datalink_supported(int datalink_type)
{
/* ****************************************************** */
int ndpi_is_datalink_supported(int datalink_type) {
/* Keep in sync with the similar switch in ndpi_workflow_process_packet */
switch(datalink_type) {
case DLT_NULL:
+56 -28
View File
@@ -50,9 +50,9 @@ extern "C" {
/*
In case a custom DGA function is used, the fucntion
below must be overwritten,
*/
*/
extern ndpi_custom_dga_predict_fctn ndpi_dga_function;
/**
* Check if a string is encoded with punycode
* ( https://tools.ietf.org/html/rfc3492 )
@@ -276,7 +276,7 @@ extern "C" {
void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_struct,
const NDPI_PROTOCOL_BITMASK * detection_bitmask);
/**
/**
* Function to be called before we give up with detection for a given flow.
* This function reduces the NDPI_UNKNOWN_PROTOCOL detection
*
@@ -375,7 +375,7 @@ extern "C" {
* @par l4_protocol_return = protocol of the layer 4 data if return value == 0, undefined otherwise
* @par flags = limit operation on ipv4 or ipv6 packets. Possible values: NDPI_DETECTION_ONLY_IPV4 - NDPI_DETECTION_ONLY_IPV6 - 0 (any)
* @return 0 if layer 4 data could be found correctly;
else != 0
else != 0
*
*/
u_int8_t ndpi_detection_get_l4(const u_int8_t *l3, u_int16_t l3_len, const u_int8_t **l4_return, u_int16_t *l4_len_return,
@@ -489,9 +489,9 @@ extern "C" {
*
*/
void ndpi_exclude_protocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int16_t master_protocol_id,
const char *_file, const char *_func,int _line);
struct ndpi_flow_struct *flow,
u_int16_t master_protocol_id,
const char *_file, const char *_func,int _line);
/**
* Check if the string -bigram_to_match- match with a bigram of -automa-
*
@@ -705,7 +705,29 @@ extern "C" {
int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod,
const char* path);
/**
/**
* Add an IP-address based risk mask
*
* @par ndpi_mod = the detection module
* @par ip = the IP address for which you wanna set the mask
* @par mask = the IP risk mask
* @return 0 if the rule is loaded correctly;
* -1 else
*/
int ndpi_add_ip_risk_mask(struct ndpi_detection_module_struct *ndpi_mod, char *ip, ndpi_risk mask);
/**
* Add a host-address based risk mask
*
* @par ndpi_mod = the detection module
* @par host = the hostname/domain for which you wanna set the mask
* @par mask = the host risk mask
* @return 0 if the rule is loaded correctly;
* -1 else
*/
int ndpi_add_host_risk_mask(struct ndpi_detection_module_struct *ndpi_mod, char *host, ndpi_risk mask);
/**
* Read a file and load the categories
*
* @par ndpi_mod = the detection module
@@ -715,7 +737,7 @@ extern "C" {
*/
int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str, const char* path);
/**
/**
* Read a file and load the list of risky domains
*
* @par ndpi_mod = the detection module
@@ -725,7 +747,7 @@ extern "C" {
*/
int ndpi_load_risk_domain_file(struct ndpi_detection_module_struct *ndpi_str, const char* path);
/**
/**
* Read a file and load the list of malicious JA3 signatures
*
* @par ndpi_mod = the detection module
@@ -771,7 +793,7 @@ extern "C" {
void ndpi_set_automa(struct ndpi_detection_module_struct *ndpi_struct,
void* automa);
/* NDPI_PROTOCOL_HTTP */
/* NDPI_PROTOCOL_HTTP */
/**
* Retrieve information for HTTP flows
*
@@ -779,7 +801,7 @@ extern "C" {
* @par flow = the detected flow
* @return the HTTP method information about the flow
*
*/
*/
ndpi_http_method ndpi_get_http_method(struct ndpi_detection_module_struct *ndpi_mod,
struct ndpi_flow_struct *flow);
@@ -790,7 +812,7 @@ extern "C" {
* @par flow = the detected flow
* @return the HTTP method information about the flow
*
*/
*/
char* ndpi_get_http_url(struct ndpi_detection_module_struct *ndpi_mod,
struct ndpi_flow_struct *flow);
@@ -801,11 +823,11 @@ extern "C" {
* @par flow = the detected flow
* @return the HTTP method information about the flow
*
*/
*/
char* ndpi_get_http_content_type(struct ndpi_detection_module_struct *ndpi_mod,
struct ndpi_flow_struct *flow);
/* NDPI_PROTOCOL_TOR */
/* NDPI_PROTOCOL_TOR */
/**
* Check if the flow could be detected as TOR protocol
*
@@ -876,16 +898,16 @@ extern "C" {
int ndpi_match_string(void *_automa, char *string_to_match);
int ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_struct,
const char *ip_address_and_mask, ndpi_protocol_category_t category);
const char *ip_address_and_mask, ndpi_protocol_category_t category);
int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct,
const char *name_to_add, ndpi_protocol_category_t category);
const char *name_to_add, ndpi_protocol_category_t category);
int ndpi_load_category(struct ndpi_detection_module_struct *ndpi_struct,
const char *ip_or_name, ndpi_protocol_category_t category);
const char *ip_or_name, ndpi_protocol_category_t category);
int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct);
int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
u_int32_t saddr,
u_int32_t daddr,
ndpi_protocol *ret);
u_int32_t saddr,
u_int32_t daddr,
ndpi_protocol *ret);
int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, u_int name_len, ndpi_protocol_category_t *id);
void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
@@ -904,7 +926,7 @@ extern "C" {
u_int16_t ndpi_get_lower_proto(ndpi_protocol proto);
u_int16_t ndpi_get_upper_proto(ndpi_protocol proto);
ndpi_proto_defaults_t* ndpi_get_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod);
u_int ndpi_get_ndpi_num_supported_protocols(struct ndpi_detection_module_struct *ndpi_mod);
u_int ndpi_get_ndpi_num_custom_protocols(struct ndpi_detection_module_struct *ndpi_mod);
@@ -1524,7 +1546,7 @@ extern "C" {
ndpi_risk_info* ndpi_risk2severity(ndpi_risk_enum risk);
u_int16_t ndpi_risk2score(ndpi_risk risk,
u_int16_t *client_score, u_int16_t *server_score);
/* ******************************* */
/* HyperLogLog cardinality estimator */
@@ -1571,14 +1593,20 @@ extern "C" {
/* ******************************* */
int ndpi_load_geoip(struct ndpi_detection_module_struct *ndpi_str,
const char *ip_city_data, const char *ip_as_data);
void ndpi_free_geoip(struct ndpi_detection_module_struct *ndpi_str);
const char *ip_city_data, const char *ip_as_data);
void ndpi_free_geoip(struct ndpi_detection_module_struct *ndpi_str);
int ndpi_get_geoip_asn(struct ndpi_detection_module_struct *ndpi_str,
char *ip, u_int32_t *asn);
int ndpi_get_geoip_country_continent(struct ndpi_detection_module_struct *ndpi_str, char *ip,
char *country_code, u_int8_t country_code_len,
char *continent, u_int8_t continent_len);
int ndpi_get_geoip_country_continent(struct ndpi_detection_module_struct *ndpi_str, char *ip,
char *country_code, u_int8_t country_code_len,
char *continent, u_int8_t continent_len);
/* ******************************* */
char* ndpi_get_flow_name(struct ndpi_flow_struct *flow);
/* ******************************* */
#ifdef __cplusplus
}
#endif
+4 -1
View File
@@ -1160,9 +1160,12 @@ struct ndpi_detection_module_struct {
content_automa, /* Used for HTTP subprotocol_detection */
subprotocol_automa, /* Used for HTTP subprotocol_detection */
risky_domain_automa, tls_cert_subject_automa,
malicious_ja3_automa, malicious_sha1_automa;
malicious_ja3_automa, malicious_sha1_automa,
host_risk_mask_automa;
/* IMPORTANT: please update ndpi_finalize_initialization() whenever you add a new automa */
void *ip_risk_mask_ptree;
struct {
ndpi_automa hostnames, hostnames_shadow;
void *ipAddresses, *ipAddresses_shadow; /* Patricia */
+277 -99
View File
@@ -555,11 +555,11 @@ static u_int8_t ndpi_is_middle_string_char(char c) {
/*******************************************************/
static const u_int8_t ndpi_domain_level_automat[4][4]= {
/* symbol,'.','-',inc */
{ 2,1,2,0 }, // start state
{ 2,0,0,0 }, // first char is '.'; disable .. or .-
{ 2,3,2,0 }, // part of domain name
{ 2,0,0,1 } // next level domain name; disable .. or .-
/* symbol,'.','-',inc */
{ 2,1,2,0 }, // start state
{ 2,0,0,0 }, // first char is '.'; disable .. or .-
{ 2,3,2,0 }, // part of domain name
{ 2,0,0,1 } // next level domain name; disable .. or .-
};
/*
@@ -570,15 +570,15 @@ static const u_int8_t ndpi_domain_level_automat[4][4]= {
*/
static u_int8_t ndpi_domain_level(const char *name) {
u_int8_t level = 1, state = 0;
char c;
while((c = *name++) != '\0') {
c = c == '-' ? 2 : (c == '.' ? 1:0);
level += ndpi_domain_level_automat[state][3];
state = ndpi_domain_level_automat[state][(uint8_t)c];
if(!state) break;
}
return state >= 2 ? level:0;
u_int8_t level = 1, state = 0;
char c;
while((c = *name++) != '\0') {
c = c == '-' ? 2 : (c == '.' ? 1:0);
level += ndpi_domain_level_automat[state][3];
state = ndpi_domain_level_automat[state][(uint8_t)c];
if(!state) break;
}
return state >= 2 ? level:0;
}
/* ****************************************************** */
@@ -603,7 +603,7 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_str,
value_dup = ndpi_strdup(value);
if(!value_dup)
return(-1);
return(-1);
memset(&ac_pattern, 0, sizeof(ac_pattern));
@@ -627,9 +627,10 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_str,
rc = ac_automata_add(ac_automa, &ac_pattern);
if(rc != ACERR_SUCCESS) {
ndpi_free(value_dup);
if(rc != ACERR_DUPLICATE_PATTERN)
return (-2);
ndpi_free(value_dup);
if(rc != ACERR_DUPLICATE_PATTERN)
return (-2);
}
return(0);
@@ -646,7 +647,7 @@ static int ndpi_add_host_url_subprotocol(struct ndpi_detection_module_struct *nd
#endif
return ndpi_string_to_automa(ndpi_str, (AC_AUTOMATA_t *)ndpi_str->host_automa.ac_automa,
value, protocol_id, category, breed, level, 1);
value, protocol_id, category, breed, level, 1);
}
@@ -717,18 +718,18 @@ static unsigned int trigrams_bitmap[(XGRAMS_C*XGRAMS_C*XGRAMS_C+31)/32];
static void ndpi_xgrams_init(unsigned int *dst,size_t dn, const char **src,size_t sn,int l) {
unsigned int i,j,c;
for(i=0;i < sn && src[i]; i++) {
for(j=0,c=0; j < l; j++) {
unsigned char a = (unsigned char)src[i][j];
if(a < 'a' || a > 'z') { printf("%u: c%u %c\n",i,j,a); abort(); }
c *= XGRAMS_C;
c += a - 'a';
}
if(src[i][l]) { printf("%u: c[%d] != 0\n",i,l); abort(); }
if((c >> 3) >= dn) abort();
dst[c >> 5] |= 1u << (c & 0x1f);
}
unsigned int i,j,c;
for(i=0;i < sn && src[i]; i++) {
for(j=0,c=0; j < l; j++) {
unsigned char a = (unsigned char)src[i][j];
if(a < 'a' || a > 'z') { printf("%u: c%u %c\n",i,j,a); abort(); }
c *= XGRAMS_C;
c += a - 'a';
}
if(src[i][l]) { printf("%u: c[%d] != 0\n",i,l); abort(); }
if((c >> 3) >= dn) abort();
dst[c >> 5] |= 1u << (c & 0x1f);
}
}
static void init_string_based_protocols(struct ndpi_detection_module_struct *ndpi_str) {
@@ -764,12 +765,12 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
if(!ndpi_xgrams_inited) {
ndpi_xgrams_inited = 1;
ndpi_xgrams_init(bigrams_bitmap,sizeof(bigrams_bitmap),
ndpi_en_bigrams,sizeof(ndpi_en_bigrams)/sizeof(ndpi_en_bigrams[0]), 2);
ndpi_en_bigrams,sizeof(ndpi_en_bigrams)/sizeof(ndpi_en_bigrams[0]), 2);
ndpi_xgrams_init(imposible_bigrams_bitmap,sizeof(imposible_bigrams_bitmap),
ndpi_en_impossible_bigrams,sizeof(ndpi_en_impossible_bigrams)/sizeof(ndpi_en_impossible_bigrams[0]), 2);
ndpi_en_impossible_bigrams,sizeof(ndpi_en_impossible_bigrams)/sizeof(ndpi_en_impossible_bigrams[0]), 2);
ndpi_xgrams_init(trigrams_bitmap,sizeof(trigrams_bitmap),
ndpi_en_trigrams,sizeof(ndpi_en_trigrams)/sizeof(ndpi_en_trigrams[0]), 3);
ndpi_en_trigrams,sizeof(ndpi_en_trigrams)/sizeof(ndpi_en_trigrams[0]), 3);
}
}
@@ -1732,9 +1733,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_AVAST_SECUREDNS,
"AVAST SecureDNS", NDPI_PROTOCOL_CATEGORY_NETWORK,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0), /* TCP */
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0)); /* UDP */
"AVAST SecureDNS", NDPI_PROTOCOL_CATEGORY_NETWORK,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0), /* TCP */
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0)); /* UDP */
#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_main.c"
@@ -1768,40 +1769,40 @@ static int ac_domain_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *matc
* Skip shorter (less precise) templates.
*/
if(!(m->match_map & (1 << i)))
continue;
continue;
start = end - pattern->length;
MATCH_DEBUG_INFO("[NDPI] Searching: [to search: %.*s/%u][pattern: %s%.*s%s/%u l:%u] %d-%d\n",
txt->length, txt->astring,(unsigned int) txt->length,
m->patterns[0].rep.from_start ? "^":"",
(unsigned int) pattern->length, pattern->astring,
m->patterns[0].rep.at_end ? "$":"", (unsigned int) pattern->length,m->patterns[0].rep.level,
start,end);
txt->length, txt->astring,(unsigned int) txt->length,
m->patterns[0].rep.from_start ? "^":"",
(unsigned int) pattern->length, pattern->astring,
m->patterns[0].rep.at_end ? "$":"", (unsigned int) pattern->length,m->patterns[0].rep.level,
start,end);
if(start == 0 && end == txt->length) {
*match = pattern->rep; txt->match.last = pattern;
MATCH_DEBUG_INFO("[NDPI] Searching: Found exact match. Proto %d \n",pattern->rep.number);
return 1;
*match = pattern->rep; txt->match.last = pattern;
MATCH_DEBUG_INFO("[NDPI] Searching: Found exact match. Proto %d \n",pattern->rep.number);
return 1;
}
/* pattern is DOMAIN.NAME and string x.DOMAIN.NAME ? */
if(start > 1 && !ndpi_is_middle_string_char(pattern->astring[0]) && pattern->rep.dot) {
/*
The patch below allows in case of pattern ws.amazon.com
to avoid matching aws.amazon.com whereas a.ws.amazon.com
has to match
*/
if(ndpi_is_middle_string_char(txt->astring[start-1])) {
if(!txt->match.last || txt->match.last->rep.level < pattern->rep.level) {
txt->match.last = pattern; *match = pattern->rep;
MATCH_DEBUG_INFO("[NDPI] Searching: Found domain match. Proto %d \n",pattern->rep.number);
}
}
continue;
/*
The patch below allows in case of pattern ws.amazon.com
to avoid matching aws.amazon.com whereas a.ws.amazon.com
has to match
*/
if(ndpi_is_middle_string_char(txt->astring[start-1])) {
if(!txt->match.last || txt->match.last->rep.level < pattern->rep.level) {
txt->match.last = pattern; *match = pattern->rep;
MATCH_DEBUG_INFO("[NDPI] Searching: Found domain match. Proto %d \n",pattern->rep.number);
}
}
continue;
}
if(!txt->match.last || txt->match.last->rep.level < pattern->rep.level) {
txt->match.last = pattern; *match = pattern->rep;
MATCH_DEBUG_INFO("[NDPI] Searching: matched. Proto %d \n",pattern->rep.number);
txt->match.last = pattern; *match = pattern->rep;
MATCH_DEBUG_INFO("[NDPI] Searching: matched. Proto %d \n",pattern->rep.number);
}
}
return 0;
@@ -2316,6 +2317,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
if((ndpi_str->protocols_ptree = ndpi_patricia_new(32 /* IPv4 */)) != NULL)
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, host_protocol_list, prefs & ndpi_dont_load_tor_hosts);
ndpi_str->ip_risk_mask_ptree = ndpi_patricia_new(32 /* IPv4 */);
NDPI_BITMASK_RESET(ndpi_str->detection_bitmask);
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
ndpi_str->user_data = NULL;
@@ -2341,6 +2344,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->host_automa.ac_automa = ac_automata_init(ac_domain_match_handler);
ndpi_str->content_automa.ac_automa = ac_automata_init(ac_domain_match_handler);
ndpi_str->host_risk_mask_automa.ac_automa = ac_automata_init(ac_domain_match_handler);
ndpi_str->tls_cert_subject_automa.ac_automa = ac_automata_init(NULL);
ndpi_str->malicious_ja3_automa.ac_automa = NULL; /* Initialized on demand */
ndpi_str->malicious_sha1_automa.ac_automa = NULL; /* Initialized on demand */
@@ -2359,33 +2363,42 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->custom_categories.ipAddresses_shadow = ndpi_patricia_new(32 /* IPv4 */);
if(ndpi_str->host_automa.ac_automa)
ac_automata_feature(ndpi_str->host_automa.ac_automa,AC_FEATURE_LC);
ac_automata_feature(ndpi_str->host_automa.ac_automa,AC_FEATURE_LC);
if(ndpi_str->custom_categories.hostnames.ac_automa)
ac_automata_feature(ndpi_str->custom_categories.hostnames.ac_automa,AC_FEATURE_LC);
ac_automata_feature(ndpi_str->custom_categories.hostnames.ac_automa,AC_FEATURE_LC);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa,AC_FEATURE_LC);
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa,AC_FEATURE_LC);
if(ndpi_str->tls_cert_subject_automa.ac_automa)
ac_automata_feature(ndpi_str->tls_cert_subject_automa.ac_automa,AC_FEATURE_LC);
ac_automata_feature(ndpi_str->tls_cert_subject_automa.ac_automa,AC_FEATURE_LC);
if(ndpi_str->content_automa.ac_automa)
ac_automata_feature(ndpi_str->content_automa.ac_automa,AC_FEATURE_LC);
ac_automata_feature(ndpi_str->content_automa.ac_automa,AC_FEATURE_LC);
if(ndpi_str->host_risk_mask_automa.ac_automa)
ac_automata_feature(ndpi_str->host_risk_mask_automa.ac_automa,AC_FEATURE_LC);
/* ahocorasick debug */
/* Needed ac_automata_enable_debug(1) for show debug */
if(ndpi_str->host_automa.ac_automa)
ac_automata_name(ndpi_str->host_automa.ac_automa,"host",AC_FEATURE_DEBUG);
if(ndpi_str->custom_categories.hostnames.ac_automa)
ac_automata_name(ndpi_str->custom_categories.hostnames.ac_automa,"ccat",0);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
ac_automata_name(ndpi_str->custom_categories.hostnames_shadow.ac_automa,"ccat_sh",0);
if(ndpi_str->tls_cert_subject_automa.ac_automa)
ac_automata_name(ndpi_str->tls_cert_subject_automa.ac_automa,"tls_cert",AC_FEATURE_DEBUG);
if(ndpi_str->content_automa.ac_automa)
ac_automata_name(ndpi_str->content_automa.ac_automa,"content",AC_FEATURE_DEBUG);
ac_automata_name(ndpi_str->host_automa.ac_automa,"host",AC_FEATURE_DEBUG);
if(ndpi_str->custom_categories.hostnames.ac_automa)
ac_automata_name(ndpi_str->custom_categories.hostnames.ac_automa,"ccat",0);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
ac_automata_name(ndpi_str->custom_categories.hostnames_shadow.ac_automa,"ccat_sh",0);
if(ndpi_str->tls_cert_subject_automa.ac_automa)
ac_automata_name(ndpi_str->tls_cert_subject_automa.ac_automa,"tls_cert",AC_FEATURE_DEBUG);
if(ndpi_str->content_automa.ac_automa)
ac_automata_name(ndpi_str->content_automa.ac_automa,"content",AC_FEATURE_DEBUG);
if(ndpi_str->host_risk_mask_automa.ac_automa)
ac_automata_name(ndpi_str->host_risk_mask_automa.ac_automa,"content",AC_FEATURE_DEBUG);
if((ndpi_str->custom_categories.ipAddresses == NULL) || (ndpi_str->custom_categories.ipAddresses_shadow == NULL)) {
NDPI_LOG_ERR(ndpi_str, "[NDPI] Error allocating Patricia trees\n");
@@ -2432,13 +2445,17 @@ void ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str)
automa = &ndpi_str->malicious_sha1_automa;
break;
case 5:
automa = &ndpi_str->host_risk_mask_automa;
break;
default:
ndpi_str->ac_automa_finalized = 1;
return;
}
if(automa && automa->ac_automa)
ac_automata_finalize((AC_AUTOMATA_t *) automa->ac_automa);
ac_automata_finalize((AC_AUTOMATA_t *) automa->ac_automa);
}
}
@@ -2531,7 +2548,8 @@ int ndpi_match_string(void *_automa, char *string_to_match) {
if(!string_to_match)
return(-2);
rc = ndpi_match_string_common(_automa,string_to_match,strlen(string_to_match),
rc = ndpi_match_string_common(_automa, string_to_match,
strlen(string_to_match),
&proto_id, NULL, NULL);
if(rc < 0) return rc;
@@ -2555,13 +2573,13 @@ int ndpi_match_string_protocol_id(void *automa, char *string_to_match,
/* ****************************************************** */
int ndpi_match_string_value(void *automa, char *string_to_match,
u_int match_len, u_int32_t *num) {
u_int match_len, u_int32_t *num) {
int rc = ndpi_match_string_common((AC_AUTOMATA_t *)automa, string_to_match,
match_len, num, NULL, NULL);
match_len, num, NULL, NULL);
if(rc < 0) return rc;
return rc ? 0 : -1;
}
}
/* *********************************************** */
@@ -2571,7 +2589,7 @@ int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_str,
ndpi_protocol_category_t *category) {
u_int32_t id;
int rc = ndpi_match_string_common(ndpi_str->custom_categories.hostnames.ac_automa,
name, name_len, &id, category, NULL);
name, name_len, &id, category, NULL);
if(rc < 0) return rc;
return(id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1);
}
@@ -2662,6 +2680,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
if(ndpi_str->protocols_ptree)
ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->protocols_ptree, free_ptree_data);
if(ndpi_str->ip_risk_mask_ptree)
ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->ip_risk_mask_ptree, free_ptree_data);
if(ndpi_str->udpRoot != NULL)
ndpi_tdestroy(ndpi_str->udpRoot, ndpi_free);
if(ndpi_str->tcpRoot != NULL)
@@ -2702,6 +2723,10 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
if(ndpi_str->custom_categories.ipAddresses_shadow != NULL)
ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses_shadow, free_ptree_data);
if(ndpi_str->host_risk_mask_automa.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->host_risk_mask_automa.ac_automa,
1 /* free patterns strings memory */);
#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/ndpi_exit_detection_module.c"
@@ -2876,6 +2901,86 @@ char *strsep(char **sp, char *sep) {
}
#endif
/* ********************************************************************************* */
static u_int64_t ndpi_host_ip_risk_ptree_match(struct ndpi_detection_module_struct *ndpi_str,
struct in_addr *pin /* network byte order */) {
ndpi_prefix_t prefix;
ndpi_patricia_node_t *node;
/* Make sure all in network byte order otherwise compares wont work */
ndpi_fill_prefix_v4(&prefix, pin, 32, ((ndpi_patricia_tree_t *) ndpi_str->protocols_ptree)->maxbits);
node = ndpi_patricia_search_best(ndpi_str->ip_risk_mask_ptree, &prefix);
if(node)
return(node->value.u.uv64);
else
return((u_int64_t)-1);
}
/* ******************************************************************** */
int ndpi_add_ip_risk_mask(struct ndpi_detection_module_struct *ndpi_str,
char *ip, ndpi_risk mask) {
char *saveptr, *addr = strtok_r(ip, "/", &saveptr);
if(addr) {
char *cidr = strtok_r(NULL, "\n", &saveptr);
struct in_addr pin;
ndpi_patricia_node_t *node;
pin.s_addr = inet_addr(addr);
/* FIX: Add IPv6 support */
if((node = add_to_ptree(ndpi_str->ip_risk_mask_ptree, AF_INET,
&pin, cidr ? atoi(cidr) : 32 /* bits */)) != NULL) {
node->value.u.uv64 = (u_int64_t)mask;
return(0);
} else
return(-1);
} else
return(-2);
}
/* ******************************************************************** */
int ndpi_add_host_risk_mask(struct ndpi_detection_module_struct *ndpi_str,
char *host, ndpi_risk mask) {
/* host_risk_mask_automa */
AC_PATTERN_t ac_pattern;
AC_ERROR_t rc;
u_int len;
char *host_dup = NULL;
if((ndpi_str->host_risk_mask_automa.ac_automa == NULL) || (host == NULL))
return(-2);
host_dup = ndpi_strdup(host);
if(!host_dup)
return(-1);
memset(&ac_pattern, 0, sizeof(ac_pattern));
len = strlen(host);
ac_pattern.astring = host_dup;
ac_pattern.length = len;
ac_pattern.rep.number64 = (ndpi_risk)mask;
ac_pattern.rep.level = ndpi_domain_level(host);
ac_pattern.rep.at_end = 0;
ac_pattern.rep.dot = memchr(host,'.',len) != NULL;
rc = ac_automata_add(ndpi_str->host_risk_mask_automa.ac_automa, &ac_pattern);
if(rc != ACERR_SUCCESS) {
ndpi_free(host_dup);
if(rc != ACERR_DUPLICATE_PATTERN)
return (-2);
}
return(0);
}
/* ******************************************************************** */
int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, char *rule, u_int8_t do_add) {
@@ -2885,7 +2990,34 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, char *rule,
at = strrchr(rule, '@');
if(at == NULL) {
NDPI_LOG_ERR(ndpi_str, "Invalid rule '%s'\n", rule);
/* This looks like a mask rule or an invalid rule */
char _rule[256], *rule_type, *key;
snprintf(_rule, sizeof(_rule), "%s", rule);
rule_type = strtok(rule, ":");
if(!rule_type) {
NDPI_LOG_ERR(ndpi_str, "Invalid rule '%s'\n", rule);
return(-1);
}
key = strtok(NULL, "=");
if(key) {
char *mask = strtok(NULL, "=");
if(mask) {
ndpi_risk risk_mask = (ndpi_risk)atoll(mask);
if(!strcmp(rule_type, "ip_risk_mask")) {
return(ndpi_add_ip_risk_mask(ndpi_str, key, risk_mask));
} else if(!strcmp(rule_type, "host_risk_mask")) {
return(ndpi_add_host_risk_mask(ndpi_str, key, risk_mask));
}
}
}
NDPI_LOG_ERR(ndpi_str, "Unknown rule '%s'\n", rule);
return(-1);
} else
at[0] = 0, proto = &at[1];
@@ -2985,7 +3117,7 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, char *rule,
} else {
if(do_add)
ndpi_add_host_url_subprotocol(ndpi_str, value, subprotocol_id, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED,
NDPI_PROTOCOL_ACCEPTABLE,0);
NDPI_PROTOCOL_ACCEPTABLE, 0);
else
ndpi_remove_host_url_subprotocol(ndpi_str, value, subprotocol_id);
}
@@ -3057,14 +3189,14 @@ static int ndpi_load_risky_domain(struct ndpi_detection_module_struct *ndpi_str,
ndpi_str->risky_domain_automa.ac_automa = ac_automata_init(ac_domain_match_handler);
if(!ndpi_str->risky_domain_automa.ac_automa) return -1;
ac_automata_feature(ndpi_str->risky_domain_automa.ac_automa,AC_FEATURE_LC);
ac_automata_name(ndpi_str->risky_domain_automa.ac_automa,"risky",0);
ac_automata_name(ndpi_str->risky_domain_automa.ac_automa, "risky", 0);
}
if(!ndpi_str->risky_domain_automa.ac_automa)
return -1;
return ndpi_string_to_automa(ndpi_str, (AC_AUTOMATA_t *)ndpi_str->risky_domain_automa.ac_automa,
domain_name, 1, 0, 0, 0, 1); /* domain, protocol, category, breed, level , at_end */
domain_name, 1, 0, 0, 0, 1); /* domain, protocol, category, breed, level , at_end */
}
/* ******************************************************************** */
@@ -3130,7 +3262,7 @@ int ndpi_load_malicious_ja3_file(struct ndpi_detection_module_struct *ndpi_str,
if(ndpi_str->malicious_ja3_automa.ac_automa == NULL)
ndpi_str->malicious_ja3_automa.ac_automa = ac_automata_init(NULL);
if(ndpi_str->malicious_ja3_automa.ac_automa)
ac_automata_name(ndpi_str->malicious_ja3_automa.ac_automa,"ja3",0);
ac_automata_name(ndpi_str->malicious_ja3_automa.ac_automa,"ja3",0);
fd = fopen(path, "r");
@@ -3193,7 +3325,7 @@ int ndpi_load_malicious_sha1_file(struct ndpi_detection_module_struct *ndpi_str,
if (ndpi_str->malicious_sha1_automa.ac_automa == NULL)
ndpi_str->malicious_sha1_automa.ac_automa = ac_automata_init(NULL);
if(ndpi_str->malicious_sha1_automa.ac_automa)
ac_automata_name(ndpi_str->malicious_sha1_automa.ac_automa,"sha1",0);
ac_automata_name(ndpi_str->malicious_sha1_automa.ac_automa,"sha1",0);
fd = fopen(path, "r");
@@ -4602,6 +4734,49 @@ u_int32_t ndpi_check_flow_func(struct ndpi_detection_module_struct *ndpi_str,
/* ********************************************************************************* */
void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow) {
char *host;
if(flow->risk == 0) return; /* Nothing to do */
host = ndpi_get_flow_name(flow);
if(host) {
/* Check host exception */
ndpi_automa *automa = &ndpi_str->host_risk_mask_automa;
if(automa->ac_automa) {
AC_TEXT_t ac_input_text;
AC_REP_t match;
ac_input_text.astring = host, ac_input_text.length = strlen(host);
ac_input_text.option = 0;
if(ac_automata_search(automa->ac_automa, &ac_input_text, &match) > 0)
flow->risk &= match.number64;
}
}
/* TODO: add IPv6 support */
if(flow->packet.iph) {
struct ndpi_packet_struct *packet = &flow->packet;
struct in_addr pin;
if(flow->risk) {
pin.s_addr = packet->iph->saddr;
flow->risk &= ndpi_host_ip_risk_ptree_match(ndpi_str, &pin);
}
if(flow->risk) {
pin.s_addr = packet->iph->daddr;
flow->risk &= ndpi_host_ip_risk_ptree_match(ndpi_str, &pin);
}
}
}
/* ********************************************************************************* */
u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow) {
u_int16_t ret = NDPI_PROTOCOL_UNKNOWN;
@@ -4859,7 +5034,7 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_str, c
return(-1);
return ndpi_string_to_automa(ndpi_str,(AC_AUTOMATA_t *)ndpi_str->custom_categories.hostnames_shadow.ac_automa,
name_to_add,category,category, 0, 0, 1); /* at_end */
name_to_add,category,category, 0, 0, 1); /* at_end */
}
/* ********************************************************************************* */
@@ -4902,8 +5077,8 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
/* Realloc */
ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_domain_match_handler);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa) {
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa,AC_FEATURE_LC);
ac_automata_name(ndpi_str->custom_categories.hostnames_shadow.ac_automa,"ccat_sh",0);
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa,AC_FEATURE_LC);
ac_automata_name(ndpi_str->custom_categories.hostnames_shadow.ac_automa,"ccat_sh",0);
}
if(ndpi_str->custom_categories.ipAddresses != NULL)
@@ -5379,6 +5554,8 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
ret.app_protocol = flow->guessed_host_protocol_id;
}
ndpi_reconcile_protocols(ndpi_str, flow, &ret);
if((!flow->risk_checked) && (ret.master_protocol != NDPI_PROTOCOL_UNKNOWN)) {
ndpi_default_ports_tree_node_t *found;
u_int16_t *default_ports, sport, dport;
@@ -5432,11 +5609,12 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
}
}
ndpi_handle_risk_exceptions(ndpi_str, flow);
flow->risk_checked = 1;
}
ndpi_reconcile_protocols(ndpi_str, flow, &ret);
// printf("===> RISK=%llu\n", flow->risk);
if(num_calls == 0)
flow->fail_with_unknown = 1;
@@ -6684,8 +6862,8 @@ int ndpi_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_str,
return(NDPI_PROTOCOL_UNKNOWN);
rc = ndpi_match_string_common(((AC_AUTOMATA_t *) automa->ac_automa),
string_to_match,string_to_match_len, &ret_match->protocol_id,
&ret_match->protocol_category, &ret_match->protocol_breed);
string_to_match,string_to_match_len, &ret_match->protocol_id,
&ret_match->protocol_category, &ret_match->protocol_breed);
return rc < 0 ? rc : ret_match->protocol_id;
}
@@ -6720,7 +6898,7 @@ static u_int16_t ndpi_automa_match_string_subprotocol(struct ndpi_detection_modu
ndpi_match_string_subprotocol(ndpi_str, string_to_match, string_to_match_len, ret_match, is_host_match);
if(matching_protocol_id < 0)
return NDPI_PROTOCOL_UNKNOWN;
return NDPI_PROTOCOL_UNKNOWN;
#ifdef DEBUG
{
@@ -6795,7 +6973,7 @@ u_int16_t ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_
if(ndpi_str->risky_domain_automa.ac_automa != NULL) {
u_int32_t proto_id;
u_int16_t rc1 = ndpi_match_string_common(ndpi_str->risky_domain_automa.ac_automa,
string_to_match,string_to_match_len, &proto_id, NULL, NULL);
string_to_match,string_to_match_len, &proto_id, NULL, NULL);
if(rc1 > 0)
ndpi_set_risk(flow, NDPI_RISKY_DOMAIN);
}
@@ -6844,10 +7022,10 @@ u_int16_t ndpi_match_content_subprotocol(struct ndpi_detection_module_struct *nd
static inline int ndpi_match_xgram(unsigned int *map,int l,const char *str) {
unsigned int i,c;
for(i=0,c=0; *str && i < l; i++) {
unsigned char a = (unsigned char)(*str++);
if(a < 'a' || a > 'z') return 0;
c *= XGRAMS_C;
c += a-'a';
unsigned char a = (unsigned char)(*str++);
if(a < 'a' || a > 'z') return 0;
c *= XGRAMS_C;
c += a-'a';
}
return (map[c >> 5] & (1u << (c & 0x1f))) != 0;
}
+15
View File
@@ -2050,3 +2050,18 @@ float ndpi_calculate_entropy(u_int8_t const * const buf, size_t len)
entropy *= -1.0f;
return entropy;
}
/* ******************************************* */
char* ndpi_get_flow_name(struct ndpi_flow_struct *flow) {
if(!flow) goto no_flow_info;
if(flow->host_server_name[0] != '\0')
return((char*)flow->host_server_name);
if(flow->protos.tls_quic_stun.tls_quic.client_requested_server_name[0] != '\0')
return(flow->protos.tls_quic_stun.tls_quic.client_requested_server_name);
no_flow_info:
return((char*)"");
}
+1
View File
@@ -54,6 +54,7 @@ typedef char AC_ALPHABET_t;
**/
typedef struct {
uint32_t number; /* Often used to store procotolId */
uint64_t number64;
uint16_t breed, category;
uint16_t level, /* Domain level for comparison */
from_start:1, /* match from start of string */