Domain Classification Improvements (#2396)
* Added size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize); size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize); used to compress short strings such as domain names. This code is based on https://github.com/Ed-von-Schleck/shoco * Major code rewrite for ndpi_hash and ndpi_domain_classify * Improvements to make sure custom categories are loaded and enabled * Fixed string encoding * Extended SalesForce/Cloudflare domains list
This commit is contained in:
+56
-37
@@ -954,7 +954,7 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
|
||||
|
||||
/* ************************ */
|
||||
|
||||
ndpi_enable_loaded_categories(ndpi_str);
|
||||
//ndpi_enable_loaded_categories(ndpi_str);
|
||||
|
||||
if(!ndpi_xgrams_inited) {
|
||||
ndpi_xgrams_inited = 1;
|
||||
@@ -3454,11 +3454,17 @@ static int is_ip_list_enabled(struct ndpi_detection_module_struct *ndpi_str, int
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* *********************************************** */
|
||||
|
||||
int ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
u_int i;
|
||||
|
||||
if(!ndpi_str)
|
||||
return -1;
|
||||
|
||||
if(!ndpi_str->custom_categories.categories_loaded)
|
||||
ndpi_enable_loaded_categories(ndpi_str);
|
||||
|
||||
if(ndpi_str->finalized) /* Already finalized */
|
||||
return 0;
|
||||
|
||||
@@ -4038,15 +4044,18 @@ int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_str,
|
||||
return(id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1);
|
||||
#else
|
||||
char buf[128];
|
||||
u_int8_t class_id;
|
||||
u_int16_t class_id;
|
||||
u_int max_len = sizeof(buf)-1;
|
||||
|
||||
if(!ndpi_str->custom_categories.categories_loaded)
|
||||
ndpi_enable_loaded_categories(ndpi_str);
|
||||
|
||||
if(name_len > max_len) name_len = max_len;
|
||||
memcpy(buf, name, name_len);
|
||||
buf[name_len] = '\0';
|
||||
|
||||
if(ndpi_domain_classify_contains(ndpi_str->custom_categories.sc_hostnames,
|
||||
&class_id, buf)) {
|
||||
if(ndpi_domain_classify_hostname(ndpi_str, ndpi_str->custom_categories.sc_hostnames,
|
||||
&class_id, buf)) {
|
||||
*category = (ndpi_protocol_category_t)class_id;
|
||||
return(0);
|
||||
} else
|
||||
@@ -4066,9 +4075,11 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
|
||||
ndpi_patricia_node_t *node;
|
||||
u_int cp_len = ndpi_min(sizeof(ipbuf) - 1, name_len);
|
||||
|
||||
*id = 0;
|
||||
|
||||
if(!ndpi_str->custom_categories.categories_loaded)
|
||||
return(-1);
|
||||
|
||||
ndpi_enable_loaded_categories(ndpi_str);
|
||||
|
||||
if(cp_len > 0) {
|
||||
memcpy(ipbuf, name_or_ip, cp_len);
|
||||
ipbuf[cp_len] = '\0';
|
||||
@@ -4085,7 +4096,8 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
|
||||
/* Search IPv4 */
|
||||
|
||||
/* Make sure all in network byte order otherwise compares wont work */
|
||||
ndpi_fill_prefix_v4(&prefix, &pin, 32, ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses)->maxbits);
|
||||
ndpi_fill_prefix_v4(&prefix, &pin, 32,
|
||||
((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses)->maxbits);
|
||||
node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses, &prefix);
|
||||
|
||||
if(node) {
|
||||
@@ -4095,7 +4107,8 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
|
||||
return(-1);
|
||||
} else if(inet_pton(AF_INET6, ipbuf, &pin6) == 1) {
|
||||
/* Search IPv6 */
|
||||
ndpi_fill_prefix_v6(&prefix, &pin6, 128, ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits);
|
||||
ndpi_fill_prefix_v6(&prefix, &pin6, 128,
|
||||
((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits);
|
||||
node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses6, &prefix);
|
||||
|
||||
if(node) {
|
||||
@@ -4196,10 +4209,10 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->tls_cert_subject_automa.ac_automa, 0);
|
||||
|
||||
if(ndpi_str->malicious_ja3_hashmap != NULL)
|
||||
ndpi_hash_free(&ndpi_str->malicious_ja3_hashmap, NULL);
|
||||
ndpi_hash_free(&ndpi_str->malicious_ja3_hashmap);
|
||||
|
||||
if(ndpi_str->malicious_sha1_hashmap != NULL)
|
||||
ndpi_hash_free(&ndpi_str->malicious_sha1_hashmap, NULL);
|
||||
ndpi_hash_free(&ndpi_str->malicious_sha1_hashmap);
|
||||
|
||||
#ifdef USE_LEGACY_AHO_CORASICK
|
||||
if(ndpi_str->custom_categories.hostnames.ac_automa != NULL)
|
||||
@@ -4260,7 +4273,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
ndpi_free(ndpi_str->callback_buffer_tcp_payload);
|
||||
|
||||
if(ndpi_str->public_domain_suffixes)
|
||||
ndpi_domain_classify_free(ndpi_str->public_domain_suffixes);
|
||||
ndpi_hash_free(&ndpi_str->public_domain_suffixes);
|
||||
|
||||
ndpi_free(ndpi_str);
|
||||
}
|
||||
@@ -4865,6 +4878,8 @@ int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str,
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
int load_categories_file_fd(struct ndpi_detection_module_struct *ndpi_str,
|
||||
FILE *fd, void *user_data) {
|
||||
char buffer[512], *line, *name, *category, *saveptr;
|
||||
@@ -5064,10 +5079,10 @@ int ndpi_load_categories_dir(struct ndpi_detection_module_struct *ndpi_str,
|
||||
|
||||
if(failed_files)
|
||||
return(-1 * failed_files);
|
||||
|
||||
return(num_loaded);
|
||||
}
|
||||
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static int ndpi_load_risky_domain(struct ndpi_detection_module_struct *ndpi_str,
|
||||
@@ -5209,7 +5224,7 @@ int load_malicious_ja3_file_fd(struct ndpi_detection_module_struct *ndpi_str, FI
|
||||
continue;
|
||||
}
|
||||
|
||||
if(ndpi_hash_add_entry(&ndpi_str->malicious_ja3_hashmap, line, len, NULL) == 0)
|
||||
if(ndpi_hash_add_entry(&ndpi_str->malicious_ja3_hashmap, line, len, 0) == 0)
|
||||
num++;
|
||||
}
|
||||
|
||||
@@ -5287,7 +5302,8 @@ int load_malicious_sha1_file_fd(struct ndpi_detection_module_struct *ndpi_str, F
|
||||
for (i = 0; i < 40; ++i)
|
||||
first_comma[i] = toupper(first_comma[i]);
|
||||
|
||||
if(ndpi_hash_add_entry(&ndpi_str->malicious_sha1_hashmap, first_comma, second_comma - first_comma, NULL) == 0)
|
||||
if(ndpi_hash_add_entry(&ndpi_str->malicious_sha1_hashmap, first_comma,
|
||||
second_comma - first_comma, 0) == 0)
|
||||
num++;
|
||||
}
|
||||
|
||||
@@ -6225,42 +6241,42 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
/* ******************************************************************** */
|
||||
|
||||
static inline int ndpi_proto_cb_tcp_payload(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static inline int ndpi_proto_cb_tcp_nopayload(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0
|
||||
&& (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0;
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0
|
||||
&& (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static inline int ndpi_proto_cb_udp(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static inline int ndpi_proto_cb_other(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP)) == 0
|
||||
||
|
||||
(ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) != 0;
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP)) == 0
|
||||
||
|
||||
(ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) != 0;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
@@ -7896,7 +7912,7 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_str,
|
||||
if(ndpi_str->custom_categories.sc_hostnames_shadow == NULL)
|
||||
return(-1);
|
||||
|
||||
return(ndpi_domain_classify_add(ndpi_str->custom_categories.sc_hostnames_shadow,
|
||||
return(ndpi_domain_classify_add(ndpi_str, ndpi_str->custom_categories.sc_hostnames_shadow,
|
||||
(u_int16_t)category, (char*)name_to_add) ? 0 : -1);
|
||||
#endif
|
||||
}
|
||||
@@ -7931,6 +7947,9 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
|
||||
int i;
|
||||
static char *built_in = "built-in";
|
||||
|
||||
if(ndpi_str->custom_categories.categories_loaded)
|
||||
return(-1); /* Already loaded */
|
||||
|
||||
/* First add the nDPI known categories matches */
|
||||
for(i = 0; category_match[i].string_to_match != NULL; i++)
|
||||
ndpi_load_category(ndpi_str, category_match[i].string_to_match,
|
||||
@@ -7956,7 +7975,7 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
|
||||
}
|
||||
#else
|
||||
ndpi_domain_classify_free(ndpi_str->custom_categories.sc_hostnames);
|
||||
ndpi_domain_classify_finalize(ndpi_str->custom_categories.sc_hostnames_shadow);
|
||||
// ndpi_domain_classify_finalize(ndpi_str->custom_categories.sc_hostnames_shadow);
|
||||
ndpi_str->custom_categories.sc_hostnames = ndpi_str->custom_categories.sc_hostnames_shadow;
|
||||
ndpi_str->custom_categories.sc_hostnames_shadow = ndpi_domain_classify_alloc();
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user