Domain Classification Improvements (#2396)

* Added
size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize);
size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize);

used to compress short strings such as domain names. This code is based on
https://github.com/Ed-von-Schleck/shoco

* Major code rewrite for ndpi_hash and ndpi_domain_classify

* Improvements to make sure custom categories are loaded and enabled

* Fixed string encoding

* Extended SalesForce/Cloudflare domains list
This commit is contained in:
Luca Deri
2024-04-18 23:21:40 +02:00
committed by GitHub
parent 108b8331d5
commit ad117bfaab
65 changed files with 1194 additions and 612 deletions
+56 -37
View File
@@ -954,7 +954,7 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
/* ************************ */
ndpi_enable_loaded_categories(ndpi_str);
//ndpi_enable_loaded_categories(ndpi_str);
if(!ndpi_xgrams_inited) {
ndpi_xgrams_inited = 1;
@@ -3454,11 +3454,17 @@ static int is_ip_list_enabled(struct ndpi_detection_module_struct *ndpi_str, int
return 1;
}
/* *********************************************** */
int ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str) {
u_int i;
if(!ndpi_str)
return -1;
if(!ndpi_str->custom_categories.categories_loaded)
ndpi_enable_loaded_categories(ndpi_str);
if(ndpi_str->finalized) /* Already finalized */
return 0;
@@ -4038,15 +4044,18 @@ int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_str,
return(id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1);
#else
char buf[128];
u_int8_t class_id;
u_int16_t class_id;
u_int max_len = sizeof(buf)-1;
if(!ndpi_str->custom_categories.categories_loaded)
ndpi_enable_loaded_categories(ndpi_str);
if(name_len > max_len) name_len = max_len;
memcpy(buf, name, name_len);
buf[name_len] = '\0';
if(ndpi_domain_classify_contains(ndpi_str->custom_categories.sc_hostnames,
&class_id, buf)) {
if(ndpi_domain_classify_hostname(ndpi_str, ndpi_str->custom_categories.sc_hostnames,
&class_id, buf)) {
*category = (ndpi_protocol_category_t)class_id;
return(0);
} else
@@ -4066,9 +4075,11 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
ndpi_patricia_node_t *node;
u_int cp_len = ndpi_min(sizeof(ipbuf) - 1, name_len);
*id = 0;
if(!ndpi_str->custom_categories.categories_loaded)
return(-1);
ndpi_enable_loaded_categories(ndpi_str);
if(cp_len > 0) {
memcpy(ipbuf, name_or_ip, cp_len);
ipbuf[cp_len] = '\0';
@@ -4085,7 +4096,8 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
/* Search IPv4 */
/* Make sure all in network byte order otherwise compares wont work */
ndpi_fill_prefix_v4(&prefix, &pin, 32, ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses)->maxbits);
ndpi_fill_prefix_v4(&prefix, &pin, 32,
((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses)->maxbits);
node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses, &prefix);
if(node) {
@@ -4095,7 +4107,8 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
return(-1);
} else if(inet_pton(AF_INET6, ipbuf, &pin6) == 1) {
/* Search IPv6 */
ndpi_fill_prefix_v6(&prefix, &pin6, 128, ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits);
ndpi_fill_prefix_v6(&prefix, &pin6, 128,
((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits);
node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses6, &prefix);
if(node) {
@@ -4196,10 +4209,10 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->tls_cert_subject_automa.ac_automa, 0);
if(ndpi_str->malicious_ja3_hashmap != NULL)
ndpi_hash_free(&ndpi_str->malicious_ja3_hashmap, NULL);
ndpi_hash_free(&ndpi_str->malicious_ja3_hashmap);
if(ndpi_str->malicious_sha1_hashmap != NULL)
ndpi_hash_free(&ndpi_str->malicious_sha1_hashmap, NULL);
ndpi_hash_free(&ndpi_str->malicious_sha1_hashmap);
#ifdef USE_LEGACY_AHO_CORASICK
if(ndpi_str->custom_categories.hostnames.ac_automa != NULL)
@@ -4260,7 +4273,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
ndpi_free(ndpi_str->callback_buffer_tcp_payload);
if(ndpi_str->public_domain_suffixes)
ndpi_domain_classify_free(ndpi_str->public_domain_suffixes);
ndpi_hash_free(&ndpi_str->public_domain_suffixes);
ndpi_free(ndpi_str);
}
@@ -4865,6 +4878,8 @@ int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str,
return rc;
}
/* ******************************************************************** */
int load_categories_file_fd(struct ndpi_detection_module_struct *ndpi_str,
FILE *fd, void *user_data) {
char buffer[512], *line, *name, *category, *saveptr;
@@ -5064,10 +5079,10 @@ int ndpi_load_categories_dir(struct ndpi_detection_module_struct *ndpi_str,
if(failed_files)
return(-1 * failed_files);
return(num_loaded);
}
/* ******************************************************************** */
static int ndpi_load_risky_domain(struct ndpi_detection_module_struct *ndpi_str,
@@ -5209,7 +5224,7 @@ int load_malicious_ja3_file_fd(struct ndpi_detection_module_struct *ndpi_str, FI
continue;
}
if(ndpi_hash_add_entry(&ndpi_str->malicious_ja3_hashmap, line, len, NULL) == 0)
if(ndpi_hash_add_entry(&ndpi_str->malicious_ja3_hashmap, line, len, 0) == 0)
num++;
}
@@ -5287,7 +5302,8 @@ int load_malicious_sha1_file_fd(struct ndpi_detection_module_struct *ndpi_str, F
for (i = 0; i < 40; ++i)
first_comma[i] = toupper(first_comma[i]);
if(ndpi_hash_add_entry(&ndpi_str->malicious_sha1_hashmap, first_comma, second_comma - first_comma, NULL) == 0)
if(ndpi_hash_add_entry(&ndpi_str->malicious_sha1_hashmap, first_comma,
second_comma - first_comma, 0) == 0)
num++;
}
@@ -6225,42 +6241,42 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
/* ******************************************************************** */
static inline int ndpi_proto_cb_tcp_payload(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
}
/* ******************************************************************** */
static inline int ndpi_proto_cb_tcp_nopayload(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0
&& (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0;
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0
&& (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0;
}
/* ******************************************************************** */
static inline int ndpi_proto_cb_udp(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
}
/* ******************************************************************** */
static inline int ndpi_proto_cb_other(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP)) == 0
||
(ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) != 0;
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP)) == 0
||
(ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) != 0;
}
/* ******************************************************************** */
@@ -7896,7 +7912,7 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_str,
if(ndpi_str->custom_categories.sc_hostnames_shadow == NULL)
return(-1);
return(ndpi_domain_classify_add(ndpi_str->custom_categories.sc_hostnames_shadow,
return(ndpi_domain_classify_add(ndpi_str, ndpi_str->custom_categories.sc_hostnames_shadow,
(u_int16_t)category, (char*)name_to_add) ? 0 : -1);
#endif
}
@@ -7931,6 +7947,9 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
int i;
static char *built_in = "built-in";
if(ndpi_str->custom_categories.categories_loaded)
return(-1); /* Already loaded */
/* First add the nDPI known categories matches */
for(i = 0; category_match[i].string_to_match != NULL; i++)
ndpi_load_category(ndpi_str, category_match[i].string_to_match,
@@ -7956,7 +7975,7 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
}
#else
ndpi_domain_classify_free(ndpi_str->custom_categories.sc_hostnames);
ndpi_domain_classify_finalize(ndpi_str->custom_categories.sc_hostnames_shadow);
// ndpi_domain_classify_finalize(ndpi_str->custom_categories.sc_hostnames_shadow);
ndpi_str->custom_categories.sc_hostnames = ndpi_str->custom_categories.sc_hostnames_shadow;
ndpi_str->custom_categories.sc_hostnames_shadow = ndpi_domain_classify_alloc();
#endif