Improved hyperscan support and category definition

This commit is contained in:
Luca Deri
2018-05-10 23:46:35 +02:00
parent 3707553906
commit 50216991bd
7 changed files with 283 additions and 77 deletions
+3 -3
View File
@@ -751,9 +751,9 @@ extern "C" {
void ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_struct,
char *ip_address_and_mask, ndpi_protocol_category_t category);
void ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, ndpi_protocol_category_t category);
void ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct);
int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, ndpi_protocol_category_t category);
int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct);
/**
* Add a string to match to an automata
+14
View File
@@ -828,6 +828,14 @@ typedef struct ndpi_proto {
#define NUM_CUSTOM_CATEGORIES 5
#define CUSTOM_CATEGORY_LABEL_LEN 32
#ifdef HAVE_HYPERSCAN
struct hs_list {
char *expression;
unsigned int id;
struct hs_list *next;
};
#endif
struct ndpi_detection_module_struct {
NDPI_PROTOCOL_BITMASK detection_bitmask;
NDPI_PROTOCOL_BITMASK generic_http_packet_bitmask;
@@ -886,7 +894,13 @@ struct ndpi_detection_module_struct {
bigrams_automa, impossible_bigrams_automa; /* TOR */
struct {
#ifdef HAVE_HYPERSCAN
struct hs *hostnames;
unsigned int num_to_load;
struct hs_list *to_load;
#else
ndpi_automa hostnames, hostnames_shadow;
#endif
void *ipAddresses, *ipAddresses_shadow; /* Patricia */
} custom_categories;
+230 -49
View File
@@ -745,12 +745,36 @@ void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod,
#ifdef HAVE_HYPERSCAN
static int hyperscan_load_patterns(struct hs *hs, u_int num_patterns,
const char **expressions, unsigned int *ids) {
hs_compile_error_t *compile_err;
if(hs_compile_multi(expressions, NULL, ids,
num_patterns, HS_MODE_BLOCK, NULL,
&hs->database, &compile_err) != HS_SUCCESS) {
NDPI_LOG_ERR(ndpi_mod, "Unable to initialize hyperscan database\n");
hs_free_compile_error(compile_err);
return -1;
}
hs->scratch = NULL;
if(hs_alloc_scratch(hs->database, &hs->scratch) != HS_SUCCESS) {
NDPI_LOG_ERR(ndpi_mod, "Unable to allocate hyperscan scratch space\n");
hs_free_database(hs->database);
return -1;
}
return 0;
}
/* ******************************************************************** */
static int init_hyperscan(struct ndpi_detection_module_struct *ndpi_mod) {
u_int num_patterns = 0, i;
const char **expressions;
unsigned int *ids;
hs_compile_error_t *compile_err;
struct hs *hs;
int rc;
ndpi_mod->hyperscan = (void*)malloc(sizeof(struct hs));
if(!ndpi_mod->hyperscan) return(-1);
@@ -780,32 +804,26 @@ static int init_hyperscan(struct ndpi_detection_module_struct *ndpi_mod) {
}
}
if(hs_compile_multi(expressions, NULL, ids,
num_patterns, HS_MODE_BLOCK, NULL,
&hs->database, &compile_err) != HS_SUCCESS) {
NDPI_LOG_ERR(ndpi_mod, "Unable to initialize hyperscan database\n");
hs_free_compile_error(compile_err);
return -1;
}
rc = hyperscan_load_patterns(hs, num_patterns, expressions, ids);
free(expressions), free(ids);
if(hs_alloc_scratch(hs->database, &hs->scratch) != HS_SUCCESS) {
NDPI_LOG_ERR(ndpi_mod, "Unable to allocate hyperscan scratch space\n");
hs_free_database(hs->database);
return -1;
}
return(rc);
}
return 0;
/* ******************************************************************** */
static void free_hyperscan_memory(struct hs *h) {
if(h) {
hs_free_scratch(h->scratch);
hs_free_database(h->database);
}
}
/* ******************************************************************** */
static void destroy_hyperscan(struct ndpi_detection_module_struct *ndpi_mod) {
if(ndpi_mod->hyperscan) {
struct hs *hs = (struct hs*)ndpi_mod->hyperscan;
hs_free_scratch(hs->scratch);
hs_free_database(hs->database);
}
if(ndpi_mod->hyperscan)
free_hyperscan_memory((struct hs*)ndpi_mod->hyperscan);
}
#endif
@@ -1946,12 +1964,12 @@ static int ndpi_add_host_ip_subprotocol(struct ndpi_detection_module_struct *ndp
if(atoi(ptr)>=0 && atoi(ptr)<=32)
bits = atoi(ptr);
}
inet_pton(AF_INET, value, &pin);
if((node = add_to_ptree(ndpi_struct->protocols_ptree, AF_INET, &pin, bits)) != NULL)
node->value.user_value = protocol_id;
return 0;
}
@@ -2048,15 +2066,21 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(void) {
ndpi_str->bigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->impossible_bigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
#ifdef HAVE_HYPERSCAN
ndpi_str->custom_categories.num_to_load = 0, ndpi_str->custom_categories.to_load = NULL;
ndpi_str->custom_categories.hostnames = NULL;
#else
ndpi_str->custom_categories.hostnames.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler);
#endif
ndpi_str->custom_categories.ipAddresses = ndpi_New_Patricia(32 /* IPv4 */);
ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */);
if((ndpi_str->custom_categories.ipAddresses == NULL)
|| (ndpi_str->custom_categories.ipAddresses_shadow == NULL))
return(NULL);
ndpi_init_protocol_defaults(ndpi_str);
for(i=0; i<NUM_CUSTOM_CATEGORIES; i++)
@@ -2132,6 +2156,54 @@ int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id
/* *********************************************** */
#ifdef HAVE_HYPERSCAN
static int hyperscanCustomEventHandler(unsigned int id,
unsigned long long from,
unsigned long long to,
unsigned int flags, void *ctx) {
*((unsigned long *)ctx) = (unsigned long)id;
#ifdef DEBUG
printf("[HS] Found category %u\n", id);
#endif
return HS_SCAN_TERMINATED;
}
#endif
/* *********************************************** */
static int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, unsigned long *id) {
#ifdef HAVE_HYPERSCAN
if(ndpi_struct->custom_categories.hostnames == NULL)
return(-1);
else {
hs_error_t rc;
*id = (unsigned long)-1;
rc = hs_scan(ndpi_struct->custom_categories.hostnames->database,
name, strlen(name), 0,
ndpi_struct->custom_categories.hostnames->scratch,
hyperscanCustomEventHandler, id);
if(rc == HS_SCAN_TERMINATED) {
#ifdef DEBUG
printf("[HS] Found category %lu for %s\n", *id, name);
#endif
return(0);
} else
return(-1);
}
#else
return(ndpi_match_string_id(ndpi_struct->custom_categories.hostnames.ac_automa, name, id));
#endif
}
/* *********************************************** */
static void free_ptree_data(void *data) { ; }
/* ****************************************************** */
@@ -2172,20 +2244,30 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct
#ifdef HAVE_HYPERSCAN
destroy_hyperscan(ndpi_struct);
#endif
while(ndpi_struct->custom_categories.to_load != NULL) {
struct hs_list *next = ndpi_struct->custom_categories.to_load->next;
free(ndpi_struct->custom_categories.to_load->expression);
free(ndpi_struct->custom_categories.to_load);
ndpi_struct->custom_categories.to_load = next;
}
free_hyperscan_memory(ndpi_struct->custom_categories.hostnames);
#else
if(ndpi_struct->custom_categories.hostnames.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t*)ndpi_struct->custom_categories.hostnames.ac_automa);
if(ndpi_struct->custom_categories.hostnames_shadow.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t*)ndpi_struct->custom_categories.hostnames_shadow.ac_automa);
#endif
if(ndpi_struct->custom_categories.ipAddresses != NULL)
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->custom_categories.ipAddresses, free_ptree_data);
if(ndpi_struct->custom_categories.ipAddresses_shadow != NULL)
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->custom_categories.ipAddresses_shadow, free_ptree_data);
ndpi_free(ndpi_struct);
}
}
@@ -3754,9 +3836,9 @@ void ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_struct,
if (atoi(ptr)>=0 && atoi(ptr)<=32)
bits = atoi(ptr);
}
inet_pton(AF_INET, ip_address_and_mask, &pin);
if((node = add_to_ptree(ndpi_struct->custom_categories.ipAddresses_shadow,
AF_INET, &pin, bits)) != NULL)
node->value.user_value = (int)category;
@@ -3764,38 +3846,135 @@ void ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_struct,
/* ********************************************************************************* */
void ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct,
int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, ndpi_protocol_category_t category) {
AC_PATTERN_t ac_pattern;
if(name == NULL) return;
if(name == NULL)
return(-1);
/* printf("===> Loading %s as %u\n", name, category); */
if(ndpi_struct->custom_categories.hostnames_shadow.ac_automa == NULL) return;
#ifdef HAVE_HYPERSCAN
{
struct hs_list *h = (struct hs_list*)malloc(sizeof(struct hs_list));
if(h) {
char tmp[256];
int i, j;
for(i=0, j=0; (j<sizeof(tmp)) && (name[i] != '\0'); i++) {
if(name[i] == '.')
tmp[j++] = '\\';
tmp[j++] = name[i];
}
tmp[j] = '\0';
h->expression = strdup(name), h->id = (unsigned int)category;
if(h->expression == NULL) {
free(h);
return(-2);
}
h->next = ndpi_struct->custom_categories.to_load;
ndpi_struct->custom_categories.to_load = h;
ndpi_struct->custom_categories.num_to_load++;
} else
return(-1);
}
#else
if(ndpi_struct->custom_categories.hostnames_shadow.ac_automa == NULL)
return(-1);
ac_pattern.astring = name, ac_pattern.length = strlen(ac_pattern.astring);
ac_pattern.rep.number = (int)category;
ac_automata_add(ndpi_struct->custom_categories.hostnames_shadow.ac_automa, &ac_pattern);
#endif
return(0);
}
/* ********************************************************************************* */
void ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct) {
int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str) {
#ifdef HAVE_HYPERSCAN
if(ndpi_str->custom_categories.num_to_load > 0) {
const char **expressions;
unsigned int *ids, i;
int rc;
struct hs_list *head = ndpi_str->custom_categories.to_load;
expressions = (const char**)calloc(sizeof(char*),
ndpi_str->custom_categories.num_to_load+1);
if(!expressions) return(-1);
ids = (unsigned int*)calloc(sizeof(unsigned int),
ndpi_str->custom_categories.num_to_load+1);
if(!ids) {
free(expressions);
return(-1);
}
for(i=0; head != NULL; i++) {
#ifdef DEBUG
printf("[HS] Loading category %u for %s\n", head->id, head->expression);
#endif
expressions[i] = head->expression, ids[i] = head->id;
head = head->next;
}
free_hyperscan_memory(ndpi_str->custom_categories.hostnames);
ndpi_str->custom_categories.hostnames = (struct hs*)malloc(sizeof(struct hs));
if(ndpi_str->custom_categories.hostnames == NULL) {
free(expressions), free(ids);
return(-1); /* Failed */
}
rc = hyperscan_load_patterns(ndpi_str->custom_categories.hostnames,
ndpi_str->custom_categories.num_to_load,
expressions, ids);
free(expressions), free(ids);
head = ndpi_str->custom_categories.to_load;
while(head != NULL) {
struct hs_list *next = head->next;
free(head->expression);
free(head);
head = next;
}
ndpi_str->custom_categories.to_load = NULL;
if(rc < 0) {
free(ndpi_str->custom_categories.hostnames);
ndpi_str->custom_categories.hostnames = NULL;
ndpi_str->custom_categories.hostnames = NULL;
}
}
#else
/* Free */
ac_automata_release((AC_AUTOMATA_t*)ndpi_struct->custom_categories.hostnames.ac_automa);
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->custom_categories.ipAddresses, free_ptree_data);
ac_automata_release((AC_AUTOMATA_t*)ndpi_str->custom_categories.hostnames.ac_automa);
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_str->custom_categories.ipAddresses, free_ptree_data);
/* Finalize */
ac_automata_finalize((AC_AUTOMATA_t*)ndpi_struct->custom_categories.hostnames_shadow.ac_automa);
ac_automata_finalize((AC_AUTOMATA_t*)ndpi_str->custom_categories.hostnames_shadow.ac_automa);
/* Swap */
ndpi_struct->custom_categories.hostnames.ac_automa = ndpi_struct->custom_categories.hostnames_shadow.ac_automa;
ndpi_struct->custom_categories.ipAddresses = ndpi_struct->custom_categories.ipAddresses_shadow;
ndpi_str->custom_categories.hostnames.ac_automa = ndpi_str->custom_categories.hostnames_shadow.ac_automa;
ndpi_str->custom_categories.ipAddresses = ndpi_str->custom_categories.ipAddresses_shadow;
/* Realloc */
ndpi_struct->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler);
ndpi_struct->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */);
ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */);
#endif
return(0);
}
/* ********************************************************************************* */
@@ -3806,18 +3985,18 @@ static void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndp
if(flow->packet.iph) {
prefix_t prefix;
patricia_node_t *node;
/* Make sure all in network byte order otherwise compares wont work */
fill_prefix_v4(&prefix, (struct in_addr *)&flow->packet.iph->saddr,
32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
if(!node) {
fill_prefix_v4(&prefix, (struct in_addr *)&flow->packet.iph->daddr,
32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
}
if(node) {
ret->category = (ndpi_protocol_category_t)node->value.user_value;
return;
@@ -3826,7 +4005,7 @@ static void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndp
if(flow->host_server_name[0] != '\0') {
unsigned long id;
int rc = ndpi_match_string_id(ndpi_struct->custom_categories.hostnames.ac_automa, (char *)flow->host_server_name, &id);
int rc =ndpi_match_custom_category(ndpi_struct, (char *)flow->host_server_name, &id);
if(rc == 0) {
ret->category = (ndpi_protocol_category_t)id;
@@ -3836,7 +4015,7 @@ static void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndp
if(flow->protos.ssl.server_certificate[0] != '\0') {
unsigned long id;
int rc = ndpi_match_string_id(ndpi_struct->custom_categories.hostnames.ac_automa, (char *)flow->protos.ssl.server_certificate, &id);
int rc = ndpi_match_custom_category(ndpi_struct, (char *)flow->protos.ssl.server_certificate, &id);
if(rc == 0) {
ret->category = (ndpi_protocol_category_t)id;
@@ -5292,11 +5471,13 @@ static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_str
/* ******************************************************************** */
static int hyperscanEventHandler(unsigned int id, unsigned long long from,
unsigned long long to, unsigned int flags, void *ctx) {
unsigned long long to, unsigned int flags, void *ctx) {
*((int *)ctx) = (int)id;
return HS_SCAN_TERMINATED;
}
/* *********************************************** */
static int ndpi_automa_match_string_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
char *string_to_match, u_int string_to_match_len,