Domain Classification Improvements (#2396)

* Added
size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize);
size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize);

used to compress short strings such as domain names. This code is based on
https://github.com/Ed-von-Schleck/shoco

* Major code rewrite for ndpi_hash and ndpi_domain_classify

* Improvements to make sure custom categories are loaded and enabled

* Fixed string encoding

* Extended SalesForce/Cloudflare domains list
This commit is contained in:
Luca Deri
2024-04-18 23:21:40 +02:00
committed by GitHub
parent 108b8331d5
commit ad117bfaab
65 changed files with 1194 additions and 612 deletions
+194 -76
View File
@@ -1,7 +1,7 @@
/*
* ndpi_utils.c
*
* Copyright (C) 2011-23 - ntop.org and contributors
* Copyright (C) 2011-24 - ntop.org and contributors
*
* This file is part of nDPI, an open source deep packet inspection
* library based on the OpenDPI and PACE technology by ipoque GmbH
@@ -38,6 +38,7 @@
#include "ahocorasick.h"
#include "libcache.h"
#include "shoco.h"
#include <time.h>
#ifndef WIN32
@@ -72,21 +73,11 @@ struct pcre2_struct {
};
#endif
/*
* Please keep this strcture in sync with
* `struct ndpi_str_hash` in src/include/ndpi_typedefs.h
*/
typedef struct ndpi_str_hash_private {
unsigned int hash;
void *value;
// u_int8_t private_data[1]; /* Avoid error C2466 and do not initiate private data with 0 */
typedef struct {
char *key;
u_int16_t value16;
UT_hash_handle hh;
} ndpi_str_hash_private;
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
_Static_assert(sizeof(struct ndpi_str_hash) == sizeof(struct ndpi_str_hash_private) - sizeof(UT_hash_handle),
"Please keep `struct ndpi_str_hash` and `struct ndpi_str_hash_private` syncd.");
#endif
} ndpi_str_hash_priv;
/* ****************************************** */
@@ -1246,15 +1237,15 @@ static void ndpi_tls2json(ndpi_serializer *serializer, struct ndpi_flow_struct *
static char* print_ndpi_address_port(ndpi_address_port *ap, char *buf, u_int buf_len) {
char ipbuf[INET6_ADDRSTRLEN];
if(ap->is_ipv6) {
inet_ntop(AF_INET6, &ap->address, ipbuf, sizeof(ipbuf));
} else {
inet_ntop(AF_INET, &ap->address, ipbuf, sizeof(ipbuf));
}
snprintf(buf, buf_len, "%s:%u", ipbuf, ap->port);
return(buf);
}
@@ -2280,12 +2271,9 @@ ndpi_http_method ndpi_http_str2method(const char* method, u_int16_t method_len)
/* ******************************************************************** */
int ndpi_hash_init(ndpi_str_hash **h)
{
int ndpi_hash_init(ndpi_str_hash **h) {
if (h == NULL)
{
return 1;
}
*h = NULL;
return 0;
@@ -2293,77 +2281,69 @@ int ndpi_hash_init(ndpi_str_hash **h)
/* ******************************************************************** */
void ndpi_hash_free(ndpi_str_hash **h, void (*cleanup_func)(ndpi_str_hash *h))
{
struct ndpi_str_hash_private *h_priv;
struct ndpi_str_hash_private *current, *tmp;
if (h == NULL)
{
return;
}
h_priv = *(struct ndpi_str_hash_private **)h;
HASH_ITER(hh, h_priv, current, tmp) {
HASH_DEL(h_priv, current);
if (cleanup_func != NULL)
{
cleanup_func((ndpi_str_hash *)current);
void ndpi_hash_free(ndpi_str_hash **h) {
if(h != NULL) {
ndpi_str_hash_priv *h_priv = *((ndpi_str_hash_priv **)h);
ndpi_str_hash_priv *current, *tmp;
HASH_ITER(hh, h_priv, current, tmp) {
HASH_DEL(h_priv, current);
ndpi_free(current->key);
ndpi_free(current);
}
ndpi_free(current);
*h = NULL;
}
*h = NULL;
}
/* ******************************************************************** */
int ndpi_hash_find_entry(ndpi_str_hash *h, char *key, u_int key_len, void **value)
{
struct ndpi_str_hash_private *h_priv = (struct ndpi_str_hash_private *)h;
struct ndpi_str_hash_private *found;
unsigned int hash_value;
int ndpi_hash_find_entry(ndpi_str_hash *h, char *key, u_int key_len, u_int16_t *value) {
ndpi_str_hash_priv *h_priv = (ndpi_str_hash_priv *)h;
ndpi_str_hash_priv *item;
HASH_FIND(hh, h_priv, key, key_len, item);
if (item != NULL) {
if(value != NULL)
*value = item->value16;
HASH_VALUE(key, key_len, hash_value);
HASH_FIND_INT(h_priv, &hash_value, found);
if (found != NULL)
{
if (value != NULL)
{
*value = found->value;
}
return 0;
} else {
} else
return 1;
}
}
/* ******************************************************************** */
int ndpi_hash_add_entry(ndpi_str_hash **h, char *key, u_int8_t key_len, void *value)
{
struct ndpi_str_hash_private **h_priv = (struct ndpi_str_hash_private **)h;
struct ndpi_str_hash_private *new = ndpi_calloc(1, sizeof(*new));
struct ndpi_str_hash_private *found;
unsigned int hash_value;
int ndpi_hash_add_entry(ndpi_str_hash **h, char *key, u_int8_t key_len, u_int16_t value) {
ndpi_str_hash_priv *h_priv = (ndpi_str_hash_priv *)*h;
ndpi_str_hash_priv *item;
if (new == NULL)
{
return 1;
HASH_FIND(hh, h_priv, key, key_len, item);
if(item != NULL) {
item->value16 = value;
return(1); /* Entry already present */
}
HASH_VALUE(key, key_len, hash_value);
new->hash = hash_value;
new->value = value;
HASH_ADD_INT(*h_priv, hash, new);
item = ndpi_calloc(1, sizeof(ndpi_str_hash_priv));
if(item == NULL)
return(2);
HASH_FIND_INT(*h_priv, &hash_value, found);
if (found == NULL) /* The insertion failed (because of a memory allocation error) */
{
ndpi_free(new);
return 1;
item->key = ndpi_malloc(key_len+1);
if(item->key == NULL) {
ndpi_free(item);
return(1);
} else {
memcpy(item->key, key, key_len);
item->key[key_len] = '\0';
}
item->value16 = value;
HASH_ADD(hh, *((ndpi_str_hash_priv **)h), key[0], key_len, item);
return 0;
}
@@ -2502,7 +2482,7 @@ void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str,
if(host && (host[0] != '\0')) {
/* Check host exception */
ndpi_check_hostname_risk_exception(ndpi_str, flow, host);
if(flow->risk_mask == 0) {
u_int i;
@@ -2555,7 +2535,7 @@ void ndpi_set_risk(struct ndpi_flow_struct *flow, ndpi_risk_enum r,
/* In case there is an exception set, take it into account */
if(flow->host_risk_mask_evaluated)
v &= flow->risk_mask;
// NDPI_SET_BIT(flow->risk, (u_int32_t)r);
flow->risk |= v;
@@ -3149,3 +3129,141 @@ const char *ndpi_lru_cache_idx_to_name(lru_cache_type idx)
return "unknown";
return names[idx];
}
/* ******************************************* */
size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize) {
size_t ret = shoco_compress(in, len, out, bufsize);
if(ret > bufsize)
return(0); /* Better not to compress data (it is longer than the uncompressed data) */
return(ret);
}
/* ******************************************* */
size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize) {
return(shoco_decompress(in, len, out, bufsize));
}
/* ******************************************* */
static u_char ndpi_domain_mapper[256];
static bool ndpi_domain_mapper_initialized = false;
#define IGNORE_CHAR 0xFF
#define NUM_BITS_NIBBLE 6 /* each 'nibble' is encoded with 6 bits */
#define NIBBLE_ELEM_OFFSET 24
/* Used fo encoding domain names 8 bits -> 6 bits */
static void ndpi_domain_mapper_init() {
u_int i;
u_char idx = 1 /* start from 1 to make sure 0 is no ambiguous */;
memset(ndpi_domain_mapper, IGNORE_CHAR, 256);
for(i='a'; i<= 'z'; i++)
ndpi_domain_mapper[i] = idx++;
for(i='0'; i<= '9'; i++)
ndpi_domain_mapper[i] = idx++;
ndpi_domain_mapper['-'] = idx++;
ndpi_domain_mapper['_'] = idx++;
ndpi_domain_mapper['.'] = idx++;
}
/* ************************************************ */
u_int ndpi_encode_domain(struct ndpi_detection_module_struct *ndpi_str,
char *domain, char *out, u_int out_len) {
u_int out_idx = 0, i, buf_shift = 0, domain_buf_len, compressed_len, suffix_len, domain_len;
u_int32_t value = 0;
u_char domain_buf[256], compressed[128];
u_int16_t domain_id = 0;
const char *suffix;
if(!ndpi_domain_mapper_initialized) {
ndpi_domain_mapper_init();
ndpi_domain_mapper_initialized = true;
}
domain_len = strlen(domain);
if(domain_len >= (out_len-3))
return(0);
if(domain_len <= 4)
return((u_int)snprintf(out, out_len, "%s", domain)); /* Too short */
/* [1] Encode the domain in 6 bits */
suffix = ndpi_get_host_domain_suffix(ndpi_str, domain, &domain_id);
if(suffix == NULL)
return((u_int)snprintf(out, out_len, "%s", domain)); /* Unknown suffix */
snprintf((char*)domain_buf, sizeof(domain_buf), "%s", domain);
domain_buf_len = strlen((char*)domain_buf), suffix_len = strlen(suffix);
if(domain_buf_len > suffix_len) {
snprintf((char*)domain_buf, sizeof(domain_buf), "%s", domain);
domain_buf_len = domain_buf_len-suffix_len-1;
domain_buf[domain_buf_len] = '\0';
for(i=0; domain_buf[i] != '\0'; i++) {
u_int32_t mapped_idx = ndpi_domain_mapper[domain_buf[i]];
if(mapped_idx != IGNORE_CHAR) {
mapped_idx <<= buf_shift;
value |= mapped_idx, buf_shift += NUM_BITS_NIBBLE;
if(buf_shift == NIBBLE_ELEM_OFFSET) {
memcpy(&out[out_idx], &value, 3);
out_idx += 3;
buf_shift = 0; /* Move to the next buffer */
value = 0;
}
}
}
if(buf_shift != 0) {
u_int bytes = buf_shift / NUM_BITS_NIBBLE;
memcpy(&out[out_idx], &value, bytes);
out_idx += bytes;
}
}
/* [2] Check if compressing the string is more efficient */
compressed_len = ndpi_compress_str((char*)domain_buf, domain_buf_len,
(char*)compressed, sizeof(compressed));
if((compressed_len > 0) && ((out_idx == 0) || (compressed_len < out_idx))) {
if(compressed_len >= domain_len) {
/* Compression creates a longer buffer */
return((u_int)snprintf(out, out_len, "%s", domain));
} else {
compressed_len = ndpi_min(ndpi_min(compressed_len, sizeof(compressed)), out_len-3);
memcpy(out, compressed, compressed_len);
out_idx = compressed_len;
}
}
/* Add trailer domainId value */
out[out_idx++] = (domain_id >> 8) & 0xFF;
out[out_idx++] = domain_id & 0xFF;
#ifdef DEBUG
{
u_int i;
fprintf(stdout, "%s [len: %u][", domain, out_idx);
for(i=0; i<out_idx; i++) fprintf(stdout, "%02X", out[i] & 0xFF);
fprintf(stdout, "]\n");
}
#endif
return(out_idx);
}