Remove classification "by-ip" from protocol stack (#1743)

Basically:
* "classification by-ip" (i.e. `flow->guessed_protocol_id_by_ip` is
NEVER returned in the protocol stack (i.e.
`flow->detected_protocol_stack[]`);
* if the application is interested into such information, it can access
`ndpi_protocol->protocol_by_ip` itself.

There are mainly 4 points in the code that set the "classification
by-ip" in the protocol stack:  the generic `ndpi_set_detected_protocol()`/
`ndpi_detection_giveup()` functions and the HTTP/STUN  dissectors.

In the unit tests output, a print about `ndpi_protocol->protocol_by_ip`
has been added for each flow: the huge diff of this commit is mainly due
to that.

Strictly speaking, this change is NOT an API/ABI breakage, but there are
important differences in the classification results. For examples:
* TLS flows without the initial handshake (or without a matching
SNI/certificate) are simply classified as `TLS`;
* similar for HTTP or QUIC flows;
* DNS flows without a matching request domain are simply classified as
`DNS`; we don't have `DNS/Google` anymore just because the server is
8.8.8.8 (that was an outrageous behaviour...);
* flows previusoly classified only "by-ip" are now classified as
`NDPI_PROTOCOL_UNKNOWN`.

See #1425 for other examples of why adding the "classification by-ip" in
the protocol stack is a bad idea.

Please, note that IPV6 is not supported :(  (long standing issue in nDPI) i.e.
`ndpi_protocol->protocol_by_ip` wil be always `NDPI_PROTOCOL_UNKNOWN` for
IPv6 flows.

Define `NDPI_CONFIDENCE_MATCH_BY_IP` has been removed.

Close #1687
This commit is contained in:
Ivan Nardi
2022-09-20 22:24:47 +02:00
committed by GitHub
parent 174cd739db
commit a7c2734b38
379 changed files with 9022 additions and 9061 deletions
+10 -10
View File
@@ -916,14 +916,14 @@ typedef struct {
} ndpi_port_range;
typedef enum {
NDPI_CONFIDENCE_UNKNOWN = 0, /* Unknown classification */
NDPI_CONFIDENCE_MATCH_BY_PORT, /* Classification obtained looking only at the L4 ports */
NDPI_CONFIDENCE_MATCH_BY_IP, /* Classification obtained looking only at the L3 addresses */
NDPI_CONFIDENCE_DPI_PARTIAL, /* Classification results based on partial/incomplete DPI information */
NDPI_CONFIDENCE_DPI_PARTIAL_CACHE, /* Classification results based on some LRU cache with partial/incomplete DPI information */
NDPI_CONFIDENCE_DPI_CACHE, /* Classification results based on some LRU cache (i.e. correlation among sessions) */
NDPI_CONFIDENCE_DPI, /* Deep packet inspection */
NDPI_CONFIDENCE_NBPF, /* PF_RING nBPF (custom protocol) */
/* Try to have "stable" values (across releases/changes) */
NDPI_CONFIDENCE_UNKNOWN = 0, /* Unknown classification */
NDPI_CONFIDENCE_MATCH_BY_PORT = 10, /* Classification obtained looking only at the L4 ports */
NDPI_CONFIDENCE_NBPF = 50, /* PF_RING nBPF (custom protocol) */
NDPI_CONFIDENCE_DPI_PARTIAL = 100, /* Classification results based on partial/incomplete DPI information */
NDPI_CONFIDENCE_DPI_PARTIAL_CACHE = 110, /* Classification results based on some LRU cache with partial/incomplete DPI information */
NDPI_CONFIDENCE_DPI_CACHE = 200, /* Classification results based on some LRU cache (i.e. correlation among sessions) */
NDPI_CONFIDENCE_DPI = 210, /* Deep packet inspection */
/*
IMPORTANT
@@ -1075,12 +1075,12 @@ typedef struct ndpi_proto {
below we do not use ndpi_protocol_id_t as users can define their own
custom protocols and thus the typedef could be too short in size.
*/
u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */;
u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */, protocol_by_ip;
ndpi_protocol_category_t category;
void *custom_category_userdata;
} ndpi_protocol;
#define NDPI_PROTOCOL_NULL { NDPI_PROTOCOL_UNKNOWN , NDPI_PROTOCOL_UNKNOWN , NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL }
#define NDPI_PROTOCOL_NULL { NDPI_PROTOCOL_UNKNOWN , NDPI_PROTOCOL_UNKNOWN , NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL }
#define NUM_CUSTOM_CATEGORIES 5
#define CUSTOM_CATEGORY_LABEL_LEN 32
+14 -64
View File
@@ -5741,8 +5741,8 @@ static void ndpi_add_connection_as_zoom(struct ndpi_detection_module_struct *ndp
ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow,
u_int8_t enable_guess, u_int8_t *protocol_was_guessed) {
ndpi_protocol ret = {NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL};
u_int16_t guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN, guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN;
ndpi_protocol ret = NDPI_PROTOCOL_NULL;
u_int16_t guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN;
/* *** We can't access ndpi_str->packet from this function!! *** */
@@ -5753,6 +5753,7 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
/* Init defaults */
ret.master_protocol = flow->detected_protocol_stack[1], ret.app_protocol = flow->detected_protocol_stack[0];
ret.protocol_by_ip = flow->guessed_protocol_id_by_ip;
ret.category = flow->category;
/* Ensure that we don't change our mind if detection is already complete */
@@ -5783,38 +5784,22 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
ndpi_set_detected_protocol(ndpi_str, flow, flow->guessed_protocol_id, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL);
} else if(enable_guess) {
guessed_protocol_id = flow->guessed_protocol_id, guessed_host_protocol_id = flow->guessed_protocol_id_by_ip;
if((guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN) &&
((flow->l4_proto == IPPROTO_UDP) &&
NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_host_protocol_id) &&
is_udp_guessable_protocol(guessed_host_protocol_id)))
guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN;
guessed_protocol_id = flow->guessed_protocol_id;
/* Ignore guessed protocol if they have been discarded */
if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
// && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)
&& (flow->l4_proto == IPPROTO_UDP) &&
NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_protocol_id) &&
is_udp_guessable_protocol(guessed_protocol_id))
flow->guessed_protocol_id = guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN;
if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) || (guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)) {
ndpi_confidence_t confidence;
if(guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
confidence = NDPI_CONFIDENCE_MATCH_BY_PORT;
if(guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)
confidence = NDPI_CONFIDENCE_MATCH_BY_IP;
if((guessed_protocol_id == 0) && (flow->stun.num_binding_requests > 0) &&
(flow->stun.num_processed_pkts > 0)) {
guessed_protocol_id = NDPI_PROTOCOL_STUN;
confidence = NDPI_CONFIDENCE_DPI_PARTIAL;
}
if(guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN) {
*protocol_was_guessed = 1;
ndpi_set_detected_protocol(ndpi_str, flow, guessed_host_protocol_id, guessed_protocol_id, confidence);
ndpi_set_detected_protocol(ndpi_str, flow, guessed_protocol_id, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_MATCH_BY_PORT);
} else if(flow->stun.num_binding_requests > 0 &&
flow->stun.num_processed_pkts > 0) {
*protocol_was_guessed = 1;
ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL);
}
}
@@ -5824,26 +5809,13 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
flow->detected_protocol_stack[1] = flow->guessed_protocol_id;
flow->confidence = NDPI_CONFIDENCE_MATCH_BY_PORT;
}
if(flow->guessed_protocol_id_by_ip != NDPI_PROTOCOL_UNKNOWN) {
*protocol_was_guessed = 1;
flow->detected_protocol_stack[0] = flow->guessed_protocol_id_by_ip;
flow->confidence = NDPI_CONFIDENCE_MATCH_BY_IP;
}
if((flow->detected_protocol_stack[1] == flow->detected_protocol_stack[0]) &&
(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN)) {
*protocol_was_guessed = 1;
flow->detected_protocol_stack[1] = flow->guessed_protocol_id_by_ip;
flow->confidence = NDPI_CONFIDENCE_MATCH_BY_IP;
}
}
if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) &&
(flow->guessed_protocol_id == NDPI_PROTOCOL_STUN)) {
check_stun_export:
*protocol_was_guessed = 1;
ndpi_set_detected_protocol(ndpi_str, flow, flow->guessed_protocol_id_by_ip, NDPI_PROTOCOL_STUN, NDPI_CONFIDENCE_DPI_PARTIAL);
ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL);
}
ret.master_protocol = flow->detected_protocol_stack[1], ret.app_protocol = flow->detected_protocol_stack[0];
@@ -6200,7 +6172,7 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
if(flow->guessed_protocol_id >= NDPI_MAX_SUPPORTED_PROTOCOLS) {
/* This is a custom protocol and it has priority over everything else */
ret->master_protocol = NDPI_PROTOCOL_UNKNOWN,
ret->app_protocol = flow->guessed_protocol_id ? flow->guessed_protocol_id : flow->guessed_protocol_id_by_ip;
ret->app_protocol = flow->guessed_protocol_id;
flow->confidence = NDPI_CONFIDENCE_MATCH_BY_PORT; /* TODO */
ndpi_fill_protocol_category(ndpi_str, flow, ret);
return(-1);
@@ -6210,7 +6182,6 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
if(flow->guessed_protocol_id_by_ip != NDPI_PROTOCOL_UNKNOWN) {
u_int8_t protocol_was_guessed;
/* ret->master_protocol = flow->guessed_protocol_id , ret->app_protocol = flow->guessed_protocol_id_by_ip; /\* ****** *\/ */
*ret = ndpi_detection_giveup(ndpi_str, flow, 0, &protocol_was_guessed);
}
@@ -6243,7 +6214,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
struct ndpi_packet_struct *packet = &ndpi_str->packet;
NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_packet;
u_int32_t num_calls = 0;
ndpi_protocol ret = { flow->detected_protocol_stack[1], flow->detected_protocol_stack[0], flow->category, NULL };
ndpi_protocol ret = { flow->detected_protocol_stack[1], flow->detected_protocol_stack[0], flow->guessed_protocol_id_by_ip, flow->category, NULL };
NDPI_LOG_DBG(ndpi_str, "[%d/%d] START packet processing\n",
flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
@@ -6398,15 +6369,6 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
ret = ndpi_detection_giveup(ndpi_str, flow, 0, &protocol_was_guessed);
}
#if 0
/* See https://github.com/ntop/nDPI/pull/1425 */
if((ret.master_protocol == NDPI_PROTOCOL_UNKNOWN) && (ret.app_protocol != NDPI_PROTOCOL_UNKNOWN) &&
(flow->guessed_protocol_id_by_ip != NDPI_PROTOCOL_UNKNOWN)) {
ret.master_protocol = ret.app_protocol;
ret.app_protocol = flow->guessed_protocol_id_by_ip;
}
#endif
if((!flow->risk_checked)
&& ((ret.master_protocol != NDPI_PROTOCOL_UNKNOWN) || (ret.app_protocol != NDPI_PROTOCOL_UNKNOWN))
) {
@@ -7208,16 +7170,6 @@ static void ndpi_int_change_protocol(struct ndpi_detection_module_struct *ndpi_s
if(upper_detected_protocol == lower_detected_protocol)
lower_detected_protocol = NDPI_PROTOCOL_UNKNOWN;
if((upper_detected_protocol != NDPI_PROTOCOL_UNKNOWN) && (lower_detected_protocol == NDPI_PROTOCOL_UNKNOWN)) {
if((flow->guessed_protocol_id_by_ip != NDPI_PROTOCOL_UNKNOWN) &&
(upper_detected_protocol != flow->guessed_protocol_id_by_ip)) {
if(ndpi_str->proto_defaults[upper_detected_protocol].subprotocol_count > 0) {
lower_detected_protocol = upper_detected_protocol;
upper_detected_protocol = flow->guessed_protocol_id_by_ip;
}
}
}
ndpi_int_change_flow_protocol(ndpi_str, flow, upper_detected_protocol, lower_detected_protocol, confidence);
}
@@ -7413,7 +7365,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
u_int32_t dhost /* host byte order */, u_int16_t dport) {
u_int32_t rc;
struct in_addr addr;
ndpi_protocol ret = {NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL};
ndpi_protocol ret = NDPI_PROTOCOL_NULL;
u_int8_t user_defined_proto;
#ifdef BITTORRENT_CACHE_DEBUG
@@ -7592,8 +7544,6 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)
return "Unknown";
case NDPI_CONFIDENCE_MATCH_BY_PORT:
return "Match by port";
case NDPI_CONFIDENCE_MATCH_BY_IP:
return "Match by IP";
case NDPI_CONFIDENCE_DPI_PARTIAL:
return "DPI (partial)";
case NDPI_CONFIDENCE_DPI_PARTIAL_CACHE:
+1 -1
View File
@@ -63,7 +63,7 @@ static void set_ajp_detected(struct ndpi_detection_module_struct *ndpi_struct,
/* If no custom protocol has been detected */
/* if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) */
ndpi_int_reset_protocol(flow);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_AJP, flow->guessed_protocol_id_by_ip, NDPI_CONFIDENCE_DPI);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_AJP, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
}
}
+1 -1
View File
@@ -30,7 +30,7 @@ static void ndpi_int_alicloud_add_connection(struct ndpi_detection_module_struct
{
NDPI_LOG_INFO(ndpi_struct, "found alicloud\n");
ndpi_set_detected_protocol(ndpi_struct, flow, flow->guessed_protocol_id_by_ip, NDPI_PROTOCOL_ALICLOUD,
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_ALICLOUD, NDPI_PROTOCOL_UNKNOWN,
NDPI_CONFIDENCE_DPI);
}
+2 -12
View File
@@ -335,22 +335,12 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd
struct ndpi_flow_struct *flow,
u_int16_t http_protocol,
ndpi_protocol_category_t category) {
u_int16_t master_protocol, app_protocol;
u_int16_t master_protocol;
#ifdef HTTP_DEBUG
printf("=> %s()\n", __FUNCTION__);
#endif
app_protocol = flow->guessed_protocol_id_by_ip;
/* If no custom protocol has been detected */
if((app_protocol == NDPI_PROTOCOL_UNKNOWN)
|| ((http_protocol != NDPI_PROTOCOL_HTTP) &&
(http_protocol != NDPI_PROTOCOL_HTTP_CONNECT) &&
(http_protocol != NDPI_PROTOCOL_HTTP_PROXY))
)
app_protocol = http_protocol;
// ndpi_int_reset_protocol(flow);
master_protocol = NDPI_PROTOCOL_HTTP;
if(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN)
master_protocol = flow->detected_protocol_stack[1];
@@ -363,7 +353,7 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd
sub-protocol via the (content-matched) subprotocols logic (i.e.
MPEGDASH, SOAP, ....) */
if(flow->detected_protocol_stack[1] == 0)
ndpi_set_detected_protocol(ndpi_struct, flow, app_protocol,
ndpi_set_detected_protocol(ndpi_struct, flow, http_protocol,
master_protocol,
NDPI_CONFIDENCE_DPI);
+1 -1
View File
@@ -58,7 +58,7 @@ static void set_mongodb_detected(struct ndpi_detection_module_struct *ndpi_struc
/* If no custom protocol has been detected */
/* if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) */
ndpi_int_reset_protocol(flow);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MONGODB, flow->guessed_protocol_id_by_ip, NDPI_CONFIDENCE_DPI);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MONGODB, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
}
}
+7 -5
View File
@@ -51,10 +51,12 @@ static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *nd
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
ndpi_confidence_t confidence = NDPI_CONFIDENCE_DPI;
if(app_proto == NDPI_PROTOCOL_GOOGLE)
app_proto = NDPI_PROTOCOL_HANGOUT_DUO;
else if(app_proto == NDPI_PROTOCOL_FACEBOOK)
app_proto = NDPI_PROTOCOL_FACEBOOK_VOIP;
if(app_proto == NDPI_PROTOCOL_UNKNOWN) {
if(flow->guessed_protocol_id_by_ip == NDPI_PROTOCOL_GOOGLE)
app_proto = NDPI_PROTOCOL_HANGOUT_DUO;
else if(flow->guessed_protocol_id_by_ip == NDPI_PROTOCOL_FACEBOOK)
app_proto = NDPI_PROTOCOL_FACEBOOK_VOIP;
}
if(ndpi_struct->stun_cache == NULL)
ndpi_struct->stun_cache = ndpi_lru_cache_init(1024);
@@ -424,7 +426,7 @@ void ndpi_search_stun(struct ndpi_detection_module_struct *ndpi_struct, struct n
NDPI_LOG_DBG(ndpi_struct, "search stun\n");
app_proto = flow->guessed_protocol_id_by_ip;
app_proto = NDPI_PROTOCOL_UNKNOWN;
if(packet->tcp) {
/* STUN may be encapsulated in TCP packets */
+2 -2
View File
@@ -309,7 +309,7 @@ static void checkTLSSubprotocol(struct ndpi_detection_module_struct *ndpi_struct
if(ndpi_lru_find_cache(ndpi_struct->tls_cert_cache, key,
&cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
ndpi_protocol ret = { __get_master(ndpi_struct, flow), cached_proto, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL};
ndpi_protocol ret = { __get_master(ndpi_struct, flow), cached_proto, NDPI_PROTOCOL_UNKNOWN /* unused */, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL};
ndpi_set_detected_protocol(ndpi_struct, flow, cached_proto, __get_master(ndpi_struct, flow), NDPI_CONFIDENCE_DPI_CACHE);
flow->category = ndpi_get_proto_category(ndpi_struct, ret);
@@ -689,7 +689,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
if(rc == 0) {
/* Match found */
u_int16_t proto_id = (u_int16_t)val;
ndpi_protocol ret = { __get_master(ndpi_struct, flow), proto_id, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL};
ndpi_protocol ret = { __get_master(ndpi_struct, flow), proto_id, NDPI_PROTOCOL_UNKNOWN /* unused */, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NULL};
ndpi_set_detected_protocol(ndpi_struct, flow, proto_id, __get_master(ndpi_struct, flow), NDPI_CONFIDENCE_DPI);
flow->category = ndpi_get_proto_category(ndpi_struct, ret);
+1 -1
View File
@@ -53,7 +53,7 @@ static void set_websocket_detected(struct ndpi_detection_module_struct *ndpi_str
ndpi_search_tcp_or_udp(ndpi_struct, flow);
ndpi_int_reset_protocol(flow);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WEBSOCKET, flow->guessed_protocol_id_by_ip, NDPI_CONFIDENCE_DPI);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WEBSOCKET, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
}
}