Implemented STUN cache to enhance matching of STUN-based protocols

This commit is contained in:
Luca
2019-08-12 14:16:28 +02:00
parent 51ac583d83
commit fdeecf5d41
8 changed files with 143 additions and 44 deletions
+3 -2
View File
@@ -774,8 +774,9 @@ extern "C" {
/* LRU cache */
struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries);
void ndpi_lru_free_cache(struct ndpi_lru_cache *c);
u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found);
void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key);
u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key,
u_int16_t *value, u_int8_t clean_key_when_found);
void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int16_t value);
/**
* Add a string to match to an automata
+10 -1
View File
@@ -411,8 +411,14 @@ typedef enum {
NDPI_HTTP_METHOD_CONNECT
} ndpi_http_method;
struct ndpi_lru_cache_entry {
u_int32_t key; /* Store the whole key to avoid ambiguities */
u_int32_t is_full:1, value:16, pad:15;
};
struct ndpi_lru_cache {
u_int32_t num_entries, *entries;
u_int32_t num_entries;
struct ndpi_lru_cache_entry *entries;
};
struct ndpi_id_struct {
@@ -1057,6 +1063,9 @@ struct ndpi_detection_module_struct {
/* NDPI_PROTOCOL_TINC */
struct cache *tinc_cache;
/* NDPI_PROTOCOL_STUN and subprotocols */
struct ndpi_lru_cache *stun_cache;
ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
u_int8_t http_dont_dissect_response:1, dns_dont_dissect_response:1,
+18 -8
View File
@@ -51,6 +51,9 @@
#define NDPI_CONST_GENERIC_PROTOCOL_NAME "GenericProtocol"
/* stun.c */
extern u_int32_t get_stun_lru_key(struct ndpi_flow_struct *flow);
static int _ndpi_debug_callbacks = 0;
/* #define MATCH_DEBUG 1 */
@@ -2338,6 +2341,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct
if(ndpi_struct->ookla_cache)
ndpi_lru_free_cache(ndpi_struct->ookla_cache);
if(ndpi_struct->stun_cache)
ndpi_lru_free_cache(ndpi_struct->stun_cache);
if(ndpi_struct->protocols_ptree)
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->protocols_ptree, free_ptree_data);
@@ -4103,10 +4109,11 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
// if(/* (flow->protos.stun_ssl.stun.num_processed_pkts >= NDPI_MIN_NUM_STUN_DETECTION) */
if(flow->protos.stun_ssl.stun.num_processed_pkts && flow->protos.stun_ssl.stun.is_skype) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SKYPE_CALL, NDPI_PROTOCOL_SKYPE);
} else
} else {
ndpi_set_detected_protocol(ndpi_struct, flow,
flow->guessed_host_protocol_id,
NDPI_PROTOCOL_STUN);
}
}
}
@@ -6181,7 +6188,8 @@ struct ndpi_lru_cache* ndpi_lru_cache_init(u_int32_t num_entries) {
if(!c) return(NULL);
c->entries = (u_int32_t*)calloc(num_entries, sizeof(u_int32_t));
c->entries = (struct ndpi_lru_cache_entry*)calloc(num_entries,
sizeof(struct ndpi_lru_cache_entry));
if(!c->entries) {
free(c);
@@ -6197,21 +6205,23 @@ void ndpi_lru_free_cache(struct ndpi_lru_cache *c) {
free(c);
}
u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int8_t clean_key_when_found) {
u_int8_t ndpi_lru_find_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int16_t *value, u_int8_t clean_key_when_found) {
u_int32_t slot = key % c->num_entries;
if(c->entries[slot] == key) {
if(clean_key_when_found) c->entries[slot] = 0;
if(c->entries[slot].is_full) {
*value = c->entries[slot].value;
if(clean_key_when_found) c->entries[slot].is_full = 0;
return(1);
} else
return(0);
}
void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key) {
void ndpi_lru_add_to_cache(struct ndpi_lru_cache *c, u_int32_t key, u_int16_t value) {
u_int32_t slot = key % c->num_entries;
c->entries[slot] = key;
c->entries[slot].is_full = 1,
c->entries[slot].key = key,
c->entries[slot].value = value;
}
/* ******************************************************************** */
+25 -3
View File
@@ -22,8 +22,12 @@
#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_HANGOUT_DUO
/* #define DEBUG_LRU 1 */
#include "ndpi_api.h"
/* stun.c */
extern u_int32_t get_stun_lru_key(struct ndpi_flow_struct *flow);
/* https://support.google.com/a/answer/1279090?hl=en */
#define HANGOUT_UDP_LOW_PORT 19302
@@ -85,11 +89,29 @@ void ndpi_search_hangout(struct ndpi_detection_module_struct *ndpi_struct,
if((packet->payload_packet_len > 24) && is_google_flow(ndpi_struct, flow)) {
if(
((packet->udp != NULL) && (isHangoutUDPPort(ntohs(packet->udp->source)) || isHangoutUDPPort(ntohs(packet->udp->dest))))
((packet->udp != NULL) && (isHangoutUDPPort(ntohs(packet->udp->source))
|| isHangoutUDPPort(ntohs(packet->udp->dest))))
||
((packet->tcp != NULL) && (isHangoutTCPPort(ntohs(packet->tcp->source)) || isHangoutTCPPort(ntohs(packet->tcp->dest))))) {
((packet->tcp != NULL) && (isHangoutTCPPort(ntohs(packet->tcp->source))
|| isHangoutTCPPort(ntohs(packet->tcp->dest))))) {
NDPI_LOG_INFO(ndpi_struct, "found Hangout\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HANGOUT_DUO, NDPI_PROTOCOL_UNKNOWN);
/* Hangout is over STUN hence the LRU cache is shared */
if(ndpi_struct->stun_cache == NULL)
ndpi_struct->stun_cache = ndpi_lru_cache_init(1024);
if(ndpi_struct->stun_cache && flow->packet.iph && flow->packet.udp) {
u_int32_t key = get_stun_lru_key(flow);
#ifdef DEBUG_LRU
printf("[LRU] ADDING %u / %u.%u\n", key, NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_HANGOUT_DUO);
#endif
ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, NDPI_PROTOCOL_HANGOUT_DUO);
}
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HANGOUT_DUO,
NDPI_PROTOCOL_STUN);
return;
}
}
+11 -11
View File
@@ -35,7 +35,7 @@ static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *nd
printf("[%s] [http_dont_dissect_response: %u]->> %s\n", __FUNCTION__,
ndpi_struct->http_dont_dissect_response, flow->http.response_status_code);
#endif
if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
/* This is HTTP and it is not a sub protocol (e.g. skype or dropbox) */
@@ -157,7 +157,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
#endif
/* Leave the statement below commented necessary in case of call to ndpi_get_partial_detection() */
/* if(!ndpi_struct->http_dont_dissect_response) */ {
if((flow->http.url == NULL)
&& (packet->http_url_name.len > 0)
@@ -320,7 +320,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
/**
check result of host subprotocol detection
if "detected" in flow == 0 then "detected" = "guess"
else "guess" = "detected"
**/
@@ -338,7 +338,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
if(flow->detected_protocol_stack[0] != flow->guessed_host_protocol_id)
flow->guessed_host_protocol_id = flow->detected_protocol_stack[0];
}
if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
&& ((ndpi_struct->http_dont_dissect_response) || flow->http_detected)
&& (packet->http_origin.len > 0)) {
@@ -480,17 +480,17 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
if(packet->payload_packet_len >= 12) {
char buf[4];
/* Set server HTTP response code */
strncpy(buf, (char*)&packet->payload[9], 3);
buf[3] = '\0';
flow->http.response_status_code = atoi(buf);
/* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */
if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509))
flow->http.response_status_code = 0; /* Out of range */
}
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
check_content_type_and_change_protocol(ndpi_struct, flow);
return;
@@ -533,14 +533,14 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
if(ndpi_struct->ookla_cache == NULL)
ndpi_struct->ookla_cache = ndpi_lru_cache_init(1024);
if(packet->iph != NULL && ndpi_struct->ookla_cache != NULL) {
if(packet->tcp->source == htons(8080))
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr);
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */);
else
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr);
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */);
}
return;
}
+3 -1
View File
@@ -38,7 +38,9 @@ void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct
goto ookla_exclude;
if(ndpi_struct->ookla_cache != NULL) {
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, 0 /* Don't remove it as it can be used for other connections */)) {
u_int16_t dummy;
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, &dummy, 0 /* Don't remove it as it can be used for other connections */)) {
NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN);
return;
+72 -17
View File
@@ -32,6 +32,8 @@
/* #define DEBUG_STUN 1 */
/* #define DEBUG_LRU 1 */
struct stun_packet_header {
u_int16_t msg_type, msg_len;
u_int32_t cookie;
@@ -40,9 +42,41 @@ struct stun_packet_header {
/* ************************************************************ */
u_int32_t get_stun_lru_key(struct ndpi_flow_struct *flow) {
return(flow->packet.iph->saddr + flow->packet.udp->source);
}
/* ************************************************************ */
static void ndpi_int_stun_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
u_int proto, struct ndpi_flow_struct *flow) {
ndpi_set_detected_protocol(ndpi_struct, flow, proto, NDPI_PROTOCOL_UNKNOWN);
struct ndpi_flow_struct *flow,
u_int app_proto, u_int proto) {
if(ndpi_struct->stun_cache == NULL)
ndpi_struct->stun_cache = ndpi_lru_cache_init(1024);
if(ndpi_struct->stun_cache
&& flow->packet.iph
&& flow->packet.udp
&& (app_proto != NDPI_PROTOCOL_UNKNOWN)
) /* Cache flow sender info */ {
u_int32_t key = get_stun_lru_key(flow);
u_int16_t cached_proto;
if(ndpi_lru_find_cache(ndpi_struct->stun_cache, key, &cached_proto, 0 /* Don't remove it as it can be used for other connections */)) {
#ifdef DEBUG_LRU
printf("[LRU] FOUND %u / %u: no need to cache %u.%u\n", key, cached_proto, proto, app_proto);
#endif
app_proto = cached_proto, proto = NDPI_PROTOCOL_STUN;
} else {
#ifdef DEBUG_LRU
printf("[LRU] ADDING %u / %u.%u\n", key, proto, app_proto);
#endif
ndpi_lru_add_to_cache(ndpi_struct->stun_cache, key, app_proto);
}
}
ndpi_set_detected_protocol(ndpi_struct, flow, app_proto, proto);
}
typedef enum {
@@ -103,7 +137,7 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
*is_whatsapp = 0, *is_messenger = 0, *is_duo = 0;
if(payload_length < sizeof(struct stun_packet_header)) {
/* This looks like an invlid packet */
/* This looks like an invalid packet */
if(flow->protos.stun_ssl.stun.num_udp_pkts > 0) {
*is_whatsapp = 1;
@@ -124,6 +158,28 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
if(msg_type > 0x000C)
return(NDPI_IS_NOT_STUN);
if(ndpi_struct->stun_cache) {
u_int16_t proto;
u_int32_t key = get_stun_lru_key(flow);
if(ndpi_lru_find_cache(ndpi_struct->stun_cache, key, &proto, 0 /* Don't remove it as it can be used for other connections */)) {
#ifdef DEBUG_LRU
printf("[LRU] FOUND %u / %u\n", key, proto);
#endif
flow->guessed_host_protocol_id = proto, flow->guessed_protocol_id = NDPI_PROTOCOL_STUN;
return(NDPI_IS_STUN);
} else {
#ifdef DEBUG_LRU
printf("[LRU] NOT FOUND %u\n", key);
#endif
}
} else {
#ifdef DEBUG_LRU
printf("[LRU] NO/EMPTY CACHE\n");
#endif
}
if(msg_type == 0x01 /* Binding Request */) {
flow->protos.stun_ssl.stun.num_binding_requests++;
@@ -229,7 +285,7 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
if(strstr((char*)flow->host_server_name, "google.com") != NULL) {
*is_duo = 1;
flow->guessed_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO, flow->guessed_protocol_id = NDPI_PROTOCOL_STUN;
return(NDPI_IS_STUN);
}
}
@@ -245,8 +301,7 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
*is_duo = 1;
if(1) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
flow->guessed_protocol_id = NDPI_PROTOCOL_STUN;
flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO, flow->guessed_protocol_id = NDPI_PROTOCOL_STUN;
return(NDPI_IS_NOT_STUN); /* This case is found also with signal traffic */
} else
return(NDPI_IS_STUN);
@@ -386,20 +441,21 @@ void ndpi_search_stun(struct ndpi_detection_module_struct *ndpi_struct, struct n
if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) flow->guessed_protocol_id = NDPI_PROTOCOL_STUN;
if(is_messenger) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MESSENGER, NDPI_PROTOCOL_STUN);
ndpi_int_stun_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MESSENGER, NDPI_PROTOCOL_STUN);
return;
} else if(is_duo) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HANGOUT_DUO, NDPI_PROTOCOL_STUN);
ndpi_int_stun_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HANGOUT_DUO, NDPI_PROTOCOL_STUN);
return;
} else if(flow->protos.stun_ssl.stun.is_skype) {
NDPI_LOG_INFO(ndpi_struct, "found Skype\n");
if((flow->protos.stun_ssl.stun.num_processed_pkts >= 8) || (flow->protos.stun_ssl.stun.num_binding_requests >= 4))
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SKYPE_CALL, NDPI_PROTOCOL_SKYPE);
ndpi_int_stun_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SKYPE_CALL, NDPI_PROTOCOL_SKYPE);
} else {
NDPI_LOG_INFO(ndpi_struct, "found UDP stun\n"); /* Ummmmm we're in the TCP branch. This code looks bad */
ndpi_int_stun_add_connection(ndpi_struct,
is_whatsapp ? (is_whatsapp == 1 ? NDPI_PROTOCOL_WHATSAPP_VOICE : NDPI_PROTOCOL_WHATSAPP_VIDEO) : NDPI_PROTOCOL_STUN, flow);
ndpi_int_stun_add_connection(ndpi_struct, flow,
is_whatsapp ? (is_whatsapp == 1 ? NDPI_PROTOCOL_WHATSAPP_VOICE : NDPI_PROTOCOL_WHATSAPP_VIDEO) : NDPI_PROTOCOL_STUN,
NDPI_PROTOCOL_UNKNOWN);
}
return;
@@ -414,23 +470,22 @@ void ndpi_search_stun(struct ndpi_detection_module_struct *ndpi_struct, struct n
if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN) flow->guessed_protocol_id = NDPI_PROTOCOL_STUN;
if(is_messenger) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MESSENGER, NDPI_PROTOCOL_STUN);
ndpi_int_stun_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_MESSENGER, NDPI_PROTOCOL_STUN);
return;
} else if(is_duo) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HANGOUT_DUO, NDPI_PROTOCOL_STUN);
ndpi_int_stun_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HANGOUT_DUO, NDPI_PROTOCOL_STUN);
return;
} else if(flow->protos.stun_ssl.stun.is_skype) {
NDPI_LOG_INFO(ndpi_struct, "Found Skype\n");
/* flow->protos.stun_ssl.stun.num_binding_requests < 4) ? NDPI_PROTOCOL_SKYPE_CALL_IN : NDPI_PROTOCOL_SKYPE_CALL_OUT */
if((flow->protos.stun_ssl.stun.num_processed_pkts >= 8) || (flow->protos.stun_ssl.stun.num_binding_requests >= 4))
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SKYPE_CALL, NDPI_PROTOCOL_SKYPE);
ndpi_int_stun_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_SKYPE_CALL, NDPI_PROTOCOL_SKYPE);
} else {
NDPI_LOG_INFO(ndpi_struct, "found UDP stun\n");
ndpi_int_stun_add_connection(ndpi_struct,
ndpi_int_stun_add_connection(ndpi_struct, flow,
is_whatsapp ? (is_whatsapp == 1 ? NDPI_PROTOCOL_WHATSAPP_VOICE : NDPI_PROTOCOL_WHATSAPP_VIDEO)
: NDPI_PROTOCOL_STUN,
flow);
: NDPI_PROTOCOL_STUN, NDPI_PROTOCOL_UNKNOWN);
}
return;
+1 -1
View File
@@ -1,3 +1,3 @@
GoogleHangoutDuo 19 2774 1
1 UDP 74.125.134.127:19305 -> 10.89.61.13:56406 [proto: 201/GoogleHangoutDuo][cat: VoIP/10][19 pkts/2774 bytes -> 0 pkts/0 bytes][PLAIN TEXT (sdiKGkw)]
1 UDP 74.125.134.127:19305 -> 10.89.61.13:56406 [proto: 78.201/STUN.GoogleHangoutDuo][cat: VoIP/10][19 pkts/2774 bytes -> 0 pkts/0 bytes][PLAIN TEXT (sdiKGkw)]