Improve/add several protocols (#1383)

Improve Microsoft, GMail, Likee, Whatsapp, DisneyPlus and Tiktok
detection.
Add Vimeo, Fuze, Alibaba and Firebase Crashlytics detection.

Try to differentiate between Messenger/Signal standard flows (i.e chat)
and their VOIP (video)calls (like we already do for Whatsapp and
Snapchat).

Add a partial list of some ADS/Tracking stuff.

Fix Cassandra, Radius and GTP false positives.
Fix DNS, Syslog and SIP false negatives.

Improve GTP (sub)classification: differentiate among GTP-U, GTP_C and
GTP_PRIME.

Fix 3 LGTM warnings.
This commit is contained in:
Ivan Nardi
2021-12-18 13:24:51 +01:00
committed by GitHub
parent 58b33dcb2d
commit 7153b8933c
38 changed files with 390 additions and 175 deletions
+6
View File
@@ -100,6 +100,11 @@ static bool ndpi_check_valid_cassandra_opcode(uint8_t opcode)
return false;
}
static bool ndpi_check_valid_cassandra_flags(uint8_t flags)
{
return (flags & 0xF0) == 0;
}
void ndpi_search_cassandra(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow)
{
@@ -108,6 +113,7 @@ void ndpi_search_cassandra(struct ndpi_detection_module_struct *ndpi_struct,
if (packet->tcp) {
if (packet->payload_packet_len >= CASSANDRA_HEADER_LEN &&
ndpi_check_valid_cassandra_version(get_u_int8_t(packet->payload, 0)) &&
ndpi_check_valid_cassandra_flags(get_u_int8_t(packet->payload, 1)) &&
ndpi_check_valid_cassandra_opcode(get_u_int8_t(packet->payload, 4)) &&
get_u_int32_t(packet->payload, 5) <= CASSANDRA_MAX_BODY_SIZE &&
get_u_int32_t(packet->payload, 5) >= (uint32_t) (packet->payload_packet_len - CASSANDRA_HEADER_LEN)) {
+2 -1
View File
@@ -206,10 +206,11 @@ static int search_valid_dns(struct ndpi_detection_module_struct *ndpi_struct,
if(*is_query) {
/* DNS Request */
if((dns_header->num_queries > 0) && (dns_header->num_queries <= NDPI_MAX_DNS_REQUESTS)
if((dns_header->num_queries <= NDPI_MAX_DNS_REQUESTS)
// && (dns_header->num_answers == 0)
&& (((dns_header->flags & 0x2800) == 0x2800 /* Dynamic DNS Update */)
|| ((dns_header->flags & 0xFCF0) == 0x00) /* Standard Query */
|| ((dns_header->flags & 0xFCFF) == 0x0800) /* Inverse query */
|| ((dns_header->num_answers == 0) && (dns_header->authority_rrs == 0)))) {
/* This is a good query */
while(x+2 < packet->payload_packet_len) {
+10 -5
View File
@@ -82,28 +82,33 @@ static void ndpi_check_gtp(struct ndpi_detection_module_struct *ndpi_struct, str
(payload_len >= HEADER_LEN_GTP_U) &&
(message_len <= (payload_len - HEADER_LEN_GTP_U))) {
NDPI_LOG_INFO(ndpi_struct, "found gtp-u\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP, NDPI_PROTOCOL_UNKNOWN);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP_U, NDPI_PROTOCOL_GTP);
return;
}
}
if((packet->udp->source == gtp_c) || (packet->udp->dest == gtp_c)) {
if(((version == 1) &&
(payload_len >= HEADER_LEN_GTP_C_V1) &&
(message_len == (payload_len - HEADER_LEN_GTP_C_V1))) ||
(message_len == (payload_len - HEADER_LEN_GTP_C_V1)) &&
(message_len >= 4 * (!!(gtp->flags & 0x07))) &&
(gtp->message_type > 0 && gtp->message_type <= 129)) || /* Loose check based on TS 29.060 7.1 */
((version == 2) &&
/* payload_len is always valid, because HEADER_LEN_GTP_C_V2 == sizeof(struct gtp_header_generic) */
(message_len <= (payload_len - HEADER_LEN_GTP_C_V2)))) {
NDPI_LOG_INFO(ndpi_struct, "found gtp-c\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP, NDPI_PROTOCOL_UNKNOWN);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP_C, NDPI_PROTOCOL_GTP);
return;
}
}
if((packet->udp->source == gtp_prime) || (packet->udp->dest == gtp_prime)) {
if((pt == 0) &&
((gtp->flags & 0x0E) >> 1 == 0x7) && /* Spare bits */
(payload_len >= HEADER_LEN_GTP_PRIME) &&
(message_len <= (payload_len - HEADER_LEN_GTP_PRIME))) {
(message_len <= (payload_len - HEADER_LEN_GTP_PRIME)) &&
((gtp->message_type > 0 && gtp->message_type <= 7) || /* Check based on TS 32.295 6.2.1 */
gtp->message_type == 240 || gtp->message_type == 241)) {
NDPI_LOG_INFO(ndpi_struct, "found gtp-prime\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP, NDPI_PROTOCOL_UNKNOWN);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_GTP_PRIME, NDPI_PROTOCOL_GTP);
return;
}
}
+8 -1
View File
@@ -24,6 +24,10 @@
#include "ndpi_api.h"
#define RADIUS_PORT 1812
#define RADIUS_PORT_ACC 1813
#define RADIUS_PORT_ACC_ALTERNATIVE 18013
struct radius_header {
u_int8_t code;
@@ -37,7 +41,10 @@ static void ndpi_check_radius(struct ndpi_detection_module_struct *ndpi_struct,
// const u_int8_t *packet_payload = packet->payload;
u_int32_t payload_len = packet->payload_packet_len;
if(packet->udp != NULL) {
if(packet->udp != NULL &&
(packet->udp->dest == htons(RADIUS_PORT) || packet->udp->source == htons(RADIUS_PORT) ||
packet->udp->dest == htons(RADIUS_PORT_ACC) || packet->udp->source == htons(RADIUS_PORT_ACC) ||
packet->udp->dest == htons(RADIUS_PORT_ACC_ALTERNATIVE) || packet->udp->source == htons(RADIUS_PORT_ACC_ALTERNATIVE))) {
struct radius_header *h = (struct radius_header*)packet->payload;
/* RFC2865: The minimum length is 20 and maximum length is 4096. */
if((payload_len < 20) || (payload_len > 4096)) {
+14
View File
@@ -153,6 +153,20 @@ void ndpi_search_sip_handshake(struct ndpi_detection_module_struct
ndpi_int_sip_add_connection(ndpi_struct, flow, 0);
return;
}
if((memcmp(packet_payload, "REFER ", 6) == 0 || memcmp(packet_payload, "refer ", 6) == 0)
&& (memcmp(&packet_payload[6], "SIP:", 4) == 0 || memcmp(&packet_payload[6], "sip:", 4) == 0)) {
NDPI_LOG_INFO(ndpi_struct, "found sip REFER\n");
ndpi_int_sip_add_connection(ndpi_struct, flow, 0);
return;
}
if((memcmp(packet_payload, "PRACK ", 6) == 0 || memcmp(packet_payload, "prack ", 6) == 0)
&& (memcmp(&packet_payload[6], "SIP:", 4) == 0 || memcmp(&packet_payload[6], "sip:", 4) == 0)) {
NDPI_LOG_INFO(ndpi_struct, "found sip REFER\n");
ndpi_int_sip_add_connection(ndpi_struct, flow, 0);
return;
}
}
/* add bitmask for tcp only, some stupid udp programs
+14 -6
View File
@@ -264,6 +264,8 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
if(!msg_len && flow->guessed_host_protocol_id == NDPI_PROTOCOL_GOOGLE)
flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
else if(flow->guessed_host_protocol_id == NDPI_PROTOCOL_FACEBOOK)
flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
else
flow->guessed_protocol_id = NDPI_PROTOCOL_STUN;
@@ -272,6 +274,10 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
return(NDPI_IS_NOT_STUN); /* This to keep analyzing STUN instead of giving up */
}
}
if(msg_type == 0x03 /* Allocate Request */) {
if(flow->guessed_host_protocol_id == NDPI_PROTOCOL_FACEBOOK)
flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
}
if(!msg_len && flow->guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN) {
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
@@ -294,7 +300,8 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
else {
switch(flow->guessed_protocol_id) {
case NDPI_PROTOCOL_HANGOUT_DUO:
case NDPI_PROTOCOL_MESSENGER:
case NDPI_PROTOCOL_FACEBOOK_VOIP:
case NDPI_PROTOCOL_SIGNAL_VOIP:
case NDPI_PROTOCOL_WHATSAPP_CALL:
/* Don't overwrite the protocol with sub-STUN protocols */
break;
@@ -360,11 +367,12 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
if(strstr(flow->host_server_name, "google.com") != NULL) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
return(NDPI_IS_STUN);
} else if(strstr(flow->host_server_name, "whispersystems.org") != NULL) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_SIGNAL;
} else if(strstr(flow->host_server_name, "whispersystems.org") != NULL ||
(strstr(flow->host_server_name, "signal.org") != NULL)) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_SIGNAL_VOIP;
return(NDPI_IS_STUN);
} else if(strstr(flow->host_server_name, "facebook") != NULL) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_MESSENGER;
flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
return(NDPI_IS_STUN);
}
}
@@ -374,7 +382,7 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
case 0xC057: /* Messeger */
if(msg_type == 0x0001) {
if((msg_len == 100) || (msg_len == 104)) {
flow->guessed_host_protocol_id = NDPI_PROTOCOL_MESSENGER;
flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
return(NDPI_IS_STUN);
} else if(msg_len == 76) {
#if 0
@@ -473,7 +481,7 @@ static ndpi_int_stun_t ndpi_int_check_stun(struct ndpi_detection_module_struct *
if(packet->iph) { /* TODO: ipv6 */
if(is_messenger_ip_address(ntohl(packet->iph->saddr)) || is_messenger_ip_address(ntohl(packet->iph->daddr)))
flow->guessed_host_protocol_id = NDPI_PROTOCOL_MESSENGER;
flow->guessed_host_protocol_id = NDPI_PROTOCOL_FACEBOOK_VOIP;
else if(is_google_ip_address(ntohl(packet->iph->saddr)) || is_google_ip_address(ntohl(packet->iph->daddr)))
flow->guessed_host_protocol_id = NDPI_PROTOCOL_HANGOUT_DUO;
}
+19 -45
View File
@@ -42,7 +42,9 @@ void ndpi_search_syslog(struct ndpi_detection_module_struct
NDPI_LOG_DBG(ndpi_struct, "search syslog\n");
if (packet->payload_packet_len > 20 && packet->payload_packet_len <= 1024 && packet->payload[0] == '<') {
if (packet->payload_packet_len > 20 && packet->payload[0] == '<') {
int j;
NDPI_LOG_DBG2(ndpi_struct, "checked len>20 and <1024 and first symbol=<\n");
for (i = 1; i <= 3; i++) {
@@ -55,7 +57,7 @@ void ndpi_search_syslog(struct ndpi_detection_module_struct
if (packet->payload[i++] != '>') {
NDPI_LOG_DBG(ndpi_struct, "excluded, there is no > following the number\n");
NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_SYSLOG);
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
return;
} else {
NDPI_LOG_DBG2(ndpi_struct, "a > following the number\n");
@@ -68,51 +70,23 @@ void ndpi_search_syslog(struct ndpi_detection_module_struct
NDPI_LOG_DBG2(ndpi_struct, "no blank following the >: do nothing\n");
}
/* check for "last message repeated" */
if (i + sizeof("last message") - 1 <= packet->payload_packet_len &&
memcmp(packet->payload + i, "last message", sizeof("last message") - 1) == 0) {
NDPI_LOG_INFO(ndpi_struct, "found syslog by 'last message' string\n");
ndpi_int_syslog_add_connection(ndpi_struct, flow);
return;
} else if (i + sizeof("snort: ") - 1 <= packet->payload_packet_len &&
memcmp(packet->payload + i, "snort: ", sizeof("snort: ") - 1) == 0) {
/* snort events */
NDPI_LOG_INFO(ndpi_struct, "found syslog by 'snort: ' string\n");
ndpi_int_syslog_add_connection(ndpi_struct, flow);
return;
/* Even if there are 2 RFCs (3164, 5424), syslog format after "<NUMBER>" is
not standard. The only common pattern seems to be that the entire
payload is made by printable characters */
/* TODO: check only the first N bytes to avoid touching the entire payload? */
for (j = 0; j < packet->payload_packet_len - i; j++) {
if (!(ndpi_isprint(packet->payload[i + j]) ||
ndpi_isspace(packet->payload[i + j]))) {
NDPI_LOG_DBG2(ndpi_struct, "no printable char 0x%x [i/j %d/%d]\n",
packet->payload[i + j], i, j);
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
return;
}
}
if (memcmp(&packet->payload[i], "Jan", 3) != 0
&& memcmp(&packet->payload[i], "Feb", 3) != 0
&& memcmp(&packet->payload[i], "Mar", 3) != 0
&& memcmp(&packet->payload[i], "Apr", 3) != 0
&& memcmp(&packet->payload[i], "May", 3) != 0
&& memcmp(&packet->payload[i], "Jun", 3) != 0
&& memcmp(&packet->payload[i], "Jul", 3) != 0
&& memcmp(&packet->payload[i], "Aug", 3) != 0
&& memcmp(&packet->payload[i], "Sep", 3) != 0
&& memcmp(&packet->payload[i], "Oct", 3) != 0
&& memcmp(&packet->payload[i], "Nov", 3) != 0 && memcmp(&packet->payload[i], "Dec", 3) != 0) {
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
return;
} else {
NDPI_LOG_INFO(ndpi_struct, "found syslog\n");
ndpi_int_syslog_add_connection(ndpi_struct, flow);
return;
}
NDPI_LOG_INFO(ndpi_struct, "found syslog\n");
ndpi_int_syslog_add_connection(ndpi_struct, flow);
return;
}
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
}
+2 -1
View File
@@ -2115,7 +2115,8 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
flow->protos.tls_quic.encrypted_sni.esni = (char*)ndpi_malloc(e_sni_len*2+1);
if(flow->protos.tls_quic.encrypted_sni.esni) {
u_int16_t i, off;
u_int16_t off;
int i;
for(i=e_offset, off=0; i<(e_offset+e_sni_len); i++) {
int rc = sprintf(&flow->protos.tls_quic.encrypted_sni.esni[off], "%02X", packet->payload[i] & 0XFF);