Improved WeChat support

Removed Musical.ly protocol as the service has been merged with another one and thus it is no longer used
Improved guess for UDP protocols
This commit is contained in:
Luca Deri
2018-11-19 18:24:33 +01:00
parent 1537669178
commit 582da8d4bb
10 changed files with 108 additions and 61 deletions
+1
View File
@@ -911,6 +911,7 @@ static void node_print_known_proto_walker(const void *node,
static u_int16_t node_guess_undetected_protocol(u_int16_t thread_id, struct ndpi_flow_info *flow) {
flow->detected_protocol = ndpi_guess_undetected_protocol(ndpi_thread_info[thread_id].workflow->ndpi_struct,
NULL,
flow->protocol,
ntohl(flow->src_ip),
ntohs(flow->src_port),
+2
View File
@@ -326,6 +326,7 @@ extern "C" {
* Search and return the protocol guessed that is undetected
*
* @par ndpi_struct = the detection module
* @par flow = the flow we're trying to guess, NULL if not available
* @par proto = the l4 protocol number
* @par shost = source address in host byte order
* @par sport = source port number
@@ -335,6 +336,7 @@ extern "C" {
*
*/
ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t proto,
u_int32_t shost,
u_int16_t sport,
+1
View File
@@ -119,6 +119,7 @@ extern "C" {
u_int16_t ndpi_get_proto_by_name(struct ndpi_detection_module_struct *ndpi_mod, const char *name);
extern u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t proto, u_int16_t sport, u_int16_t dport,
u_int8_t *user_defined_proto);
+1 -1
View File
@@ -73,7 +73,7 @@ typedef enum {
NDPI_PROTOCOL_EDONKEY = 36, /* Tomasz Bujlow <tomasz@skatnet.dk> */
NDPI_PROTOCOL_BITTORRENT = 37,
NDPI_PROTOCOL_SKYPE_CALL_OUT = 38,
NDPI_PROTOCOL_MUSICALLY = 39,
NDPI_PROTOCOL_FREE_39 = 39, /* Free */
NDPI_PROTOCOL_MEMCACHED = 40, /* Memcached - Darryl Sokoloski <darryl@egloo.ca> */
NDPI_PROTOCOL_SMBV23 = 41, /* SMB version 2/3 */
NDPI_PROTOCOL_MINING = 42, /* Bitcoin, Ethereum, ZCash, Monero */
+1 -1
View File
@@ -826,7 +826,7 @@ typedef enum {
ndpi_pref_dns_dissect_response,
ndpi_pref_direction_detect_disable,
ndpi_pref_disable_metadata_export,
ndpi_pref_enable_category_substring_match,
ndpi_pref_enable_category_substring_match
} ndpi_detection_preference;
/* ntop extensions */
+6 -7
View File
@@ -21,7 +21,6 @@
/* ****************************************************** */
static ndpi_network host_protocol_list[] = {
/*
SoundCloud
*/
@@ -111,11 +110,14 @@ static ndpi_network host_protocol_list[] = {
WeChat
origin AS132203, AS132591, AS45090
*/
{ 0xCBCD93AB /* 203.205.147.171/32 */, 32, NDPI_PROTOCOL_WECHAT },
{ 0xCBCD93AD /* 203.205.147.173/32 */, 32, NDPI_PROTOCOL_WECHAT },
{ 0xCBCD97A2 /* 203.205.151.162/32 */, 32, NDPI_PROTOCOL_WECHAT },
{ 0x67071E25 /* 103.7.30.37/32 */, 32, NDPI_PROTOCOL_WECHAT },
{ 0x67071E25 /* 103.7.30.37/32 */, 32, NDPI_PROTOCOL_WECHAT },
{ 0XCBCDB000 /* 203.205.176.0/20 */, 20, NDPI_PROTOCOL_WECHAT },
{ 0XCBCDC000 /* 203.205.192.0/18 */, 18, NDPI_PROTOCOL_WECHAT },
/*
OpenDNS, LLC
@@ -8327,10 +8329,7 @@ ndpi_protocol_match host_match[] = {
/* Detected "slack-assets2.s3-us-west-2.amazonaws.com.". Omitted "*amazonaws.com" CDN, but no generic pattern to use on first part */
{ "slack-assets2.s3-", NULL, "slack-assets2\\.s3-", "Slack", NDPI_PROTOCOL_SLACK, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
{ "zhiliaoapp.com", NULL, "zhiliaoapp" TLD, "Musical.ly", NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
{ "muscdn.com", NULL, "muscdn" TLD, "Musical.ly", NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
{ "livelycdn.com", NULL, "livelycdn" TLD, "Musical.ly", NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
{ "direct.ly", NULL, "direct\\.ly$", "Musical.ly", NDPI_PROTOCOL_MUSICALLY, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
{ "wechat.com", NULL, "wechat\\.com" TLD, "WeChat", NDPI_PROTOCOL_WECHAT, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_FUN },
{ "github.com", NULL, "github" TLD, "Github", NDPI_PROTOCOL_GITHUB, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
{ ".github.com", NULL, "\\.github" TLD, "Github", NDPI_PROTOCOL_GITHUB, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE },
+58 -37
View File
@@ -517,7 +517,6 @@ void ndpi_exclude_protocol(struct ndpi_detection_module_struct *ndpi_struct,
(*(ndpi_struct->ndpi_debug_printf))(protocol_id, ndpi_struct, NDPI_LOG_DEBUG,
_file, _func, _line, "exclude %s\n",ndpi_get_proto_name(ndpi_struct, protocol_id));
}
#endif
NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, protocol_id);
@@ -1205,9 +1204,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
no_master, "Teredo", NDPI_PROTOCOL_CATEGORY_NETWORK,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 3544, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_MUSICALLY,
no_master,
no_master, "Musical.ly", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK,
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_WECHAT,
no_master, /* wechat.com */
no_master, "WeChat", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_MEMCACHED,
@@ -1245,7 +1244,12 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_196,
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_39,
no_master,
no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_FREE_39,
no_master,
no_master, "Free", NDPI_PROTOCOL_CATEGORY_CUSTOM_1 /* dummy */,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
@@ -2523,6 +2527,7 @@ static ndpi_default_ports_tree_node_t* ndpi_get_guessed_protocol_id(struct ndpi_
/* ****************************************************** */
u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t proto, u_int16_t sport, u_int16_t dport,
u_int8_t *user_defined_proto) {
@@ -2531,8 +2536,17 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_struc
ndpi_default_ports_tree_node_t *found = ndpi_get_guessed_protocol_id(ndpi_struct, proto, sport, dport);
if(found != NULL) {
*user_defined_proto = found->customUserProto;
return(found->proto->protoId);
u_int16_t guessed_proto = found->proto->protoId;
/* We need to check if the guessed protocol isn't excluded by nDPI */
if(flow
&& (proto == IPPROTO_UDP)
&& (NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, guessed_proto)))
return(NDPI_PROTOCOL_UNKNOWN);
else {
*user_defined_proto = found->customUserProto;
return(guessed_proto);
}
}
} else {
/* No TCP/UDP */
@@ -3956,18 +3970,17 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
&& (flow->l4.tcp.ssl_stage > 1))
flow->guessed_protocol_id = NDPI_PROTOCOL_SSL_NO_CERT;
guessed_protocol_id = flow->guessed_protocol_id,
guessed_host_protocol_id = flow->guessed_host_protocol_id;
guessed_protocol_id = flow->guessed_protocol_id, guessed_host_protocol_id = flow->guessed_host_protocol_id;
if((guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)
&& (NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_host_protocol_id)))
guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN;
&& ((flow->packet.l4_protocol == IPPROTO_UDP) && NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_host_protocol_id)))
flow->guessed_host_protocol_id = guessed_host_protocol_id = NDPI_PROTOCOL_UNKNOWN;
/* Ignore guessed protocol if they have been discarded */
if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
&& (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)
&& (NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_protocol_id)))
guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN;
// && (guessed_host_protocol_id == NDPI_PROTOCOL_UNKNOWN)
&& (flow->packet.l4_protocol == IPPROTO_UDP) && NDPI_ISSET(&flow->excluded_protocol_bitmask, guessed_protocol_id))
flow->guessed_protocol_id = guessed_protocol_id = NDPI_PROTOCOL_UNKNOWN;
if((guessed_protocol_id != NDPI_PROTOCOL_UNKNOWN)
|| (guessed_host_protocol_id != NDPI_PROTOCOL_UNKNOWN)) {
@@ -4394,7 +4407,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
else sport = dport = 0;
/* guess protocol */
flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_struct, protocol, sport, dport, &user_defined_proto);
flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_struct, flow, protocol, sport, dport, &user_defined_proto);
flow->guessed_host_protocol_id = ndpi_guess_host_protocol_id(ndpi_struct, flow);
if(flow->guessed_protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS-1)) {
@@ -5356,6 +5369,7 @@ u_int16_t ndpi_get_lower_proto(ndpi_protocol proto) {
/* ****************************************************** */
ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t proto,
u_int32_t shost /* host byte order */, u_int16_t sport,
u_int32_t dhost /* host byte order */, u_int16_t dport) {
@@ -5365,33 +5379,40 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
u_int8_t user_defined_proto;
if((proto == IPPROTO_TCP) || (proto == IPPROTO_UDP)) {
rc = ndpi_search_tcp_or_udp_raw(ndpi_struct, NULL, proto,
shost, dhost, sport, dport);
rc = ndpi_search_tcp_or_udp_raw(ndpi_struct, NULL, proto, shost, dhost, sport, dport);
if(rc != NDPI_PROTOCOL_UNKNOWN) {
ret.app_protocol = rc,
ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, proto, sport,
dport, &user_defined_proto);
if(ret.app_protocol == ret.master_protocol)
ret.master_protocol = NDPI_PROTOCOL_UNKNOWN;
ret.category = ndpi_get_proto_category(ndpi_struct, ret);
return(ret);
}
rc = ndpi_guess_protocol_id(ndpi_struct, proto, sport, dport, &user_defined_proto);
if(rc != NDPI_PROTOCOL_UNKNOWN) {
ret.app_protocol = rc;
if(rc == NDPI_PROTOCOL_SSL)
goto check_guessed_skype;
if(flow && (proto == IPPROTO_UDP) && NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, rc))
;
else {
ret.app_protocol = rc,
ret.master_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport,
dport, &user_defined_proto);
if(ret.app_protocol == ret.master_protocol)
ret.master_protocol = NDPI_PROTOCOL_UNKNOWN;
ret.category = ndpi_get_proto_category(ndpi_struct, ret);
return(ret);
}
}
rc = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport, dport, &user_defined_proto);
if(rc != NDPI_PROTOCOL_UNKNOWN) {
if(flow && (proto == IPPROTO_UDP) && NDPI_COMPARE_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, rc))
;
else {
ret.app_protocol = rc;
if(rc == NDPI_PROTOCOL_SSL)
goto check_guessed_skype;
else {
ret.category = ndpi_get_proto_category(ndpi_struct, ret);
return(ret);
}
}
}
check_guessed_skype:
addr.s_addr = htonl(shost);
if(ndpi_network_ptree_match(ndpi_struct, &addr) == NDPI_PROTOCOL_SKYPE) {
@@ -5402,10 +5423,10 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
ret.app_protocol = NDPI_PROTOCOL_SKYPE;
}
} else
ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, proto, sport,
ret.app_protocol = ndpi_guess_protocol_id(ndpi_struct, NULL, proto, sport,
dport, &user_defined_proto);
ret.category = ndpi_get_proto_category(ndpi_struct, ret);
ret.category = ndpi_get_proto_category(ndpi_struct, ret);
return(ret);
}
+18 -8
View File
@@ -48,14 +48,24 @@ static void ndpi_check_dropbox(struct ndpi_detection_module_struct *ndpi_struct,
if(packet->udp != NULL) {
u_int16_t dropbox_port = htons(DB_LSP_PORT);
if((packet->udp->source == dropbox_port)
&& (packet->udp->dest == dropbox_port)) {
if(payload_len > 2) {
if(strstr((const char *)packet->payload, "\"host_int\"") != NULL) {
NDPI_LOG_INFO(ndpi_struct, "found dropbox\n");
ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0);
return;
if(packet->udp->dest == dropbox_port) {
if(packet->udp->source == dropbox_port) {
if(payload_len > 10) {
if(ndpi_strnstr((const char *)packet->payload, "\"host_int\"", payload_len) != NULL) {
NDPI_LOG_INFO(ndpi_struct, "found dropbox\n");
ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0);
return;
}
}
} else {
if(payload_len > 10) {
if(ndpi_strnstr((const char *)packet->payload, "Bus17Cmd", payload_len) != NULL) {
NDPI_LOG_INFO(ndpi_struct, "found dropbox\n");
ndpi_int_dropbox_add_connection(ndpi_struct, flow, 0);
return;
}
}
}
}
+7 -3
View File
@@ -53,12 +53,15 @@ void ndpi_search_eaq(struct ndpi_detection_module_struct *ndpi_struct, struct nd
if(packet->udp != NULL) {
u_int32_t seq = (packet->payload[0] * 1000) + (packet->payload[1] * 100) + (packet->payload[2] * 10) + packet->payload[3];
if(flow->l4.udp.eaq_pkt_id == 0)
flow->l4.udp.eaq_sequence = seq;
else {
if( (flow->l4.udp.eaq_sequence != seq) &&
((flow->l4.udp.eaq_sequence+1) != seq)) break;
((flow->l4.udp.eaq_sequence+1) != seq))
break;
else
flow->l4.udp.eaq_sequence = seq;
}
if(++flow->l4.udp.eaq_pkt_id == 4) {
@@ -66,7 +69,8 @@ void ndpi_search_eaq(struct ndpi_detection_module_struct *ndpi_struct, struct nd
NDPI_LOG_INFO(ndpi_struct, "found eaq\n");
ndpi_int_eaq_add_connection(ndpi_struct, flow);
return;
}
} else
return;
}
} while(0);
+13 -4
View File
@@ -31,13 +31,16 @@ static void ndpi_skype_report_protocol(struct ndpi_detection_module_struct *ndpi
NDPI_LOG_INFO(ndpi_struct, "found skype\n");
ndpi_set_detected_protocol(ndpi_struct, flow, proto, NDPI_PROTOCOL_SKYPE);
}
static int is_port(u_int16_t a, u_int16_t b, u_int16_t c) {
return(((a == c) || (b == c)) ? 1 : 0);
}
static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
struct ndpi_packet_struct *packet = &flow->packet;
// const u_int8_t *packet_payload = packet->payload;
u_int32_t payload_len = packet->payload_packet_len;
if(flow->host_server_name[0] != '\0')
return;
@@ -46,10 +49,15 @@ static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, s
flow->l4.udp.skype_packet_id++;
if(flow->l4.udp.skype_packet_id < 5) {
u_int16_t sport = ntohs(packet->udp->source);
u_int16_t dport = ntohs(packet->udp->dest);
/* skype-to-skype */
if(dport != 1119) /* It can be confused with battle.net */ {
if(is_port(sport, dport, 1119) /* It can be confused with battle.net */
|| is_port(sport, dport, 80) /* No HTTP-like protocols UDP/80 */
) {
;
} else {
if(((payload_len == 3) && ((packet->payload[2] & 0x0F)== 0x0d)) ||
((payload_len >= 16)
&& (packet->payload[0] != 0x30) /* Avoid invalid SNMP detection */
@@ -57,11 +65,12 @@ static void ndpi_check_skype(struct ndpi_detection_module_struct *ndpi_struct, s
ndpi_skype_report_protocol(ndpi_struct, flow);
}
}
return;
}
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
return;
// TCP check
} else if(packet->tcp != NULL) {
flow->l4.tcp.skype_packet_id++;