Merge branch 'dev' of https://github.com/ntop/nDPI into dev

This commit is contained in:
Luca Deri
2019-03-08 15:59:26 +00:00
10 changed files with 141 additions and 79 deletions
+4 -1
View File
@@ -740,8 +740,11 @@ extern "C" {
char *name, ndpi_protocol_category_t category);
int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct);
int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
const struct ndpi_iphdr *iph,
u_int32_t saddr,
u_int32_t daddr,
ndpi_protocol *ret);
int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, unsigned long *id);
void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
ndpi_protocol *ret);
+1 -1
View File
@@ -85,7 +85,7 @@ typedef enum {
NDPI_PROTOCOL_XBOX = 47,
NDPI_PROTOCOL_QQ = 48,
NDPI_PROTOCOL_FREE_49 = 49, /* Free */
NDPI_PROTOCOL_TIKTOK = 49,
NDPI_PROTOCOL_RTSP = 50,
NDPI_PROTOCOL_MAIL_IMAPS = 51,
NDPI_PROTOCOL_ICECAST = 52,
+2
View File
@@ -8420,6 +8420,8 @@ ndpi_protocol_match host_match[] = {
{ "signal.org", NULL, "signal\\.org" TLD, "Signal", NDPI_PROTOCOL_SIGNAL, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE },
{ "whispersystems.org", NULL, "whispersystems\\.org" TLD, "Signal", NDPI_PROTOCOL_SIGNAL, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE },
{ "musical.ly", NULL, "musical\\.ly" TLD, "TikTok", NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_ACCEPTABLE },
{ "muscdn.com", NULL, "muscndl\\.com" TLD, "TikTok", NDPI_PROTOCOL_TIKTOK, NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK, NDPI_PROTOCOL_ACCEPTABLE },
{ NULL, NULL, NULL, 0 }
};
+46 -23
View File
@@ -1198,14 +1198,9 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
no_master, "SkypeCall", NDPI_PROTOCOL_CATEGORY_VOIP,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_FREE_49,
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TIKTOK,
0 /* can_have_a_subprotocol */, no_master,
no_master, "Free_49", NDPI_PROTOCOL_CATEGORY_VOIP,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_FREE_49,
0 /* can_have_a_subprotocol */, no_master,
no_master, "SkypeCall", NDPI_PROTOCOL_CATEGORY_VOIP,
no_master, "TikTok", NDPI_PROTOCOL_CATEGORY_SOCIAL_NETWORK,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_mod, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_TEREDO,
@@ -1991,21 +1986,42 @@ static int ac_match_handler(AC_MATCH_t *m, AC_TEXT_t *txt, AC_REP_t *match) {
m->match_num, m->patterns->astring);
#endif
{
char *whatfound = strstr(buf, m->patterns->astring);
#ifdef MATCH_DEBUG
printf("[NDPI] %s() [searching=%s][pattern=%s][%s][%c]\n",
__FUNCTION__, buf, m->patterns->astring,
whatfound ? whatfound : "<NULL>",
whatfound[-1]);
#endif
/*
The patch below allows in case of pattern ws.amazon.com
to avoid matching aws.amazon.com whereas a.ws.amazon.com
has to match
*/
if(whatfound && (whatfound != buf)
&& (m->patterns->astring[0] != '.') /* The searched patter does not start with . */
&& strchr(m->patterns->astring, '.') /* The matched pattern has a . (e.g. numeric or sym IPs) */
&& (whatfound[-1] != '.')
)
return(0);
}
/*
Return 1 for stopping to the first match.
We might consider searching for the more
specific match, paying more cpu cycles.
*/
memcpy(match, &m->patterns[0].rep, sizeof(AC_REP_t));
if(((buf_len >= min_len) && (strncmp(&buf[buf_len-min_len], m->patterns->astring, min_len) == 0))
|| (strncmp(buf, m->patterns->astring, min_len) == 0) /* begins with */
)
{
) {
#ifdef MATCH_DEBUG
printf("Found match [%s][%s] [len: %u][proto_id: %u]\n",
buf, m->patterns->astring, min_len, *matching_protocol_id);
buf, m->patterns->astring, min_len , *matching_protocol_id);
#endif
return(1); /* If the pattern found matches the string at the beginning we stop here */
} else
@@ -2301,8 +2317,8 @@ int ndpi_match_string(void *_automa, char *string_to_match) {
int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id) {
AC_TEXT_t ac_input_text;
AC_AUTOMATA_t *automa = (AC_AUTOMATA_t*)_automa;
AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED };
AC_REP_t match = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED };
*id = -1;
if((automa == NULL)
|| (string_to_match == NULL)
@@ -2338,11 +2354,14 @@ static int hyperscanCustomEventHandler(unsigned int id,
/* *********************************************** */
static int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, unsigned long *id) {
/* printf("[NDPI] %s(%s)\n", __FUNCTION__, name); */
#ifdef DEBUG
printf("[NDPI] %s(%s) [enable_category_substring_match: %u]\n",
__FUNCTION__, name, ndpi_struct->enable_category_substring_match);
#endif
if(!ndpi_struct->enable_category_substring_match) {
if(ndpi_struct->enable_category_substring_match == 0) {
if(ndpi_struct->custom_categories.hostnames_hash == NULL)
return(-1);
else {
@@ -2731,7 +2750,8 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_mod,
if(sscanf(value, "%u-%u", (u_int32_t *)&range.port_low, (u_int32_t *)&range.port_high) != 2)
range.port_low = range.port_high = atoi(&elem[4]);
if(do_add)
addDefaultPort(ndpi_mod, &range, def, 1 /* Custom user proto */, is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot, __FUNCTION__,__LINE__);
addDefaultPort(ndpi_mod, &range, def, 1 /* Custom user proto */,
is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot, __FUNCTION__,__LINE__);
else
removeDefaultPort(&range, def, is_tcp ? &ndpi_mod->tcpRoot : &ndpi_mod->udpRoot);
} else if(is_ip) {
@@ -4358,19 +4378,20 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
/* ********************************************************************************* */
int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
const struct ndpi_iphdr *iph,
u_int32_t saddr,
u_int32_t daddr,
ndpi_protocol *ret) {
if(ndpi_struct->custom_categories.categories_loaded) {
prefix_t prefix;
patricia_node_t *node;
/* Make sure all in network byte order otherwise compares wont work */
fill_prefix_v4(&prefix, (struct in_addr *)&iph->saddr,
fill_prefix_v4(&prefix, (struct in_addr *)&saddr,
32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
if(!node) {
fill_prefix_v4(&prefix, (struct in_addr *)&iph->daddr,
fill_prefix_v4(&prefix, (struct in_addr *)&daddr,
32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
}
@@ -4385,12 +4406,14 @@ int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_str
return 0;
}
/* ********************************************************************************* */
void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
ndpi_protocol *ret) {
if(ndpi_struct->custom_categories.categories_loaded) {
if(flow->packet.iph) {
if(ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph, ret)) {
if(ndpi_fill_ip_protocol_category(ndpi_struct, flow->packet.iph->saddr, flow->packet.iph->daddr, ret)) {
flow->category = ret->category;
return;
}
@@ -4435,7 +4458,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
if(ndpi_struct->ndpi_log_level >= NDPI_LOG_TRACE)
NDPI_LOG(flow ? flow->detected_protocol_stack[0]:NDPI_PROTOCOL_UNKNOWN,
ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n");
ndpi_struct, NDPI_LOG_TRACE, "START packet processing\n");
if(flow == NULL)
return(ret);
+8 -6
View File
@@ -63,7 +63,7 @@ static u_int getNameLength(u_int i, const u_int8_t *payload, u_int payloadLen) {
/* *********************************************** */
void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
int x;
int x, payload_offset;
u_int8_t is_query;
u_int16_t s_port = 0, d_port = 0;
@@ -72,16 +72,18 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
if(flow->packet.udp != NULL) {
s_port = ntohs(flow->packet.udp->source);
d_port = ntohs(flow->packet.udp->dest);
x = 0;
payload_offset = 0;
} else if(flow->packet.tcp != NULL) /* pkt size > 512 bytes */ {
s_port = ntohs(flow->packet.tcp->source);
d_port = ntohs(flow->packet.tcp->dest);
x = 2;
payload_offset = 2;
} else {
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
return;
}
x = payload_offset;
if((s_port == 53 || d_port == 53 || d_port == 5355)
&& (flow->packet.payload_packet_len > sizeof(struct ndpi_dns_packet_header)+x)) {
struct ndpi_dns_packet_header dns_header;
@@ -175,7 +177,8 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
}
}
}
}
} else
invalid = 1;
}
if(invalid) {
@@ -184,8 +187,7 @@ void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, struct nd
}
/* extract host name server */
int j = 0, max_len = sizeof(flow->host_server_name)-1, off = sizeof(struct ndpi_dns_packet_header) + 1;
int j = 0, max_len = sizeof(flow->host_server_name)-1, off = sizeof(struct ndpi_dns_packet_header) + 1 + payload_offset;
while(off < flow->packet.payload_packet_len && flow->packet.payload[off] != '\0') {
flow->host_server_name[j] = flow->packet.payload[off];
if(j < max_len) {
+12 -17
View File
@@ -63,17 +63,15 @@ static int ndpi_int_check_mdns_payload(struct ndpi_detection_module_struct
struct ndpi_packet_struct *packet = &flow->packet;
struct mdns_header *h = (struct mdns_header*)packet->payload;
u_int16_t questions = ntohs(h->questions), answers = ntohs(h->answers);
if((questions > NDPI_MAX_MDNS_REQUESTS)
|| (answers > NDPI_MAX_MDNS_REQUESTS))
return(0);
if(((packet->payload[2] & 0x80) == 0)
&& (questions <= NDPI_MAX_MDNS_REQUESTS)
&& (answers <= NDPI_MAX_MDNS_REQUESTS)) {
if((packet->payload[2] & 0x80) == 0) {
NDPI_LOG_INFO(ndpi_struct, "found MDNS with question query\n");
return 1;
}
else if(((packet->payload[2] & 0x80) != 0)
&& (questions == 0)
&& (answers <= NDPI_MAX_MDNS_REQUESTS)
&& (answers != 0)) {
} else if((packet->payload[2] & 0x80) != 0) {
char answer[256];
int i, j, len;
@@ -100,8 +98,6 @@ static int ndpi_int_check_mdns_payload(struct ndpi_detection_module_struct
void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct *packet = &flow->packet;
u_int16_t dport;
NDPI_LOG_DBG(ndpi_struct, "search MDNS\n");
/**
@@ -111,15 +107,13 @@ void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct n
/* check if UDP packet */
if(packet->udp != NULL) {
/* read destination port */
dport = ntohs(packet->udp->dest);
u_int16_t sport = ntohs(packet->udp->source);
u_int16_t dport = ntohs(packet->udp->dest);
/* check standard MDNS ON port 5353 */
if(dport == 5353 && packet->payload_packet_len >= 12) {
/* mdns protocol must have destination address 224.0.0.251 */
if(packet->iph != NULL /* && ntohl(packet->iph->daddr) == 0xe00000fb */) {
NDPI_LOG_INFO(ndpi_struct, "found MDNS with destination address 224.0.0.251 (=0xe00000fb)\n");
if(((dport == 5353) || (sport == 5353))
&& (packet->payload_packet_len >= 12)) {
if(packet->iph != NULL) {
if(ndpi_int_check_mdns_payload(ndpi_struct, flow) == 1) {
ndpi_int_mdns_add_connection(ndpi_struct, flow);
return;
@@ -141,6 +135,7 @@ void ndpi_search_mdns(struct ndpi_detection_module_struct *ndpi_struct, struct n
#endif
}
}
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
}
+12 -6
View File
@@ -44,9 +44,12 @@ void ndpi_search_mining_tcp(struct ndpi_detection_module_struct *ndpi_struct,
if((*to_match == magic) || (*to_match == magic1)) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN);
}
} if(ndpi_strnstr((const char *)packet->payload, "\"eth1.0\"", packet->payload_packet_len)
|| ndpi_strnstr((const char *)packet->payload, "\"worker\"", packet->payload_packet_len)
|| ndpi_strnstr((const char *)packet->payload, "\"id\"", packet->payload_packet_len)) {
} if(ndpi_strnstr((const char *)packet->payload, "{", packet->payload_packet_len)
&& (
ndpi_strnstr((const char *)packet->payload, "\"eth1.0\"", packet->payload_packet_len)
|| ndpi_strnstr((const char *)packet->payload, "\"worker\":", packet->payload_packet_len)
/* || ndpi_strnstr((const char *)packet->payload, "\"id\":", packet->payload_packet_len) - Removed as too generic */
)) {
/*
Ethereum
@@ -55,9 +58,12 @@ void ndpi_search_mining_tcp(struct ndpi_detection_module_struct *ndpi_struct,
{"worker": "", "jsonrpc": "2.0", "params": [], "id": 3, "method": "eth_getWork"}
*/
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MINING, NDPI_PROTOCOL_UNKNOWN);
} else if(ndpi_strnstr((const char *)packet->payload, "\"method\"", packet->payload_packet_len)
|| ndpi_strnstr((const char *)packet->payload, "\"blob\"", packet->payload_packet_len)
|| ndpi_strnstr((const char *)packet->payload, "\"id\"", packet->payload_packet_len)) {
} else if(ndpi_strnstr((const char *)packet->payload, "{", packet->payload_packet_len)
&& (ndpi_strnstr((const char *)packet->payload, "\"method\":", packet->payload_packet_len)
|| ndpi_strnstr((const char *)packet->payload, "\"blob\":", packet->payload_packet_len)
/* || ndpi_strnstr((const char *)packet->payload, "\"id\":", packet->payload_packet_len) - Removed as too generic */
)
) {
/*
ZCash