Improved DGA detection with trigrams. Disadvantage: slower startup time

Reworked Tor dissector embedded in TLS (fixes #1141)
Removed false positive on HTTP User-Agent
This commit is contained in:
Luca Deri
2021-03-03 00:41:07 +01:00
parent 4c00ff89df
commit 56bfb439f8
29 changed files with 266 additions and 265 deletions
+1
View File
@@ -153,6 +153,7 @@ extern "C" {
int ndpi_handle_ipv6_extension_headers(struct ndpi_detection_module_struct *ndpi_str,
const u_int8_t ** l4ptr, u_int16_t * l4len,
u_int8_t * nxt_hdr);
void ndpi_set_risk(struct ndpi_flow_struct *flow, ndpi_risk_enum r);
#ifdef __cplusplus
}
#endif
-1
View File
@@ -159,7 +159,6 @@ void init_telegram_dissector(struct ndpi_detection_module_struct *ndpi_struct, u
void init_telnet_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_tftp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_thunder_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_tor_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_tvuplayer_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_usenet_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_wsd_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
+1 -1
View File
@@ -1100,7 +1100,7 @@ struct ndpi_detection_module_struct {
ndpi_automa host_automa, /* Used for DNS/HTTPS */
content_automa, /* Used for HTTP subprotocol_detection */
subprotocol_automa, /* Used for HTTP subprotocol_detection */
bigrams_automa, impossible_bigrams_automa, /* TOR */
bigrams_automa, trigrams_automa, impossible_bigrams_automa, /* TOR */
risky_domain_automa, tls_cert_subject_automa,
malicious_ja3_automa, malicious_sha1_automa;
/* IMPORTANT: please update ndpi_finalize_initialization() whenever you add a new automa */
File diff suppressed because one or more lines are too long
+131 -25
View File
@@ -645,6 +645,9 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
for(i = 0; ndpi_en_bigrams[i] != NULL; i++)
ndpi_string_to_automa(ndpi_str, &ndpi_str->bigrams_automa, (char *) ndpi_en_bigrams[i], 1, 1, 1, 0, 0);
for(i = 0; ndpi_en_trigrams[i] != NULL; i++)
ndpi_string_to_automa(ndpi_str, &ndpi_str->trigrams_automa, (char *) ndpi_en_trigrams[i], 1, 1, 1, 0, 0);
for(i = 0; ndpi_en_impossible_bigrams[i] != NULL; i++)
ndpi_string_to_automa(ndpi_str, &ndpi_str->impossible_bigrams_automa, (char *) ndpi_en_impossible_bigrams[i], 1,
1, 1, 0, 0);
@@ -2190,6 +2193,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->host_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->content_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->bigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->trigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->impossible_bigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->tls_cert_subject_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->malicious_ja3_automa.ac_automa = NULL; /* Initialized on demand */
@@ -2259,6 +2263,10 @@ void ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str)
automa = &ndpi_str->malicious_sha1_automa;
break;
case 7:
automa = &ndpi_str->trigrams_automa;
break;
default:
return;
}
@@ -2512,6 +2520,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
if(ndpi_str->bigrams_automa.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->bigrams_automa.ac_automa, 0);
if(ndpi_str->trigrams_automa.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->trigrams_automa.ac_automa, 0);
if(ndpi_str->impossible_bigrams_automa.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->impossible_bigrams_automa.ac_automa, 0);
@@ -2652,7 +2663,7 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str,
/* Run some basic consistency tests */
if(flow->packet.payload_packet_len < sizeof(struct ndpi_icmphdr))
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
else {
u_int8_t icmp_type = (u_int8_t)flow->packet.payload[0];
u_int8_t icmp_code = (u_int8_t)flow->packet.payload[1];
@@ -2660,7 +2671,7 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str,
/* https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xhtml */
if(((icmp_type >= 44) && (icmp_type <= 252))
|| (icmp_code > 15))
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
}
}
return(NDPI_PROTOCOL_IP_ICMP);
@@ -2685,7 +2696,7 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str,
/* Run some basic consistency tests */
if(flow->packet.payload_packet_len < sizeof(struct ndpi_icmphdr))
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
else {
u_int8_t icmp6_type = (u_int8_t)flow->packet.payload[0];
u_int8_t icmp6_code = (u_int8_t)flow->packet.payload[1];
@@ -2694,7 +2705,7 @@ u_int16_t ndpi_guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str,
if(((icmp6_type >= 5) && (icmp6_type <= 127))
|| (icmp6_type >= 156)
|| ((icmp6_code > 7) && (icmp6_type != 255)))
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
}
}
return(NDPI_PROTOCOL_IP_ICMPV6);
@@ -3523,9 +3534,6 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
/* TEAMSPEAK */
init_teamspeak_dissector(ndpi_str, &a, detection_bitmask);
/* TOR */
init_tor_dissector(ndpi_str, &a, detection_bitmask);
/* SKINNY */
init_skinny_dissector(ndpi_str, &a, detection_bitmask);
@@ -4997,18 +5005,21 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
static void ndpi_reconcile_protocols(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
ndpi_protocol *ret) {
/*
Skype for a host doing MS Teams means MS Teams
(MS Teams uses Skype as transport protocol for voice/video)
*/
#if 0
if(flow) {
/* Do not go for DNS when there is an application protocol. Example DNS.Apple */
if((flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN)
&& (flow->detected_protocol_stack[0] /* app */ != flow->detected_protocol_stack[1] /* major */))
NDPI_CLR_BIT(flow->risk, NDPI_SUSPICIOUS_DGA_DOMAIN);
}
#endif
switch(ret->app_protocol) {
/*
Skype for a host doing MS Teams means MS Teams
(MS Teams uses Skype as transport protocol for voice/video)
*/
case NDPI_PROTOCOL_MSTEAMS:
if(flow->packet.iph && flow->packet.tcp) {
// printf("====>> NDPI_PROTOCOL_MSTEAMS\n");
@@ -5053,7 +5064,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
case NDPI_PROTOCOL_UNSAFE:
case NDPI_PROTOCOL_POTENTIALLY_DANGEROUS:
case NDPI_PROTOCOL_DANGEROUS:
NDPI_SET_BIT(flow->risk, NDPI_UNSAFE_PROTOCOL);
ndpi_set_risk(flow, NDPI_UNSAFE_PROTOCOL);
break;
default:
/* Nothing to do */
@@ -5341,7 +5352,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
// printf("******** %u / %u\n", found->proto->protoId, ret.master_protocol);
if(!ndpi_check_protocol_port_mismatch_exceptions(ndpi_str, flow, found, &ret))
NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
ndpi_set_risk(flow, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
} else if((!ndpi_is_ntop_protocol(&ret)) && default_ports && (default_ports[0] != 0)) {
u_int8_t found = 0, i, num_loops = 0;
@@ -5365,7 +5376,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
if(!found) {
// printf("******** Invalid default port\n");
NDPI_SET_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
ndpi_set_risk(flow, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
}
}
@@ -6721,7 +6732,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
u_int16_t rc1 = ndpi_match_string(ndpi_str->risky_domain_automa.ac_automa, buf);
if(rc1 > 0)
NDPI_SET_BIT(flow->risk, NDPI_RISKY_DOMAIN);
ndpi_set_risk(flow, NDPI_RISKY_DOMAIN);
}
return(rc);
@@ -6799,6 +6810,50 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
/* ****************************************************** */
int ndpi_match_trigram(struct ndpi_detection_module_struct *ndpi_str,
ndpi_automa *automa, char *trigram_to_match) {
AC_TEXT_t ac_input_text;
AC_REP_t match = {NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_UNRATED};
int rc;
if((automa->ac_automa == NULL) || (trigram_to_match == NULL))
return(-1);
if(!automa->ac_automa_finalized) {
#if 1
ndpi_finalize_initialization(ndpi_str);
#else
printf("[%s:%d] [NDPI] Internal error: please call ndpi_finalize_initialization()\n", __FILE__, __LINE__);
return(0); /* No matches */
#endif
}
ac_input_text.astring = trigram_to_match, ac_input_text.length = 3;
rc = ac_automata_search(((AC_AUTOMATA_t *) automa->ac_automa), &ac_input_text, &match);
/*
As ac_automata_search can detect partial matches and continue the search process
in case rc == 0 (i.e. no match), we need to check if there is a partial match
and in this case return it
*/
if((rc == 0) && (match.number != 0))
rc = 1;
#ifdef TRIGRAM_CHECK
if(rc && match.number) {
printf("[%s:%d] [NDPI] Trigram %c%c%c\n",
__FILE__, __LINE__,
trigram_to_match[0],
trigram_to_match[1],
trigram_to_match[2]);
}
#endif
return(rc ? match.number : 0);
}
/* ****************************************************** */
void ndpi_free_flow(struct ndpi_flow_struct *flow) {
if(flow) {
ndpi_free_flow_data(flow);
@@ -7195,6 +7250,33 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
/* ******************************************************************** */
static int ndpi_is_trigram_char(char c) {
if(isdigit(c) || (c == '.') || (c == '-'))
return(0);
else
return(1);
}
/* ******************************************************************** */
static int ndpi_is_vowel(char c) {
switch(c) {
case 'a':
case 'e':
case 'i':
case 'o':
case 'u':
case 'y': // Not a real vowel...
return(1);
break;
default:
return(0);
}
}
/* ******************************************************************** */
int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
char *name, u_int8_t is_hostname) {
@@ -7211,7 +7293,9 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
len = strlen(name);
if(len >= 5) {
int i, j, num_found = 0, num_impossible = 0, num_bigram_checks = 0, num_digits = 0, num_vowels = 0, num_words = 0;
int i, j, num_found = 0, num_impossible = 0, num_bigram_checks = 0,
num_trigram_found = 0, num_trigram_checked = 0,
num_digits = 0, num_vowels = 0, num_words = 0;
char tmp[128], *word, *tok_tmp;
u_int max_tmp_len = sizeof(tmp)-1;
@@ -7273,6 +7357,9 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
j++;
}
if(num_dots < 2) /* At least XXX.YYY.ZZZ */
return(0);
if(curr_domain_element_len > max_domain_element_len)
max_domain_element_len = curr_domain_element_len;
@@ -7298,7 +7385,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
*/
|| (max_domain_element_len >= 19 /* word too long. Example bbcbedxhgjmdobdprmen.com */)
) {
if(flow) NDPI_SET_BIT(flow->risk, NDPI_SUSPICIOUS_DGA_DOMAIN);
if(flow) ndpi_set_risk(flow, NDPI_SUSPICIOUS_DGA_DOMAIN);
#ifdef DGA_DEBUG
printf("[DGA] Found!");
#endif
@@ -7348,6 +7435,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
break;
}
#if 0
switch(word[i]) {
case 'a':
case 'e':
@@ -7357,7 +7445,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
num_vowels++;
break;
}
#endif
if(isdigit(word[i+1])) {
num_digits++;
// num_impossible++;
@@ -7380,21 +7468,36 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
} else if(ndpi_match_bigram(ndpi_str, &ndpi_str->bigrams_automa, &word[i])) {
num_found++;
}
if((i > 0) && (word[0] != '_') && (word[i+2] != '\0')) {
if(ndpi_is_trigram_char(word[i]) && ndpi_is_trigram_char(word[i+1]) && ndpi_is_trigram_char(word[i+2])) {
num_trigram_checked++;
if(ndpi_match_trigram(ndpi_str, &ndpi_str->trigrams_automa, &word[i])) {
num_trigram_found++;
}
}
/* Count vowels */
num_vowels += ndpi_is_vowel(word[i]) + ndpi_is_vowel(word[i+1]) + ndpi_is_vowel(word[i+2]);
}
} /* for */
} /* for */
#ifdef DGA_DEBUG
printf("[num_found: %u][num_impossible: %u][num_digits: %u][num_bigram_checks: %u][num_vowels: %u/%u]\n",
num_found, num_impossible, num_digits, num_bigram_checks, num_vowels, j-num_vowels);
printf("[%s][num_found: %u][num_impossible: %u][num_digits: %u][num_bigram_checks: %u][num_vowels: %u/%u][num_trigram_found: %u/%u][vowels: %u]\n",
name, num_found, num_impossible, num_digits, num_bigram_checks, num_vowels, j-num_vowels,
num_trigram_checked, num_trigram_found, num_vowels);
#endif
if(num_bigram_checks
&& ((num_found == 0) || ((num_digits > 5) && (num_words <= 3)) || enough(num_found, num_impossible)))
&& ((num_found == 0) || ((num_digits > 5) && (num_words <= 3))
|| enough(num_found, num_impossible) || ((num_trigram_checked > 3) && (num_trigram_found < (num_trigram_checked/2)))))
rc = 1;
if(rc && flow)
NDPI_SET_BIT(flow->risk, NDPI_SUSPICIOUS_DGA_DOMAIN);
if(num_trigram_checked && (num_vowels == 0))
rc = 1;
#ifdef DGA_DEBUG
if(rc)
printf("DGA %s [num_found: %u][num_impossible: %u]\n",
@@ -7406,6 +7509,9 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
printf("[DGA] Result: %u", rc);
#endif
if(rc && flow)
ndpi_set_risk(flow, NDPI_SUSPICIOUS_DGA_DOMAIN);
return(rc);
}
+9
View File
@@ -2450,3 +2450,12 @@ int ndpi_hash_add_entry(ndpi_str_hash *h, char *key, u_int8_t key_len, u_int8_t
return(0);
}
/* ******************************************************************** */
void ndpi_set_risk(struct ndpi_flow_struct *flow, ndpi_risk_enum r) {
u_int32_t v = 1 << r;
// NDPI_SET_BIT(flow->risk, (u_int32_t)r);
flow->risk |= v;
}
+4 -4
View File
@@ -89,7 +89,7 @@ static void ndpi_check_dns_type(struct ndpi_detection_module_struct *ndpi_struct
case 106:
case 107:
case 259:
NDPI_SET_BIT(flow->risk, NDPI_DNS_SUSPICIOUS_TRAFFIC);
ndpi_set_risk(flow, NDPI_DNS_SUSPICIOUS_TRAFFIC);
break;
}
}
@@ -194,7 +194,7 @@ static int search_valid_dns(struct ndpi_detection_module_struct *ndpi_struct,
else if((dns_header->flags & FLAGS_MASK) == 0x8000)
*is_query = 0;
else {
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
return(1 /* invalid */);
}
@@ -219,7 +219,7 @@ static int search_valid_dns(struct ndpi_detection_module_struct *ndpi_struct,
x++;
}
} else {
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
return(1 /* invalid */);
}
} else {
@@ -408,7 +408,7 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
#ifdef DNS_DEBUG
printf("[DNS] Invalid query len [%u >= %u]\n", i+4, flow->packet.payload_packet_len);
#endif
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
break;
} else
idx += tot_len+4, num_queries++;
+21 -21
View File
@@ -51,23 +51,23 @@ static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struc
static void ndpi_analyze_content_signature(struct ndpi_flow_struct *flow) {
if((flow->initial_binary_bytes_len >= 2) && (flow->initial_binary_bytes[0] == 0x4D) && (flow->initial_binary_bytes[1] == 0x5A))
NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); /* Win executable */
ndpi_set_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Win executable */
else if((flow->initial_binary_bytes_len >= 4) && (flow->initial_binary_bytes[0] == 0x7F) && (flow->initial_binary_bytes[1] == 'E')
&& (flow->initial_binary_bytes[2] == 'L') && (flow->initial_binary_bytes[3] == 'F'))
NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); /* Linux executable */
ndpi_set_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Linux executable */
else if((flow->initial_binary_bytes_len >= 4) && (flow->initial_binary_bytes[0] == 0xCF) && (flow->initial_binary_bytes[1] == 0xFA)
&& (flow->initial_binary_bytes[2] == 0xED) && (flow->initial_binary_bytes[3] == 0xFE))
NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); /* Linux executable */
ndpi_set_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Linux executable */
else if((flow->initial_binary_bytes_len >= 3)
&& (flow->initial_binary_bytes[0] == '#')
&& (flow->initial_binary_bytes[1] == '!')
&& (flow->initial_binary_bytes[2] == '/'))
NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); /* Unix script (e.g. #!/bin/sh) */
ndpi_set_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Unix script (e.g. #!/bin/sh) */
else if(flow->initial_binary_bytes_len >= 8) {
u_int8_t exec_pattern[] = { 0x64, 0x65, 0x78, 0x0A, 0x30, 0x33, 0x35, 0x00 };
if(memcmp(flow->initial_binary_bytes, exec_pattern, 8) == 0)
NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER); /* Dalvik Executable (Android) */
ndpi_set_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER); /* Dalvik Executable (Android) */
}
}
@@ -128,7 +128,7 @@ static void ndpi_http_check_human_redeable_content(struct ndpi_detection_module_
&& (content[3] == 0x00)) {
/* Looks like compressed data */
} else
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_CONTENT);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_CONTENT);
}
}
}
@@ -220,7 +220,7 @@ static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_mo
for(i = 0; cmp_mimes[i] != NULL; i++) {
if(strncasecmp(app, cmp_mimes[i], app_len_avail) == 0) {
flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT;
NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER);
ndpi_set_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER);
NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer");
return(flow->category);
}
@@ -248,7 +248,7 @@ static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_mo
if(memcmp((const char*)&packet->content_disposition_line.ptr[attachment_len],
binary_file_ext[i], ATTACHMENT_LEN) == 0) {
flow->guessed_category = flow->category = NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT;
NDPI_SET_BIT(flow->risk, NDPI_BINARY_APPLICATION_TRANSFER);
ndpi_set_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER);
NDPI_LOG_INFO(ndpi_struct, "found executable HTTP transfer");
return(flow->category);
}
@@ -367,10 +367,10 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
|| (!strncmp(ua, "<?", 2))
|| strchr(ua, '{')
|| strchr(ua, '}')
|| ndpi_check_dga_name(ndpi_struct, NULL, ua, 0)
// || ndpi_check_dga_name(ndpi_struct, NULL, ua, 0)
// || ndpi_match_bigram(ndpi_struct, &ndpi_struct->impossible_bigrams_automa, ua)
) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_USER_AGENT);
}
}
@@ -470,7 +470,7 @@ static void ndpi_check_numeric_ip(struct ndpi_detection_module_struct *ndpi_stru
ip_addr.s_addr = inet_addr(buf);
if(strcmp(inet_ntoa(ip_addr), buf) == 0)
NDPI_SET_BIT(flow->risk, NDPI_HTTP_NUMERIC_IP_HOST);
ndpi_set_risk(flow, NDPI_HTTP_NUMERIC_IP_HOST);
}
/* ************************************************************* */
@@ -781,55 +781,55 @@ static void ndpi_check_http_header(struct ndpi_detection_module_struct *ndpi_str
switch(packet->line[i].ptr[0]){
case 'A':
if(is_a_suspicious_header(suspicious_http_header_keys_A, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
break;
case 'C':
if(is_a_suspicious_header(suspicious_http_header_keys_C, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
break;
case 'M':
if(is_a_suspicious_header(suspicious_http_header_keys_M, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
break;
case 'O':
if(is_a_suspicious_header(suspicious_http_header_keys_O, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
break;
case 'R':
if(is_a_suspicious_header(suspicious_http_header_keys_R, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
break;
case 'S':
if(is_a_suspicious_header(suspicious_http_header_keys_S, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
break;
case 'T':
if(is_a_suspicious_header(suspicious_http_header_keys_T, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
break;
case 'U':
if(is_a_suspicious_header(suspicious_http_header_keys_U, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
break;
case 'X':
if(is_a_suspicious_header(suspicious_http_header_keys_X, packet->line[i])) {
NDPI_SET_BIT(flow->risk, NDPI_HTTP_SUSPICIOUS_HEADER);
ndpi_set_risk(flow, NDPI_HTTP_SUSPICIOUS_HEADER);
return;
}
@@ -1151,7 +1151,7 @@ static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struc
ndpi_http_method ndpi_get_http_method(struct ndpi_detection_module_struct *ndpi_mod,
struct ndpi_flow_struct *flow) {
if(!flow) {
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
return(NDPI_HTTP_METHOD_UNKNOWN);
} else
return(flow->http.method);
+1 -1
View File
@@ -1283,7 +1283,7 @@ static void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
/* Add check for missing SNI */
if(flow->protos.tls_quic_stun.tls_quic.client_requested_server_name[0] == '\0') {
/* This is a bit suspicious */
NDPI_SET_BIT(flow->risk, NDPI_TLS_MISSING_SNI);
ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI);
}
}
+1 -1
View File
@@ -46,7 +46,7 @@ void ndpi_search_smb_tcp(struct ndpi_detection_module_struct *ndpi_struct, struc
if(memcmp(&packet->payload[4], smbv1, sizeof(smbv1)) == 0) {
if(packet->payload[8] != 0x72) /* Skip Negotiate request */ {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SMBV1, NDPI_PROTOCOL_NETBIOS);
NDPI_SET_BIT(flow->risk, NDPI_SMB_INSECURE_VERSION);
ndpi_set_risk(flow, NDPI_SMB_INSECURE_VERSION);
}
} else
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_SMBV23, NDPI_PROTOCOL_NETBIOS);
+1 -1
View File
@@ -175,7 +175,7 @@ static void ssh_analyse_cipher(struct ndpi_detection_module_struct *ndpi_struct,
}
if(found_obsolete_cipher) {
NDPI_SET_BIT(flow->risk, (is_client_signature ? NDPI_SSH_OBSOLETE_CLIENT_VERSION_OR_CIPHER : NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER));
ndpi_set_risk(flow, (is_client_signature ? NDPI_SSH_OBSOLETE_CLIENT_VERSION_OR_CIPHER : NDPI_SSH_OBSOLETE_SERVER_VERSION_OR_CIPHER));
}
ndpi_free(cipher_copy);
+33 -14
View File
@@ -417,7 +417,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
if((time_sec < flow->protos.tls_quic_stun.tls_quic.notBefore)
|| (time_sec > flow->protos.tls_quic_stun.tls_quic.notAfter))
NDPI_SET_BIT(flow->risk, NDPI_TLS_CERTIFICATE_EXPIRED); /* Certificate expired */
ndpi_set_risk(flow, NDPI_TLS_CERTIFICATE_EXPIRED); /* Certificate expired */
}
}
}
@@ -509,7 +509,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
} /* while */
if(!matched_name)
NDPI_SET_BIT(flow->risk, NDPI_TLS_CERTIFICATE_MISMATCH); /* Certificate mismatch */
ndpi_set_risk(flow, NDPI_TLS_CERTIFICATE_MISMATCH); /* Certificate mismatch */
}
}
}
@@ -533,7 +533,7 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
if(flow->protos.tls_quic_stun.tls_quic.subjectDN && flow->protos.tls_quic_stun.tls_quic.issuerDN
&& (!strcmp(flow->protos.tls_quic_stun.tls_quic.subjectDN, flow->protos.tls_quic_stun.tls_quic.issuerDN)))
NDPI_SET_BIT(flow->risk, NDPI_TLS_SELFSIGNED_CERTIFICATE);
ndpi_set_risk(flow, NDPI_TLS_SELFSIGNED_CERTIFICATE);
#if DEBUG_TLS
printf("[TLS] %s() SubjectDN [%s]\n", __FUNCTION__, rdnSeqBuf);
@@ -561,7 +561,7 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
#endif
if((packet->payload_packet_len != (length + 4 + (is_dtls ? 8 : 0))) || (packet->payload[1] != 0x0)) {
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
return(-1); /* Invalid length */
}
@@ -570,7 +570,7 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
packet->payload[certificates_offset - 1];
if((packet->payload[certificates_offset - 3] != 0x0) || ((certificates_length+3) != length)) {
NDPI_SET_BIT(flow->risk, NDPI_MALFORMED_PACKET);
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET);
return(-2); /* Invalid length */
}
@@ -644,7 +644,7 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t rc1 = ndpi_match_string(ndpi_struct->malicious_sha1_automa.ac_automa, sha1_str);
if(rc1 > 0)
NDPI_SET_BIT(flow->risk, NDPI_MALICIOUS_SHA1);
ndpi_set_risk(flow, NDPI_MALICIOUS_SHA1);
}
processCertificateElements(ndpi_struct, flow, certificates_offset, certificate_len);
@@ -1065,7 +1065,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
tls_version = ntohs(*((u_int16_t*)&packet->payload[version_offset]));
flow->protos.tls_quic_stun.tls_quic.ssl_version = ja3.tls_handshake_version = tls_version;
if(flow->protos.tls_quic_stun.tls_quic.ssl_version < 0x0302) /* TLSv1.1 */
NDPI_SET_BIT(flow->risk, NDPI_TLS_OBSOLETE_VERSION);
ndpi_set_risk(flow, NDPI_TLS_OBSOLETE_VERSION);
if(handshake_type == 0x02 /* Server Hello */) {
int i, rc;
@@ -1090,7 +1090,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
ja3.num_cipher = 1, ja3.cipher[0] = ntohs(*((u_int16_t*)&packet->payload[offset]));
if((flow->protos.tls_quic_stun.tls_quic.server_unsafe_cipher = ndpi_is_safe_ssl_cipher(ja3.cipher[0])) == 1)
NDPI_SET_BIT(flow->risk, NDPI_TLS_WEAK_CIPHER);
ndpi_set_risk(flow, NDPI_TLS_WEAK_CIPHER);
flow->protos.tls_quic_stun.tls_quic.server_cipher = ja3.cipher[0];
@@ -1324,9 +1324,28 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_QUIC, buffer, strlen(buffer)))
flow->l4.tcp.tls.subprotocol_detected = 1;
}
if(ndpi_check_dga_name(ndpi_struct, flow,
flow->protos.tls_quic_stun.tls_quic.client_requested_server_name, 1)) {
char *sni = flow->protos.tls_quic_stun.tls_quic.client_requested_server_name;
int len = strlen(sni);
ndpi_check_dga_name(ndpi_struct, flow,
flow->protos.tls_quic_stun.tls_quic.client_requested_server_name, 1);
#ifdef DEBUG_TLS
printf("[TLS] SNI: (DGA) [%s]\n", flow->protos.tls_quic_stun.tls_quic.client_requested_server_name);
#endif
if((len >= 4)
&& strcmp(&sni[len-4], ".com") /* Check if it ends in .com or .net */
&& strcmp(&sni[len-4], ".net")
&& strncmp(sni, "www.", 4)) /* Not starting with www.... */
;
else
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TOR, NDPI_PROTOCOL_TLS);
} else {
#ifdef DEBUG_TLS
printf("[TLS] SNI: (NO DGA) [%s]\n", flow->protos.tls_quic_stun.tls_quic.client_requested_server_name);
#endif
}
} else {
#ifdef DEBUG_TLS
printf("[TLS] Extensions server len too short: %u vs %u\n",
@@ -1661,21 +1680,21 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
flow->protos.tls_quic_stun.tls_quic.ja3_client);
if(rc1 > 0)
NDPI_SET_BIT(flow->risk, NDPI_MALICIOUS_JA3);
ndpi_set_risk(flow, NDPI_MALICIOUS_JA3);
}
}
/* Before returning to the caller we need to make a final check */
if((flow->protos.tls_quic_stun.tls_quic.ssl_version >= 0x0303) /* >= TLSv1.2 */
&& (flow->protos.tls_quic_stun.tls_quic.alpn == NULL) /* No ALPN */) {
NDPI_SET_BIT(flow->risk, NDPI_TLS_NOT_CARRYING_HTTPS);
ndpi_set_risk(flow, NDPI_TLS_NOT_CARRYING_HTTPS);
}
/* Suspicious Domain Fronting:
https://github.com/SixGenInc/Noctilucent/blob/master/docs/ */
if(flow->protos.tls_quic_stun.tls_quic.encrypted_sni.esni &&
flow->protos.tls_quic_stun.tls_quic.client_requested_server_name[0] != '\0') {
NDPI_SET_BIT(flow->risk, NDPI_TLS_SUSPICIOUS_ESNI_USAGE);
ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_ESNI_USAGE);
}
/* Add check for missing SNI */
@@ -1684,7 +1703,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
&& (flow->protos.tls_quic_stun.tls_quic.encrypted_sni.esni == NULL) /* No ESNI */
) {
/* This is a bit suspicious */
NDPI_SET_BIT(flow->risk, NDPI_TLS_MISSING_SNI);
ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI);
}
return(2 /* Client Certificate */);
-106
View File
@@ -1,106 +0,0 @@
/*
* tor.c
*
* Copyright (C) 2016-18 ntop.org
* Copyright (C) 2013 Remy Mudingay <mudingay@ill.fr>
*
*/
#include "ndpi_protocol_ids.h"
#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_TOR
#include "ndpi_api.h"
static void ndpi_int_tor_add_connection(struct ndpi_detection_module_struct
*ndpi_struct, struct ndpi_flow_struct *flow) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TOR, NDPI_PROTOCOL_UNKNOWN);
}
int ndpi_is_tls_tor(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow, char *certificate) {
int len;
char dummy[48], *dot, *name;
if((certificate == NULL) || (certificate[0] == '\0'))
return(0);
else
len = strlen(certificate);
/* Check if it ends in .com or .net */
if(len>=4 && strcmp(&certificate[len-4], ".com") && strcmp(&certificate[len-4], ".net"))
return(0);
if((len < 6)
|| (!strncmp(certificate, "*.", 2)) /* Wildcard certificate */
|| (strncmp(certificate, "www.", 4)) /* Not starting with www.... */
)
return(0);
// printf("***** [SSL] %s(): %s\n", __FUNCTION__, certificate);
snprintf(dummy, sizeof(dummy), "%s", certificate);
if((dot = strrchr(dummy, '.')) == NULL) return(0);
dot[0] = '\0';
if((dot = strrchr(dummy, '.')) == NULL) return(0);
name = &dot[1];
if(ndpi_check_dga_name(ndpi_struct, flow, name, 1)) {
ndpi_int_tor_add_connection(ndpi_struct, flow);
return(1);
} else {
#ifdef PEDANTIC_TOR_CHECK
if(gethostbyname(certificate) == NULL) {
ndpi_int_tor_add_connection(ndpi_struct, flow);
return(1);
}
#endif
}
return(0);
}
/* ******************************************* */
void ndpi_search_tor(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct *packet = &flow->packet;
NDPI_LOG_DBG(ndpi_struct, "search for TOR\n");
if((packet->tcp != NULL)
&& (!packet->tls_certificate_detected)) {
u_int16_t dport, sport;
sport = ntohs(packet->tcp->source), dport = ntohs(packet->tcp->dest);
NDPI_LOG_DBG2(ndpi_struct, "calculating TOR over tcp\n");
if ((((dport == 9001) || (sport == 9001)) || ((dport == 9030) || (sport == 9030)))
&& ((packet->payload[0] == 0x17) || (packet->payload[0] == 0x16))
&& (packet->payload[1] == 0x03)
&& (packet->payload[2] == 0x01)
&& (packet->payload[3] == 0x00)) {
NDPI_LOG_INFO(ndpi_struct, "found tor\n");
ndpi_int_tor_add_connection(ndpi_struct, flow);
}
} else {
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
}
}
void init_tor_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask)
{
ndpi_set_bitmask_protocol_detection("Tor", ndpi_struct, detection_bitmask, *id,
NDPI_PROTOCOL_TOR,
ndpi_search_tor,
NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_TCP_WITH_PAYLOAD_WITHOUT_RETRANSMISSION,
SAVE_DETECTION_BITMASK_AS_UNKNOWN,
ADD_TO_DETECTION_BITMASK);
*id += 1;
}