Improved DGA detection

Before
Accuracy 66%, Precision 86%, Recall 38%

After
Accuracy 71%, Precision 89%, Recall 49%
This commit is contained in:
Luca Deri
2021-03-03 19:30:01 +01:00
parent 49843509e5
commit 0f8a994841
14 changed files with 168 additions and 138 deletions
+1 -1
View File
@@ -9606,6 +9606,6 @@ static const char *ndpi_en_impossible_bigrams[] = {
"qz", "sx", "sz", "tq", "tx", "vb", "vc", "vd", "vf", "vg", "vh", "vj", "vm", "vn", /* "vp", Removed for vpbank.com */ "bw", /* "vk", "zr" Removed for kavkazr */
"vq", "vt", "vw", "vx", "vz", "wq", "wv", "wx", "wz", /* "xb", foxbusiness.com */
"xg", "xj", "xk", "xv", "xz", "xw", "yd", /*"yp", Removed for paypal */
"yj", "yq", "yv", "yz", "yw", "zb", "zc", "zg", "zh", "zj", "zn", "zq", "zs", "zx", "wh", "wk",
"yj", "yq", "yv", "yz", "yw", "zb", "zc", "zg", "zh", "zj", "zn", "zq", "zs", "zx", /* "wh", e.g. why */ "wk",
"wb", "zk", "kp", "zk", "xy", "xx",
NULL };
+133 -106
View File
@@ -58,6 +58,8 @@ static int _ndpi_debug_callbacks = 0;
/* #define DGA_DEBUG 1 */
/* #define MATCH_DEBUG 1 */
u_int ndpi_verbose_dga_detection = 0;
/* ****************************************** */
static void *(*_ndpi_flow_malloc)(size_t size);
@@ -497,7 +499,7 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_str,
dot = len -1;
memset(&ac_pattern, 0, sizeof(ac_pattern));
if((!add_ends_with) || ndpi_is_middle_string_char(value[dot])) {
ac_pattern.length = len;
ac_pattern.astring = value;
@@ -621,14 +623,14 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
ndpi_init_protocol_match(ndpi_str, &host_match[i]);
/* ************************ */
for(i = 0; tls_certificate_match[i].string_to_match != NULL; i++) {
#if 0
printf("%s() %s / %u\n", __FUNCTION__,
tls_certificate_match[i].string_to_match,
tls_certificate_match[i].protocol_id);
#endif
ndpi_add_string_value_to_automa(ndpi_str->tls_cert_subject_automa.ac_automa,
tls_certificate_match[i].string_to_match,
tls_certificate_match[i].protocol_id);
@@ -2997,7 +2999,7 @@ int ndpi_load_malicious_ja3_file(struct ndpi_detection_module_struct *ndpi_str,
if(ndpi_str->malicious_ja3_automa.ac_automa == NULL)
ndpi_str->malicious_ja3_automa.ac_automa = ac_automata_init(ac_match_handler);
fd = fopen(path, "r");
if(fd == NULL) {
@@ -3007,7 +3009,7 @@ int ndpi_load_malicious_ja3_file(struct ndpi_detection_module_struct *ndpi_str,
while(1) {
char *comma;
line = fgets(buffer, sizeof(buffer), fd);
if(line == NULL)
@@ -5014,7 +5016,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
NDPI_CLR_BIT(flow->risk, NDPI_SUSPICIOUS_DGA_DOMAIN);
}
#endif
switch(ret->app_protocol) {
/*
Skype for a host doing MS Teams means MS Teams
@@ -6839,16 +6841,14 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
if((rc == 0) && (match.number != 0))
rc = 1;
#ifdef TRIGRAM_CHECK
if(rc && match.number) {
if(ndpi_verbose_dga_detection && rc && match.number) {
printf("[%s:%d] [NDPI] Trigram %c%c%c\n",
__FILE__, __LINE__,
trigram_to_match[0],
trigram_to_match[1],
trigram_to_match[2]);
}
#endif
return(rc ? match.number : 0);
}
@@ -7223,11 +7223,11 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
static int enough(int a, int b) {
u_int8_t percentage = 20;
if(b == 0) return(0);
if(b <= 1) return(0);
if(a == 0) return(1);
if(b > (((a+1)*percentage)/100)) return(1);
return(0);
}
@@ -7267,6 +7267,7 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
case 'o':
case 'u':
case 'y': // Not a real vowel...
case 'x': // Not a real vowel...
return(1);
break;
@@ -7274,39 +7275,45 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
return(0);
}
}
/* ******************************************************************** */
int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
char *name, u_int8_t is_hostname) {
int len, rc = 0;
u_int8_t max_num_char_repetitions = 0, last_char = 0, num_char_repetitions = 0, num_dots = 0;
int len, rc = 0, trigram_char_skip = 0;
u_int8_t max_num_char_repetitions = 0, last_char = 0, num_char_repetitions = 0, num_dots = 0, num_trigram_dots = 0;
u_int8_t max_domain_element_len = 0, curr_domain_element_len = 0, first_element_is_numeric = 1;
if(!name) return(0);
if((!name)
|| (strchr(name, '_') != NULL)
|| (endsWith(name, "in-addr.arpa", 12)))
return(0);
if(flow && (flow->packet.detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN))
return(0); /* Ignore DGA check for protocols already fully detected */
#ifdef DGA_DEBUG
printf("[DGA] %s\n", name);
#endif
if(strncmp(name, "www.", 4) == 0)
name = &name[4];
if(ndpi_verbose_dga_detection)
printf("[DGA check] %s\n", name);
len = strlen(name);
if(len >= 5) {
int i, j, num_found = 0, num_impossible = 0, num_bigram_checks = 0,
num_trigram_found = 0, num_trigram_checked = 0,
num_digits = 0, num_vowels = 0, num_words = 0;
num_trigram_found = 0, num_trigram_checked = 0, num_dash = 0,
num_digits = 0, num_vowels = 0, num_trigram_vowels = 0, num_words = 0, skip_next_bigram = 0;
char tmp[128], *word, *tok_tmp;
u_int max_tmp_len = sizeof(tmp)-1;
len = snprintf(tmp, max_tmp_len, "%s", name);
if(len < 0) {
#ifdef DGA_DEBUG
printf("[DGA] Too short");
#endif
if(ndpi_verbose_dga_detection)
printf("[DGA] Too short");
return(0);
} else
tmp[len < max_tmp_len ? len : max_tmp_len] = '\0';
@@ -7314,19 +7321,32 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
for(i=0, j=0; (i<len) && (j<max_tmp_len); i++) {
tmp[j] = tolower(name[i]);
if(tmp[j] == '.')
if(tmp[j] == '.') {
num_dots++;
else if(num_dots == 0) {
} else if(num_dots == 0) {
if(!isdigit(tmp[j]))
first_element_is_numeric = 0;
}
if(ndpi_is_vowel(tmp[j]))
num_vowels++;
if(last_char == tmp[j]) {
if(++num_char_repetitions > max_num_char_repetitions)
max_num_char_repetitions = num_char_repetitions;
} else
num_char_repetitions = 1, last_char = tmp[j];
if(isdigit(tmp[j])) {
num_digits++;
if(((j+2)<len) && isdigit(tmp[j+1]) && (tmp[j+2] == '.')) {
/* Check if there are too many digits */
if(num_digits < 4)
return(0); /* Double digits */
}
}
switch(tmp[j]) {
case '.':
case '-':
@@ -7360,22 +7380,20 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
j++;
}
if(num_dots < 2) /* At least XXX.YYY.ZZZ */
if(num_dots == 0) /* Doesn't look like a domain name */
return(0);
if(curr_domain_element_len > max_domain_element_len)
max_domain_element_len = curr_domain_element_len;
#ifdef DGA_DEBUG
printf("[DGA] [max_num_char_repetitions: %u][max_domain_element_len: %u]\n",
max_num_char_repetitions, max_domain_element_len);
#endif
if(ndpi_verbose_dga_detection)
printf("[DGA] [max_num_char_repetitions: %u][max_domain_element_len: %u]\n",
max_num_char_repetitions, max_domain_element_len);
if(
(is_hostname
&& (num_dots > 5)
&& (!first_element_is_numeric)
&& (!endsWith(tmp, "in-addr.arpa", 12))
)
|| (max_num_char_repetitions > 5 /* num or consecutive repeated chars */)
/*
@@ -7389,9 +7407,10 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
|| (max_domain_element_len >= 19 /* word too long. Example bbcbedxhgjmdobdprmen.com */)
) {
if(flow) ndpi_set_risk(flow, NDPI_SUSPICIOUS_DGA_DOMAIN);
#ifdef DGA_DEBUG
printf("[DGA] Found!");
#endif
if(ndpi_verbose_dga_detection)
printf("[DGA] Found!");
return(1);
}
@@ -7405,21 +7424,15 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
if(strlen(word) < 3) continue;
#ifdef DGA_DEBUG
printf("-> %s [%s][len: %u]\n", word, name, (unsigned int)strlen(word));
#endif
if(ndpi_verbose_dga_detection)
printf("-> word(%s) [%s][len: %u]\n", word, name, (unsigned int)strlen(word));
trigram_char_skip = 0;
for(i = 0; word[i+1] != '\0'; i++) {
if(isdigit(word[i])) {
num_digits++;
// if(!isdigit(word[i+1])) num_impossible++;
continue;
}
switch(word[i]) {
case '-':
num_dash++;
/*
Let's check for double+consecutive --
that are usually ok
@@ -7437,84 +7450,98 @@ uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
continue;
break;
}
#if 0
switch(word[i]) {
case 'a':
case 'e':
case 'i':
case 'o':
case 'u':
num_vowels++;
break;
}
#endif
if(isdigit(word[i+1])) {
num_digits++;
// num_impossible++;
continue;
}
num_bigram_checks++;
#ifdef DGA_DEBUG
printf("-> Checking %c%c\n", word[i], word[i+1]);
#endif
if(ndpi_verbose_dga_detection)
printf("-> Checking %c%c\n", word[i], word[i+1]);
if(ndpi_match_bigram(ndpi_str,
&ndpi_str->impossible_bigrams_automa,
&word[i])) {
#ifdef DGA_DEBUG
printf("IMPOSSIBLE %s\n", &word[i]);
#endif
if(ndpi_verbose_dga_detection)
printf("IMPOSSIBLE %s\n", &word[i]);
num_impossible++;
} else if(ndpi_match_bigram(ndpi_str, &ndpi_str->bigrams_automa, &word[i])) {
num_found++;
} else {
if(!skip_next_bigram) {
if(ndpi_match_bigram(ndpi_str, &ndpi_str->bigrams_automa, &word[i])) {
num_found++, skip_next_bigram = 1;
}
} else
skip_next_bigram = 0;
}
if((i > 0) && (word[0] != '_') && (word[i+2] != '\0')) {
if((num_trigram_dots < 2) && (word[i+2] != '\0')) {
if(ndpi_verbose_dga_detection)
printf("***> %s [trigram_char_skip: %u]\n", &word[i], trigram_char_skip);
if(ndpi_is_trigram_char(word[i]) && ndpi_is_trigram_char(word[i+1]) && ndpi_is_trigram_char(word[i+2])) {
num_trigram_checked++;
if(ndpi_match_trigram(ndpi_str, &ndpi_str->trigrams_automa, &word[i])) {
num_trigram_found++;
}
}
if(trigram_char_skip) {
trigram_char_skip--;
} else {
num_trigram_checked++;
/* Count vowels */
num_vowels += ndpi_is_vowel(word[i]) + ndpi_is_vowel(word[i+1]) + ndpi_is_vowel(word[i+2]);
if(ndpi_match_trigram(ndpi_str, &ndpi_str->trigrams_automa, &word[i]))
num_trigram_found++, trigram_char_skip = 2 /* 1 char overlap */;
else if(ndpi_verbose_dga_detection)
printf("[NDPI] NO Trigram %c%c%c\n", word[i], word[i+1], word[i+2]);
/* Count vowels */
num_trigram_vowels += ndpi_is_vowel(word[i]) + ndpi_is_vowel(word[i+1]) + ndpi_is_vowel(word[i+2]);
}
} else {
if(word[i] == '.')
num_trigram_dots++;
trigram_char_skip = 0;
}
}
} /* for */
} /* for */
#ifdef DGA_DEBUG
printf("[%s][num_found: %u][num_impossible: %u][num_digits: %u][num_bigram_checks: %u][num_vowels: %u/%u][num_trigram_found: %u/%u][vowels: %u]\n",
name, num_found, num_impossible, num_digits, num_bigram_checks, num_vowels, j-num_vowels,
num_trigram_checked, num_trigram_found, num_vowels);
#endif
if(ndpi_verbose_dga_detection)
printf("[%s][num_found: %u][num_impossible: %u][num_digits: %u][num_bigram_checks: %u][num_vowels: %u/%u][num_trigram_vowels: %u][num_trigram_found: %u/%u][vowels: %u][rc: %u]\n",
name, num_found, num_impossible, num_digits, num_bigram_checks, num_vowels, len, num_trigram_vowels,
num_trigram_checked, num_trigram_found, num_vowels, rc);
if(num_bigram_checks
&& ((num_found == 0) || ((num_digits > 5) && (num_words <= 3))
|| enough(num_found, num_impossible) || ((num_trigram_checked > 3) && (num_trigram_found < (num_trigram_checked/2)))))
rc = 1;
if(num_trigram_checked && (num_vowels == 0))
rc = 1;
if((len > 16) && (num_dots < 3) && ((num_vowels*4) < (len-num_dots))) {
if((num_trigram_checked > 2) && (num_trigram_vowels >= (num_trigram_found-1)))
; /* skip me */
else
rc = 1;
}
#ifdef DGA_DEBUG
if(rc)
printf("DGA %s [num_found: %u][num_impossible: %u]\n",
name, num_found, num_impossible);
#endif
if(num_bigram_checks
&& (num_dots > 0)
&& ((num_found == 0) || ((num_digits > 5) && (num_words <= 3))
|| enough(num_found, num_impossible)
|| ((num_trigram_checked > 2)
&& ((num_trigram_found < (num_trigram_checked/2))
|| ((num_trigram_vowels < (num_trigram_found-1)) && (num_dash == 0) && (num_dots > 1)))
)
)
)
rc = 1;
if((num_trigram_checked > 2) && (num_vowels == 0))
rc = 1;
if(num_dash > 2)
rc = 0;
if(ndpi_verbose_dga_detection) {
if(rc)
printf("DGA %s [num_found: %u][num_impossible: %u]\n",
name, num_found, num_impossible);
}
}
#ifdef DGA_DEBUG
printf("[DGA] Result: %u", rc);
#endif
if(ndpi_verbose_dga_detection)
printf("[DGA] Result: %u\n", rc);
if(rc && flow)
ndpi_set_risk(flow, NDPI_SUSPICIOUS_DGA_DOMAIN);
return(rc);
}