IMAP, POP3, SMTP: improve dissection (#1368)

Avoid NATS false positives
This commit is contained in:
Ivan Nardi
2021-11-11 11:55:56 +01:00
committed by GitHub
parent 9abf528c8e
commit 0f168d9150
14 changed files with 158 additions and 50 deletions
+2 -2
View File
@@ -1340,8 +1340,8 @@ struct ndpi_flow_struct {
} http;
struct {
u_int8_t auth_found:1, auth_failed:1, auth_tls:1, _pad:5;
char username[16], password[16];
u_int8_t auth_found:1, auth_failed:1, auth_tls:1, auth_done:1, _pad:4;
char username[32], password[16];
} ftp_imap_pop_smtp;
struct {
+2 -1
View File
@@ -7247,7 +7247,8 @@ u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndp
case NDPI_PROTOCOL_MAIL_IMAP:
case NDPI_PROTOCOL_MAIL_SMTP:
if(flow->protos.ftp_imap_pop_smtp.password[0] == '\0' &&
flow->protos.ftp_imap_pop_smtp.auth_tls == 0)
flow->protos.ftp_imap_pop_smtp.auth_tls == 0 &&
flow->protos.ftp_imap_pop_smtp.auth_done == 0)
return(1);
break;
+32 -33
View File
@@ -47,7 +47,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
NDPI_LOG_DBG(ndpi_struct, "search IMAP_IMAP\n");
#ifdef IMAP_DEBUG
printf("%s() [%s]\n", __FUNCTION__, packet->payload);
printf("%s() [%.*s]\n", __FUNCTION__, packet->payload_packet_len, packet->payload);
#endif
if(flow->l4.tcp.mail_imap_starttls == 2) {
@@ -66,8 +66,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else {
if(flow->l4.tcp.mail_imap_stage < 4) {
if(flow->l4.tcp.mail_imap_stage < 5) {
// search for the first space character (end of the tag)
while (i < 20 && i < packet->payload_packet_len) {
if(i > 0 && packet->payload[i] == ' ') {
@@ -121,6 +120,13 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->payload[command_start + 2] == 'D' || packet->payload[command_start + 2] == 'd')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
&& (packet->payload[command_start + 1] == 'O' || packet->payload[command_start + 1] == 'o')
&& packet->payload[command_start + 2] == ' ') {
flow->l4.tcp.mail_imap_stage += 1;
if(flow->l4.tcp.mail_imap_starttls == 1)
flow->l4.tcp.mail_imap_starttls = 2;
saw_command = 1;
}
}
if((command_start + 10) < packet->payload_packet_len) {
@@ -159,40 +165,29 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->payload[command_start + 2] == 'G' || packet->payload[command_start + 2] == 'g')
&& (packet->payload[command_start + 3] == 'I' || packet->payload[command_start + 3] == 'i')
&& (packet->payload[command_start + 4] == 'N' || packet->payload[command_start + 4] == 'n')) {
/* xxxx LOGIN "username" "password" */
char str[256], *item;
u_int len = packet->payload_packet_len >= sizeof(str) ? sizeof(str)-1 : packet->payload_packet_len;
/* xxxx LOGIN "username" "password"
xxxx LOGIN username password */
char str[256], *user, *saveptr;
u_int len = ndpi_min(packet->payload_packet_len - (command_start + 5), (int)sizeof(str) - 1);
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
strncpy(str, (const char*)packet->payload, len);
strncpy(str, (const char*)packet->payload + command_start + 5, len);
str[len] = '\0';
item = strchr(str, '"');
if(item) {
char *column;
item++;
column = strchr(item, '"');
user = strtok_r(str, " \"\r\n", &saveptr);
if(user) {
char *pwd;
if(column) {
column[0] = '\0';
snprintf(flow->protos.ftp_imap_pop_smtp.username,
sizeof(flow->protos.ftp_imap_pop_smtp.username),
"%s", item);
snprintf(flow->protos.ftp_imap_pop_smtp.username,
sizeof(flow->protos.ftp_imap_pop_smtp.username),
"%s", user);
column = strchr(&column[1], '"');
if(column) {
item = &column[1];
column = strchr(item, '"');
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
if(column) {
column[0] = '\0';
snprintf(flow->protos.ftp_imap_pop_smtp.password,
sizeof(flow->protos.ftp_imap_pop_smtp.password),
"%s", item);
}
}
pwd = strtok_r(NULL, " \"\r\n", &saveptr);
if(pwd) {
snprintf(flow->protos.ftp_imap_pop_smtp.password,
sizeof(flow->protos.ftp_imap_pop_smtp.password),
"%s", pwd);
}
}
@@ -242,6 +237,10 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->payload[command_start + 10] == 'T' || packet->payload[command_start + 10] == 't')
&& (packet->payload[command_start + 11] == 'E' || packet->payload[command_start + 11] == 'e')) {
flow->l4.tcp.mail_imap_stage += 1;
/* Authenticate phase may have multiple messages. Ignore them since they are
somehow encrypted anyway. */
flow->l4.tcp.mail_imap_starttls = 2;
flow->detected_protocol_stack[0] = NDPI_PROTOCOL_MAIL_IMAPS;
saw_command = 1;
}
}
@@ -320,7 +319,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
if((flow->l4.tcp.mail_imap_stage == 3)
|| (flow->l4.tcp.mail_imap_stage == 5)
|| (flow->l4.tcp.mail_imap_stage == 7)
) {
) {
if((flow->protos.ftp_imap_pop_smtp.username[0] != '\0')
|| (flow->l4.tcp.mail_imap_stage >= 7)) {
NDPI_LOG_INFO(ndpi_struct, "found MAIL_IMAP\n");
@@ -344,7 +343,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
// skip over possible authentication hashes etc. that cannot be identified as imap commands or responses
// if the packet count is low enough and at least one command or response was seen before
if((packet->payload_packet_len >= 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a)
&& flow->packet_counter < 6 && flow->l4.tcp.mail_imap_stage >= 1) {
&& flow->packet_counter < 8 && flow->l4.tcp.mail_imap_stage >= 1) {
NDPI_LOG_DBG2(ndpi_struct,
"no imap command or response but packet count < 6 and imap stage >= 1 -> skip\n");
return;
+86 -11
View File
@@ -38,12 +38,13 @@
#define SMTP_BIT_HELO_EHLO 0x20
#define SMTP_BIT_MAIL 0x40
#define SMTP_BIT_RCPT 0x80
#define SMTP_BIT_AUTH 0x100
#define SMTP_BIT_AUTH_LOGIN 0x100
#define SMTP_BIT_STARTTLS 0x200
#define SMTP_BIT_DATA 0x400
#define SMTP_BIT_NOOP 0x800
#define SMTP_BIT_RSET 0x1000
#define SMTP_BIT_TlRM 0x2000
#define SMTP_BIT_AUTH_PLAIN 0x4000
/* #define SMTP_DEBUG 1 */
@@ -65,6 +66,53 @@ static void smtpInitExtraPacketProcessing(struct ndpi_flow_struct *flow);
/* **************************************** */
static void get_credentials_auth_plain(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
const u_int8_t *line, u_int16_t line_len)
{
u_int8_t buf[255];
u_char *out;
size_t i, out_len;
unsigned int user_len = 0;
/* AUTH PLAIN XXXXXX */
if(line_len <= 11)
return;
line += 11;
line_len -= 11;
ndpi_user_pwd_payload_copy(buf, sizeof(buf), 0, line, line_len);
out = ndpi_base64_decode(buf, strlen((char *)buf), &out_len);
if(!out)
return;
/* No guarantee that out is null terminated:
UTF8NUL authcid UTF8NUL passwd */
for(i = 1; i < out_len; i++) {
if(out[i] == '\0')
user_len = i - 1;
}
if(user_len > 0) {
user_len = ndpi_min(user_len, sizeof(flow->protos.ftp_imap_pop_smtp.username) - 1);
memcpy(flow->protos.ftp_imap_pop_smtp.username, out + 1, user_len);
flow->protos.ftp_imap_pop_smtp.username[user_len] = '\0';
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
if(1 + user_len + 1 < out_len) {
unsigned int pwd_len;
pwd_len = ndpi_min(out_len - (1 + user_len + 1), sizeof(flow->protos.ftp_imap_pop_smtp.password) - 1);
memcpy(flow->protos.ftp_imap_pop_smtp.password, out + 1 + user_len + 1, pwd_len);
flow->protos.ftp_imap_pop_smtp.password[pwd_len] = '\0';
}
}
ndpi_free(out);
}
/* **************************************** */
void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow) {
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
@@ -102,10 +150,14 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
len = ndpi_min(len, sizeof(flow->host_server_name)-1);
strncpy((char*)flow->host_server_name, (char*)&packet->line[a].ptr[4], len);
flow->host_server_name[len] = '\0';
ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP,
(char *)flow->host_server_name,
strlen((const char *)flow->host_server_name));
if(ndpi_match_hostname_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_SMTP,
(char *)flow->host_server_name,
strlen((const char *)flow->host_server_name))) {
/* We set the protocols; we need to initialize extra dissection
to search for credentials */
NDPI_LOG_DBG(ndpi_struct, "SMTP: hostname matched\n");
smtpInitExtraPacketProcessing(flow);
}
}
}
}
@@ -139,6 +191,8 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& packet->line[a].ptr[4] == ' ') {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_MAIL;
flow->protos.ftp_imap_pop_smtp.auth_found = 0;
/* We shouldn't be here if there are credentials */
flow->protos.ftp_imap_pop_smtp.auth_done = 1;
} else if((packet->line[a].ptr[0] == 'R' || packet->line[a].ptr[0] == 'r')
&& (packet->line[a].ptr[1] == 'C' || packet->line[a].ptr[1] == 'c')
&& (packet->line[a].ptr[2] == 'P' || packet->line[a].ptr[2] == 'p')
@@ -146,6 +200,8 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& packet->line[a].ptr[4] == ' ') {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_RCPT;
flow->protos.ftp_imap_pop_smtp.auth_found = 0;
/* We shouldn't be here if there are credentials */
flow->protos.ftp_imap_pop_smtp.auth_done = 1;
} else if((packet->line[a].ptr[0] == 'A' || packet->line[a].ptr[0] == 'a')
&& (packet->line[a].ptr[1] == 'U' || packet->line[a].ptr[1] == 'u')
&& (packet->line[a].ptr[2] == 'T' || packet->line[a].ptr[2] == 't')
@@ -154,16 +210,27 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
#ifdef SMTP_DEBUG
printf("%s() AUTH [%.*s]\n", __FUNCTION__, packet->line[a].len, packet->line[a].ptr);
#endif
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_AUTH;
flow->protos.ftp_imap_pop_smtp.auth_found = 1;
if(packet->line[a].len >= 6) {
if(packet->line[a].ptr[5] == 'L' || packet->line[a].ptr[5] == 'l') {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_AUTH_LOGIN;
/* AUTH LOGIN: Username and pwd on the next messages */
} else if(packet->line[a].ptr[5] == 'P' || packet->line[a].ptr[5] == 'p') {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_AUTH_PLAIN;
/* AUTH PLAIN: username and pwd here */
get_credentials_auth_plain(ndpi_struct, flow,
packet->line[a].ptr, packet->line[a].len);
flow->protos.ftp_imap_pop_smtp.auth_done = 1;
}
}
} else {
if(packet->line[a].ptr[3] != ' ') {
#ifdef SMTP_DEBUG
printf("%s() => [%.*s]\n", __FUNCTION__, packet->line[a].len, packet->line[a].ptr);
#endif
if(flow->protos.ftp_imap_pop_smtp.auth_found) {
if(flow->protos.ftp_imap_pop_smtp.auth_found &&
(flow->l4.tcp.smtp_command_bitmask & SMTP_BIT_AUTH_LOGIN)) {
if(flow->protos.ftp_imap_pop_smtp.username[0] == '\0') {
/* Username */
u_int8_t buf[48];
@@ -214,6 +281,8 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
}
ndpi_set_risk(ndpi_struct, flow, NDPI_CLEAR_TEXT_CREDENTIALS);
flow->protos.ftp_imap_pop_smtp.auth_done = 1;
} else {
flow->host_server_name[0] = '\0';
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
@@ -234,6 +303,8 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->line[a].ptr[6] == 'L' || packet->line[a].ptr[6] == 'l')
&& (packet->line[a].ptr[7] == 'S' || packet->line[a].ptr[7] == 's')) {
flow->l4.tcp.smtp_command_bitmask |= SMTP_BIT_STARTTLS;
flow->protos.ftp_imap_pop_smtp.auth_tls = 1;
flow->protos.ftp_imap_pop_smtp.auth_done = 1;
}
}
@@ -274,8 +345,12 @@ void ndpi_search_mail_smtp_tcp(struct ndpi_detection_module_struct *ndpi_struct,
bit_count, flow->protos.ftp_imap_pop_smtp.password);
#endif
ndpi_int_mail_smtp_add_connection(ndpi_struct, flow);
smtpInitExtraPacketProcessing(flow);
/* Only if we don't have already set the protocol via hostname matching */
if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN &&
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN) {
ndpi_int_mail_smtp_add_connection(ndpi_struct, flow);
smtpInitExtraPacketProcessing(flow);
}
return;
}
@@ -324,7 +399,7 @@ static void smtpInitExtraPacketProcessing(struct ndpi_flow_struct *flow) {
#endif
flow->check_extra_packets = 1;
/* At most 7 packets should almost always be enough */
/* At most 12 packets should almost always be enough */
flow->max_extra_packets_to_check = 12;
flow->extra_packets_func = ndpi_extra_search_mail_smtp_tcp;
}
+6 -1
View File
@@ -51,10 +51,15 @@ void ndpi_search_nats_tcp(struct ndpi_detection_module_struct *ndpi_struct,
for(i=0; commands[i] != NULL; i++) {
char *match = ndpi_strnstr((const char *)packet->payload,
commands[i],
packet->payload_packet_len);
ndpi_min(strlen(commands[i]), packet->payload_packet_len));
if(!match) continue;
/* These commands are used by POP3 too. To avoid false positives, look for the other ones */
if((strcmp(commands[i], "+OK") == 0) || (strcmp(commands[i], "-ERR") == 0)) {
return;
}
if(ndpi_strnstr((const char *)match, "\r\n",
packet->payload_packet_len - ((size_t)match - (size_t)packet->payload)) != NULL) {
NDPI_LOG_INFO(ndpi_struct, "found NATS\n");
Binary file not shown.
BIN
View File
Binary file not shown.
Binary file not shown.
Binary file not shown.
+7
View File
@@ -0,0 +1,7 @@
Guessed flow protos: 0
DPI Packets (TCP): 10 (10.00 pkts/flow)
IMAPS 32 7975 1
1 TCP 192.168.17.53:49640 <-> 212.227.17.186:143 [proto: 51/IMAPS][Encrypted][cat: Email/3][18 pkts/1536 bytes <-> 14 pkts/6439 bytes][Goodput ratio: 35/88][3.02 sec][bytes ratio: -0.615 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 188/251 1486/1677 371/512][Pkt Len c2s/s2c min/avg/max/stddev: 54/60 85/460 372/1514 76/571][Risk: ** Known protocol on non standard port **][Risk Score: 50][PLAIN TEXT (CAPABILITY IMAP)][Plen Bins: 25,18,6,6,0,0,0,6,6,6,0,0,0,0,0,0,0,0,6,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,18,0,0]
+7
View File
@@ -0,0 +1,7 @@
Guessed flow protos: 0
DPI Packets (TCP): 11 (11.00 pkts/flow)
IMAP 33 3774 1
1 TCP 10.40.4.2:46045 <-> 10.40.3.2:143 [proto: 4/IMAP][ClearText][cat: Email/3][20 pkts/1507 bytes <-> 13 pkts/2267 bytes][Goodput ratio: 12/62][4.57 sec][User: samir][Pwd: pfres][bytes ratio: -0.201 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/17 39/39 15/16][Pkt Len c2s/s2c min/avg/max/stddev: 66/74 75/174 139/762 17/181][Risk: ** Unsafe Protocol **** Clear-text credentials **][Risk Score: 110][PLAIN TEXT ( OK IMAP4)][Plen Bins: 51,22,11,5,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+7
View File
@@ -0,0 +1,7 @@
Guessed flow protos: 0
DPI Packets (TCP): 10 (10.00 pkts/flow)
POP3 31 3915 1
1 TCP 143.225.229.181:35287 <-> 74.208.5.28:110 [proto: 2/POP3][ClearText][cat: Email/3][18 pkts/1269 bytes <-> 13 pkts/2646 bytes][Goodput ratio: 6/67][27.32 sec][User: cicciopernacchio@mail.com][Pwd: pippozzo][bytes ratio: -0.352 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 1792/2973 5526/5668 2204/2427][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 70/204 98/1514 8/379][Risk: ** Unsafe Protocol **** Clear-text credentials **][Risk Score: 110][PLAIN TEXT (OK POP server ready H migmxus)][Plen Bins: 60,20,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,6,0,0]
+2 -2
View File
@@ -1,6 +1,6 @@
Guessed flow protos: 1
Guessed flow protos: 0
DPI Packets (TCP): 36 (36.00 pkts/flow)
DPI Packets (TCP): 9 (9.00 pkts/flow)
Google 36 8403 1
+7
View File
@@ -0,0 +1,7 @@
Guessed flow protos: 0
DPI Packets (TCP): 11 (11.00 pkts/flow)
SMTP 95 23157 1
1 TCP 194.7.248.153:2127 <-> 172.16.114.207:25 [proto: 3/SMTP][ClearText][cat: Email/3][51 pkts/19311 bytes <-> 44 pkts/3846 bytes][Goodput ratio: 86/37][0.23 sec][Host: pigeon.eyrie.af.mil][bytes ratio: 0.668 (Upload)][IAT c2s/s2c min/avg/max/stddev: 1/1 5/6 67/68 12/15][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 379/87 1514/138 562/15][PLAIN TEXT (220 pigeon.eyrie.af.mil ESMTP S)][Plen Bins: 8,78,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,10,0,0]