Implemented IMAP data extraction

This commit is contained in:
Luca Deri
2019-10-20 22:46:08 +02:00
parent 1a5c7daaf6
commit 437e8bd706
4 changed files with 114 additions and 49 deletions
+11 -3
View File
@@ -949,9 +949,17 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
}
/* FTP */
else if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_FTP_CONTROL) {
snprintf(flow->info, sizeof(flow->info), "User: %s][Pwd: %s",
flow->ndpi_flow->protos.ftp.username,
flow->ndpi_flow->protos.ftp.password);
if(flow->ndpi_flow->protos.ftp.username[0] != '\0')
snprintf(flow->info, sizeof(flow->info), "User: %s][Pwd: %s",
flow->ndpi_flow->protos.ftp.username,
flow->ndpi_flow->protos.ftp.password);
}
/* IMAP */
else if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_MAIL_IMAP) {
if(flow->ndpi_flow->protos.imap.username[0] != '\0')
snprintf(flow->info, sizeof(flow->info), "User: %s][Pwd: %s",
flow->ndpi_flow->protos.imap.username,
flow->ndpi_flow->protos.imap.password);
}
/* KERBEROS */
else if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_KERBEROS) {
+4
View File
@@ -1238,6 +1238,10 @@ struct ndpi_flow_struct {
char username[16], password[16];
} ftp;
struct {
char username[16], password[16];
} imap;
struct {
/* Bittorrent hash */
u_char hash[20];
+7 -1
View File
@@ -628,7 +628,6 @@ static int ndpi_is_valid_char(char c) {
/* ******************************************************************** */
static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_struct,
char *str) {
char s[3];
@@ -881,6 +880,13 @@ int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_MAIL_IMAP:
ndpi_serialize_start_of_block(serializer, "imap");
ndpi_serialize_string_string(serializer, "user", flow->protos.imap.username);
ndpi_serialize_string_string(serializer, "password", flow->protos.imap.password);
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_FTP_CONTROL:
ndpi_serialize_start_of_block(serializer, "ftp");
ndpi_serialize_string_string(serializer, "user", flow->protos.ftp.username);
+92 -45
View File
@@ -28,6 +28,7 @@
#include "ndpi_api.h"
/* #define IMAP_DEBUG 1*/
static void ndpi_int_mail_imap_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
{
@@ -44,17 +45,21 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
/* const u_int8_t *command = 0; */
NDPI_LOG_DBG(ndpi_struct, "search IMAP_IMAP\n");
if (flow->l4.tcp.mail_imap_starttls == 2) {
#ifdef IMAP_DEBUG
printf("%s() [%s]\n", __FUNCTION__, packet->payload);
#endif
if(flow->l4.tcp.mail_imap_starttls == 2) {
NDPI_LOG_DBG2(ndpi_struct, "starttls detected\n");
NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_MAIL_IMAP);
NDPI_DEL_PROTOCOL_FROM_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_TLS);
return;
}
if (packet->payload_packet_len >= 4 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a) {
if(packet->payload_packet_len >= 4 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a) {
// the DONE command appears without a tag
if (packet->payload_packet_len == 6 && ((packet->payload[0] == 'D' || packet->payload[0] == 'd')
if(packet->payload_packet_len == 6 && ((packet->payload[0] == 'D' || packet->payload[0] == 'd')
&& (packet->payload[1] == 'O' || packet->payload[1] == 'o')
&& (packet->payload[2] == 'N' || packet->payload[2] == 'n')
&& (packet->payload[3] == 'E' || packet->payload[3] == 'e'))) {
@@ -62,37 +67,37 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
saw_command = 1;
} else {
if (flow->l4.tcp.mail_imap_stage < 4) {
if(flow->l4.tcp.mail_imap_stage < 4) {
// search for the first space character (end of the tag)
while (i < 20 && i < packet->payload_packet_len) {
if (i > 0 && packet->payload[i] == ' ') {
if(i > 0 && packet->payload[i] == ' ') {
space_pos = i;
break;
}
if (!((packet->payload[i] >= 'a' && packet->payload[i] <= 'z') ||
if(!((packet->payload[i] >= 'a' && packet->payload[i] <= 'z') ||
(packet->payload[i] >= 'A' && packet->payload[i] <= 'Z') ||
(packet->payload[i] >= '0' && packet->payload[i] <= '9') || packet->payload[i] == '*' || packet->payload[i] == '.')) {
goto imap_excluded;
}
i++;
}
if (space_pos == 0 || space_pos == (packet->payload_packet_len - 1)) {
if(space_pos == 0 || space_pos == (packet->payload_packet_len - 1)) {
goto imap_excluded;
}
// now walk over a possible mail number to the next space
i++;
if (i < packet->payload_packet_len && (packet->payload[i] >= '0' && packet->payload[i] <= '9')) {
if(i < packet->payload_packet_len && (packet->payload[i] >= '0' && packet->payload[i] <= '9')) {
while (i < 20 && i < packet->payload_packet_len) {
if (i > 0 && packet->payload[i] == ' ') {
if(i > 0 && packet->payload[i] == ' ') {
space_pos = i;
break;
}
if (!(packet->payload[i] >= '0' && packet->payload[i] <= '9')) {
if(!(packet->payload[i] >= '0' && packet->payload[i] <= '9')) {
goto imap_excluded;
}
i++;
}
if (space_pos == 0 || space_pos == (packet->payload_packet_len - 1)) {
if(space_pos == 0 || space_pos == (packet->payload_packet_len - 1)) {
goto imap_excluded;
}
}
@@ -103,23 +108,23 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
/* command = &(packet->payload[command_start]); */
}
if ((command_start + 3) < packet->payload_packet_len) {
if ((packet->payload[command_start] == 'O' || packet->payload[command_start] == 'o')
if((command_start + 3) < packet->payload_packet_len) {
if((packet->payload[command_start] == 'O' || packet->payload[command_start] == 'o')
&& (packet->payload[command_start + 1] == 'K' || packet->payload[command_start + 1] == 'k')
&& packet->payload[command_start + 2] == ' ') {
flow->l4.tcp.mail_imap_stage += 1;
if (flow->l4.tcp.mail_imap_starttls == 1)
if(flow->l4.tcp.mail_imap_starttls == 1)
flow->l4.tcp.mail_imap_starttls = 2;
saw_command = 1;
} else if ((packet->payload[command_start] == 'U' || packet->payload[command_start] == 'u')
} else if((packet->payload[command_start] == 'U' || packet->payload[command_start] == 'u')
&& (packet->payload[command_start + 1] == 'I' || packet->payload[command_start + 1] == 'i')
&& (packet->payload[command_start + 2] == 'D' || packet->payload[command_start + 2] == 'd')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
}
}
if ((command_start + 10) < packet->payload_packet_len) {
if ((packet->payload[command_start] == 'C' || packet->payload[command_start] == 'c')
if((command_start + 10) < packet->payload_packet_len) {
if((packet->payload[command_start] == 'C' || packet->payload[command_start] == 'c')
&& (packet->payload[command_start + 1] == 'A' || packet->payload[command_start + 1] == 'a')
&& (packet->payload[command_start + 2] == 'P' || packet->payload[command_start + 2] == 'p')
&& (packet->payload[command_start + 3] == 'A' || packet->payload[command_start + 3] == 'a')
@@ -133,8 +138,8 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
saw_command = 1;
}
}
if ((command_start + 8) < packet->payload_packet_len) {
if ((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
if((command_start + 8) < packet->payload_packet_len) {
if((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
&& (packet->payload[command_start + 1] == 'T' || packet->payload[command_start + 1] == 't')
&& (packet->payload[command_start + 2] == 'A' || packet->payload[command_start + 2] == 'a')
&& (packet->payload[command_start + 3] == 'R' || packet->payload[command_start + 3] == 'r')
@@ -148,36 +153,71 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
saw_command = 1;
}
}
if ((command_start + 5) < packet->payload_packet_len) {
if ((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
if((command_start + 5) < packet->payload_packet_len) {
if((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
&& (packet->payload[command_start + 1] == 'O' || packet->payload[command_start + 1] == 'o')
&& (packet->payload[command_start + 2] == 'G' || packet->payload[command_start + 2] == 'g')
&& (packet->payload[command_start + 3] == 'I' || packet->payload[command_start + 3] == 'i')
&& (packet->payload[command_start + 4] == 'N' || packet->payload[command_start + 4] == 'n')) {
/* xxxx LOGIN "username" "password" */
char str[256], *item;
u_int len = packet->payload_packet_len > sizeof(str) ? sizeof(str) : packet->payload_packet_len;
strncpy(str, (const char*)packet->payload, len);
str[len] = '\0';
item = strchr(str, '"');
if(item) {
char *column;
item++;
column = strchr(item, '"');
if(column) {
column[0] = '\0';
snprintf(flow->protos.imap.username,
sizeof(flow->protos.imap.username),
"%s", item);
column = strchr(&column[1], '"');
if(column) {
item = &column[1];
column = strchr(item, '"');
if(column) {
column[0] = '\0';
snprintf(flow->protos.imap.password,
sizeof(flow->protos.imap.password),
"%s", item);
}
}
}
}
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'F' || packet->payload[command_start] == 'f')
} else if((packet->payload[command_start] == 'F' || packet->payload[command_start] == 'f')
&& (packet->payload[command_start + 1] == 'E' || packet->payload[command_start + 1] == 'e')
&& (packet->payload[command_start + 2] == 'T' || packet->payload[command_start + 2] == 't')
&& (packet->payload[command_start + 3] == 'C' || packet->payload[command_start + 3] == 'c')
&& (packet->payload[command_start + 4] == 'H' || packet->payload[command_start + 4] == 'h')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'F' || packet->payload[command_start] == 'f')
} else if((packet->payload[command_start] == 'F' || packet->payload[command_start] == 'f')
&& (packet->payload[command_start + 1] == 'L' || packet->payload[command_start + 1] == 'l')
&& (packet->payload[command_start + 2] == 'A' || packet->payload[command_start + 2] == 'a')
&& (packet->payload[command_start + 3] == 'G' || packet->payload[command_start + 3] == 'g')
&& (packet->payload[command_start + 4] == 'S' || packet->payload[command_start + 4] == 's')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'C' || packet->payload[command_start] == 'c')
} else if((packet->payload[command_start] == 'C' || packet->payload[command_start] == 'c')
&& (packet->payload[command_start + 1] == 'H' || packet->payload[command_start + 1] == 'h')
&& (packet->payload[command_start + 2] == 'E' || packet->payload[command_start + 2] == 'e')
&& (packet->payload[command_start + 3] == 'C' || packet->payload[command_start + 3] == 'c')
&& (packet->payload[command_start + 4] == 'K' || packet->payload[command_start + 4] == 'k')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
} else if((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
&& (packet->payload[command_start + 1] == 'T' || packet->payload[command_start + 1] == 't')
&& (packet->payload[command_start + 2] == 'O' || packet->payload[command_start + 2] == 'o')
&& (packet->payload[command_start + 3] == 'R' || packet->payload[command_start + 3] == 'r')
@@ -186,8 +226,8 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
saw_command = 1;
}
}
if ((command_start + 12) < packet->payload_packet_len) {
if ((packet->payload[command_start] == 'A' || packet->payload[command_start] == 'a')
if((command_start + 12) < packet->payload_packet_len) {
if((packet->payload[command_start] == 'A' || packet->payload[command_start] == 'a')
&& (packet->payload[command_start + 1] == 'U' || packet->payload[command_start + 1] == 'u')
&& (packet->payload[command_start + 2] == 'T' || packet->payload[command_start + 2] == 't')
&& (packet->payload[command_start + 3] == 'H' || packet->payload[command_start + 3] == 'h')
@@ -203,8 +243,8 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
saw_command = 1;
}
}
if ((command_start + 9) < packet->payload_packet_len) {
if ((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
if((command_start + 9) < packet->payload_packet_len) {
if((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
&& (packet->payload[command_start + 1] == 'A' || packet->payload[command_start + 1] == 'a')
&& (packet->payload[command_start + 2] == 'M' || packet->payload[command_start + 2] == 'm')
&& (packet->payload[command_start + 3] == 'E' || packet->payload[command_start + 3] == 'e')
@@ -217,26 +257,26 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
saw_command = 1;
}
}
if ((command_start + 4) < packet->payload_packet_len) {
if ((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
if((command_start + 4) < packet->payload_packet_len) {
if((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
&& (packet->payload[command_start + 1] == 'S' || packet->payload[command_start + 1] == 's')
&& (packet->payload[command_start + 2] == 'U' || packet->payload[command_start + 2] == 'u')
&& (packet->payload[command_start + 3] == 'B' || packet->payload[command_start + 3] == 'b')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
} else if((packet->payload[command_start] == 'L' || packet->payload[command_start] == 'l')
&& (packet->payload[command_start + 1] == 'I' || packet->payload[command_start + 1] == 'i')
&& (packet->payload[command_start + 2] == 'S' || packet->payload[command_start + 2] == 's')
&& (packet->payload[command_start + 3] == 'T' || packet->payload[command_start + 3] == 't')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
} else if((packet->payload[command_start] == 'N' || packet->payload[command_start] == 'n')
&& (packet->payload[command_start + 1] == 'O' || packet->payload[command_start + 1] == 'o')
&& (packet->payload[command_start + 2] == 'O' || packet->payload[command_start + 2] == 'o')
&& (packet->payload[command_start + 3] == 'P' || packet->payload[command_start + 3] == 'p')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'I' || packet->payload[command_start] == 'i')
} else if((packet->payload[command_start] == 'I' || packet->payload[command_start] == 'i')
&& (packet->payload[command_start + 1] == 'D' || packet->payload[command_start + 1] == 'd')
&& (packet->payload[command_start + 2] == 'L' || packet->payload[command_start + 2] == 'l')
&& (packet->payload[command_start + 3] == 'E' || packet->payload[command_start + 3] == 'e')) {
@@ -244,8 +284,8 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
saw_command = 1;
}
}
if ((command_start + 6) < packet->payload_packet_len) {
if ((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
if((command_start + 6) < packet->payload_packet_len) {
if((packet->payload[command_start] == 'S' || packet->payload[command_start] == 's')
&& (packet->payload[command_start + 1] == 'E' || packet->payload[command_start + 1] == 'e')
&& (packet->payload[command_start + 2] == 'L' || packet->payload[command_start + 2] == 'l')
&& (packet->payload[command_start + 3] == 'E' || packet->payload[command_start + 3] == 'e')
@@ -253,7 +293,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->payload[command_start + 5] == 'T' || packet->payload[command_start + 5] == 't')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'E' || packet->payload[command_start] == 'e')
} else if((packet->payload[command_start] == 'E' || packet->payload[command_start] == 'e')
&& (packet->payload[command_start + 1] == 'X' || packet->payload[command_start + 1] == 'x')
&& (packet->payload[command_start + 2] == 'I' || packet->payload[command_start + 2] == 'i')
&& (packet->payload[command_start + 3] == 'S' || packet->payload[command_start + 3] == 's')
@@ -261,7 +301,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
&& (packet->payload[command_start + 5] == 'S' || packet->payload[command_start + 5] == 's')) {
flow->l4.tcp.mail_imap_stage += 1;
saw_command = 1;
} else if ((packet->payload[command_start] == 'A' || packet->payload[command_start] == 'a')
} else if((packet->payload[command_start] == 'A' || packet->payload[command_start] == 'a')
&& (packet->payload[command_start + 1] == 'P' || packet->payload[command_start + 1] == 'p')
&& (packet->payload[command_start + 2] == 'P' || packet->payload[command_start + 2] == 'p')
&& (packet->payload[command_start + 3] == 'E' || packet->payload[command_start + 3] == 'e')
@@ -274,16 +314,23 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
}
if (saw_command == 1) {
if (flow->l4.tcp.mail_imap_stage == 3 || flow->l4.tcp.mail_imap_stage == 5) {
NDPI_LOG_INFO(ndpi_struct, "found MAIL_IMAP\n");
ndpi_int_mail_imap_add_connection(ndpi_struct, flow);
if(saw_command == 1) {
if((flow->l4.tcp.mail_imap_stage == 3)
|| (flow->l4.tcp.mail_imap_stage == 5)
|| (flow->l4.tcp.mail_imap_stage == 7)
) {
if((flow->protos.imap.username[0] != '\0')
|| (flow->l4.tcp.mail_imap_stage >= 7)) {
NDPI_LOG_INFO(ndpi_struct, "found MAIL_IMAP\n");
ndpi_int_mail_imap_add_connection(ndpi_struct, flow);
}
return;
}
}
}
if (packet->payload_packet_len > 1 && packet->payload[packet->payload_packet_len - 1] == ' ') {
if(packet->payload_packet_len > 1 && packet->payload[packet->payload_packet_len - 1] == ' ') {
NDPI_LOG_DBG2(ndpi_struct,
"maybe a split imap command -> need next packet and imap_stage is set to 4.\n");
flow->l4.tcp.mail_imap_stage = 4;
@@ -294,7 +341,7 @@ void ndpi_search_mail_imap_tcp(struct ndpi_detection_module_struct *ndpi_struct,
// skip over possible authentication hashes etc. that cannot be identified as imap commands or responses
// if the packet count is low enough and at least one command or response was seen before
if ((packet->payload_packet_len >= 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a)
if((packet->payload_packet_len >= 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a)
&& flow->packet_counter < 6 && flow->l4.tcp.mail_imap_stage >= 1) {
NDPI_LOG_DBG2(ndpi_struct,
"no imap command or response but packet count < 6 and imap stage >= 1 -> skip\n");