Added POP3 metadata extraction

This commit is contained in:
Luca Deri
2019-10-20 23:06:05 +02:00
parent 437e8bd706
commit 58daa5ed7d
6 changed files with 183 additions and 163 deletions
+7
View File
@@ -961,6 +961,13 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
flow->ndpi_flow->protos.imap.username,
flow->ndpi_flow->protos.imap.password);
}
/* POP */
else if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_MAIL_POP) {
if(flow->ndpi_flow->protos.pop.username[0] != '\0')
snprintf(flow->info, sizeof(flow->info), "User: %s][Pwd: %s",
flow->ndpi_flow->protos.pop.username,
flow->ndpi_flow->protos.pop.password);
}
/* KERBEROS */
else if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_KERBEROS) {
if(flow->ndpi_flow->protos.kerberos.cname[0] != '\0') {
+3
View File
@@ -837,6 +837,9 @@ extern "C" {
char *outbuf, u_int outbuf_len);
char* ndpi_ssl_version2str(u_int16_t version, u_int8_t *unknown_tls_version);
void ndpi_patchIPv6Address(char *str);
void ndpi_user_pwd_payload_copy(u_int8_t *dest, u_int dest_len,
const u_int8_t *src, u_int src_len);
int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t ip_version,
+4
View File
@@ -1242,6 +1242,10 @@ struct ndpi_flow_struct {
char username[16], password[16];
} imap;
struct {
char username[16], password[16];
} pop;
struct {
/* Bittorrent hash */
u_char hash[20];
+19
View File
@@ -755,6 +755,25 @@ void ndpi_patchIPv6Address(char *str) {
/* ********************************** */
void ndpi_user_pwd_payload_copy(u_int8_t *dest, u_int dest_len,
const u_int8_t *src, u_int src_len) {
u_int i, j, k = dest_len-1;
for(i=5, j=0; i<src_len; i++) {
if((j == k) || ((src[i] == '\r')
|| (src[i] == '\n')
|| (src[i] == ' ')
))
break;
dest[j++] = src[i];
}
dest[k] = '\0';
}
/* ********************************** */
int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t ip_version,
+6 -25
View File
@@ -39,25 +39,6 @@ static void ndpi_int_ftp_control_add_connection(struct ndpi_detection_module_str
/* *************************************************************** */
static void ftp_payload_copy(u_int8_t *dest, u_int dest_len,
const u_int8_t *src, u_int src_len) {
u_int i, j, k = dest_len-1;
for(i=5, j=0; i<src_len; i++) {
if((j == k) || ((src[i] == '\r')
|| (src[i] == '\n')
|| (src[i] == ' ')
))
break;
dest[j++] = src[i];
}
dest[k] = '\0';
}
/* *************************************************************** */
static int ndpi_ftp_control_check_request(struct ndpi_flow_struct *flow,
const u_int8_t *payload,
size_t payload_len) {
@@ -66,16 +47,16 @@ static int ndpi_ftp_control_check_request(struct ndpi_flow_struct *flow,
#endif
if(ndpi_match_strprefix(payload, payload_len, "USER")) {
ftp_payload_copy((u_int8_t*)flow->protos.ftp.username,
sizeof(flow->protos.ftp.username),
payload, payload_len);
ndpi_user_pwd_payload_copy((u_int8_t*)flow->protos.ftp.username,
sizeof(flow->protos.ftp.username),
payload, payload_len);
return 1;
}
if(ndpi_match_strprefix(payload, payload_len, "PASS")) {
ftp_payload_copy((u_int8_t*)flow->protos.ftp.password,
sizeof(flow->protos.ftp.password),
payload, payload_len);
ndpi_user_pwd_payload_copy((u_int8_t*)flow->protos.ftp.password,
sizeof(flow->protos.ftp.password),
payload, payload_len);
return 1;
}
+144 -138
View File
@@ -1,8 +1,8 @@
/*
* mail_pop.c
*
* Copyright (C) 2009-2011 by ipoque GmbH
* Copyright (C) 2011-19 - ntop.org
* Copyright (C) 2009-2011 by ipoque GmbH
*
* This file is part of nDPI, an open source deep packet inspection
* library based on the OpenDPI and PACE technology by ipoque GmbH
@@ -44,163 +44,169 @@
static void ndpi_int_mail_pop_add_connection(struct ndpi_detection_module_struct
*ndpi_struct, struct ndpi_flow_struct *flow)
{
*ndpi_struct, struct ndpi_flow_struct *flow) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_MAIL_POP, NDPI_PROTOCOL_UNKNOWN);
}
static int ndpi_int_mail_pop_check_for_client_commands(struct ndpi_detection_module_struct
*ndpi_struct, struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct *packet = &flow->packet;
*ndpi_struct, struct ndpi_flow_struct *flow) {
struct ndpi_packet_struct *packet = &flow->packet;
if (packet->payload_packet_len > 4) {
if ((packet->payload[0] == 'A' || packet->payload[0] == 'a')
&& (packet->payload[1] == 'U' || packet->payload[1] == 'u')
&& (packet->payload[2] == 'T' || packet->payload[2] == 't')
&& (packet->payload[3] == 'H' || packet->payload[3] == 'h')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_AUTH;
return 1;
} else if ((packet->payload[0] == 'A' || packet->payload[0] == 'a')
&& (packet->payload[1] == 'P' || packet->payload[1] == 'p')
&& (packet->payload[2] == 'O' || packet->payload[2] == 'o')
&& (packet->payload[3] == 'P' || packet->payload[3] == 'p')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_APOP;
return 1;
} else if ((packet->payload[0] == 'U' || packet->payload[0] == 'u')
&& (packet->payload[1] == 'S' || packet->payload[1] == 's')
&& (packet->payload[2] == 'E' || packet->payload[2] == 'e')
&& (packet->payload[3] == 'R' || packet->payload[3] == 'r')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_USER;
return 1;
} else if ((packet->payload[0] == 'P' || packet->payload[0] == 'p')
&& (packet->payload[1] == 'A' || packet->payload[1] == 'a')
&& (packet->payload[2] == 'S' || packet->payload[2] == 's')
&& (packet->payload[3] == 'S' || packet->payload[3] == 's')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_PASS;
return 1;
} else if ((packet->payload[0] == 'C' || packet->payload[0] == 'c')
&& (packet->payload[1] == 'A' || packet->payload[1] == 'a')
&& (packet->payload[2] == 'P' || packet->payload[2] == 'p')
&& (packet->payload[3] == 'A' || packet->payload[3] == 'a')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_CAPA;
return 1;
} else if ((packet->payload[0] == 'L' || packet->payload[0] == 'l')
&& (packet->payload[1] == 'I' || packet->payload[1] == 'i')
&& (packet->payload[2] == 'S' || packet->payload[2] == 's')
&& (packet->payload[3] == 'T' || packet->payload[3] == 't')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_LIST;
return 1;
} else if ((packet->payload[0] == 'S' || packet->payload[0] == 's')
&& (packet->payload[1] == 'T' || packet->payload[1] == 't')
&& (packet->payload[2] == 'A' || packet->payload[2] == 'a')
&& (packet->payload[3] == 'T' || packet->payload[3] == 't')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_STAT;
return 1;
} else if ((packet->payload[0] == 'U' || packet->payload[0] == 'u')
&& (packet->payload[1] == 'I' || packet->payload[1] == 'i')
&& (packet->payload[2] == 'D' || packet->payload[2] == 'd')
&& (packet->payload[3] == 'L' || packet->payload[3] == 'l')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_UIDL;
return 1;
} else if ((packet->payload[0] == 'R' || packet->payload[0] == 'r')
&& (packet->payload[1] == 'E' || packet->payload[1] == 'e')
&& (packet->payload[2] == 'T' || packet->payload[2] == 't')
&& (packet->payload[3] == 'R' || packet->payload[3] == 'r')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_RETR;
return 1;
} else if ((packet->payload[0] == 'D' || packet->payload[0] == 'd')
&& (packet->payload[1] == 'E' || packet->payload[1] == 'e')
&& (packet->payload[2] == 'L' || packet->payload[2] == 'l')
&& (packet->payload[3] == 'E' || packet->payload[3] == 'e')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_DELE;
return 1;
} else if ((packet->payload[0] == 'S' || packet->payload[0] == 's')
&& (packet->payload[1] == 'T' || packet->payload[1] == 't')
&& (packet->payload[2] == 'L' || packet->payload[2] == 'l')
&& (packet->payload[3] == 'S' || packet->payload[3] == 's')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_STLS;
return 1;
}
}
return 0;
if(packet->payload_packet_len > 4) {
if((packet->payload[0] == 'A' || packet->payload[0] == 'a')
&& (packet->payload[1] == 'U' || packet->payload[1] == 'u')
&& (packet->payload[2] == 'T' || packet->payload[2] == 't')
&& (packet->payload[3] == 'H' || packet->payload[3] == 'h')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_AUTH;
return 1;
} else if((packet->payload[0] == 'A' || packet->payload[0] == 'a')
&& (packet->payload[1] == 'P' || packet->payload[1] == 'p')
&& (packet->payload[2] == 'O' || packet->payload[2] == 'o')
&& (packet->payload[3] == 'P' || packet->payload[3] == 'p')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_APOP;
return 1;
} else if((packet->payload[0] == 'U' || packet->payload[0] == 'u')
&& (packet->payload[1] == 'S' || packet->payload[1] == 's')
&& (packet->payload[2] == 'E' || packet->payload[2] == 'e')
&& (packet->payload[3] == 'R' || packet->payload[3] == 'r')) {
ndpi_user_pwd_payload_copy((u_int8_t*)flow->protos.pop.username,
sizeof(flow->protos.pop.username),
packet->payload, packet->payload_packet_len);
flow->l4.tcp.pop_command_bitmask |= POP_BIT_USER;
return 1;
} else if((packet->payload[0] == 'P' || packet->payload[0] == 'p')
&& (packet->payload[1] == 'A' || packet->payload[1] == 'a')
&& (packet->payload[2] == 'S' || packet->payload[2] == 's')
&& (packet->payload[3] == 'S' || packet->payload[3] == 's')) {
ndpi_user_pwd_payload_copy((u_int8_t*)flow->protos.pop.password,
sizeof(flow->protos.pop.password),
packet->payload, packet->payload_packet_len);
flow->l4.tcp.pop_command_bitmask |= POP_BIT_PASS;
return 1;
} else if((packet->payload[0] == 'C' || packet->payload[0] == 'c')
&& (packet->payload[1] == 'A' || packet->payload[1] == 'a')
&& (packet->payload[2] == 'P' || packet->payload[2] == 'p')
&& (packet->payload[3] == 'A' || packet->payload[3] == 'a')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_CAPA;
return 1;
} else if((packet->payload[0] == 'L' || packet->payload[0] == 'l')
&& (packet->payload[1] == 'I' || packet->payload[1] == 'i')
&& (packet->payload[2] == 'S' || packet->payload[2] == 's')
&& (packet->payload[3] == 'T' || packet->payload[3] == 't')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_LIST;
return 1;
} else if((packet->payload[0] == 'S' || packet->payload[0] == 's')
&& (packet->payload[1] == 'T' || packet->payload[1] == 't')
&& (packet->payload[2] == 'A' || packet->payload[2] == 'a')
&& (packet->payload[3] == 'T' || packet->payload[3] == 't')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_STAT;
return 1;
} else if((packet->payload[0] == 'U' || packet->payload[0] == 'u')
&& (packet->payload[1] == 'I' || packet->payload[1] == 'i')
&& (packet->payload[2] == 'D' || packet->payload[2] == 'd')
&& (packet->payload[3] == 'L' || packet->payload[3] == 'l')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_UIDL;
return 1;
} else if((packet->payload[0] == 'R' || packet->payload[0] == 'r')
&& (packet->payload[1] == 'E' || packet->payload[1] == 'e')
&& (packet->payload[2] == 'T' || packet->payload[2] == 't')
&& (packet->payload[3] == 'R' || packet->payload[3] == 'r')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_RETR;
return 1;
} else if((packet->payload[0] == 'D' || packet->payload[0] == 'd')
&& (packet->payload[1] == 'E' || packet->payload[1] == 'e')
&& (packet->payload[2] == 'L' || packet->payload[2] == 'l')
&& (packet->payload[3] == 'E' || packet->payload[3] == 'e')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_DELE;
return 1;
} else if((packet->payload[0] == 'S' || packet->payload[0] == 's')
&& (packet->payload[1] == 'T' || packet->payload[1] == 't')
&& (packet->payload[2] == 'L' || packet->payload[2] == 'l')
&& (packet->payload[3] == 'S' || packet->payload[3] == 's')) {
flow->l4.tcp.pop_command_bitmask |= POP_BIT_STLS;
return 1;
}
}
return 0;
}
void ndpi_search_mail_pop_tcp(struct ndpi_detection_module_struct
*ndpi_struct, struct ndpi_flow_struct *flow)
*ndpi_struct, struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct *packet = &flow->packet;
u_int8_t a = 0;
u_int8_t bit_count = 0;
struct ndpi_packet_struct *packet = &flow->packet;
u_int8_t a = 0;
u_int8_t bit_count = 0;
NDPI_LOG_DBG(ndpi_struct, "search mail_pop\n");
NDPI_LOG_DBG(ndpi_struct, "search mail_pop\n");
if((packet->payload_packet_len > 3
&& (packet->payload[0] == '+' && (packet->payload[1] == 'O' || packet->payload[1] == 'o')
&& (packet->payload[2] == 'K' || packet->payload[2] == 'k')))
|| (packet->payload_packet_len > 4
&& (packet->payload[0] == '-' && (packet->payload[1] == 'E' || packet->payload[1] == 'e')
&& (packet->payload[2] == 'R' || packet->payload[2] == 'r')
&& (packet->payload[3] == 'R' || packet->payload[3] == 'r')))) {
// +OK or -ERR seen
flow->l4.tcp.mail_pop_stage += 1;
} else if(!ndpi_int_mail_pop_check_for_client_commands(ndpi_struct, flow)) {
goto maybe_split_pop;
}
if(packet->payload_packet_len > 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a) {
// count the bits set in the bitmask
if(flow->l4.tcp.pop_command_bitmask != 0) {
for (a = 0; a < 16; a++) {
bit_count += (flow->l4.tcp.pop_command_bitmask >> a) & 0x01;
}
}
NDPI_LOG_DBG2(ndpi_struct,
"mail_pop +OK/-ERR responses: %u, unique commands: %u\n",
flow->l4.tcp.mail_pop_stage, bit_count);
if((bit_count + flow->l4.tcp.mail_pop_stage) >= 3) {
if(flow->l4.tcp.mail_pop_stage > 0) {
NDPI_LOG_INFO(ndpi_struct, "mail_pop identified\n");
// printf("==> %u\n", flow->l4.tcp.mail_pop_stage);
if((flow->protos.pop.password[0] != '\0')
|| (flow->l4.tcp.mail_pop_stage > 3))
ndpi_int_mail_pop_add_connection(ndpi_struct, flow);
}
return;
} else
return;
} else {
// first part of a split packet
NDPI_LOG_DBG2(ndpi_struct,
"mail_pop command without line ending -> skip\n");
return;
}
maybe_split_pop:
if ((packet->payload_packet_len > 3
&& (packet->payload[0] == '+' && (packet->payload[1] == 'O' || packet->payload[1] == 'o')
&& (packet->payload[2] == 'K' || packet->payload[2] == 'k')))
|| (packet->payload_packet_len > 4
&& (packet->payload[0] == '-' && (packet->payload[1] == 'E' || packet->payload[1] == 'e')
&& (packet->payload[2] == 'R' || packet->payload[2] == 'r')
&& (packet->payload[3] == 'R' || packet->payload[3] == 'r')))) {
// +OK or -ERR seen
flow->l4.tcp.mail_pop_stage += 1;
} else if (!ndpi_int_mail_pop_check_for_client_commands(ndpi_struct, flow)) {
goto maybe_split_pop;
}
if(((packet->payload_packet_len > 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a)
|| flow->l4.tcp.pop_command_bitmask != 0 || flow->l4.tcp.mail_pop_stage != 0) && flow->packet_counter < 12) {
// maybe part of a split pop packet
NDPI_LOG_DBG2(ndpi_struct,
"maybe part of split mail_pop packet -> skip\n");
return;
}
if (packet->payload_packet_len > 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a) {
// count the bits set in the bitmask
if (flow->l4.tcp.pop_command_bitmask != 0) {
for (a = 0; a < 16; a++) {
bit_count += (flow->l4.tcp.pop_command_bitmask >> a) & 0x01;
}
}
NDPI_LOG_DBG2(ndpi_struct,
"mail_pop +OK/-ERR responses: %u, unique commands: %u\n", flow->l4.tcp.mail_pop_stage, bit_count);
if ((bit_count + flow->l4.tcp.mail_pop_stage) >= 3) {
if (flow->l4.tcp.mail_pop_stage > 0) {
NDPI_LOG_INFO(ndpi_struct, "mail_pop identified\n");
ndpi_int_mail_pop_add_connection(ndpi_struct, flow);
return;
} else {
return;
}
} else {
return;
}
} else {
// first part of a split packet
NDPI_LOG_DBG2(ndpi_struct,
"mail_pop command without line ending -> skip\n");
return;
}
maybe_split_pop:
if (((packet->payload_packet_len > 2 && ntohs(get_u_int16_t(packet->payload, packet->payload_packet_len - 2)) == 0x0d0a)
|| flow->l4.tcp.pop_command_bitmask != 0 || flow->l4.tcp.mail_pop_stage != 0) && flow->packet_counter < 12) {
// maybe part of a split pop packet
NDPI_LOG_DBG2(ndpi_struct,
"maybe part of split mail_pop packet -> skip\n");
return;
}
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
}
void init_mail_pop_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask)
{
void init_mail_pop_dissector(struct ndpi_detection_module_struct *ndpi_struct,
u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) {
ndpi_set_bitmask_protocol_detection("MAIL_POP", ndpi_struct, detection_bitmask, *id,
NDPI_PROTOCOL_MAIL_POP,
ndpi_search_mail_pop_tcp,