Improvements for dta dissection

This commit is contained in:
Luca Deri
2019-07-18 11:24:44 +02:00
parent f394102b80
commit 20bac32e1f
5 changed files with 93 additions and 71 deletions
+7 -25
View File
@@ -82,7 +82,7 @@ static u_int8_t live_capture = 0;
static u_int8_t undetected_flows_deleted = 0;
/** User preferences **/
u_int8_t enable_protocol_guess = 1;
static u_int8_t verbose = 0, json_flag = 0;
u_int8_t verbose = 0, json_flag = 0;
int nDPI_LogLevel = 0;
char *_debug_protocols = NULL;
static u_int8_t stats_flag = 0, bpf_filter_flag = 0;
@@ -833,15 +833,8 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
//fprintf(out, "[Num_Packt_Human_Readable_String: %d]", flow->n_pckt_human_readable_string);
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_HTTP ) && (flow->n_pckt_human_readable_string == 0) ) printf("!WARNING!");
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_FTP_CONTROL) && (flow->n_pckt_human_readable_string == 0) ) printf("!WARNING!");
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_NTP ) && (flow->n_pckt_human_readable_string != 0) ) printf("!WARNING!");
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_IP_ICMP ) && (flow->n_pckt_human_readable_string != 0) ) printf("!WARNING!");
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_VNC ) && (flow->n_pckt_human_readable_string != 0 ) ) printf("!WARNING!");
if(flow->has_human_readeable_strings) fprintf(out, "[PLAIN TEXT (%s)]", flow->human_readeable_string_buffer);
fprintf(out, "\n");
} else {
#ifdef HAVE_JSON_C
@@ -2804,20 +2797,9 @@ static void ndpi_process_packet(u_char *args,
/* allocate an exact size buffer to check overflows */
uint8_t *packet_checked = malloc(header->caplen);
ndpi_thread_info[thread_id].workflow->hrs = 0;
if (ndpi_has_human_readeable_string(ndpi_info_mod, (char*)packet, header->caplen) == 1) {
ndpi_thread_info[thread_id].workflow->hrs = 1;
}
memcpy(packet_checked, packet, header->caplen);
p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked);
if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) {
if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL)
pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle);
return;
}
if(!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
@@ -2840,7 +2822,9 @@ static void ndpi_process_packet(u_char *args,
ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
}
if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0;
if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots)
ndpi_thread_info[thread_id].idle_scan_idx = 0;
ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].workflow->last_time;
}
}
@@ -2874,8 +2858,7 @@ static void ndpi_process_packet(u_char *args,
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
crc = (uint32_t*)&extcap_buf[h.caplen+sizeof(struct ndpi_packet_trailer)];
*crc = ethernet_crc32((const void*)extcap_buf, h.caplen+sizeof(struct ndpi_packet_trailer));
h.caplen += delta;
h.len += delta;
h.caplen += delta, h.len += delta;
#ifdef DEBUG_TRACE
if(trace) fprintf(trace, "Dumping %u bytes packet\n", h.caplen);
@@ -2921,7 +2904,6 @@ static void ndpi_process_packet(u_char *args,
free(packet_checked);
}
/**
* @brief Call pcap_loop() to process packets from a live capture or savefile
*/
+45 -26
View File
@@ -75,6 +75,7 @@
#include "ndpi_util.h"
extern u_int8_t enable_protocol_guess;
extern u_int8_t verbose;
/* ***************************************************** */
@@ -459,32 +460,11 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
*src = newflow->src_id, *dst = newflow->dst_id;
if(workflow->hrs == 1){
/* count if no SSL protocol */
if((newflow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSL)
&& (newflow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSL)){
newflow->n_pckt_human_readable_string++;
}
}
return newflow;
}
} else {
struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret;
if(workflow->hrs == 1){
/* count if no SSL protocol */
if((flow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSL)
&& (flow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSL)){
flow->n_pckt_human_readable_string++;
}
}
if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)){
flow->n_pckt_human_readable_string = 0;
}
if(is_changed) {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
@@ -631,7 +611,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
const struct ndpi_iphdr *iph,
struct ndpi_ipv6hdr *iph6,
u_int16_t ip_offset,
u_int16_t ipsize, u_int16_t rawsize) {
u_int16_t ipsize, u_int16_t rawsize,
const struct pcap_pkthdr *header,
const u_char *packet) {
struct ndpi_id_struct *src, *dst;
struct ndpi_flow_info *flow = NULL;
struct ndpi_flow_struct *ndpi_flow = NULL;
@@ -668,6 +650,42 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
flow->dst2src_packets++, flow->dst2src_bytes += rawsize;
flow->last_seen = time;
if(verbose) {
}
if(!flow->has_human_readeable_strings) {
u_int8_t skip = 0;
if((iph->protocol == IPPROTO_TCP)
&& (
(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
) {
if((flow->src2dst_packets+flow->dst2src_packets) < 10 /* MIN_NUM_ENCRYPT_SKIP_PACKETS */)
skip = 1;
}
if(!skip) {
char outbuf[64] = { '\0' };
if(ndpi_has_human_readeable_string(workflow->ndpi_struct, (char*)packet, header->caplen, 8,
flow->human_readeable_string_buffer,
sizeof(flow->human_readeable_string_buffer)) == 1)
flow->has_human_readeable_strings = 1;
}
} else {
if((iph->protocol == IPPROTO_TCP)
&& (
(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
)
flow->has_human_readeable_strings = 0;
}
} else { // flow is NULL
workflow->stats.total_discarded_bytes++;
return(nproto);
@@ -965,20 +983,20 @@ iph_check:
ip_len = sizeof(struct ndpi_ipv6hdr);
if(proto == IPPROTO_DSTOPTS /* IPv6 destination option */) {
u_int8_t *options = (u_int8_t*)&packet[ip_offset+ip_len];
proto = options[0];
ip_len += 8 * (options[1] + 1);
}
iph = NULL;
} else {
static u_int8_t ipv4_warning_used = 0;
v4_warning:
if(ipv4_warning_used == 0) {
if(!workflow->prefs.quiet_mode)
NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_DEBUG, "\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n");
NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_DEBUG,
"\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n");
ipv4_warning_used = 1;
}
workflow->stats.total_discarded_bytes += header->len;
@@ -1053,7 +1071,8 @@ iph_check:
/* process the packet */
return(packet_processing(workflow, time, vlan_id, iph, iph6,
ip_offset, header->caplen - ip_offset, header->caplen));
ip_offset, header->caplen - ip_offset,
header->caplen, header, packet));
}
/* ********************************************************** */
+4 -6
View File
@@ -128,8 +128,9 @@ typedef struct ndpi_flow_info {
u_int64_t last_seen;
u_int64_t src2dst_bytes, dst2src_bytes;
u_int32_t src2dst_packets, dst2src_packets;
u_int32_t n_pckt_human_readable_string;
u_int32_t has_human_readeable_strings;
char human_readeable_string_buffer[32];
// result only, not used for flow identification
ndpi_protocol detected_protocol;
@@ -200,10 +201,7 @@ typedef struct ndpi_workflow {
void **ndpi_flows_root;
struct ndpi_detection_module_struct *ndpi_struct;
u_int32_t num_allocated_flows;
int hrs;
} ndpi_workflow_t;
} ndpi_workflow_t;
/* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */
+3 -1
View File
@@ -812,7 +812,9 @@ extern "C" {
u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);
int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_struct,
char *buffer, u_int buffer_size);
char *buffer, u_int buffer_size,
u_int8_t min_string_match_len, /* Will return 0 if no string > min_string_match_len have been found */
char *outbuf, u_int outbuf_len);
char* ndpi_ssl_version2str(u_int16_t version);
/* Serializer */
+34 -13
View File
@@ -608,16 +608,21 @@ const char* ndpi_cipher2str(u_int32_t cipher) {
static int ndpi_is_other_char(char c) {
return((c == '.')
|| (c == ' ')
|| (c == '@')
|| (c == '/')
);
}
/* ******************************************************************** */
static int ndpi_is_valid_char(char c) {
return(isdigit(c)
|| isalpha(c)
|| ndpi_is_other_char(c));
if(ispunct(c) && (!ndpi_is_other_char(c)))
return(0);
else
return(isdigit(c)
|| isalpha(c)
|| ndpi_is_other_char(c));
}
/* ******************************************************************** */
@@ -628,11 +633,11 @@ static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_s
char s[3];
if((isdigit(str[0]) && isdigit(str[1]))
|| (ndpi_is_other_char(str[0]) || ndpi_is_other_char(str[1]))
|| (ndpi_is_other_char(str[0]) || ndpi_is_other_char(str[1]))
|| ndpi_is_other_char(str[0])
|| ndpi_is_other_char(str[1])
)
return(1);
s[0] = tolower(str[0]), s[1] = tolower(str[1]), s[2] = '\0';
return(ndpi_match_bigram(ndpi_struct, &ndpi_struct->bigrams_automa, s));
@@ -643,13 +648,17 @@ static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_s
/* #define PRINT_STRINGS 1 */
int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_struct,
char *buffer, u_int buffer_size) {
u_int ret = 0, i = 0, do_cr = 0, len = 0;
const u_int8_t NDPI_MIN_VALID_STRING_LEN = 4; /* Will return 0 if no string > NDPI_MIN_VALID_STRING_LEN have been found */
char *buffer, u_int buffer_size,
u_int8_t min_string_match_len,
char *outbuf, u_int outbuf_len) {
u_int ret = 0, i = 0, do_cr = 0, len = 0, o_idx = 0, being_o_idx = 0;
if(buffer_size <= 0)
return(0);
outbuf_len--;
outbuf[outbuf_len] = '\0';
for(i=0; i<buffer_size-2; i++) {
if(ndpi_is_valid_char(buffer[i])
&& ndpi_is_valid_char(buffer[i+1])
@@ -657,23 +666,35 @@ int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_st
#ifdef PRINT_STRINGS
printf("%c%c", buffer[i], buffer[i+1]);
#endif
if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i];
if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i+1];
do_cr = 1, i += 1, len += 2;
} else {
if(ndpi_is_valid_char(buffer[i]) && do_cr) {
#ifdef PRINT_STRINGS
printf("%c", buffer[i]);
#endif
if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i];
len += 1;
}
// printf("->> %c%c\n", isprint(buffer[i]) ? buffer[i] : '.', isprint(buffer[i+1]) ? buffer[i+1] : '.');
if(do_cr) {
#ifdef PRINT_STRINGS
printf(" [len: %u]\n", len);
#endif
if(len > NDPI_MIN_VALID_STRING_LEN)
if(len > min_string_match_len)
ret = 1;
else {
o_idx = being_o_idx;
being_o_idx = o_idx;
outbuf[o_idx] = '\0';
}
#ifdef PRINT_STRINGS
printf(" [len: %u]%s\n", len, ret ? "<-- HIT" : "");
#endif
if(ret)
break;
do_cr = 0, len = 0;
}
}