Improvements for dta dissection
This commit is contained in:
+7
-25
@@ -82,7 +82,7 @@ static u_int8_t live_capture = 0;
|
||||
static u_int8_t undetected_flows_deleted = 0;
|
||||
/** User preferences **/
|
||||
u_int8_t enable_protocol_guess = 1;
|
||||
static u_int8_t verbose = 0, json_flag = 0;
|
||||
u_int8_t verbose = 0, json_flag = 0;
|
||||
int nDPI_LogLevel = 0;
|
||||
char *_debug_protocols = NULL;
|
||||
static u_int8_t stats_flag = 0, bpf_filter_flag = 0;
|
||||
@@ -833,15 +833,8 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
|
||||
if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
|
||||
|
||||
//fprintf(out, "[Num_Packt_Human_Readable_String: %d]", flow->n_pckt_human_readable_string);
|
||||
|
||||
|
||||
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_HTTP ) && (flow->n_pckt_human_readable_string == 0) ) printf("!WARNING!");
|
||||
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_FTP_CONTROL) && (flow->n_pckt_human_readable_string == 0) ) printf("!WARNING!");
|
||||
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_NTP ) && (flow->n_pckt_human_readable_string != 0) ) printf("!WARNING!");
|
||||
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_IP_ICMP ) && (flow->n_pckt_human_readable_string != 0) ) printf("!WARNING!");
|
||||
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_VNC ) && (flow->n_pckt_human_readable_string != 0 ) ) printf("!WARNING!");
|
||||
|
||||
if(flow->has_human_readeable_strings) fprintf(out, "[PLAIN TEXT (%s)]", flow->human_readeable_string_buffer);
|
||||
|
||||
fprintf(out, "\n");
|
||||
} else {
|
||||
#ifdef HAVE_JSON_C
|
||||
@@ -2804,20 +2797,9 @@ static void ndpi_process_packet(u_char *args,
|
||||
/* allocate an exact size buffer to check overflows */
|
||||
uint8_t *packet_checked = malloc(header->caplen);
|
||||
|
||||
ndpi_thread_info[thread_id].workflow->hrs = 0;
|
||||
if (ndpi_has_human_readeable_string(ndpi_info_mod, (char*)packet, header->caplen) == 1) {
|
||||
ndpi_thread_info[thread_id].workflow->hrs = 1;
|
||||
}
|
||||
|
||||
memcpy(packet_checked, packet, header->caplen);
|
||||
p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked);
|
||||
|
||||
if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) {
|
||||
if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL)
|
||||
pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle);
|
||||
return;
|
||||
}
|
||||
|
||||
if(!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
|
||||
pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
|
||||
|
||||
@@ -2840,7 +2822,9 @@ static void ndpi_process_packet(u_char *args,
|
||||
ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
|
||||
}
|
||||
|
||||
if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0;
|
||||
if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots)
|
||||
ndpi_thread_info[thread_id].idle_scan_idx = 0;
|
||||
|
||||
ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].workflow->last_time;
|
||||
}
|
||||
}
|
||||
@@ -2874,8 +2858,7 @@ static void ndpi_process_packet(u_char *args,
|
||||
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
|
||||
crc = (uint32_t*)&extcap_buf[h.caplen+sizeof(struct ndpi_packet_trailer)];
|
||||
*crc = ethernet_crc32((const void*)extcap_buf, h.caplen+sizeof(struct ndpi_packet_trailer));
|
||||
h.caplen += delta;
|
||||
h.len += delta;
|
||||
h.caplen += delta, h.len += delta;
|
||||
|
||||
#ifdef DEBUG_TRACE
|
||||
if(trace) fprintf(trace, "Dumping %u bytes packet\n", h.caplen);
|
||||
@@ -2921,7 +2904,6 @@ static void ndpi_process_packet(u_char *args,
|
||||
free(packet_checked);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @brief Call pcap_loop() to process packets from a live capture or savefile
|
||||
*/
|
||||
|
||||
+45
-26
@@ -75,6 +75,7 @@
|
||||
#include "ndpi_util.h"
|
||||
|
||||
extern u_int8_t enable_protocol_guess;
|
||||
extern u_int8_t verbose;
|
||||
|
||||
/* ***************************************************** */
|
||||
|
||||
@@ -459,32 +460,11 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
|
||||
|
||||
*src = newflow->src_id, *dst = newflow->dst_id;
|
||||
|
||||
if(workflow->hrs == 1){
|
||||
/* count if no SSL protocol */
|
||||
if((newflow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSL)
|
||||
&& (newflow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSL)){
|
||||
newflow->n_pckt_human_readable_string++;
|
||||
}
|
||||
}
|
||||
|
||||
return newflow;
|
||||
}
|
||||
} else {
|
||||
struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret;
|
||||
|
||||
if(workflow->hrs == 1){
|
||||
/* count if no SSL protocol */
|
||||
if((flow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSL)
|
||||
&& (flow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSL)){
|
||||
flow->n_pckt_human_readable_string++;
|
||||
}
|
||||
|
||||
}
|
||||
if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|
||||
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)){
|
||||
flow->n_pckt_human_readable_string = 0;
|
||||
}
|
||||
|
||||
if(is_changed) {
|
||||
if(flow->src_ip == iph->saddr
|
||||
&& flow->dst_ip == iph->daddr
|
||||
@@ -631,7 +611,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
const struct ndpi_iphdr *iph,
|
||||
struct ndpi_ipv6hdr *iph6,
|
||||
u_int16_t ip_offset,
|
||||
u_int16_t ipsize, u_int16_t rawsize) {
|
||||
u_int16_t ipsize, u_int16_t rawsize,
|
||||
const struct pcap_pkthdr *header,
|
||||
const u_char *packet) {
|
||||
struct ndpi_id_struct *src, *dst;
|
||||
struct ndpi_flow_info *flow = NULL;
|
||||
struct ndpi_flow_struct *ndpi_flow = NULL;
|
||||
@@ -668,6 +650,42 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
flow->dst2src_packets++, flow->dst2src_bytes += rawsize;
|
||||
|
||||
flow->last_seen = time;
|
||||
|
||||
if(verbose) {
|
||||
}
|
||||
|
||||
if(!flow->has_human_readeable_strings) {
|
||||
u_int8_t skip = 0;
|
||||
|
||||
if((iph->protocol == IPPROTO_TCP)
|
||||
&& (
|
||||
(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|
||||
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
|
||||
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
|
||||
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
|
||||
) {
|
||||
if((flow->src2dst_packets+flow->dst2src_packets) < 10 /* MIN_NUM_ENCRYPT_SKIP_PACKETS */)
|
||||
skip = 1;
|
||||
}
|
||||
|
||||
if(!skip) {
|
||||
char outbuf[64] = { '\0' };
|
||||
|
||||
if(ndpi_has_human_readeable_string(workflow->ndpi_struct, (char*)packet, header->caplen, 8,
|
||||
flow->human_readeable_string_buffer,
|
||||
sizeof(flow->human_readeable_string_buffer)) == 1)
|
||||
flow->has_human_readeable_strings = 1;
|
||||
}
|
||||
} else {
|
||||
if((iph->protocol == IPPROTO_TCP)
|
||||
&& (
|
||||
(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|
||||
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
|
||||
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
|
||||
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
|
||||
)
|
||||
flow->has_human_readeable_strings = 0;
|
||||
}
|
||||
} else { // flow is NULL
|
||||
workflow->stats.total_discarded_bytes++;
|
||||
return(nproto);
|
||||
@@ -965,20 +983,20 @@ iph_check:
|
||||
ip_len = sizeof(struct ndpi_ipv6hdr);
|
||||
|
||||
if(proto == IPPROTO_DSTOPTS /* IPv6 destination option */) {
|
||||
|
||||
u_int8_t *options = (u_int8_t*)&packet[ip_offset+ip_len];
|
||||
proto = options[0];
|
||||
ip_len += 8 * (options[1] + 1);
|
||||
}
|
||||
|
||||
iph = NULL;
|
||||
|
||||
} else {
|
||||
static u_int8_t ipv4_warning_used = 0;
|
||||
|
||||
v4_warning:
|
||||
if(ipv4_warning_used == 0) {
|
||||
if(!workflow->prefs.quiet_mode)
|
||||
NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_DEBUG, "\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n");
|
||||
NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_DEBUG,
|
||||
"\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n");
|
||||
ipv4_warning_used = 1;
|
||||
}
|
||||
workflow->stats.total_discarded_bytes += header->len;
|
||||
@@ -1053,7 +1071,8 @@ iph_check:
|
||||
|
||||
/* process the packet */
|
||||
return(packet_processing(workflow, time, vlan_id, iph, iph6,
|
||||
ip_offset, header->caplen - ip_offset, header->caplen));
|
||||
ip_offset, header->caplen - ip_offset,
|
||||
header->caplen, header, packet));
|
||||
}
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
+4
-6
@@ -128,8 +128,9 @@ typedef struct ndpi_flow_info {
|
||||
u_int64_t last_seen;
|
||||
u_int64_t src2dst_bytes, dst2src_bytes;
|
||||
u_int32_t src2dst_packets, dst2src_packets;
|
||||
u_int32_t n_pckt_human_readable_string;
|
||||
|
||||
u_int32_t has_human_readeable_strings;
|
||||
char human_readeable_string_buffer[32];
|
||||
|
||||
// result only, not used for flow identification
|
||||
ndpi_protocol detected_protocol;
|
||||
|
||||
@@ -200,10 +201,7 @@ typedef struct ndpi_workflow {
|
||||
void **ndpi_flows_root;
|
||||
struct ndpi_detection_module_struct *ndpi_struct;
|
||||
u_int32_t num_allocated_flows;
|
||||
|
||||
int hrs;
|
||||
|
||||
} ndpi_workflow_t;
|
||||
} ndpi_workflow_t;
|
||||
|
||||
|
||||
/* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */
|
||||
|
||||
@@ -812,7 +812,9 @@ extern "C" {
|
||||
u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow);
|
||||
int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
char *buffer, u_int buffer_size);
|
||||
char *buffer, u_int buffer_size,
|
||||
u_int8_t min_string_match_len, /* Will return 0 if no string > min_string_match_len have been found */
|
||||
char *outbuf, u_int outbuf_len);
|
||||
char* ndpi_ssl_version2str(u_int16_t version);
|
||||
|
||||
/* Serializer */
|
||||
|
||||
+34
-13
@@ -608,16 +608,21 @@ const char* ndpi_cipher2str(u_int32_t cipher) {
|
||||
|
||||
static int ndpi_is_other_char(char c) {
|
||||
return((c == '.')
|
||||
|| (c == ' ')
|
||||
|| (c == '@')
|
||||
|| (c == '/')
|
||||
);
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static int ndpi_is_valid_char(char c) {
|
||||
return(isdigit(c)
|
||||
|| isalpha(c)
|
||||
|| ndpi_is_other_char(c));
|
||||
if(ispunct(c) && (!ndpi_is_other_char(c)))
|
||||
return(0);
|
||||
else
|
||||
return(isdigit(c)
|
||||
|| isalpha(c)
|
||||
|| ndpi_is_other_char(c));
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
@@ -628,11 +633,11 @@ static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_s
|
||||
char s[3];
|
||||
|
||||
if((isdigit(str[0]) && isdigit(str[1]))
|
||||
|| (ndpi_is_other_char(str[0]) || ndpi_is_other_char(str[1]))
|
||||
|| (ndpi_is_other_char(str[0]) || ndpi_is_other_char(str[1]))
|
||||
|| ndpi_is_other_char(str[0])
|
||||
|| ndpi_is_other_char(str[1])
|
||||
)
|
||||
return(1);
|
||||
|
||||
|
||||
s[0] = tolower(str[0]), s[1] = tolower(str[1]), s[2] = '\0';
|
||||
|
||||
return(ndpi_match_bigram(ndpi_struct, &ndpi_struct->bigrams_automa, s));
|
||||
@@ -643,13 +648,17 @@ static int ndpi_find_non_eng_bigrams(struct ndpi_detection_module_struct *ndpi_s
|
||||
/* #define PRINT_STRINGS 1 */
|
||||
|
||||
int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
char *buffer, u_int buffer_size) {
|
||||
u_int ret = 0, i = 0, do_cr = 0, len = 0;
|
||||
const u_int8_t NDPI_MIN_VALID_STRING_LEN = 4; /* Will return 0 if no string > NDPI_MIN_VALID_STRING_LEN have been found */
|
||||
char *buffer, u_int buffer_size,
|
||||
u_int8_t min_string_match_len,
|
||||
char *outbuf, u_int outbuf_len) {
|
||||
u_int ret = 0, i = 0, do_cr = 0, len = 0, o_idx = 0, being_o_idx = 0;
|
||||
|
||||
if(buffer_size <= 0)
|
||||
return(0);
|
||||
|
||||
outbuf_len--;
|
||||
outbuf[outbuf_len] = '\0';
|
||||
|
||||
for(i=0; i<buffer_size-2; i++) {
|
||||
if(ndpi_is_valid_char(buffer[i])
|
||||
&& ndpi_is_valid_char(buffer[i+1])
|
||||
@@ -657,23 +666,35 @@ int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_st
|
||||
#ifdef PRINT_STRINGS
|
||||
printf("%c%c", buffer[i], buffer[i+1]);
|
||||
#endif
|
||||
if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i];
|
||||
if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i+1];
|
||||
do_cr = 1, i += 1, len += 2;
|
||||
} else {
|
||||
if(ndpi_is_valid_char(buffer[i]) && do_cr) {
|
||||
#ifdef PRINT_STRINGS
|
||||
printf("%c", buffer[i]);
|
||||
#endif
|
||||
if(o_idx < outbuf_len) outbuf[o_idx++] = buffer[i];
|
||||
len += 1;
|
||||
}
|
||||
|
||||
// printf("->> %c%c\n", isprint(buffer[i]) ? buffer[i] : '.', isprint(buffer[i+1]) ? buffer[i+1] : '.');
|
||||
if(do_cr) {
|
||||
#ifdef PRINT_STRINGS
|
||||
printf(" [len: %u]\n", len);
|
||||
#endif
|
||||
if(len > NDPI_MIN_VALID_STRING_LEN)
|
||||
if(len > min_string_match_len)
|
||||
ret = 1;
|
||||
else {
|
||||
o_idx = being_o_idx;
|
||||
being_o_idx = o_idx;
|
||||
outbuf[o_idx] = '\0';
|
||||
}
|
||||
|
||||
#ifdef PRINT_STRINGS
|
||||
printf(" [len: %u]%s\n", len, ret ? "<-- HIT" : "");
|
||||
#endif
|
||||
|
||||
if(ret)
|
||||
break;
|
||||
|
||||
do_cr = 0, len = 0;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user