Improvements for dta dissection

This commit is contained in:
Luca Deri
2019-07-18 11:24:44 +02:00
parent f394102b80
commit 20bac32e1f
5 changed files with 93 additions and 71 deletions
+7 -25
View File
@@ -82,7 +82,7 @@ static u_int8_t live_capture = 0;
static u_int8_t undetected_flows_deleted = 0;
/** User preferences **/
u_int8_t enable_protocol_guess = 1;
static u_int8_t verbose = 0, json_flag = 0;
u_int8_t verbose = 0, json_flag = 0;
int nDPI_LogLevel = 0;
char *_debug_protocols = NULL;
static u_int8_t stats_flag = 0, bpf_filter_flag = 0;
@@ -833,15 +833,8 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
//fprintf(out, "[Num_Packt_Human_Readable_String: %d]", flow->n_pckt_human_readable_string);
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_HTTP ) && (flow->n_pckt_human_readable_string == 0) ) printf("!WARNING!");
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_FTP_CONTROL) && (flow->n_pckt_human_readable_string == 0) ) printf("!WARNING!");
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_NTP ) && (flow->n_pckt_human_readable_string != 0) ) printf("!WARNING!");
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_IP_ICMP ) && (flow->n_pckt_human_readable_string != 0) ) printf("!WARNING!");
//if( (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_VNC ) && (flow->n_pckt_human_readable_string != 0 ) ) printf("!WARNING!");
if(flow->has_human_readeable_strings) fprintf(out, "[PLAIN TEXT (%s)]", flow->human_readeable_string_buffer);
fprintf(out, "\n");
} else {
#ifdef HAVE_JSON_C
@@ -2804,20 +2797,9 @@ static void ndpi_process_packet(u_char *args,
/* allocate an exact size buffer to check overflows */
uint8_t *packet_checked = malloc(header->caplen);
ndpi_thread_info[thread_id].workflow->hrs = 0;
if (ndpi_has_human_readeable_string(ndpi_info_mod, (char*)packet, header->caplen) == 1) {
ndpi_thread_info[thread_id].workflow->hrs = 1;
}
memcpy(packet_checked, packet, header->caplen);
p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked);
if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) {
if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL)
pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle);
return;
}
if(!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
@@ -2840,7 +2822,9 @@ static void ndpi_process_packet(u_char *args,
ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
}
if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0;
if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots)
ndpi_thread_info[thread_id].idle_scan_idx = 0;
ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].workflow->last_time;
}
}
@@ -2874,8 +2858,7 @@ static void ndpi_process_packet(u_char *args,
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
crc = (uint32_t*)&extcap_buf[h.caplen+sizeof(struct ndpi_packet_trailer)];
*crc = ethernet_crc32((const void*)extcap_buf, h.caplen+sizeof(struct ndpi_packet_trailer));
h.caplen += delta;
h.len += delta;
h.caplen += delta, h.len += delta;
#ifdef DEBUG_TRACE
if(trace) fprintf(trace, "Dumping %u bytes packet\n", h.caplen);
@@ -2921,7 +2904,6 @@ static void ndpi_process_packet(u_char *args,
free(packet_checked);
}
/**
* @brief Call pcap_loop() to process packets from a live capture or savefile
*/
+45 -26
View File
@@ -75,6 +75,7 @@
#include "ndpi_util.h"
extern u_int8_t enable_protocol_guess;
extern u_int8_t verbose;
/* ***************************************************** */
@@ -459,32 +460,11 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
*src = newflow->src_id, *dst = newflow->dst_id;
if(workflow->hrs == 1){
/* count if no SSL protocol */
if((newflow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSL)
&& (newflow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSL)){
newflow->n_pckt_human_readable_string++;
}
}
return newflow;
}
} else {
struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret;
if(workflow->hrs == 1){
/* count if no SSL protocol */
if((flow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSL)
&& (flow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSL)){
flow->n_pckt_human_readable_string++;
}
}
if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)){
flow->n_pckt_human_readable_string = 0;
}
if(is_changed) {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
@@ -631,7 +611,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
const struct ndpi_iphdr *iph,
struct ndpi_ipv6hdr *iph6,
u_int16_t ip_offset,
u_int16_t ipsize, u_int16_t rawsize) {
u_int16_t ipsize, u_int16_t rawsize,
const struct pcap_pkthdr *header,
const u_char *packet) {
struct ndpi_id_struct *src, *dst;
struct ndpi_flow_info *flow = NULL;
struct ndpi_flow_struct *ndpi_flow = NULL;
@@ -668,6 +650,42 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
flow->dst2src_packets++, flow->dst2src_bytes += rawsize;
flow->last_seen = time;
if(verbose) {
}
if(!flow->has_human_readeable_strings) {
u_int8_t skip = 0;
if((iph->protocol == IPPROTO_TCP)
&& (
(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
) {
if((flow->src2dst_packets+flow->dst2src_packets) < 10 /* MIN_NUM_ENCRYPT_SKIP_PACKETS */)
skip = 1;
}
if(!skip) {
char outbuf[64] = { '\0' };
if(ndpi_has_human_readeable_string(workflow->ndpi_struct, (char*)packet, header->caplen, 8,
flow->human_readeable_string_buffer,
sizeof(flow->human_readeable_string_buffer)) == 1)
flow->has_human_readeable_strings = 1;
}
} else {
if((iph->protocol == IPPROTO_TCP)
&& (
(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSL)
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH))
)
flow->has_human_readeable_strings = 0;
}
} else { // flow is NULL
workflow->stats.total_discarded_bytes++;
return(nproto);
@@ -965,20 +983,20 @@ iph_check:
ip_len = sizeof(struct ndpi_ipv6hdr);
if(proto == IPPROTO_DSTOPTS /* IPv6 destination option */) {
u_int8_t *options = (u_int8_t*)&packet[ip_offset+ip_len];
proto = options[0];
ip_len += 8 * (options[1] + 1);
}
iph = NULL;
} else {
static u_int8_t ipv4_warning_used = 0;
v4_warning:
if(ipv4_warning_used == 0) {
if(!workflow->prefs.quiet_mode)
NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_DEBUG, "\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n");
NDPI_LOG(0, workflow->ndpi_struct, NDPI_LOG_DEBUG,
"\n\nWARNING: only IPv4/IPv6 packets are supported in this demo (nDPI supports both IPv4 and IPv6), all other packets will be discarded\n\n");
ipv4_warning_used = 1;
}
workflow->stats.total_discarded_bytes += header->len;
@@ -1053,7 +1071,8 @@ iph_check:
/* process the packet */
return(packet_processing(workflow, time, vlan_id, iph, iph6,
ip_offset, header->caplen - ip_offset, header->caplen));
ip_offset, header->caplen - ip_offset,
header->caplen, header, packet));
}
/* ********************************************************** */
+4 -6
View File
@@ -128,8 +128,9 @@ typedef struct ndpi_flow_info {
u_int64_t last_seen;
u_int64_t src2dst_bytes, dst2src_bytes;
u_int32_t src2dst_packets, dst2src_packets;
u_int32_t n_pckt_human_readable_string;
u_int32_t has_human_readeable_strings;
char human_readeable_string_buffer[32];
// result only, not used for flow identification
ndpi_protocol detected_protocol;
@@ -200,10 +201,7 @@ typedef struct ndpi_workflow {
void **ndpi_flows_root;
struct ndpi_detection_module_struct *ndpi_struct;
u_int32_t num_allocated_flows;
int hrs;
} ndpi_workflow_t;
} ndpi_workflow_t;
/* TODO: remove wrappers parameters and use ndpi global, when their initialization will be fixed... */