Integrate root flows free and little changes

This commit is contained in:
emanuele-f
2016-04-20 15:36:44 +02:00
parent 4e1317cd0f
commit 5386aede36
3 changed files with 79 additions and 87 deletions
+18 -48
View File
@@ -125,11 +125,6 @@ typedef struct ndpi_id {
struct ndpi_id_struct *ndpi_id; // nDpi worker structure
} ndpi_id_t;
static u_int32_t size_id_struct = 0; // ID tracking structure size
static u_int32_t size_flow_struct = 0;
static void help(u_int long_help) {
printf("ndpiReader -i <file|device> [-f <filter>][-s <duration>]\n"
" [-p <protos>][-l <loops> [-q][-d][-h][-t][-v <level>]\n"
@@ -383,11 +378,11 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) {
fprintf(out, "\t%s %s%s%s:%u <-> %s%s%s:%u ",
ipProto2Name(flow->protocol),
(flow->ip_version == 6) ? "[" : "",
flow->lower_name,
flow->lower_name,
(flow->ip_version == 6) ? "]" : "",
ntohs(flow->lower_port),
(flow->ip_version == 6) ? "[" : "",
flow->upper_name,
flow->upper_name,
(flow->ip_version == 6) ? "]" : "",
ntohs(flow->upper_port));
@@ -472,24 +467,6 @@ static void printFlow(u_int16_t thread_id, struct ndpi_flow_info *flow) {
/* ***************************************************** */
static void free_ndpi_flow_info(struct ndpi_flow_info *flow) {
if(flow->ndpi_flow) { ndpi_free_flow(flow->ndpi_flow); flow->ndpi_flow = NULL; }
if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; }
if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; }
}
/* ***************************************************** */
static void ndpi_flow_info_freer(void *node) {
struct ndpi_flow_info *flow = (struct ndpi_flow_info*)node;
free_ndpi_flow_info(flow);
ndpi_free(flow);
}
/* ***************************************************** */
static void node_print_unknown_proto_walker(const void *node, ndpi_VISIT which, int depth, void *user_data) {
struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)node;
u_int16_t thread_id = *((u_int16_t*)user_data);
@@ -569,7 +546,7 @@ static void node_idle_scan_walker(const void *node, ndpi_VISIT which, int depth,
if((flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) && !undetected_flows_deleted)
undetected_flows_deleted = 1;
free_ndpi_flow_info(flow);
ndpi_free_flow_info_half(flow);
ndpi_thread_info[thread_id].workflow->stats.ndpi_flow_count--;
/* adding to a queue (we can't delete it from the tree inline ) */
@@ -584,7 +561,7 @@ static void on_protocol_discovered(struct ndpi_workflow * workflow,
struct ndpi_flow_info * flow,
void * udata) {
const u_int16_t thread_id = *((u_int16_t *)udata);
if(verbose > 1) {
if(enable_protocol_guess) {
if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) {
@@ -638,7 +615,7 @@ static void debug_printf(u_int32_t protocol, void *id_struct,
static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
NDPI_PROTOCOL_BITMASK all;
struct ndpi_workflow_prefs prefs;
memset(&prefs, 0, sizeof(prefs));
prefs.decode_tunnels = decode_tunnels;
@@ -650,7 +627,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
memset(&ndpi_thread_info[thread_id], 0, sizeof(ndpi_thread_info[thread_id]));
ndpi_thread_info[thread_id].workflow = ndpi_workflow_init(&prefs, pcap_handle, malloc_wrapper, free_wrapper, debug_printf);
/* ndpi_thread_info[thread_id].workflow->ndpi_struct->http_dont_dissect_response = 1; */
ndpi_workflow_set_flow_detected_callback(ndpi_thread_info[thread_id].workflow, on_protocol_discovered, (void *)&thread_id);
// enable all protocols
@@ -669,13 +646,6 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
/* ***************************************************** */
static void terminateDetection(u_int16_t thread_id) {
int i;
for(i=0; i<NUM_ROOTS; i++) {
ndpi_tdestroy(ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i], ndpi_flow_info_freer);
ndpi_thread_info[thread_id].workflow->ndpi_flows_root[i] = NULL;
}
ndpi_workflow_free(ndpi_thread_info[thread_id].workflow);
}
@@ -810,7 +780,7 @@ static void printResults(u_int64_t tot_usec) {
if(!quiet_mode) {
printf("\nnDPI Memory statistics:\n");
printf("\tnDPI Memory (once): %-13s\n", formatBytes(sizeof(struct ndpi_detection_module_struct), buf, sizeof(buf)));
printf("\tFlow Memory (per flow): %-13s\n", formatBytes(size_flow_struct, buf, sizeof(buf)));
printf("\tFlow Memory (per flow): %-13s\n", formatBytes(sizeof(struct ndpi_flow_struct), buf, sizeof(buf)));
printf("\tActual Memory: %-13s\n", formatBytes(current_ndpi_memory, buf, sizeof(buf)));
printf("\tPeak Memory: %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf)));
@@ -1112,7 +1082,7 @@ static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_fi
signal(SIGALRM, sigproc);
#endif
}
return pcap_handle;
}
@@ -1122,24 +1092,24 @@ static void pcap_packet_callback_checked(u_char *args,
const struct pcap_pkthdr *header,
const u_char *packet) {
u_int16_t thread_id = *((u_int16_t*)args);
/* allocate an exact size buffer to check overflows */
uint8_t *packet_checked = malloc(header->caplen);
memcpy(packet_checked, packet, header->caplen);
ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked);
if((capture_until != 0) && (header->ts.tv_sec >= capture_until)) {
if(ndpi_thread_info[thread_id].workflow->pcap_handle != NULL)
pcap_breakloop(ndpi_thread_info[thread_id].workflow->pcap_handle);
return;
}
/* Check if capture is live or not */
if (!live_capture) {
if (!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
}
/* Idle flows cleanup */
if(live_capture) {
if(ndpi_thread_info[thread_id].last_idle_scan_time + IDLE_SCAN_PERIOD < ndpi_thread_info[thread_id].workflow->last_time) {
@@ -1148,22 +1118,22 @@ static void pcap_packet_callback_checked(u_char *args,
/* remove idle flows (unfortunately we cannot do this inline) */
while (ndpi_thread_info[thread_id].num_idle_flows > 0) {
/* search and delete the idle flow from the "ndpi_flow_root" (see struct reader thread) - here flows are the node of a b-tree */
ndpi_tdelete(ndpi_thread_info[thread_id].idle_flows[--ndpi_thread_info[thread_id].num_idle_flows],
&ndpi_thread_info[thread_id].workflow->ndpi_flows_root[ndpi_thread_info[thread_id].idle_scan_idx],
ndpi_workflow_node_cmp);
/* free the memory associated to idle flow in "idle_flows" - (see struct reader thread)*/
free_ndpi_flow_info(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
ndpi_free_flow_info_half(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
ndpi_free(ndpi_thread_info[thread_id].idle_flows[ndpi_thread_info[thread_id].num_idle_flows]);
}
if(++ndpi_thread_info[thread_id].idle_scan_idx == ndpi_thread_info[thread_id].workflow->prefs.num_roots) ndpi_thread_info[thread_id].idle_scan_idx = 0;
ndpi_thread_info[thread_id].last_idle_scan_time = ndpi_thread_info[thread_id].workflow->last_time;
}
}
/* check for buffer changes */
if(memcmp(packet, packet_checked, header->caplen) != 0)
printf("INTERNAL ERROR: ingress packet was nodified by nDPI: this should not happen [thread_id=%u, packetId=%lu]\n",
@@ -1211,7 +1181,7 @@ void *processing_thread(void *_thread_id) {
goto pcap_loop;
}
}
return NULL;
}
+14 -8
View File
@@ -24,13 +24,13 @@
/**
* This module contains routines to help setup a simple nDPI program.
*
*
* If you concern about performance or have to integrate nDPI in your
* application, you could need to reimplement them yourself.
*
*
* WARNING: this API is unstable! Use it at your own risk!
*/
#ifndef __NDPI_UTIL_H__
#define __NDPI_UTIL_H__
@@ -93,15 +93,15 @@ typedef void (*ndpi_workflow_callback_ptr) (struct ndpi_workflow *, struct ndpi_
typedef struct ndpi_workflow {
u_int64_t last_time;
struct ndpi_workflow_prefs prefs;
struct ndpi_stats stats;
ndpi_workflow_callback_ptr __flow_detected_callback;
void * __flow_detected_udata;
ndpi_workflow_callback_ptr __flow_giveup_callback;
void * __flow_giveup_udata;
/* outside referencies */
pcap_t *pcap_handle;
@@ -116,14 +116,20 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre
void * (*malloc_wrapper)(size_t),
void (*free_wrapper)(void*),
ndpi_debug_function_ptr ndpi_debug_printf);
void ndpi_workflow_free(struct ndpi_workflow * workflow);
/** Free flow_info ndpi support structures but not the flow_info itself
*
* TODO remove! Half freeing things is bad!
*/
void ndpi_free_flow_info_half(struct ndpi_flow_info *flow);
/** Process a @packet and update the @workflow. */
void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
const struct pcap_pkthdr *header,
const u_char *packet);
/* flow callbacks: ndpi_flow_info will be freed right after */
static inline void ndpi_workflow_set_flow_detected_callback(struct ndpi_workflow * workflow,
ndpi_workflow_callback_ptr callback,
+47 -31
View File
@@ -21,7 +21,7 @@
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdlib.h>
#ifdef WIN32
@@ -71,7 +71,7 @@
/* ***************************************************** */
static void free_ndpi_flow_info(struct ndpi_flow_info *flow) {
void ndpi_free_flow_info_half(struct ndpi_flow_info *flow) {
if(flow->ndpi_flow) { ndpi_free_flow(flow->ndpi_flow); flow->ndpi_flow = NULL; }
if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; }
if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; }
@@ -91,29 +91,45 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre
void * (*malloc_wrapper)(size_t),
void (*free_wrapper)(void*),
ndpi_debug_function_ptr ndpi_debug_printf) {
/* TODO: just needed here to init ndpi malloc wrapper */
struct ndpi_detection_module_struct * module = ndpi_init_detection_module(
prefs->detection_tick_resolution, malloc_wrapper, free_wrapper, ndpi_debug_printf);
struct ndpi_workflow * workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
removeme_free_wrapper = free_wrapper;
workflow->pcap_handle = pcap_handle;
workflow->prefs = *prefs;
workflow->ndpi_struct = module;
if(workflow->ndpi_struct == NULL) {
NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
exit(-1);
}
workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *));
return workflow;
}
/* ***************************************************** */
static void ndpi_flow_info_freer(void *node) {
struct ndpi_flow_info *flow = (struct ndpi_flow_info*)node;
ndpi_free_flow_info_half(flow);
ndpi_free(flow);
}
/* ***************************************************** */
void ndpi_workflow_free(struct ndpi_workflow * workflow) {
int i;
for(i=0; i<workflow->prefs.num_roots; i++)
ndpi_tdestroy(workflow->ndpi_flows_root[i], ndpi_flow_info_freer);
ndpi_exit_detection_module(workflow->ndpi_struct, removeme_free_wrapper);
free(workflow->ndpi_flows_root);
free(workflow);
@@ -217,7 +233,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
// tcp
*tcph = (struct ndpi_tcphdr *)l4;
*sport = ntohs((*tcph)->source), *dport = ntohs((*tcph)->dest);
if(iph->saddr < iph->daddr) {
lower_port = (*tcph)->source, upper_port = (*tcph)->dest;
*src_to_dst_direction = 1;
@@ -235,7 +251,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
}
}
}
tcp_len = ndpi_min(4*(*tcph)->doff, l4_packet_len);
*payload = &l4[tcp_len];
*payload_len = ndpi_max(0, l4_packet_len-4*(*tcph)->doff);
@@ -244,7 +260,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
workflow->stats.udp_count++;
*udph = (struct ndpi_udphdr *)l4;
*sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest);
*sport = ntohs((*udph)->source), *dport = ntohs((*udph)->dest);
*payload = &l4[sizeof(struct ndpi_udphdr)];
*payload_len = ndpi_max(0, l4_packet_len-sizeof(struct ndpi_udphdr));
@@ -259,7 +275,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
if(iph->saddr == iph->daddr) {
if(lower_port > upper_port) {
u_int16_t p = lower_port;
lower_port = upper_port;
upper_port = p;
}
@@ -406,17 +422,17 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow,
u_int16_t sport, dport, payload_len;
u_int8_t *payload;
u_int8_t src_to_dst_direction= 1;
if(iph)
flow = get_ndpi_flow_info(workflow, 4, vlan_id, iph, NULL,
ip_offset, ipsize,
ntohs(iph->tot_len) - (iph->ihl * 4),
&tcph, &udph, &sport, &dport,
&tcph, &udph, &sport, &dport,
&src, &dst, &proto,
&payload, &payload_len, &src_to_dst_direction);
else
flow = get_ndpi_flow_info6(workflow, vlan_id, iph6, ip_offset,
&tcph, &udph, &sport, &dport,
&tcph, &udph, &sport, &dport,
&src, &dst, &proto,
&payload, &payload_len, &src_to_dst_direction);
@@ -436,7 +452,7 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow,
flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
if((flow->detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN)
|| ((proto == IPPROTO_UDP) && (flow->packets > 8))
|| ((proto == IPPROTO_TCP) && (flow->packets > 10))) {
@@ -450,15 +466,15 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow,
if(flow->detected_protocol.protocol == NDPI_PROTOCOL_BITTORRENT) {
int i, j, n = 0;
for(i=0, j = 0; i<20; i++) {
sprintf(&flow->bittorent_hash[j], "%02x", flow->ndpi_flow->bittorent_hash[i]);
j += 2, n += flow->ndpi_flow->bittorent_hash[i];
}
if(n == 0) flow->bittorent_hash[0] = '\0';
}
if((proto == IPPROTO_TCP) && (flow->detected_protocol.protocol != NDPI_PROTOCOL_DNS)) {
snprintf(flow->ssl.client_certificate, sizeof(flow->ssl.client_certificate), "%s", flow->ndpi_flow->protos.ssl.client_certificate);
snprintf(flow->ssl.server_certificate, sizeof(flow->ssl.server_certificate), "%s", flow->ndpi_flow->protos.ssl.server_certificate);
@@ -466,17 +482,17 @@ static unsigned int packet_processing(struct ndpi_workflow * workflow,
if(flow->detected_protocol.protocol == NDPI_PROTOCOL_UNKNOWN) {
flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct, flow->ndpi_flow);
if (workflow->__flow_giveup_callback != NULL)
workflow->__flow_giveup_callback(workflow, flow, workflow->__flow_giveup_udata);
} else {
if (workflow->__flow_detected_callback != NULL)
workflow->__flow_detected_callback(workflow, flow, workflow->__flow_detected_udata);
}
free_ndpi_flow_info(flow);
ndpi_free_flow_info_half(flow);
}
return 0;
}
@@ -487,7 +503,7 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
/*
* Declare pointers to packet headers
*/
/* --- Ethernet header --- */
const struct ndpi_ethhdr *ethernet;
/* --- Ethernet II header --- */
@@ -576,7 +592,7 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
ip_offset = sizeof(struct ndpi_chdlc); /* CHDLC_OFF = 4 */
type = ntohs(chdlc->proto_code);
break;
/* IEEE 802.3 Ethernet - 1 */
case DLT_EN10MB :
ethernet = (struct ndpi_ethhdr *) &packet[eth_offset];
@@ -654,13 +670,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
type = (packet[ip_offset+2] << 8) + packet[ip_offset+3];
ip_offset += 4;
vlan_packet = 1;
} else if(type == MPLS_UNI || type == MPLS_MULTI) {
} else if(type == MPLS_UNI || type == MPLS_MULTI) {
mpls = (struct ndpi_mpls_header *) &packet[ip_offset];
label = ntohl(mpls->label);
/* label = ntohl(*((u_int32_t*)&packet[ip_offset])); */
workflow->stats.mpls_count++;
type = ETH_P_IP, ip_offset += 4;
while((label & 0x100) != 0x100) {
ip_offset += 4;
label = ntohl(mpls->label);
@@ -676,13 +692,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
else if(type == CISCO_D_PROTO) {
cdp = (struct ndpi_cdp *) &packet[ip_offset];
cdp_pkts++;
}
}
else if(type == PPPoE) {
workflow->stats.pppoe_count++;
type = ETH_P_IP;
ip_offset += 8;
}
workflow->stats.vlan_count += vlan_packet;
iph_check:
@@ -787,13 +803,13 @@ void ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
if((version == 1) && (type == 0) && (encapsulates == 1)) {
u_int8_t stop = 0;
offset += 4;
while((!stop) && (offset < header->caplen)) {
u_int8_t tag_type = packet[offset];
u_int8_t tag_len;
switch(tag_type) {
case 0: /* PADDING Tag */
tag_len = 1;