Remove struct ndpi_id_struct (#1427)
Remove the last uses of `struct ndpi_id_struct`. That code is not really used and it has not been updated for a very long time: see #1279 for details. Correlation among flows is achieved via LRU caches. This change allows to further reduce memory consumption (see also91bb77a8). At nDPI 4.0 (more precisly, ata6b10cf, because memory stats were wrong until that commit): ``` nDPI Memory statistics: nDPI Memory (once): 221.15 KB Flow Memory (per flow): 2.94 KB ``` Now: ``` nDPI Memory statistics: nDPI Memory (once): 235.27 KB Flow Memory (per flow): 688 B <-------- ``` i.e. memory usage per flow has been reduced by 77%. Close #1279
This commit is contained in:
@@ -2911,7 +2911,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
|
||||
if(!quiet_mode) {
|
||||
printf("\nnDPI Memory statistics:\n");
|
||||
printf("\tnDPI Memory (once): %-13s\n", formatBytes(ndpi_get_ndpi_detection_module_size(), buf, sizeof(buf)));
|
||||
printf("\tFlow Memory (per flow): %-13s\n", formatBytes( ndpi_detection_get_sizeof_ndpi_flow_struct() + 2*ndpi_detection_get_sizeof_ndpi_id_struct(), buf, sizeof(buf)));
|
||||
printf("\tFlow Memory (per flow): %-13s\n", formatBytes( ndpi_detection_get_sizeof_ndpi_flow_struct(), buf, sizeof(buf)));
|
||||
printf("\tActual Memory: %-13s\n", formatBytes(current_ndpi_memory, buf, sizeof(buf)));
|
||||
printf("\tPeak Memory: %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf)));
|
||||
printf("\tSetup Time: %lu msec\n", (unsigned long)(setup_time_usec/1000));
|
||||
|
||||
@@ -85,8 +85,6 @@ struct nDPI_flow_info {
|
||||
struct ndpi_proto guessed_protocol;
|
||||
|
||||
struct ndpi_flow_struct * ndpi_flow;
|
||||
struct ndpi_id_struct * ndpi_src;
|
||||
struct ndpi_id_struct * ndpi_dst;
|
||||
};
|
||||
|
||||
struct nDPI_workflow {
|
||||
@@ -190,8 +188,6 @@ static void ndpi_flow_info_freer(void * const node)
|
||||
{
|
||||
struct nDPI_flow_info * const flow = (struct nDPI_flow_info *)node;
|
||||
|
||||
ndpi_free(flow->ndpi_dst);
|
||||
ndpi_free(flow->ndpi_src);
|
||||
ndpi_flow_free(flow->ndpi_flow);
|
||||
ndpi_free(flow);
|
||||
}
|
||||
@@ -505,10 +501,6 @@ static void ndpi_process_packet(uint8_t * const args,
|
||||
void * tree_result;
|
||||
struct nDPI_flow_info * flow_to_process;
|
||||
|
||||
int direction_changed = 0;
|
||||
struct ndpi_id_struct * ndpi_src;
|
||||
struct ndpi_id_struct * ndpi_dst;
|
||||
|
||||
const struct ndpi_ethhdr * ethernet;
|
||||
const struct ndpi_iphdr * ip;
|
||||
struct ndpi_ipv6hdr * ip6;
|
||||
@@ -753,9 +745,6 @@ static void ndpi_process_packet(uint8_t * const args,
|
||||
flow.dst_port = orig_src_port;
|
||||
|
||||
tree_result = ndpi_tfind(&flow, &workflow->ndpi_flows_active[hashed_index], ndpi_workflow_node_cmp);
|
||||
if (tree_result != NULL) {
|
||||
direction_changed = 1;
|
||||
}
|
||||
|
||||
flow.ip_tuple.u32.src[0] = orig_src_ip[0];
|
||||
flow.ip_tuple.u32.src[1] = orig_src_ip[1];
|
||||
@@ -800,20 +789,6 @@ static void ndpi_process_packet(uint8_t * const args,
|
||||
}
|
||||
memset(flow_to_process->ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
|
||||
|
||||
flow_to_process->ndpi_src = (struct ndpi_id_struct *)ndpi_calloc(1, SIZEOF_ID_STRUCT);
|
||||
if (flow_to_process->ndpi_src == NULL) {
|
||||
fprintf(stderr, "[%8llu, %d, %4u] Not enough memory for src id struct\n",
|
||||
workflow->packets_captured, reader_thread->array_index, flow_to_process->flow_id);
|
||||
return;
|
||||
}
|
||||
|
||||
flow_to_process->ndpi_dst = (struct ndpi_id_struct *)ndpi_calloc(1, SIZEOF_ID_STRUCT);
|
||||
if (flow_to_process->ndpi_dst == NULL) {
|
||||
fprintf(stderr, "[%8llu, %d, %4u] Not enough memory for dst id struct\n",
|
||||
workflow->packets_captured, reader_thread->array_index, flow_to_process->flow_id);
|
||||
return;
|
||||
}
|
||||
|
||||
printf("[%8llu, %d, %4u] new %sflow\n", workflow->packets_captured, thread_index,
|
||||
flow_to_process->flow_id,
|
||||
(flow_to_process->is_midstream_flow != 0 ? "midstream-" : ""));
|
||||
@@ -821,19 +796,8 @@ static void ndpi_process_packet(uint8_t * const args,
|
||||
/* Possible Leak, but should not happen as we'd abort earlier. */
|
||||
return;
|
||||
}
|
||||
|
||||
ndpi_src = flow_to_process->ndpi_src;
|
||||
ndpi_dst = flow_to_process->ndpi_dst;
|
||||
} else {
|
||||
flow_to_process = *(struct nDPI_flow_info **)tree_result;
|
||||
|
||||
if (direction_changed != 0) {
|
||||
ndpi_src = flow_to_process->ndpi_dst;
|
||||
ndpi_dst = flow_to_process->ndpi_src;
|
||||
} else {
|
||||
ndpi_src = flow_to_process->ndpi_src;
|
||||
ndpi_dst = flow_to_process->ndpi_dst;
|
||||
}
|
||||
}
|
||||
|
||||
flow_to_process->packets_processed++;
|
||||
@@ -884,7 +848,7 @@ static void ndpi_process_packet(uint8_t * const args,
|
||||
flow_to_process->detected_l7_protocol =
|
||||
ndpi_detection_process_packet(workflow->ndpi_struct, flow_to_process->ndpi_flow,
|
||||
ip != NULL ? (uint8_t *)ip : (uint8_t *)ip6,
|
||||
ip_size, time_ms, ndpi_src, ndpi_dst);
|
||||
ip_size, time_ms);
|
||||
|
||||
if (ndpi_is_protocol_detected(workflow->ndpi_struct,
|
||||
flow_to_process->detected_l7_protocol) != 0 &&
|
||||
|
||||
+8
-41
@@ -300,8 +300,6 @@ void ndpi_report_payload_stats() {
|
||||
|
||||
void ndpi_free_flow_info_half(struct ndpi_flow_info *flow) {
|
||||
if(flow->ndpi_flow) { ndpi_flow_free(flow->ndpi_flow); flow->ndpi_flow = NULL; }
|
||||
if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; }
|
||||
if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; }
|
||||
}
|
||||
|
||||
/* ***************************************************** */
|
||||
@@ -699,8 +697,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
|
||||
struct ndpi_tcphdr **tcph,
|
||||
struct ndpi_udphdr **udph,
|
||||
u_int16_t *sport, u_int16_t *dport,
|
||||
struct ndpi_id_struct **src,
|
||||
struct ndpi_id_struct **dst,
|
||||
u_int8_t *proto,
|
||||
u_int8_t **payload,
|
||||
u_int16_t *payload_len,
|
||||
@@ -878,30 +874,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
|
||||
} else
|
||||
memset(newflow->ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
|
||||
|
||||
if((newflow->src_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) {
|
||||
LOG(NDPI_LOG_ERROR, "[NDPI] %s(3): not enough memory\n", __FUNCTION__);
|
||||
#ifdef DIRECTION_BINS
|
||||
ndpi_free_bin(&newflow->payload_len_bin_src2dst), ndpi_free_bin(&newflow->payload_len_bin_dst2src);
|
||||
#else
|
||||
ndpi_free_bin(&newflow->payload_len_bin);
|
||||
#endif
|
||||
ndpi_free(newflow);
|
||||
return(NULL);
|
||||
} else
|
||||
memset(newflow->src_id, 0, SIZEOF_ID_STRUCT);
|
||||
|
||||
if((newflow->dst_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) {
|
||||
LOG(NDPI_LOG_ERROR, "[NDPI] %s(4): not enough memory\n", __FUNCTION__);
|
||||
#ifdef DIRECTION_BINS
|
||||
ndpi_free_bin(&newflow->payload_len_bin_src2dst), ndpi_free_bin(&newflow->payload_len_bin_dst2src);
|
||||
#else
|
||||
ndpi_free_bin(&newflow->payload_len_bin);
|
||||
#endif
|
||||
ndpi_free(newflow);
|
||||
return(NULL);
|
||||
} else
|
||||
memset(newflow->dst_id, 0, SIZEOF_ID_STRUCT);
|
||||
|
||||
ndpi_tsearch(newflow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp); /* Add */
|
||||
workflow->stats.ndpi_flow_count++;
|
||||
if(*proto == IPPROTO_TCP)
|
||||
@@ -911,8 +883,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
|
||||
else
|
||||
workflow->stats.flow_count[2]++;
|
||||
|
||||
*src = newflow->src_id, *dst = newflow->dst_id;
|
||||
|
||||
if(enable_flow_stats) {
|
||||
newflow->entropy = ndpi_calloc(1, sizeof(struct ndpi_entropy));
|
||||
newflow->last_entropy = ndpi_calloc(1, sizeof(struct ndpi_entropy));
|
||||
@@ -939,9 +909,9 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
|
||||
&& rflow->src_port == htons(*sport)
|
||||
&& rflow->dst_port == htons(*dport)
|
||||
)
|
||||
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
|
||||
*src_to_dst_direction = 0, rflow->bidirectional = 1;
|
||||
else
|
||||
*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
|
||||
*src_to_dst_direction = 1;
|
||||
}
|
||||
else {
|
||||
if(rflow->src_ip == iph->saddr
|
||||
@@ -949,9 +919,9 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
|
||||
&& rflow->src_port == htons(*sport)
|
||||
&& rflow->dst_port == htons(*dport)
|
||||
)
|
||||
*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
|
||||
*src_to_dst_direction = 1;
|
||||
else
|
||||
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
|
||||
*src_to_dst_direction = 0, rflow->bidirectional = 1;
|
||||
}
|
||||
if(enable_flow_stats) {
|
||||
if(src_to_dst_direction) {
|
||||
@@ -997,8 +967,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
|
||||
struct ndpi_tcphdr **tcph,
|
||||
struct ndpi_udphdr **udph,
|
||||
u_int16_t *sport, u_int16_t *dport,
|
||||
struct ndpi_id_struct **src,
|
||||
struct ndpi_id_struct **dst,
|
||||
u_int8_t *proto,
|
||||
u_int8_t **payload,
|
||||
u_int16_t *payload_len,
|
||||
@@ -1024,7 +992,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
|
||||
&iph, iph6, ip_offset, ipsize,
|
||||
ip_len, l4ptr - (const u_int8_t *)iph6,
|
||||
tcph, udph, sport, dport,
|
||||
src, dst, proto, payload,
|
||||
proto, payload,
|
||||
payload_len, src_to_dst_direction, when));
|
||||
}
|
||||
|
||||
@@ -1347,7 +1315,6 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
pkt_timeval when,
|
||||
ndpi_risk *flow_risk,
|
||||
FILE * csv_fp) {
|
||||
struct ndpi_id_struct *src, *dst;
|
||||
struct ndpi_flow_info *flow = NULL;
|
||||
struct ndpi_flow_struct *ndpi_flow = NULL;
|
||||
u_int8_t proto;
|
||||
@@ -1369,13 +1336,13 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
ntohs(iph->tot_len) - (iph->ihl * 4),
|
||||
iph->ihl * 4,
|
||||
&tcph, &udph, &sport, &dport,
|
||||
&src, &dst, &proto,
|
||||
&proto,
|
||||
&payload, &payload_len, &src_to_dst_direction, when);
|
||||
else
|
||||
flow = get_ndpi_flow_info6(workflow, vlan_id,
|
||||
tunnel_type, iph6, ip_offset, ipsize,
|
||||
&tcph, &udph, &sport, &dport,
|
||||
&src, &dst, &proto,
|
||||
&proto,
|
||||
&payload, &payload_len, &src_to_dst_direction, when);
|
||||
|
||||
if(flow != NULL) {
|
||||
@@ -1563,7 +1530,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
|
||||
flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
|
||||
iph ? (uint8_t *)iph : (uint8_t *)iph6,
|
||||
ipsize, time_ms, src, dst);
|
||||
ipsize, time_ms);
|
||||
|
||||
if(enough_packets || (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN)) {
|
||||
if((!enough_packets)
|
||||
|
||||
Reference in New Issue
Block a user