Remove struct ndpi_id_struct (#1427)

Remove the last uses of `struct ndpi_id_struct`.
That code is not really used and it has not been updated for a very long
time: see #1279 for details.

Correlation among flows is achieved via LRU caches.

This change allows to further reduce memory consumption (see also
91bb77a8).

At nDPI 4.0 (more precisly, at a6b10cf, because memory stats
were wrong until that commit):
```
nDPI Memory statistics:
	nDPI Memory (once):      221.15 KB
	Flow Memory (per flow):  2.94 KB
```
Now:
```
nDPI Memory statistics:
	nDPI Memory (once):      235.27 KB
	Flow Memory (per flow):  688 B        <--------
```
i.e. memory usage per flow has been reduced by 77%.

Close #1279
This commit is contained in:
Ivan Nardi
2022-01-30 19:18:12 +01:00
committed by GitHub
parent 1fdcddb33d
commit 5bb5bec477
14 changed files with 31 additions and 843 deletions
+1 -1
View File
@@ -2911,7 +2911,7 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
if(!quiet_mode) {
printf("\nnDPI Memory statistics:\n");
printf("\tnDPI Memory (once): %-13s\n", formatBytes(ndpi_get_ndpi_detection_module_size(), buf, sizeof(buf)));
printf("\tFlow Memory (per flow): %-13s\n", formatBytes( ndpi_detection_get_sizeof_ndpi_flow_struct() + 2*ndpi_detection_get_sizeof_ndpi_id_struct(), buf, sizeof(buf)));
printf("\tFlow Memory (per flow): %-13s\n", formatBytes( ndpi_detection_get_sizeof_ndpi_flow_struct(), buf, sizeof(buf)));
printf("\tActual Memory: %-13s\n", formatBytes(current_ndpi_memory, buf, sizeof(buf)));
printf("\tPeak Memory: %-13s\n", formatBytes(max_ndpi_memory, buf, sizeof(buf)));
printf("\tSetup Time: %lu msec\n", (unsigned long)(setup_time_usec/1000));
+1 -37
View File
@@ -85,8 +85,6 @@ struct nDPI_flow_info {
struct ndpi_proto guessed_protocol;
struct ndpi_flow_struct * ndpi_flow;
struct ndpi_id_struct * ndpi_src;
struct ndpi_id_struct * ndpi_dst;
};
struct nDPI_workflow {
@@ -190,8 +188,6 @@ static void ndpi_flow_info_freer(void * const node)
{
struct nDPI_flow_info * const flow = (struct nDPI_flow_info *)node;
ndpi_free(flow->ndpi_dst);
ndpi_free(flow->ndpi_src);
ndpi_flow_free(flow->ndpi_flow);
ndpi_free(flow);
}
@@ -505,10 +501,6 @@ static void ndpi_process_packet(uint8_t * const args,
void * tree_result;
struct nDPI_flow_info * flow_to_process;
int direction_changed = 0;
struct ndpi_id_struct * ndpi_src;
struct ndpi_id_struct * ndpi_dst;
const struct ndpi_ethhdr * ethernet;
const struct ndpi_iphdr * ip;
struct ndpi_ipv6hdr * ip6;
@@ -753,9 +745,6 @@ static void ndpi_process_packet(uint8_t * const args,
flow.dst_port = orig_src_port;
tree_result = ndpi_tfind(&flow, &workflow->ndpi_flows_active[hashed_index], ndpi_workflow_node_cmp);
if (tree_result != NULL) {
direction_changed = 1;
}
flow.ip_tuple.u32.src[0] = orig_src_ip[0];
flow.ip_tuple.u32.src[1] = orig_src_ip[1];
@@ -800,20 +789,6 @@ static void ndpi_process_packet(uint8_t * const args,
}
memset(flow_to_process->ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
flow_to_process->ndpi_src = (struct ndpi_id_struct *)ndpi_calloc(1, SIZEOF_ID_STRUCT);
if (flow_to_process->ndpi_src == NULL) {
fprintf(stderr, "[%8llu, %d, %4u] Not enough memory for src id struct\n",
workflow->packets_captured, reader_thread->array_index, flow_to_process->flow_id);
return;
}
flow_to_process->ndpi_dst = (struct ndpi_id_struct *)ndpi_calloc(1, SIZEOF_ID_STRUCT);
if (flow_to_process->ndpi_dst == NULL) {
fprintf(stderr, "[%8llu, %d, %4u] Not enough memory for dst id struct\n",
workflow->packets_captured, reader_thread->array_index, flow_to_process->flow_id);
return;
}
printf("[%8llu, %d, %4u] new %sflow\n", workflow->packets_captured, thread_index,
flow_to_process->flow_id,
(flow_to_process->is_midstream_flow != 0 ? "midstream-" : ""));
@@ -821,19 +796,8 @@ static void ndpi_process_packet(uint8_t * const args,
/* Possible Leak, but should not happen as we'd abort earlier. */
return;
}
ndpi_src = flow_to_process->ndpi_src;
ndpi_dst = flow_to_process->ndpi_dst;
} else {
flow_to_process = *(struct nDPI_flow_info **)tree_result;
if (direction_changed != 0) {
ndpi_src = flow_to_process->ndpi_dst;
ndpi_dst = flow_to_process->ndpi_src;
} else {
ndpi_src = flow_to_process->ndpi_src;
ndpi_dst = flow_to_process->ndpi_dst;
}
}
flow_to_process->packets_processed++;
@@ -884,7 +848,7 @@ static void ndpi_process_packet(uint8_t * const args,
flow_to_process->detected_l7_protocol =
ndpi_detection_process_packet(workflow->ndpi_struct, flow_to_process->ndpi_flow,
ip != NULL ? (uint8_t *)ip : (uint8_t *)ip6,
ip_size, time_ms, ndpi_src, ndpi_dst);
ip_size, time_ms);
if (ndpi_is_protocol_detected(workflow->ndpi_struct,
flow_to_process->detected_l7_protocol) != 0 &&
+8 -41
View File
@@ -300,8 +300,6 @@ void ndpi_report_payload_stats() {
void ndpi_free_flow_info_half(struct ndpi_flow_info *flow) {
if(flow->ndpi_flow) { ndpi_flow_free(flow->ndpi_flow); flow->ndpi_flow = NULL; }
if(flow->src_id) { ndpi_free(flow->src_id); flow->src_id = NULL; }
if(flow->dst_id) { ndpi_free(flow->dst_id); flow->dst_id = NULL; }
}
/* ***************************************************** */
@@ -699,8 +697,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
struct ndpi_tcphdr **tcph,
struct ndpi_udphdr **udph,
u_int16_t *sport, u_int16_t *dport,
struct ndpi_id_struct **src,
struct ndpi_id_struct **dst,
u_int8_t *proto,
u_int8_t **payload,
u_int16_t *payload_len,
@@ -878,30 +874,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
} else
memset(newflow->ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
if((newflow->src_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) {
LOG(NDPI_LOG_ERROR, "[NDPI] %s(3): not enough memory\n", __FUNCTION__);
#ifdef DIRECTION_BINS
ndpi_free_bin(&newflow->payload_len_bin_src2dst), ndpi_free_bin(&newflow->payload_len_bin_dst2src);
#else
ndpi_free_bin(&newflow->payload_len_bin);
#endif
ndpi_free(newflow);
return(NULL);
} else
memset(newflow->src_id, 0, SIZEOF_ID_STRUCT);
if((newflow->dst_id = ndpi_malloc(SIZEOF_ID_STRUCT)) == NULL) {
LOG(NDPI_LOG_ERROR, "[NDPI] %s(4): not enough memory\n", __FUNCTION__);
#ifdef DIRECTION_BINS
ndpi_free_bin(&newflow->payload_len_bin_src2dst), ndpi_free_bin(&newflow->payload_len_bin_dst2src);
#else
ndpi_free_bin(&newflow->payload_len_bin);
#endif
ndpi_free(newflow);
return(NULL);
} else
memset(newflow->dst_id, 0, SIZEOF_ID_STRUCT);
ndpi_tsearch(newflow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp); /* Add */
workflow->stats.ndpi_flow_count++;
if(*proto == IPPROTO_TCP)
@@ -911,8 +883,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
else
workflow->stats.flow_count[2]++;
*src = newflow->src_id, *dst = newflow->dst_id;
if(enable_flow_stats) {
newflow->entropy = ndpi_calloc(1, sizeof(struct ndpi_entropy));
newflow->last_entropy = ndpi_calloc(1, sizeof(struct ndpi_entropy));
@@ -939,9 +909,9 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
&& rflow->src_port == htons(*sport)
&& rflow->dst_port == htons(*dport)
)
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
*src_to_dst_direction = 0, rflow->bidirectional = 1;
else
*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
*src_to_dst_direction = 1;
}
else {
if(rflow->src_ip == iph->saddr
@@ -949,9 +919,9 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
&& rflow->src_port == htons(*sport)
&& rflow->dst_port == htons(*dport)
)
*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
*src_to_dst_direction = 1;
else
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
*src_to_dst_direction = 0, rflow->bidirectional = 1;
}
if(enable_flow_stats) {
if(src_to_dst_direction) {
@@ -997,8 +967,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
struct ndpi_tcphdr **tcph,
struct ndpi_udphdr **udph,
u_int16_t *sport, u_int16_t *dport,
struct ndpi_id_struct **src,
struct ndpi_id_struct **dst,
u_int8_t *proto,
u_int8_t **payload,
u_int16_t *payload_len,
@@ -1024,7 +992,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
&iph, iph6, ip_offset, ipsize,
ip_len, l4ptr - (const u_int8_t *)iph6,
tcph, udph, sport, dport,
src, dst, proto, payload,
proto, payload,
payload_len, src_to_dst_direction, when));
}
@@ -1347,7 +1315,6 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
pkt_timeval when,
ndpi_risk *flow_risk,
FILE * csv_fp) {
struct ndpi_id_struct *src, *dst;
struct ndpi_flow_info *flow = NULL;
struct ndpi_flow_struct *ndpi_flow = NULL;
u_int8_t proto;
@@ -1369,13 +1336,13 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
ntohs(iph->tot_len) - (iph->ihl * 4),
iph->ihl * 4,
&tcph, &udph, &sport, &dport,
&src, &dst, &proto,
&proto,
&payload, &payload_len, &src_to_dst_direction, when);
else
flow = get_ndpi_flow_info6(workflow, vlan_id,
tunnel_type, iph6, ip_offset, ipsize,
&tcph, &udph, &sport, &dport,
&src, &dst, &proto,
&proto,
&payload, &payload_len, &src_to_dst_direction, when);
if(flow != NULL) {
@@ -1563,7 +1530,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time_ms, src, dst);
ipsize, time_ms);
if(enough_packets || (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN)) {
if((!enough_packets)