Reduce memory used by ndpiReader (#1371)
`ndpiReader` is only an example, aiming to show nDPI capabilities
and integration, without any claim about performances.
Nonetheless its memory usage per flow is *huge*, limiting the kinds
of traces that we can test on a "normal" hardware (example: scan
attacks).
The key reason of that behaviour is that we preallocate all the memory
needed for *all* the available features.
Try to reduce memory usage simply allocating some structures only
when they are really needed. Most significant example: JOY algorithms.
This way we should use a lot less memory in the two most common
user-cases:
* `ndpiReader` invoked without any particular flag (i.e `ndpiReader -i
$FILENAME_OR_IFACE`)
* internal unit tests
Before (on x86_64):
```
struct ndpi_flow_info {
[...]
/* size: 7320, cachelines: 115, members: 72 */
```
After:
```
struct ndpi_flow_info {
[...]
/* size: 2128, cachelines: 34, members: 75 */
```
This commit is contained in:
+25
-22
@@ -365,47 +365,50 @@ flowGetBDMeanandVariance(struct ndpi_flow_info* flow) {
|
||||
uint32_t tmp[256], i;
|
||||
unsigned int num_bytes;
|
||||
double mean = 0.0, variance = 0.0;
|
||||
struct ndpi_entropy last_entropy = flow->last_entropy;
|
||||
struct ndpi_entropy *last_entropy = flow->last_entropy;
|
||||
|
||||
fflush(out);
|
||||
|
||||
if(!last_entropy)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Sum up the byte_count array for outbound and inbound flows,
|
||||
* if this flow is bidirectional
|
||||
*/
|
||||
if (!flow->bidirectional) {
|
||||
array = last_entropy.src2dst_byte_count;
|
||||
num_bytes = last_entropy.src2dst_l4_bytes;
|
||||
array = last_entropy->src2dst_byte_count;
|
||||
num_bytes = last_entropy->src2dst_l4_bytes;
|
||||
for (i=0; i<256; i++) {
|
||||
tmp[i] = last_entropy.src2dst_byte_count[i];
|
||||
tmp[i] = last_entropy->src2dst_byte_count[i];
|
||||
}
|
||||
|
||||
if (last_entropy.src2dst_num_bytes != 0) {
|
||||
mean = last_entropy.src2dst_bd_mean;
|
||||
variance = last_entropy.src2dst_bd_variance/(last_entropy.src2dst_num_bytes - 1);
|
||||
if (last_entropy->src2dst_num_bytes != 0) {
|
||||
mean = last_entropy->src2dst_bd_mean;
|
||||
variance = last_entropy->src2dst_bd_variance/(last_entropy->src2dst_num_bytes - 1);
|
||||
variance = sqrt(variance);
|
||||
|
||||
if (last_entropy.src2dst_num_bytes == 1) {
|
||||
if (last_entropy->src2dst_num_bytes == 1) {
|
||||
variance = 0.0;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (i=0; i<256; i++) {
|
||||
tmp[i] = last_entropy.src2dst_byte_count[i] + last_entropy.dst2src_byte_count[i];
|
||||
tmp[i] = last_entropy->src2dst_byte_count[i] + last_entropy->dst2src_byte_count[i];
|
||||
}
|
||||
array = tmp;
|
||||
num_bytes = last_entropy.src2dst_l4_bytes + last_entropy.dst2src_l4_bytes;
|
||||
num_bytes = last_entropy->src2dst_l4_bytes + last_entropy->dst2src_l4_bytes;
|
||||
|
||||
if (last_entropy.src2dst_num_bytes + last_entropy.dst2src_num_bytes != 0) {
|
||||
mean = ((double)last_entropy.src2dst_num_bytes)/((double)(last_entropy.src2dst_num_bytes+last_entropy.dst2src_num_bytes))*last_entropy.src2dst_bd_mean +
|
||||
((double)last_entropy.dst2src_num_bytes)/((double)(last_entropy.dst2src_num_bytes+last_entropy.src2dst_num_bytes))*last_entropy.dst2src_bd_mean;
|
||||
if (last_entropy->src2dst_num_bytes + last_entropy->dst2src_num_bytes != 0) {
|
||||
mean = ((double)last_entropy->src2dst_num_bytes)/((double)(last_entropy->src2dst_num_bytes+last_entropy->dst2src_num_bytes))*last_entropy->src2dst_bd_mean +
|
||||
((double)last_entropy->dst2src_num_bytes)/((double)(last_entropy->dst2src_num_bytes+last_entropy->src2dst_num_bytes))*last_entropy->dst2src_bd_mean;
|
||||
|
||||
variance = ((double)last_entropy.src2dst_num_bytes)/((double)(last_entropy.src2dst_num_bytes+last_entropy.dst2src_num_bytes))*last_entropy.src2dst_bd_variance +
|
||||
((double)last_entropy.dst2src_num_bytes)/((double)(last_entropy.dst2src_num_bytes+last_entropy.src2dst_num_bytes))*last_entropy.dst2src_bd_variance;
|
||||
variance = ((double)last_entropy->src2dst_num_bytes)/((double)(last_entropy->src2dst_num_bytes+last_entropy->dst2src_num_bytes))*last_entropy->src2dst_bd_variance +
|
||||
((double)last_entropy->dst2src_num_bytes)/((double)(last_entropy->dst2src_num_bytes+last_entropy->src2dst_num_bytes))*last_entropy->dst2src_bd_variance;
|
||||
|
||||
variance = variance/((double)(last_entropy.src2dst_num_bytes + last_entropy.dst2src_num_bytes - 1));
|
||||
variance = variance/((double)(last_entropy->src2dst_num_bytes + last_entropy->dst2src_num_bytes - 1));
|
||||
variance = sqrt(variance);
|
||||
if (last_entropy.src2dst_num_bytes + last_entropy.dst2src_num_bytes == 1) {
|
||||
if (last_entropy->src2dst_num_bytes + last_entropy->dst2src_num_bytes == 1) {
|
||||
variance = 0.0;
|
||||
}
|
||||
}
|
||||
@@ -1378,7 +1381,7 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
/* Print entropy values for monitored flows. */
|
||||
flowGetBDMeanandVariance(flow);
|
||||
fflush(out);
|
||||
fprintf(out, "[score: %.4f]", flow->entropy.score);
|
||||
fprintf(out, "[score: %.4f]", flow->entropy->score);
|
||||
}
|
||||
|
||||
if(csv_fp) fprintf(csv_fp, "\n");
|
||||
@@ -1416,8 +1419,8 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
else
|
||||
fprintf(out, "[< 1 sec]");
|
||||
|
||||
if(flow->telnet.username[0] != '\0') fprintf(out, "[Username: %s]", flow->telnet.username);
|
||||
if(flow->telnet.password[0] != '\0') fprintf(out, "[Password: %s]", flow->telnet.password);
|
||||
if(flow->telnet.username) fprintf(out, "[Username: %s]", flow->telnet.username);
|
||||
if(flow->telnet.password) fprintf(out, "[Password: %s]", flow->telnet.password);
|
||||
if(flow->host_server_name[0] != '\0') fprintf(out, "[Host: %s]", flow->host_server_name);
|
||||
|
||||
if(flow->info[0] != '\0') fprintf(out, "[%s]", flow->info);
|
||||
@@ -1532,9 +1535,9 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
|
||||
if(flow->ssh_tls.server_cipher != '\0') fprintf(out, "[Cipher: %s]",
|
||||
ndpi_cipher2str(flow->ssh_tls.server_cipher));
|
||||
if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]",
|
||||
if(flow->bittorent_hash) fprintf(out, "[BT Hash: %s]",
|
||||
flow->bittorent_hash);
|
||||
if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]",
|
||||
if(flow->dhcp_fingerprint) fprintf(out, "[DHCP Fingerprint: %s]",
|
||||
flow->dhcp_fingerprint);
|
||||
|
||||
if(flow->has_human_readeable_strings) fprintf(out, "[PLAIN TEXT (%s)]",
|
||||
|
||||
Reference in New Issue
Block a user