Reduce memory used by ndpiReader (#1371)

`ndpiReader` is only an example, aiming to show nDPI capabilities
and integration, without any claim about performances.

Nonetheless its memory usage per flow is *huge*, limiting the kinds
of traces that we can test on a "normal" hardware (example: scan
attacks).

The key reason of that behaviour is that we preallocate all the memory
needed for *all* the available features.

Try to reduce memory usage simply allocating some structures only
when they are really needed. Most significant example: JOY algorithms.

This way we should use a lot less memory in the two most common
user-cases:
 * `ndpiReader` invoked without any particular flag (i.e `ndpiReader -i
$FILENAME_OR_IFACE`)
 * internal unit tests

Before (on x86_64):
```
struct ndpi_flow_info {
[...]
	/* size: 7320, cachelines: 115, members: 72 */
```
After:
```
struct ndpi_flow_info {
[...]
	/* size: 2128, cachelines: 34, members: 75 */
```
This commit is contained in:
Ivan Nardi
2021-11-11 12:37:25 +01:00
committed by GitHub
parent 3e5491fa10
commit acb1de69aa
3 changed files with 156 additions and 111 deletions
+6 -6
View File
@@ -134,7 +134,6 @@ struct flow_metrics {
struct ndpi_entropy {
// Entropy fields
pkt_timeval src2dst_last_pkt_time, dst2src_last_pkt_time, flow_last_pkt_time;
u_int16_t src2dst_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
pkt_timeval src2dst_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */
u_int16_t dst2src_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
@@ -192,14 +191,15 @@ typedef struct ndpi_flow_info {
ndpi_protocol detected_protocol;
// Flow data analysis
pkt_timeval src2dst_last_pkt_time, dst2src_last_pkt_time, flow_last_pkt_time;
struct ndpi_analyze_struct *iat_c_to_s, *iat_s_to_c, *iat_flow,
*pktlen_c_to_s, *pktlen_s_to_c;
char info[255];
char flow_extra_info[16];
char host_server_name[240];
char bittorent_hash[41];
char dhcp_fingerprint[48];
char *bittorent_hash;
char *dhcp_fingerprint;
ndpi_risk risk;
struct {
@@ -229,13 +229,13 @@ typedef struct ndpi_flow_info {
} http;
struct {
char username[32], password[32];
char *username, *password;
} telnet;
void *src_id, *dst_id;
struct ndpi_entropy entropy;
struct ndpi_entropy last_entropy;
struct ndpi_entropy *entropy;
struct ndpi_entropy *last_entropy;
/* Payload lenght bins */
#ifdef DIRECTION_BINS