Reduce memory used by ndpiReader (#1371)

`ndpiReader` is only an example, aiming to show nDPI capabilities
and integration, without any claim about performances.

Nonetheless its memory usage per flow is *huge*, limiting the kinds
of traces that we can test on a "normal" hardware (example: scan
attacks).

The key reason of that behaviour is that we preallocate all the memory
needed for *all* the available features.

Try to reduce memory usage simply allocating some structures only
when they are really needed. Most significant example: JOY algorithms.

This way we should use a lot less memory in the two most common
user-cases:
 * `ndpiReader` invoked without any particular flag (i.e `ndpiReader -i
$FILENAME_OR_IFACE`)
 * internal unit tests

Before (on x86_64):
```
struct ndpi_flow_info {
[...]
	/* size: 7320, cachelines: 115, members: 72 */
```
After:
```
struct ndpi_flow_info {
[...]
	/* size: 2128, cachelines: 34, members: 75 */
```
This commit is contained in:
Ivan Nardi
2021-11-11 12:37:25 +01:00
committed by GitHub
parent 3e5491fa10
commit acb1de69aa
3 changed files with 156 additions and 111 deletions
+25 -22
View File
@@ -365,47 +365,50 @@ flowGetBDMeanandVariance(struct ndpi_flow_info* flow) {
uint32_t tmp[256], i;
unsigned int num_bytes;
double mean = 0.0, variance = 0.0;
struct ndpi_entropy last_entropy = flow->last_entropy;
struct ndpi_entropy *last_entropy = flow->last_entropy;
fflush(out);
if(!last_entropy)
return;
/*
* Sum up the byte_count array for outbound and inbound flows,
* if this flow is bidirectional
*/
if (!flow->bidirectional) {
array = last_entropy.src2dst_byte_count;
num_bytes = last_entropy.src2dst_l4_bytes;
array = last_entropy->src2dst_byte_count;
num_bytes = last_entropy->src2dst_l4_bytes;
for (i=0; i<256; i++) {
tmp[i] = last_entropy.src2dst_byte_count[i];
tmp[i] = last_entropy->src2dst_byte_count[i];
}
if (last_entropy.src2dst_num_bytes != 0) {
mean = last_entropy.src2dst_bd_mean;
variance = last_entropy.src2dst_bd_variance/(last_entropy.src2dst_num_bytes - 1);
if (last_entropy->src2dst_num_bytes != 0) {
mean = last_entropy->src2dst_bd_mean;
variance = last_entropy->src2dst_bd_variance/(last_entropy->src2dst_num_bytes - 1);
variance = sqrt(variance);
if (last_entropy.src2dst_num_bytes == 1) {
if (last_entropy->src2dst_num_bytes == 1) {
variance = 0.0;
}
}
} else {
for (i=0; i<256; i++) {
tmp[i] = last_entropy.src2dst_byte_count[i] + last_entropy.dst2src_byte_count[i];
tmp[i] = last_entropy->src2dst_byte_count[i] + last_entropy->dst2src_byte_count[i];
}
array = tmp;
num_bytes = last_entropy.src2dst_l4_bytes + last_entropy.dst2src_l4_bytes;
num_bytes = last_entropy->src2dst_l4_bytes + last_entropy->dst2src_l4_bytes;
if (last_entropy.src2dst_num_bytes + last_entropy.dst2src_num_bytes != 0) {
mean = ((double)last_entropy.src2dst_num_bytes)/((double)(last_entropy.src2dst_num_bytes+last_entropy.dst2src_num_bytes))*last_entropy.src2dst_bd_mean +
((double)last_entropy.dst2src_num_bytes)/((double)(last_entropy.dst2src_num_bytes+last_entropy.src2dst_num_bytes))*last_entropy.dst2src_bd_mean;
if (last_entropy->src2dst_num_bytes + last_entropy->dst2src_num_bytes != 0) {
mean = ((double)last_entropy->src2dst_num_bytes)/((double)(last_entropy->src2dst_num_bytes+last_entropy->dst2src_num_bytes))*last_entropy->src2dst_bd_mean +
((double)last_entropy->dst2src_num_bytes)/((double)(last_entropy->dst2src_num_bytes+last_entropy->src2dst_num_bytes))*last_entropy->dst2src_bd_mean;
variance = ((double)last_entropy.src2dst_num_bytes)/((double)(last_entropy.src2dst_num_bytes+last_entropy.dst2src_num_bytes))*last_entropy.src2dst_bd_variance +
((double)last_entropy.dst2src_num_bytes)/((double)(last_entropy.dst2src_num_bytes+last_entropy.src2dst_num_bytes))*last_entropy.dst2src_bd_variance;
variance = ((double)last_entropy->src2dst_num_bytes)/((double)(last_entropy->src2dst_num_bytes+last_entropy->dst2src_num_bytes))*last_entropy->src2dst_bd_variance +
((double)last_entropy->dst2src_num_bytes)/((double)(last_entropy->dst2src_num_bytes+last_entropy->src2dst_num_bytes))*last_entropy->dst2src_bd_variance;
variance = variance/((double)(last_entropy.src2dst_num_bytes + last_entropy.dst2src_num_bytes - 1));
variance = variance/((double)(last_entropy->src2dst_num_bytes + last_entropy->dst2src_num_bytes - 1));
variance = sqrt(variance);
if (last_entropy.src2dst_num_bytes + last_entropy.dst2src_num_bytes == 1) {
if (last_entropy->src2dst_num_bytes + last_entropy->dst2src_num_bytes == 1) {
variance = 0.0;
}
}
@@ -1378,7 +1381,7 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
/* Print entropy values for monitored flows. */
flowGetBDMeanandVariance(flow);
fflush(out);
fprintf(out, "[score: %.4f]", flow->entropy.score);
fprintf(out, "[score: %.4f]", flow->entropy->score);
}
if(csv_fp) fprintf(csv_fp, "\n");
@@ -1416,8 +1419,8 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
else
fprintf(out, "[< 1 sec]");
if(flow->telnet.username[0] != '\0') fprintf(out, "[Username: %s]", flow->telnet.username);
if(flow->telnet.password[0] != '\0') fprintf(out, "[Password: %s]", flow->telnet.password);
if(flow->telnet.username) fprintf(out, "[Username: %s]", flow->telnet.username);
if(flow->telnet.password) fprintf(out, "[Password: %s]", flow->telnet.password);
if(flow->host_server_name[0] != '\0') fprintf(out, "[Host: %s]", flow->host_server_name);
if(flow->info[0] != '\0') fprintf(out, "[%s]", flow->info);
@@ -1532,9 +1535,9 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
if(flow->ssh_tls.server_cipher != '\0') fprintf(out, "[Cipher: %s]",
ndpi_cipher2str(flow->ssh_tls.server_cipher));
if(flow->bittorent_hash[0] != '\0') fprintf(out, "[BT Hash: %s]",
if(flow->bittorent_hash) fprintf(out, "[BT Hash: %s]",
flow->bittorent_hash);
if(flow->dhcp_fingerprint[0] != '\0') fprintf(out, "[DHCP Fingerprint: %s]",
if(flow->dhcp_fingerprint) fprintf(out, "[DHCP Fingerprint: %s]",
flow->dhcp_fingerprint);
if(flow->has_human_readeable_strings) fprintf(out, "[PLAIN TEXT (%s)]",
+125 -83
View File
@@ -447,6 +447,25 @@ void ndpi_flow_info_freer(void *node) {
static void ndpi_free_flow_tls_data(struct ndpi_flow_info *flow) {
if(flow->dhcp_fingerprint) {
ndpi_free(flow->dhcp_fingerprint);
flow->dhcp_fingerprint = NULL;
}
if(flow->bittorent_hash) {
ndpi_free(flow->bittorent_hash);
flow->bittorent_hash = NULL;
}
if(flow->telnet.username) {
ndpi_free(flow->telnet.username);
flow->telnet.username = NULL;
}
if(flow->telnet.password) {
ndpi_free(flow->telnet.password);
flow->telnet.password = NULL;
}
if(flow->ssh_tls.server_names) {
ndpi_free(flow->ssh_tls.server_names);
flow->ssh_tls.server_names = NULL;
@@ -488,6 +507,9 @@ static void ndpi_free_flow_data_analysis(struct ndpi_flow_info *flow) {
if(flow->pktlen_s_to_c) ndpi_free_data_analysis(flow->pktlen_s_to_c, 1);
if(flow->iat_flow) ndpi_free_data_analysis(flow->iat_flow, 1);
if(flow->entropy) ndpi_free(flow->entropy);
if(flow->last_entropy) ndpi_free(flow->last_entropy);
}
/* ***************************************************** */
@@ -576,14 +598,14 @@ ndpi_flow_update_byte_count(struct ndpi_flow_info *flow, const void *x,
* the 4000th octet has been seen for a flow.
*/
if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
if((flow->entropy->src2dst_pkt_count+flow->entropy->dst2src_pkt_count) <= max_num_packets_per_flow) {
/* octet count was already incremented before processing this payload */
u_int32_t current_count;
if(src_to_dst_direction) {
current_count = flow->entropy.src2dst_l4_bytes - len;
current_count = flow->entropy->src2dst_l4_bytes - len;
} else {
current_count = flow->entropy.dst2src_l4_bytes - len;
current_count = flow->entropy->dst2src_l4_bytes - len;
}
if(current_count < ETTA_MIN_OCTETS) {
@@ -592,9 +614,9 @@ ndpi_flow_update_byte_count(struct ndpi_flow_info *flow, const void *x,
for(i=0; i<len; i++) {
if(src_to_dst_direction) {
flow->entropy.src2dst_byte_count[data[i]]++;
flow->entropy->src2dst_byte_count[data[i]]++;
} else {
flow->entropy.dst2src_byte_count[data[i]]++;
flow->entropy->dst2src_byte_count[data[i]]++;
}
current_count++;
if(current_count >= ETTA_MIN_OCTETS) {
@@ -619,22 +641,22 @@ ndpi_flow_update_byte_dist_mean_var(ndpi_flow_info_t *flow, const void *x,
unsigned int len, u_int8_t src_to_dst_direction) {
const unsigned char *data = x;
if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
if((flow->entropy->src2dst_pkt_count+flow->entropy->dst2src_pkt_count) <= max_num_packets_per_flow) {
unsigned int i;
for(i=0; i<len; i++) {
double delta;
if(src_to_dst_direction) {
flow->entropy.src2dst_num_bytes += 1;
delta = ((double)data[i] - flow->entropy.src2dst_bd_mean);
flow->entropy.src2dst_bd_mean += delta/((double)flow->entropy.src2dst_num_bytes);
flow->entropy.src2dst_bd_variance += delta*((double)data[i] - flow->entropy.src2dst_bd_mean);
flow->entropy->src2dst_num_bytes += 1;
delta = ((double)data[i] - flow->entropy->src2dst_bd_mean);
flow->entropy->src2dst_bd_mean += delta/((double)flow->entropy->src2dst_num_bytes);
flow->entropy->src2dst_bd_variance += delta*((double)data[i] - flow->entropy->src2dst_bd_mean);
} else {
flow->entropy.dst2src_num_bytes += 1;
delta = ((double)data[i] - flow->entropy.dst2src_bd_mean);
flow->entropy.dst2src_bd_mean += delta/((double)flow->entropy.dst2src_num_bytes);
flow->entropy.dst2src_bd_variance += delta*((double)data[i] - flow->entropy.dst2src_bd_mean);
flow->entropy->dst2src_num_bytes += 1;
delta = ((double)data[i] - flow->entropy->dst2src_bd_mean);
flow->entropy->dst2src_bd_mean += delta/((double)flow->entropy->dst2src_num_bytes);
flow->entropy->dst2src_bd_variance += delta*((double)data[i] - flow->entropy->dst2src_bd_mean);
}
}
}
@@ -886,16 +908,21 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
workflow->stats.flow_count[2]++;
*src = newflow->src_id, *dst = newflow->dst_id;
newflow->entropy.src2dst_pkt_len[newflow->entropy.src2dst_pkt_count] = l4_data_len;
newflow->entropy.src2dst_pkt_time[newflow->entropy.src2dst_pkt_count] = when;
if(newflow->entropy.src2dst_pkt_count == 0) {
newflow->entropy.src2dst_start = when;
}
newflow->entropy.src2dst_pkt_count++;
// Non zero app data.
if(l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
newflow->entropy.src2dst_opackets++;
newflow->entropy.src2dst_l4_bytes += l4_data_len;
if(enable_joy_stats) {
newflow->entropy = ndpi_calloc(1, sizeof(struct ndpi_entropy));
newflow->last_entropy = ndpi_calloc(1, sizeof(struct ndpi_entropy));
newflow->entropy->src2dst_pkt_len[newflow->entropy->src2dst_pkt_count] = l4_data_len;
newflow->entropy->src2dst_pkt_time[newflow->entropy->src2dst_pkt_count] = when;
if(newflow->entropy->src2dst_pkt_count == 0) {
newflow->entropy->src2dst_start = when;
}
newflow->entropy->src2dst_pkt_count++;
// Non zero app data.
if(l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
newflow->entropy->src2dst_opackets++;
newflow->entropy->src2dst_l4_bytes += l4_data_len;
}
}
return newflow;
}
@@ -922,30 +949,32 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
else
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
}
if(src_to_dst_direction) {
if(rflow->entropy.src2dst_pkt_count < max_num_packets_per_flow) {
rflow->entropy.src2dst_pkt_len[rflow->entropy.src2dst_pkt_count] = l4_data_len;
rflow->entropy.src2dst_pkt_time[rflow->entropy.src2dst_pkt_count] = when;
rflow->entropy.src2dst_l4_bytes += l4_data_len;
rflow->entropy.src2dst_pkt_count++;
}
// Non zero app data.
if(l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
rflow->entropy.src2dst_opackets++;
}
} else {
if(rflow->entropy.dst2src_pkt_count < max_num_packets_per_flow) {
rflow->entropy.dst2src_pkt_len[rflow->entropy.dst2src_pkt_count] = l4_data_len;
rflow->entropy.dst2src_pkt_time[rflow->entropy.dst2src_pkt_count] = when;
if(rflow->entropy.dst2src_pkt_count == 0) {
rflow->entropy.dst2src_start = when;
if(enable_joy_stats) {
if(src_to_dst_direction) {
if(rflow->entropy->src2dst_pkt_count < max_num_packets_per_flow) {
rflow->entropy->src2dst_pkt_len[rflow->entropy->src2dst_pkt_count] = l4_data_len;
rflow->entropy->src2dst_pkt_time[rflow->entropy->src2dst_pkt_count] = when;
rflow->entropy->src2dst_l4_bytes += l4_data_len;
rflow->entropy->src2dst_pkt_count++;
}
// Non zero app data.
if(l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
rflow->entropy->src2dst_opackets++;
}
} else {
if(rflow->entropy->dst2src_pkt_count < max_num_packets_per_flow) {
rflow->entropy->dst2src_pkt_len[rflow->entropy->dst2src_pkt_count] = l4_data_len;
rflow->entropy->dst2src_pkt_time[rflow->entropy->dst2src_pkt_count] = when;
if(rflow->entropy->dst2src_pkt_count == 0) {
rflow->entropy->dst2src_start = when;
}
rflow->entropy->dst2src_l4_bytes += l4_data_len;
rflow->entropy->dst2src_pkt_count++;
}
// Non zero app data.
if(l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
rflow->entropy->dst2src_opackets++;
}
rflow->entropy.dst2src_l4_bytes += l4_data_len;
rflow->entropy.dst2src_pkt_count++;
}
// Non zero app data.
if(l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
rflow->entropy.dst2src_opackets++;
}
}
@@ -1041,18 +1070,21 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
flow->risk = flow->ndpi_flow->risk;
if(is_ndpi_proto(flow, NDPI_PROTOCOL_DHCP)) {
snprintf(flow->dhcp_fingerprint, sizeof(flow->dhcp_fingerprint), "%s", flow->ndpi_flow->protos.dhcp.fingerprint);
if(flow->ndpi_flow->protos.dhcp.fingerprint[0] != '\0')
flow->dhcp_fingerprint = ndpi_strdup(flow->ndpi_flow->protos.dhcp.fingerprint);
} else if(is_ndpi_proto(flow, NDPI_PROTOCOL_BITTORRENT)) {
u_int j, n = 0;
u_int j;
for(i=0, j = 0; j < sizeof(flow->bittorent_hash)-1; i++) {
sprintf(&flow->bittorent_hash[j], "%02x",
flow->ndpi_flow->protos.bittorrent.hash[i]);
if(flow->ndpi_flow->protos.bittorrent.hash[0] != '\0') {
flow->bittorent_hash = ndpi_malloc(sizeof(flow->ndpi_flow->protos.bittorrent.hash) * 2 + 1);
for(i=0, j = 0; i < sizeof(flow->ndpi_flow->protos.bittorrent.hash); i++) {
sprintf(&flow->bittorent_hash[j], "%02x",
flow->ndpi_flow->protos.bittorrent.hash[i]);
j += 2, n += flow->ndpi_flow->protos.bittorrent.hash[i];
j += 2;
}
flow->bittorent_hash[j] = '\0';
}
if(n == 0) flow->bittorent_hash[0] = '\0';
}
/* DNS */
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_DNS)) {
@@ -1118,8 +1150,10 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
snprintf(flow->http.user_agent, sizeof(flow->http.user_agent), "%s", flow->ndpi_flow->http.user_agent ? flow->ndpi_flow->http.user_agent : "");
}
} else if(is_ndpi_proto(flow, NDPI_PROTOCOL_TELNET)) {
snprintf(flow->telnet.username, sizeof(flow->telnet.username), "%s", flow->ndpi_flow->protos.telnet.username);
snprintf(flow->telnet.password, sizeof(flow->telnet.password), "%s", flow->ndpi_flow->protos.telnet.password);
if(flow->ndpi_flow->protos.telnet.username[0] != '\0')
flow->telnet.username = ndpi_strdup(flow->ndpi_flow->protos.telnet.username);
if(flow->ndpi_flow->protos.telnet.password[0] != '\0')
flow->telnet.password = ndpi_strdup(flow->ndpi_flow->protos.telnet.password);
} else if(is_ndpi_proto(flow, NDPI_PROTOCOL_SSH)) {
snprintf(flow->ssh_tls.client_requested_server_name,
sizeof(flow->ssh_tls.client_requested_server_name), "%s",
@@ -1241,9 +1275,11 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
*/
static void
ndpi_clear_entropy_stats(struct ndpi_flow_info *flow) {
if(flow->entropy.src2dst_pkt_count + flow->entropy.dst2src_pkt_count == max_num_packets_per_flow) {
memcpy(&flow->last_entropy, &flow->entropy, sizeof(struct ndpi_entropy));
memset(&flow->entropy, 0x00, sizeof(struct ndpi_entropy));
if(enable_joy_stats) {
if(flow->entropy->src2dst_pkt_count + flow->entropy->dst2src_pkt_count == max_num_packets_per_flow) {
memcpy(flow->last_entropy, flow->entropy, sizeof(struct ndpi_entropy));
memset(flow->entropy, 0x00, sizeof(struct ndpi_entropy));
}
}
}
@@ -1355,8 +1391,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if((tcph != NULL) && (tcph->fin || tcph->rst || tcph->syn))
begin_or_end_tcp = 1;
if(flow->entropy.flow_last_pkt_time.tv_sec) {
ndpi_timer_sub(&when, &flow->entropy.flow_last_pkt_time, &tdiff);
if(flow->flow_last_pkt_time.tv_sec) {
ndpi_timer_sub(&when, &flow->flow_last_pkt_time, &tdiff);
if(flow->iat_flow
&& (tdiff.tv_sec >= 0) /* Discard backward time */
@@ -1368,11 +1404,11 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
}
}
memcpy(&flow->entropy.flow_last_pkt_time, &when, sizeof(when));
if(src_to_dst_direction) {
if(flow->entropy.src2dst_last_pkt_time.tv_sec) {
ndpi_timer_sub(&when, &flow->entropy.src2dst_last_pkt_time, &tdiff);
memcpy(&flow->flow_last_pkt_time, &when, sizeof(when));
if(src_to_dst_direction) {
if(flow->src2dst_last_pkt_time.tv_sec) {
ndpi_timer_sub(&when, &flow->src2dst_last_pkt_time, &tdiff);
if(flow->iat_c_to_s
&& (tdiff.tv_sec >= 0) /* Discard backward time */
@@ -1385,15 +1421,15 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
ndpi_data_add_value(flow->pktlen_c_to_s, rawsize);
flow->src2dst_packets++, flow->src2dst_bytes += rawsize, flow->src2dst_goodput_bytes += payload_len;
memcpy(&flow->entropy.src2dst_last_pkt_time, &when, sizeof(when));
memcpy(&flow->src2dst_last_pkt_time, &when, sizeof(when));
#ifdef DIRECTION_BINS
if(payload_len && (flow->src2dst_packets < MAX_NUM_BIN_PKTS))
ndpi_inc_bin(&flow->payload_len_bin_src2dst, plen2slot(payload_len));
#endif
} else {
if(flow->entropy.dst2src_last_pkt_time.tv_sec && (!begin_or_end_tcp)) {
ndpi_timer_sub(&when, &flow->entropy.dst2src_last_pkt_time, &tdiff);
if(flow->dst2src_last_pkt_time.tv_sec && (!begin_or_end_tcp)) {
ndpi_timer_sub(&when, &flow->dst2src_last_pkt_time, &tdiff);
if(flow->iat_s_to_c) {
u_int32_t ms = ndpi_timeval_to_milliseconds(tdiff);
@@ -1403,7 +1439,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
}
ndpi_data_add_value(flow->pktlen_s_to_c, rawsize);
flow->dst2src_packets++, flow->dst2src_bytes += rawsize, flow->dst2src_goodput_bytes += payload_len;
memcpy(&flow->entropy.dst2src_last_pkt_time, &when, sizeof(when));
memcpy(&flow->dst2src_last_pkt_time, &when, sizeof(when));
#ifdef DIRECTION_BINS
if(payload_len && (flow->dst2src_packets < MAX_NUM_BIN_PKTS))
@@ -1431,24 +1467,24 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
ndpi_flow_update_byte_count(flow, payload, payload_len, src_to_dst_direction);
ndpi_flow_update_byte_dist_mean_var(flow, payload, payload_len, src_to_dst_direction);
/* Update SPLT scores for first 32 packets. */
if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
if((flow->entropy->src2dst_pkt_count+flow->entropy->dst2src_pkt_count) <= max_num_packets_per_flow) {
if(flow->bidirectional)
flow->entropy.score = ndpi_classify(flow->entropy.src2dst_pkt_len, flow->entropy.src2dst_pkt_time,
flow->entropy.dst2src_pkt_len, flow->entropy.dst2src_pkt_time,
flow->entropy.src2dst_start, flow->entropy.dst2src_start,
flow->entropy->score = ndpi_classify(flow->entropy->src2dst_pkt_len, flow->entropy->src2dst_pkt_time,
flow->entropy->dst2src_pkt_len, flow->entropy->dst2src_pkt_time,
flow->entropy->src2dst_start, flow->entropy->dst2src_start,
max_num_packets_per_flow, flow->src_port, flow->dst_port,
flow->src2dst_packets, flow->dst2src_packets,
flow->entropy.src2dst_opackets, flow->entropy.dst2src_opackets,
flow->entropy.src2dst_l4_bytes, flow->entropy.dst2src_l4_bytes, 1,
flow->entropy.src2dst_byte_count, flow->entropy.dst2src_byte_count);
flow->entropy->src2dst_opackets, flow->entropy->dst2src_opackets,
flow->entropy->src2dst_l4_bytes, flow->entropy->dst2src_l4_bytes, 1,
flow->entropy->src2dst_byte_count, flow->entropy->dst2src_byte_count);
else
flow->entropy.score = ndpi_classify(flow->entropy.src2dst_pkt_len, flow->entropy.src2dst_pkt_time,
NULL, NULL, flow->entropy.src2dst_start, flow->entropy.src2dst_start,
flow->entropy->score = ndpi_classify(flow->entropy->src2dst_pkt_len, flow->entropy->src2dst_pkt_time,
NULL, NULL, flow->entropy->src2dst_start, flow->entropy->src2dst_start,
max_num_packets_per_flow, flow->src_port, flow->dst_port,
flow->src2dst_packets, 0,
flow->entropy.src2dst_opackets, 0,
flow->entropy.src2dst_l4_bytes, 0, 1,
flow->entropy.src2dst_byte_count, NULL);
flow->entropy->src2dst_opackets, 0,
flow->entropy->src2dst_l4_bytes, 0, 1,
flow->entropy->src2dst_byte_count, NULL);
}
}
@@ -1459,6 +1495,12 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
/* Copy packets entropy if num packets count == 10 */
ndpi_clear_entropy_stats(flow);
/* Reset IAT reeference times (see https://github.com/ntop/nDPI/pull/1316) */
if(((flow->src2dst_packets + flow->dst2src_packets) % max_num_packets_per_flow) == 0) {
memset(&flow->src2dst_last_pkt_time, '\0', sizeof(flow->src2dst_last_pkt_time));
memset(&flow->dst2src_last_pkt_time, '\0', sizeof(flow->dst2src_last_pkt_time));
memset(&flow->flow_last_pkt_time, '\0', sizeof(flow->flow_last_pkt_time));
}
if((human_readeable_string_len != 0) && (!flow->has_human_readeable_strings)) {
u_int8_t skip = 0;
+6 -6
View File
@@ -134,7 +134,6 @@ struct flow_metrics {
struct ndpi_entropy {
// Entropy fields
pkt_timeval src2dst_last_pkt_time, dst2src_last_pkt_time, flow_last_pkt_time;
u_int16_t src2dst_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
pkt_timeval src2dst_pkt_time[MAX_NUM_PKTS]; /*!< array of arrival times */
u_int16_t dst2src_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
@@ -192,14 +191,15 @@ typedef struct ndpi_flow_info {
ndpi_protocol detected_protocol;
// Flow data analysis
pkt_timeval src2dst_last_pkt_time, dst2src_last_pkt_time, flow_last_pkt_time;
struct ndpi_analyze_struct *iat_c_to_s, *iat_s_to_c, *iat_flow,
*pktlen_c_to_s, *pktlen_s_to_c;
char info[255];
char flow_extra_info[16];
char host_server_name[240];
char bittorent_hash[41];
char dhcp_fingerprint[48];
char *bittorent_hash;
char *dhcp_fingerprint;
ndpi_risk risk;
struct {
@@ -229,13 +229,13 @@ typedef struct ndpi_flow_info {
} http;
struct {
char username[32], password[32];
char *username, *password;
} telnet;
void *src_id, *dst_id;
struct ndpi_entropy entropy;
struct ndpi_entropy last_entropy;
struct ndpi_entropy *entropy;
struct ndpi_entropy *last_entropy;
/* Payload lenght bins */
#ifdef DIRECTION_BINS