Manual merge of pull #769
This commit is contained in:
+124
-102
@@ -109,7 +109,7 @@ struct payload_stats {
|
||||
|
||||
|
||||
struct payload_stats *pstats = NULL;
|
||||
u_int32_t max_num_packets_per_flow = 32;
|
||||
u_int32_t max_num_packets_per_flow = 10; /* ETTA requires min 10 pkts for record. */
|
||||
u_int32_t max_packet_payload_dissection = 128;
|
||||
u_int32_t max_num_reported_top_payloads = 25;
|
||||
u_int16_t min_pattern_len = 4;
|
||||
@@ -185,7 +185,7 @@ void ndpi_payload_analyzer(struct ndpi_flow_info *flow,
|
||||
u_int16_t i, j;
|
||||
u_int16_t scan_len = ndpi_min(max_packet_payload_dissection, payload_len);
|
||||
|
||||
if((flow->src2dst_pkt_count+flow->dst2src_pkt_count) < max_num_packets_per_flow) {
|
||||
if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
|
||||
#ifdef DEBUG_PAYLOAD
|
||||
printf("[hashval: %u][proto: %u][vlan: %u][%s:%u <-> %s:%u][direction: %s][payload_len: %u]\n",
|
||||
flow->hashval, flow->protocol, flow->vlan_id,
|
||||
@@ -543,23 +543,25 @@ ndpi_flow_update_byte_count(struct ndpi_flow_info *flow, const void *x,
|
||||
* the 4000th octet has been seen for a flow.
|
||||
*/
|
||||
|
||||
/* octet count was already incremented before processing this payload */
|
||||
if(src_to_dst_direction) {
|
||||
current_count = flow->src2dst_l4_bytes - len;
|
||||
} else {
|
||||
current_count = flow->dst2src_l4_bytes - len;
|
||||
}
|
||||
if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
|
||||
/* octet count was already incremented before processing this payload */
|
||||
if(src_to_dst_direction) {
|
||||
current_count = flow->entropy.src2dst_l4_bytes - len;
|
||||
} else {
|
||||
current_count = flow->entropy.dst2src_l4_bytes - len;
|
||||
}
|
||||
|
||||
if(current_count < ETTA_MIN_OCTETS) {
|
||||
for(i=0; i<len; i++) {
|
||||
if(src_to_dst_direction) {
|
||||
flow->src2dst_byte_count[data[i]]++;
|
||||
} else {
|
||||
flow->dst2src_byte_count[data[i]]++;
|
||||
}
|
||||
current_count++;
|
||||
if(current_count >= ETTA_MIN_OCTETS) {
|
||||
break;
|
||||
if(current_count < ETTA_MIN_OCTETS) {
|
||||
for(i=0; i<len; i++) {
|
||||
if(src_to_dst_direction) {
|
||||
flow->entropy.src2dst_byte_count[data[i]]++;
|
||||
} else {
|
||||
flow->entropy.dst2src_byte_count[data[i]]++;
|
||||
}
|
||||
current_count++;
|
||||
if(current_count >= ETTA_MIN_OCTETS) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -581,17 +583,19 @@ ndpi_flow_update_byte_dist_mean_var(ndpi_flow_info_t *flow, const void *x,
|
||||
double delta;
|
||||
unsigned int i;
|
||||
|
||||
for(i=0; i<len; i++) {
|
||||
if(src_to_dst_direction) {
|
||||
flow->src2dst_num_bytes += 1;
|
||||
delta = ((double)data[i] - flow->src2dst_bd_mean);
|
||||
flow->src2dst_bd_mean += delta/((double)flow->src2dst_num_bytes);
|
||||
flow->src2dst_bd_variance += delta*((double)data[i] - flow->src2dst_bd_mean);
|
||||
} else {
|
||||
flow->dst2src_num_bytes += 1;
|
||||
delta = ((double)data[i] - flow->dst2src_bd_mean);
|
||||
flow->dst2src_bd_mean += delta/((double)flow->dst2src_num_bytes);
|
||||
flow->dst2src_bd_variance += delta*((double)data[i] - flow->dst2src_bd_mean);
|
||||
if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
|
||||
for(i=0; i<len; i++) {
|
||||
if(src_to_dst_direction) {
|
||||
flow->entropy.src2dst_num_bytes += 1;
|
||||
delta = ((double)data[i] - flow->entropy.src2dst_bd_mean);
|
||||
flow->entropy.src2dst_bd_mean += delta/((double)flow->entropy.src2dst_num_bytes);
|
||||
flow->entropy.src2dst_bd_variance += delta*((double)data[i] - flow->entropy.src2dst_bd_mean);
|
||||
} else {
|
||||
flow->entropy.dst2src_num_bytes += 1;
|
||||
delta = ((double)data[i] - flow->entropy.dst2src_bd_mean);
|
||||
flow->entropy.dst2src_bd_mean += delta/((double)flow->entropy.dst2src_num_bytes);
|
||||
flow->entropy.dst2src_bd_variance += delta*((double)data[i] - flow->entropy.dst2src_bd_mean);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -826,63 +830,70 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
|
||||
workflow->stats.ndpi_flow_count++;
|
||||
|
||||
*src = newflow->src_id, *dst = newflow->dst_id;
|
||||
newflow->src2dst_pkt_len[newflow->src2dst_pkt_count] = l4_packet_len;
|
||||
newflow->src2dst_pkt_time[newflow->src2dst_pkt_count] = when;
|
||||
if(newflow->src2dst_pkt_count == 0) {
|
||||
newflow->src2dst_start = when;
|
||||
newflow->entropy.src2dst_pkt_len[newflow->entropy.src2dst_pkt_count] = l4_data_len;
|
||||
newflow->entropy.src2dst_pkt_time[newflow->entropy.src2dst_pkt_count] = when;
|
||||
if (newflow->entropy.src2dst_pkt_count == 0) {
|
||||
newflow->entropy.src2dst_start = when;
|
||||
}
|
||||
newflow->src2dst_pkt_count++;
|
||||
if(l4_data_len != 0XFEEDFACE) {
|
||||
newflow->src2dst_opackets++;
|
||||
newflow->entropy.src2dst_pkt_count++;
|
||||
// Non zero app data.
|
||||
if (l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
|
||||
newflow->entropy.src2dst_opackets++;
|
||||
newflow->entropy.src2dst_l4_bytes += l4_data_len;
|
||||
}
|
||||
return newflow;
|
||||
}
|
||||
} else {
|
||||
struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret;
|
||||
struct ndpi_flow_info *rflow = *(struct ndpi_flow_info**)ret;
|
||||
|
||||
if(is_changed) {
|
||||
if(flow->src_ip == iph->saddr
|
||||
&& flow->dst_ip == iph->daddr
|
||||
&& flow->src_port == htons(*sport)
|
||||
&& flow->dst_port == htons(*dport)
|
||||
if(rflow->src_ip == iph->saddr
|
||||
&& rflow->dst_ip == iph->daddr
|
||||
&& rflow->src_port == htons(*sport)
|
||||
&& rflow->dst_port == htons(*dport)
|
||||
)
|
||||
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
|
||||
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
|
||||
else
|
||||
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
|
||||
*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
|
||||
}
|
||||
else {
|
||||
if(flow->src_ip == iph->saddr
|
||||
&& flow->dst_ip == iph->daddr
|
||||
&& flow->src_port == htons(*sport)
|
||||
&& flow->dst_port == htons(*dport)
|
||||
if(rflow->src_ip == iph->saddr
|
||||
&& rflow->dst_ip == iph->daddr
|
||||
&& rflow->src_port == htons(*sport)
|
||||
&& rflow->dst_port == htons(*dport)
|
||||
)
|
||||
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
|
||||
*src = rflow->src_id, *dst = rflow->dst_id, *src_to_dst_direction = 1;
|
||||
else
|
||||
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
|
||||
*src = rflow->dst_id, *dst = rflow->src_id, *src_to_dst_direction = 0, rflow->bidirectional = 1;
|
||||
}
|
||||
if(src_to_dst_direction) {
|
||||
if(flow->src2dst_pkt_count < MAX_NUM_PKTS) {
|
||||
flow->src2dst_pkt_len[flow->src2dst_pkt_count] = l4_packet_len;
|
||||
flow->src2dst_pkt_time[flow->src2dst_pkt_count] = when;
|
||||
flow->src2dst_pkt_count++;
|
||||
if (src_to_dst_direction) {
|
||||
if (rflow->entropy.src2dst_pkt_count < max_num_packets_per_flow) {
|
||||
rflow->entropy.src2dst_pkt_len[rflow->entropy.src2dst_pkt_count] = l4_data_len;
|
||||
rflow->entropy.src2dst_pkt_time[rflow->entropy.src2dst_pkt_count] = when;
|
||||
rflow->entropy.src2dst_l4_bytes += l4_data_len;
|
||||
rflow->entropy.src2dst_pkt_count++;
|
||||
}
|
||||
if(l4_data_len != 0XFEEDFACE) {
|
||||
flow->src2dst_opackets++;
|
||||
// Non zero app data.
|
||||
if (l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
|
||||
rflow->entropy.src2dst_opackets++;
|
||||
}
|
||||
} else {
|
||||
if(flow->dst2src_pkt_count < MAX_NUM_PKTS) {
|
||||
flow->dst2src_pkt_len[flow->dst2src_pkt_count] = l4_packet_len;
|
||||
flow->dst2src_pkt_time[flow->dst2src_pkt_count] = when;
|
||||
if(flow->dst2src_pkt_count == 0) {
|
||||
flow->dst2src_start = when;
|
||||
if (rflow->entropy.dst2src_pkt_count < max_num_packets_per_flow) {
|
||||
rflow->entropy.dst2src_pkt_len[rflow->entropy.dst2src_pkt_count] = l4_data_len;
|
||||
rflow->entropy.dst2src_pkt_time[rflow->entropy.dst2src_pkt_count] = when;
|
||||
if (rflow->entropy.dst2src_pkt_count == 0) {
|
||||
rflow->entropy.dst2src_start = when;
|
||||
}
|
||||
flow->dst2src_pkt_count++;
|
||||
rflow->entropy.dst2src_l4_bytes += l4_data_len;
|
||||
rflow->entropy.dst2src_pkt_count++;
|
||||
}
|
||||
if(l4_data_len != 0XFEEDFACE) {
|
||||
flow->dst2src_opackets++;
|
||||
// Non zero app data.
|
||||
if (l4_data_len != 0XFEEDFACE && l4_data_len != 0) {
|
||||
rflow->entropy.dst2src_opackets++;
|
||||
}
|
||||
}
|
||||
return flow;
|
||||
|
||||
return(rflow);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -927,27 +938,6 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
|
||||
/* ****************************************************** */
|
||||
|
||||
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) {
|
||||
if(enable_joy_stats) {
|
||||
/* Update SPLT scores. */
|
||||
|
||||
if(flow->bidirectional)
|
||||
flow->score = ndpi_classify(flow->src2dst_pkt_len, flow->src2dst_pkt_time,
|
||||
flow->dst2src_pkt_len, flow->dst2src_pkt_time,
|
||||
flow->src2dst_start, flow->dst2src_start,
|
||||
MAX_NUM_PKTS, flow->src_port, flow->dst_port,
|
||||
flow->src2dst_packets, flow->dst2src_packets,
|
||||
flow->src2dst_opackets, flow->dst2src_opackets,
|
||||
flow->src2dst_l4_bytes, flow->dst2src_l4_bytes, 1,
|
||||
flow->src2dst_byte_count, flow->dst2src_byte_count);
|
||||
else
|
||||
flow->score = ndpi_classify(flow->src2dst_pkt_len, flow->src2dst_pkt_time,
|
||||
NULL, NULL, flow->src2dst_start, flow->src2dst_start,
|
||||
MAX_NUM_PKTS, flow->src_port, flow->dst_port,
|
||||
flow->src2dst_packets, 0,
|
||||
flow->src2dst_opackets, 0,
|
||||
flow->src2dst_l4_bytes, 0, 1,
|
||||
flow->src2dst_byte_count, NULL);
|
||||
}
|
||||
|
||||
if(!flow->ndpi_flow) return;
|
||||
|
||||
@@ -1035,6 +1025,19 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
|
||||
|
||||
/* ****************************************************** */
|
||||
|
||||
/**
|
||||
* @brief Clear entropy stats if it meets prereq.
|
||||
*/
|
||||
static void
|
||||
ndpi_clear_entropy_stats(struct ndpi_flow_info *flow)
|
||||
{
|
||||
if(flow->entropy.src2dst_pkt_count + flow->entropy.dst2src_pkt_count == max_num_packets_per_flow) {
|
||||
memcpy(&flow->last_entropy, &flow->entropy, sizeof(struct ndpi_entropy));
|
||||
memset(&flow->entropy, 0x00, sizeof(struct ndpi_entropy));
|
||||
}
|
||||
}
|
||||
|
||||
/* ****************************************************** */
|
||||
/**
|
||||
Function to process the packet:
|
||||
determine the flow of a packet and try to decode it
|
||||
@@ -1088,9 +1091,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
if((tcph != NULL) && (tcph->fin || tcph->rst || tcph->syn))
|
||||
begin_or_end_tcp = 1;
|
||||
|
||||
if(flow->flow_last_pkt_time.tv_sec) {
|
||||
ndpi_timer_sub(&when, &flow->flow_last_pkt_time, &tdiff);
|
||||
|
||||
if(flow->entropy.flow_last_pkt_time.tv_sec) {
|
||||
ndpi_timer_sub(&when, &flow->entropy.flow_last_pkt_time, &tdiff);
|
||||
|
||||
if(flow->iat_flow) {
|
||||
u_int32_t ms = ndpi_timeval_to_milliseconds(tdiff);
|
||||
|
||||
@@ -1098,11 +1101,11 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
ndpi_data_add_value(flow->iat_flow, ms);
|
||||
}
|
||||
}
|
||||
memcpy(&flow->flow_last_pkt_time, &when, sizeof(when));
|
||||
memcpy(&flow->entropy.flow_last_pkt_time, &when, sizeof(when));
|
||||
|
||||
if(src_to_dst_direction) {
|
||||
if(flow->src2dst_last_pkt_time.tv_sec && (!begin_or_end_tcp)) {
|
||||
ndpi_timer_sub(&when, &flow->src2dst_last_pkt_time, &tdiff);
|
||||
if(flow->entropy.src2dst_last_pkt_time.tv_sec) {
|
||||
ndpi_timer_sub(&when, &flow->entropy.src2dst_last_pkt_time, &tdiff);
|
||||
|
||||
if(flow->iat_c_to_s) {
|
||||
u_int32_t ms = ndpi_timeval_to_milliseconds(tdiff);
|
||||
@@ -1113,11 +1116,10 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
|
||||
ndpi_data_add_value(flow->pktlen_c_to_s, rawsize);
|
||||
flow->src2dst_packets++, flow->src2dst_bytes += rawsize;
|
||||
flow->src2dst_l4_bytes += payload_len;
|
||||
memcpy(&flow->src2dst_last_pkt_time, &when, sizeof(when));
|
||||
memcpy(&flow->entropy.src2dst_last_pkt_time, &when, sizeof(when));
|
||||
} else {
|
||||
if(flow->dst2src_last_pkt_time.tv_sec && (!begin_or_end_tcp)) {
|
||||
ndpi_timer_sub(&when, &flow->dst2src_last_pkt_time, &tdiff);
|
||||
if(flow->entropy.dst2src_last_pkt_time.tv_sec && (!begin_or_end_tcp)) {
|
||||
ndpi_timer_sub(&when, &flow->entropy.dst2src_last_pkt_time, &tdiff);
|
||||
|
||||
if(flow->iat_s_to_c) {
|
||||
u_int32_t ms = ndpi_timeval_to_milliseconds(tdiff);
|
||||
@@ -1128,8 +1130,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
|
||||
ndpi_data_add_value(flow->pktlen_s_to_c, rawsize);
|
||||
flow->dst2src_packets++, flow->dst2src_bytes += rawsize;
|
||||
flow->dst2src_l4_bytes += payload_len;
|
||||
memcpy(&flow->dst2src_last_pkt_time, &when, sizeof(when));
|
||||
memcpy(&flow->entropy.dst2src_last_pkt_time, &when, sizeof(when));
|
||||
}
|
||||
|
||||
if(enable_payload_analyzer && (payload_len > 0))
|
||||
@@ -1141,6 +1142,26 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
/* Update BD, distribution and mean. */
|
||||
ndpi_flow_update_byte_count(flow, payload, payload_len, src_to_dst_direction);
|
||||
ndpi_flow_update_byte_dist_mean_var(flow, payload, payload_len, src_to_dst_direction);
|
||||
/* Update SPLT scores for first 32 packets. */
|
||||
if((flow->entropy.src2dst_pkt_count+flow->entropy.dst2src_pkt_count) <= max_num_packets_per_flow) {
|
||||
if(flow->bidirectional)
|
||||
flow->entropy.score = ndpi_classify(flow->entropy.src2dst_pkt_len, flow->entropy.src2dst_pkt_time,
|
||||
flow->entropy.dst2src_pkt_len, flow->entropy.dst2src_pkt_time,
|
||||
flow->entropy.src2dst_start, flow->entropy.dst2src_start,
|
||||
max_num_packets_per_flow, flow->src_port, flow->dst_port,
|
||||
flow->src2dst_packets, flow->dst2src_packets,
|
||||
flow->entropy.src2dst_opackets, flow->entropy.dst2src_opackets,
|
||||
flow->entropy.src2dst_l4_bytes, flow->entropy.dst2src_l4_bytes, 1,
|
||||
flow->entropy.src2dst_byte_count, flow->entropy.dst2src_byte_count);
|
||||
else
|
||||
flow->entropy.score = ndpi_classify(flow->entropy.src2dst_pkt_len, flow->entropy.src2dst_pkt_time,
|
||||
NULL, NULL, flow->entropy.src2dst_start, flow->entropy.src2dst_start,
|
||||
max_num_packets_per_flow, flow->src_port, flow->dst_port,
|
||||
flow->src2dst_packets, 0,
|
||||
flow->entropy.src2dst_opackets, 0,
|
||||
flow->entropy.src2dst_l4_bytes, 0, 1,
|
||||
flow->entropy.src2dst_byte_count, NULL);
|
||||
}
|
||||
}
|
||||
|
||||
if(flow->first_seen == 0)
|
||||
@@ -1148,6 +1169,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
|
||||
flow->last_seen = time;
|
||||
|
||||
/* Copy packets entropy if num packets count == 10 */
|
||||
ndpi_clear_entropy_stats(flow);
|
||||
|
||||
if(!flow->has_human_readeable_strings) {
|
||||
u_int8_t skip = 0;
|
||||
|
||||
@@ -1163,8 +1187,6 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
}
|
||||
|
||||
if(!skip) {
|
||||
char outbuf[64] = { '\0' };
|
||||
|
||||
if(ndpi_has_human_readeable_string(workflow->ndpi_struct, (char*)packet, header->caplen,
|
||||
human_readeable_string_len,
|
||||
flow->human_readeable_string_buffer,
|
||||
@@ -1527,10 +1549,10 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
|
||||
/* https://en.wikipedia.org/wiki/TZSP */
|
||||
u_int offset = ip_offset+ip_len+sizeof(struct ndpi_udphdr);
|
||||
u_int8_t version = packet[offset];
|
||||
u_int8_t type = packet[offset+1];
|
||||
u_int8_t ts_type = packet[offset+1];
|
||||
u_int16_t encapsulates = ntohs(*((u_int16_t*)&packet[offset+2]));
|
||||
|
||||
if((version == 1) && (type == 0) && (encapsulates == 1)) {
|
||||
if((version == 1) && (ts_type == 0) && (encapsulates == 1)) {
|
||||
u_int8_t stop = 0;
|
||||
|
||||
offset += 4;
|
||||
|
||||
Reference in New Issue
Block a user