Reworked categories handling

Removed GenericProtocol and replaced with categories
Removed ndpi_pref_enable_category_substring_match option: substring matching is now default
This commit is contained in:
Luca Deri
2019-09-29 21:46:41 +02:00
parent 016d568c2b
commit f2a5bbef17
15 changed files with 1301 additions and 1314 deletions
+62 -55
View File
@@ -171,7 +171,7 @@ void ndpi_analyze_payload(struct ndpi_flow_info *flow,
if((p = (struct packet_id_stats*)calloc(1, sizeof(struct packet_id_stats))) == NULL)
return; /* OOM */
p->packet_id = packet_id;
HASH_ADD_INT(ret->packets, packet_id, p);
}
}
@@ -406,21 +406,27 @@ static int _debug_protocols_ok = 0;
struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs,
pcap_t * pcap_handle) {
struct ndpi_detection_module_struct * module;
struct ndpi_workflow * workflow;
set_ndpi_malloc(malloc_wrapper), set_ndpi_free(free_wrapper);
set_ndpi_flow_malloc(NULL), set_ndpi_flow_free(NULL);
/* TODO: just needed here to init ndpi malloc wrapper */
struct ndpi_detection_module_struct * module = ndpi_init_detection_module();
if (module == NULL) {
NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
exit(-1);
module = ndpi_init_detection_module();
if(module == NULL) {
NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
exit(-1);
}
struct ndpi_workflow * workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
if (workflow == NULL) {
NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
ndpi_free(module);
exit(-1);
workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
if(workflow == NULL) {
NDPI_LOG(0, NULL, NDPI_LOG_ERROR, "global structure initialization failed\n");
ndpi_free(module);
exit(-1);
}
workflow->pcap_handle = pcap_handle;
workflow->prefs = *prefs;
workflow->ndpi_struct = module;
@@ -435,6 +441,7 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
NDPI_BITMASK_RESET(module->debug_bitmask);
if(_debug_protocols_ok)
module->debug_bitmask = debug_bitmask;
#endif
@@ -458,7 +465,7 @@ void ndpi_flow_info_freer(void *node) {
if(flow->pktlen_s_to_c) ndpi_free_data_analysis(flow->pktlen_s_to_c);
if(flow->iat_flow) ndpi_free_data_analysis(flow->iat_flow);
ndpi_free(flow);
}
@@ -489,19 +496,19 @@ int ndpi_workflow_node_cmp(const void *a, const void *b) {
if(fa->protocol < fb->protocol ) return(-1); else { if(fa->protocol > fb->protocol ) return(1); }
if(
(
(fa->src_ip == fb->src_ip )
&& (fa->src_port == fb->src_port)
&& (fa->dst_ip == fb->dst_ip )
&& (fa->dst_port == fb->dst_port)
)
||
(
(fa->src_ip == fb->dst_ip )
&& (fa->src_port == fb->dst_port)
&& (fa->dst_ip == fb->src_ip )
&& (fa->dst_port == fb->src_port)
)
(
(fa->src_ip == fb->src_ip )
&& (fa->src_port == fb->src_port)
&& (fa->dst_ip == fb->dst_ip )
&& (fa->dst_port == fb->dst_port)
)
||
(
(fa->src_ip == fb->dst_ip )
&& (fa->src_port == fb->dst_port)
&& (fa->dst_ip == fb->src_ip )
&& (fa->dst_port == fb->src_port)
)
)
return(0);
@@ -537,21 +544,21 @@ ndpi_flow_update_byte_count(struct ndpi_flow_info *flow, const void *x,
*/
/* octet count was already incremented before processing this payload */
if (src_to_dst_direction) {
if(src_to_dst_direction) {
current_count = flow->src2dst_l4_bytes - len;
} else {
current_count = flow->dst2src_l4_bytes - len;
}
if (current_count < ETTA_MIN_OCTETS) {
for (i=0; i<len; i++) {
if (src_to_dst_direction) {
if(current_count < ETTA_MIN_OCTETS) {
for(i=0; i<len; i++) {
if(src_to_dst_direction) {
flow->src2dst_byte_count[data[i]]++;
} else {
flow->dst2src_byte_count[data[i]]++;
}
current_count++;
if (current_count >= ETTA_MIN_OCTETS) {
if(current_count >= ETTA_MIN_OCTETS) {
break;
}
}
@@ -574,8 +581,8 @@ ndpi_flow_update_byte_dist_mean_var(ndpi_flow_info_t *flow, const void *x,
double delta;
unsigned int i;
for (i=0; i<len; i++) {
if (src_to_dst_direction) {
for(i=0; i<len; i++) {
if(src_to_dst_direction) {
flow->src2dst_num_bytes += 1;
delta = ((double)data[i] - flow->src2dst_bd_mean);
flow->src2dst_bd_mean += delta/((double)flow->src2dst_num_bytes);
@@ -597,9 +604,9 @@ float ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256],
int i;
float tmp, sum = 0.0;
for (i=0; i<256; i++) {
for(i=0; i<256; i++) {
tmp = (float) byte_count[i] / (float) num_bytes;
if (tmp > FLT_EPSILON) {
if(tmp > FLT_EPSILON) {
sum -= tmp * logf(tmp);
}
}
@@ -714,7 +721,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
*payload_len = (l4_packet_len > sizeof(struct ndpi_icmphdr)) ? l4_packet_len-sizeof(struct ndpi_icmphdr) : 0;
l4_data_len = l4_packet_len - sizeof(struct ndpi_icmphdr);
*sport = *dport = 0;
} else if (*proto == IPPROTO_ICMPV6) {
} else if(*proto == IPPROTO_ICMPV6) {
*payload = (u_int8_t*)&l4[sizeof(struct ndpi_icmp6hdr)];
*payload_len = (l4_packet_len > sizeof(struct ndpi_icmp6hdr)) ? l4_packet_len-sizeof(struct ndpi_icmp6hdr) : 0;
l4_data_len = l4_packet_len - sizeof(struct ndpi_icmp6hdr);
@@ -734,7 +741,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
printf("hashval=%u [%u][%u][%u:%u][%u:%u]\n", hashval, flow.protocol, flow.vlan_id,
flow.src_ip, flow.src_port, flow.dst_ip, flow.dst_port);
#endif
idx = hashval % workflow->prefs.num_roots;
ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp);
@@ -821,11 +828,11 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
*src = newflow->src_id, *dst = newflow->dst_id;
newflow->src2dst_pkt_len[newflow->src2dst_pkt_count] = l4_packet_len;
newflow->src2dst_pkt_time[newflow->src2dst_pkt_count] = when;
if (newflow->src2dst_pkt_count == 0) {
if(newflow->src2dst_pkt_count == 0) {
newflow->src2dst_start = when;
}
newflow->src2dst_pkt_count++;
if (l4_data_len != 0XFEEDFACE) {
if(l4_data_len != 0XFEEDFACE) {
newflow->src2dst_opackets++;
}
return newflow;
@@ -838,7 +845,7 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
)
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
else
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
@@ -848,30 +855,30 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
)
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
else
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
}
if (src_to_dst_direction) {
if (flow->src2dst_pkt_count < MAX_NUM_PKTS) {
if(src_to_dst_direction) {
if(flow->src2dst_pkt_count < MAX_NUM_PKTS) {
flow->src2dst_pkt_len[flow->src2dst_pkt_count] = l4_packet_len;
flow->src2dst_pkt_time[flow->src2dst_pkt_count] = when;
flow->src2dst_pkt_count++;
}
if (l4_data_len != 0XFEEDFACE) {
if(l4_data_len != 0XFEEDFACE) {
flow->src2dst_opackets++;
}
} else {
if (flow->dst2src_pkt_count < MAX_NUM_PKTS) {
if(flow->dst2src_pkt_count < MAX_NUM_PKTS) {
flow->dst2src_pkt_len[flow->dst2src_pkt_count] = l4_packet_len;
flow->dst2src_pkt_time[flow->dst2src_pkt_count] = when;
if (flow->dst2src_pkt_count == 0) {
if(flow->dst2src_pkt_count == 0) {
flow->dst2src_start = when;
}
flow->dst2src_pkt_count++;
}
if (l4_data_len != 0XFEEDFACE) {
if(l4_data_len != 0XFEEDFACE) {
flow->dst2src_opackets++;
}
}
@@ -985,7 +992,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
else if((flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS)
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS)
|| (flow->ndpi_flow->protos.stun_ssl.ssl.ja3_client[0] != '\0')
) {
) {
flow->ssh_tls.ssl_version = flow->ndpi_flow->protos.stun_ssl.ssl.ssl_version;
snprintf(flow->ssh_tls.client_info, sizeof(flow->ssh_tls.client_info), "%s",
flow->ndpi_flow->protos.stun_ssl.ssl.client_certificate);
@@ -1073,7 +1080,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if((tcph != NULL) && (tcph->fin || tcph->rst || tcph->syn))
begin_or_end_tcp = 1;
if(flow->flow_last_pkt_time.tv_sec) {
ndpi_timer_sub(&when, &flow->flow_last_pkt_time, &tdiff);
@@ -1121,8 +1128,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if(enable_payload_analyzer && (payload_len > 0))
ndpi_payload_analyzer(flow, src_to_dst_direction,
payload, payload_len,
workflow->stats.ip_packet_count);
workflow->stats.ip_packet_count);
if(enable_joy_stats) {
/* Update BD, distribution and mean. */
ndpi_flow_update_byte_count(flow, payload, payload_len, src_to_dst_direction);
@@ -1131,7 +1138,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
if(flow->first_seen == 0)
flow->first_seen = time;
flow->last_seen = time;
if(!flow->has_human_readeable_strings) {
@@ -1280,7 +1287,7 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
datalink_type = (int)pcap_datalink(workflow->pcap_handle);
#endif
datalink_check:
datalink_check:
switch(datalink_type) {
case DLT_NULL:
if(ntohl(*((u_int32_t*)&packet[eth_offset])) == 2)
@@ -1416,7 +1423,7 @@ datalink_check:
workflow->stats.vlan_count += vlan_packet;
iph_check:
iph_check:
/* Check and set IP header size and total packet length */
iph = (struct ndpi_iphdr *) &packet[ip_offset];
@@ -1610,7 +1617,7 @@ u_int32_t ethernet_crc32(const void* data, size_t n_bytes) {
#ifdef USE_DPDK
static const struct rte_eth_conf port_conf_default = {
.rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN }
.rxmode = { .max_rx_pkt_len = ETHER_MAX_LEN }
};
/* ************************************ */
@@ -1627,13 +1634,13 @@ int dpdk_port_init(int port, struct rte_mempool *mbuf_pool) {
if(retval != 0)
return retval;
for (q = 0; q < rx_rings; q++) {
for(q = 0; q < rx_rings; q++) {
retval = rte_eth_rx_queue_setup(port, q, RX_RING_SIZE, rte_eth_dev_socket_id(port), NULL, mbuf_pool);
if(retval < 0)
return retval;
}
for (q = 0; q < tx_rings; q++) {
for(q = 0; q < tx_rings; q++) {
retval = rte_eth_tx_queue_setup(port, q, TX_RING_SIZE, rte_eth_dev_socket_id(port), NULL);
if(retval < 0)
return retval;