Implemented custom category loading.

Modified ndpiReader with -c <file> (use mining_hosts.txt as example) for loading name-based categories
This commit is contained in:
Luca
2018-04-26 21:10:59 +02:00
parent 79f3199339
commit 3adccc98f7
7 changed files with 762 additions and 249 deletions
+99 -92
View File
@@ -1,7 +1,7 @@
/*
* ndpi_util.c
*
* Copyright (C) 2011-17 - ntop.org
* Copyright (C) 2011-18 - ntop.org
*
* This file is part of nDPI, an open source deep packet inspection
* library based on the OpenDPI and PACE technology by ipoque GmbH
@@ -114,49 +114,49 @@ static uint16_t ndpi_get_proto_id(struct ndpi_detection_module_struct *ndpi_mod,
char *e;
unsigned long p = strtol(name,&e,0);
if(e && !*e) {
if(p < NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS &&
ndpi_mod->proto_defaults[p].protoName) return (uint16_t)p;
return NDPI_PROTOCOL_UNKNOWN;
if(p < NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS &&
ndpi_mod->proto_defaults[p].protoName) return (uint16_t)p;
return NDPI_PROTOCOL_UNKNOWN;
}
for(proto_id=NDPI_PROTOCOL_UNKNOWN; proto_id < NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS; proto_id++) {
if(ndpi_mod->proto_defaults[proto_id].protoName &&
!strcasecmp(ndpi_mod->proto_defaults[proto_id].protoName,name))
return proto_id;
if(ndpi_mod->proto_defaults[proto_id].protoName &&
!strcasecmp(ndpi_mod->proto_defaults[proto_id].protoName,name))
return proto_id;
}
return NDPI_PROTOCOL_UNKNOWN;
}
static NDPI_PROTOCOL_BITMASK debug_bitmask;
static char _proto_delim[] = " \t,:;";
static int parse_debug_proto(struct ndpi_detection_module_struct *ndpi_mod, char *str) {
char *n;
uint16_t proto;
char op=1;
for(n = strtok(str,_proto_delim); n && *n; n = strtok(NULL,_proto_delim)) {
if(*n == '-') {
op = 0;
n++;
} else if(*n == '+') {
op = 1;
n++;
}
if(!strcmp(n,"all")) {
if(op)
NDPI_BITMASK_SET_ALL(debug_bitmask);
else
NDPI_BITMASK_RESET(debug_bitmask);
continue;
}
proto = ndpi_get_proto_id(ndpi_mod, n);
if(proto == NDPI_PROTOCOL_UNKNOWN && strcmp(n,"unknown") && strcmp(n,"0")) {
fprintf(stderr,"Invalid protocol %s\n",n);
return 1;
}
if(op)
NDPI_BITMASK_ADD(debug_bitmask,proto);
else
NDPI_BITMASK_DEL(debug_bitmask,proto);
}
return 0;
char *n;
uint16_t proto;
char op=1;
for(n = strtok(str,_proto_delim); n && *n; n = strtok(NULL,_proto_delim)) {
if(*n == '-') {
op = 0;
n++;
} else if(*n == '+') {
op = 1;
n++;
}
if(!strcmp(n,"all")) {
if(op)
NDPI_BITMASK_SET_ALL(debug_bitmask);
else
NDPI_BITMASK_RESET(debug_bitmask);
continue;
}
proto = ndpi_get_proto_id(ndpi_mod, n);
if(proto == NDPI_PROTOCOL_UNKNOWN && strcmp(n,"unknown") && strcmp(n,"0")) {
fprintf(stderr,"Invalid protocol %s\n",n);
return 1;
}
if(op)
NDPI_BITMASK_ADD(debug_bitmask,proto);
else
NDPI_BITMASK_DEL(debug_bitmask,proto);
}
return 0;
}
/* ***************************************************** */
@@ -164,16 +164,17 @@ return 0;
extern char *_debug_protocols;
static int _debug_protocols_ok = 0;
struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle) {
struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs,
pcap_t * pcap_handle) {
set_ndpi_malloc(malloc_wrapper), set_ndpi_free(free_wrapper);
set_ndpi_flow_malloc(NULL), set_ndpi_flow_free(NULL);
/* TODO: just needed here to init ndpi malloc wrapper */
struct ndpi_detection_module_struct * module = ndpi_init_detection_module();
struct ndpi_workflow * workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
workflow->pcap_handle = pcap_handle;
workflow->prefs = *prefs;
workflow->prefs = *prefs;
workflow->ndpi_struct = module;
if(workflow->ndpi_struct == NULL) {
@@ -183,16 +184,19 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre
module->ndpi_log_level = nDPI_LogLevel;
if(_debug_protocols != NULL && ! _debug_protocols_ok) {
if(parse_debug_proto(module,_debug_protocols))
exit(-1);
_debug_protocols_ok = 1;
if(parse_debug_proto(module,_debug_protocols))
exit(-1);
_debug_protocols_ok = 1;
}
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
NDPI_BITMASK_RESET(module->debug_bitmask);
if(_debug_protocols_ok)
module->debug_bitmask = debug_bitmask;
module->debug_bitmask = debug_bitmask;
#endif
workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *));
return workflow;
}
@@ -232,20 +236,20 @@ int ndpi_workflow_node_cmp(const void *a, const void *b) {
if(fa->protocol < fb->protocol ) return(-1); else { if(fa->protocol > fb->protocol ) return(1); }
if(
(
(
(fa->src_ip == fb->src_ip )
&& (fa->src_port == fb->src_port)
&& (fa->dst_ip == fb->dst_ip )
&& (fa->dst_port == fb->dst_port)
)
||
(
||
(
(fa->src_ip == fb->dst_ip )
&& (fa->src_port == fb->dst_port)
&& (fa->dst_ip == fb->src_ip )
&& (fa->dst_port == fb->src_port)
)
)
)
return(0);
if(fa->src_ip < fb->src_ip ) return(-1); else { if(fa->src_ip > fb->src_ip ) return(1); }
@@ -367,24 +371,24 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp);
/* to avoid two nodes in one binary tree for a flow */
int is_changed = 0;
if(ret == NULL)
{
u_int32_t orig_src_ip = flow.src_ip;
u_int16_t orig_src_port = flow.src_port;
u_int32_t orig_dst_ip = flow.dst_ip;
u_int16_t orig_dst_port = flow.dst_port;
/* to avoid two nodes in one binary tree for a flow */
int is_changed = 0;
if(ret == NULL)
{
u_int32_t orig_src_ip = flow.src_ip;
u_int16_t orig_src_port = flow.src_port;
u_int32_t orig_dst_ip = flow.dst_ip;
u_int16_t orig_dst_port = flow.dst_port;
flow.src_ip = orig_dst_ip;
flow.src_port = orig_dst_port;
flow.dst_ip = orig_src_ip;
flow.dst_port = orig_src_port;
flow.src_ip = orig_dst_ip;
flow.src_port = orig_dst_port;
flow.dst_ip = orig_src_ip;
flow.dst_port = orig_src_port;
is_changed = 1;
is_changed = 1;
ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp);
}
ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp);
}
if(ret == NULL) {
if(workflow->stats.ndpi_flow_count == workflow->prefs.max_ndpi_flows) {
@@ -449,26 +453,26 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
} else {
struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret;
if (is_changed) {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
else
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
}
else {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
else
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
}
if (is_changed) {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
else
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
}
else {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
else
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
}
return flow;
}
}
@@ -514,6 +518,8 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) {
if(!flow->ndpi_flow) return;
/* printf("CATEGORY %u\n", flow->detected_protocol.category); */
snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s",
flow->ndpi_flow->host_server_name);
@@ -633,8 +639,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
}
if(ndpi_flow->num_extra_packets_checked < ndpi_flow->max_extra_packets_to_check) {
ndpi_process_extra_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
if (ndpi_flow->check_extra_packets == 0) {
flow->check_extra_packets = 0;
process_ndpi_collected_info(workflow, flow);
@@ -649,9 +655,10 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
return(flow->detected_protocol);
}
flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
flow->detected_protocol =
ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
if((flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN)
|| ((proto == IPPROTO_UDP) && ((flow->src2dst_packets + flow->dst2src_packets) > 8))
@@ -663,8 +670,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
flow->check_extra_packets = 1;
if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_UNKNOWN)
flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct,
flow->ndpi_flow);
flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct,
flow->ndpi_flow);
process_ndpi_collected_info(workflow, flow);
}
@@ -739,7 +746,7 @@ struct ndpi_proto ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
/*** check Data Link type ***/
const int datalink_type = pcap_datalink(workflow->pcap_handle);
datalink_check:
datalink_check:
switch(datalink_type) {
case DLT_NULL:
if(ntohl(*((u_int32_t*)&packet[eth_offset])) == 2)
@@ -875,7 +882,7 @@ struct ndpi_proto ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
workflow->stats.vlan_count += vlan_packet;
iph_check:
iph_check:
/* Check and set IP header size and total packet length */
iph = (struct ndpi_iphdr *) &packet[ip_offset];