Added the ability to define custom protocols with arbitrary Ids in proto.txt

Example
- ip:213.75.170.11/32:443@CustomProtocol
nDPI assigns an is that can change based on protos.txt content

- ip:213.75.170.11/32:443@CustomProtocol=9999
nDPI assigns 9999 as protocolId to CustomProtocol and won't change when
protos.txt content will chaneg
This commit is contained in:
Luca Deri
2023-03-22 00:15:56 +01:00
parent b3457c644f
commit 64ebf73b29
6 changed files with 190 additions and 44 deletions
+161 -26
View File
@@ -222,7 +222,7 @@ ndpi_custom_dga_predict_fctn ndpi_dga_function = NULL;
/* ****************************************** */
static inline uint8_t flow_is_proto(struct ndpi_flow_struct *flow, u_int16_t p) {
static inline u_int8_t flow_is_proto(struct ndpi_flow_struct *flow, u_int16_t p) {
return((flow->detected_protocol_stack[0] == p) || (flow->detected_protocol_stack[1] == p));
}
@@ -294,6 +294,73 @@ u_int16_t ndpi_get_proto_by_name(struct ndpi_detection_module_struct *ndpi_str,
return(NDPI_PROTOCOL_UNKNOWN);
}
/* ************************************************************************************* */
/* ************************************************************************************* */
void ndpi_add_user_proto_id_mapping(struct ndpi_detection_module_struct *ndpi_str,
u_int16_t ndpi_proto_id, u_int16_t user_proto_id) {
if(ndpi_proto_id < NDPI_MAX_SUPPORTED_PROTOCOLS)
return; /* Nothing to map */
#ifdef DEBUG
printf("*** %u (>= %u)-> %u\n",
ndpi_proto_id, NDPI_MAX_SUPPORTED_PROTOCOLS,
user_proto_id);
#endif
ndpi_str->ndpi_to_user_proto_id[ndpi_proto_id-NDPI_MAX_SUPPORTED_PROTOCOLS] = user_proto_id;
}
/* ************************************************************************************* */
/* Map a custom user protocol into an internal nDPI protocol id */
u_int16_t ndpi_map_user_proto_id_to_ndpi_id(struct ndpi_detection_module_struct *ndpi_str,
u_int16_t user_proto_id) {
#ifdef DEBUG
printf("***** %s(%u)\n", __FUNCTION__, user_proto_id);
#endif
if(user_proto_id < NDPI_MAX_SUPPORTED_PROTOCOLS)
return(user_proto_id);
else {
u_int idx, idx_max = ndpi_str->ndpi_num_supported_protocols - NDPI_MAX_SUPPORTED_PROTOCOLS;
/* TODO: improve it and remove linear scan */
for(idx = 0; idx < idx_max; idx++) {
if(ndpi_str->ndpi_to_user_proto_id[idx] == 0)
break;
else if(ndpi_str->ndpi_to_user_proto_id[idx] == user_proto_id) {
return(idx + NDPI_MAX_SUPPORTED_PROTOCOLS);
}
}
}
return(0);
}
/* ************************************************************************************* */
/* Map an internal nDPI protocol id to a custom user protocol */
u_int16_t ndpi_map_ndpi_id_to_user_proto_id(struct ndpi_detection_module_struct *ndpi_str,
u_int16_t ndpi_proto_id) {
#ifdef DEBUG
printf("***** %s(%u)\n", __FUNCTION__, ndpi_proto_id);
#endif
if(ndpi_proto_id < NDPI_MAX_SUPPORTED_PROTOCOLS)
return(ndpi_proto_id);
else if(ndpi_proto_id < ndpi_str->ndpi_num_supported_protocols) {
u_int id = ndpi_proto_id - NDPI_MAX_SUPPORTED_PROTOCOLS;
if(id < ndpi_str->ndpi_num_supported_protocols)
return(ndpi_str->ndpi_to_user_proto_id[id]);
}
return(0);
}
/* ************************************************************************************* */
/* ************************************************************************************* */
#ifdef CODE_UNUSED
@@ -681,7 +748,7 @@ static u_int8_t ndpi_domain_level(const char *name) {
while((c = *name++) != '\0') {
c = c == '-' ? 2 : (c == '.' ? 1:0);
level += ndpi_domain_level_automat[state][3];
state = ndpi_domain_level_automat[state][(uint8_t)c];
state = ndpi_domain_level_automat[state][(u_int8_t)c];
if(!state) break;
}
return state >= 2 ? level:0;
@@ -692,7 +759,7 @@ static u_int8_t ndpi_domain_level(const char *name) {
static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_str,
AC_AUTOMATA_t *ac_automa, const char *value,
u_int16_t protocol_id, ndpi_protocol_category_t category,
ndpi_protocol_breed_t breed, uint8_t level,
ndpi_protocol_breed_t breed, u_int8_t level,
u_int8_t add_ends_with) {
AC_PATTERN_t ac_pattern;
AC_ERROR_t rc;
@@ -747,7 +814,7 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_str,
static int ndpi_add_host_url_subprotocol(struct ndpi_detection_module_struct *ndpi_str,
char *value, int protocol_id,
ndpi_protocol_category_t category,
ndpi_protocol_breed_t breed, uint8_t level) {
ndpi_protocol_breed_t breed, u_int8_t level) {
#ifndef DEBUG
NDPI_LOG_DBG2(ndpi_str, "[NDPI] Adding [%s][%d]\n", value, protocol_id);
#endif
@@ -2504,6 +2571,8 @@ void ndpi_debug_printf(unsigned int proto, struct ndpi_detection_module_struct *
#endif
}
/* ****************************************** */
void set_ndpi_debug_function(struct ndpi_detection_module_struct *ndpi_str, ndpi_debug_function_ptr ndpi_debug_printf) {
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
ndpi_str->ndpi_debug_printf = ndpi_debug_printf;
@@ -3667,7 +3736,8 @@ int ndpi_add_trusted_issuer_dn(struct ndpi_detection_module_struct *ndpi_str, ch
}
/* ******************************************************************** */
int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, char *rule, u_int8_t do_add) {
int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str,
char *rule, u_int8_t do_add) {
char *at, *proto, *elem;
ndpi_proto_defaults_t *def;
u_int subprotocol_id, i;
@@ -3737,18 +3807,38 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, char *rule,
return(-3);
} else {
ndpi_port_range ports_a[MAX_DEFAULT_PORTS], ports_b[MAX_DEFAULT_PORTS];
char *equal = strchr(proto, '=');
u_int16_t user_proto_id = ndpi_str->ndpi_num_supported_protocols;
if(equal != NULL) {
/* PROTO=VALUE */
equal[0] = '\0';
user_proto_id = atoi(&equal[1]);
/* NOTE: ndpi_str->ndpi_num_supported_protocols >= NDPI_MAX_SUPPORTED_PROTOCOLS */
ndpi_add_user_proto_id_mapping(ndpi_str, ndpi_str->ndpi_num_supported_protocols, user_proto_id);
/* printf("***** ADDING MAPPING %s: %u -> %u\n", proto, ndpi_str->ndpi_num_supported_protocols, user_proto_id); */
} else
ndpi_add_user_proto_id_mapping(ndpi_str, ndpi_str->ndpi_num_supported_protocols,
ndpi_str->ndpi_num_supported_protocols);
if(ndpi_str->ndpi_num_custom_protocols >= (NDPI_MAX_NUM_CUSTOM_PROTOCOLS - 1)) {
NDPI_LOG_ERR(ndpi_str, "Too many protocols defined (%u): skipping protocol %s\n",
ndpi_str->ndpi_num_custom_protocols, proto);
return(-2);
}
ndpi_set_proto_defaults(ndpi_str, 1, 1, NDPI_PROTOCOL_ACCEPTABLE,
ndpi_str->ndpi_num_supported_protocols, proto,
ndpi_set_proto_defaults(ndpi_str, 1 /* is_cleartext */,
1 /* is_app_protocol */,
NDPI_PROTOCOL_ACCEPTABLE, /* breed*/
ndpi_str->ndpi_num_supported_protocols /* protoId */,
proto, /* protoName */
NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, /* TODO add protocol category support in rules */
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
def = &ndpi_str->proto_defaults[ndpi_str->ndpi_num_supported_protocols];
subprotocol_id = ndpi_str->ndpi_num_supported_protocols;
ndpi_str->ndpi_num_supported_protocols++, ndpi_str->ndpi_num_custom_protocols++;
@@ -6504,7 +6594,9 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
u_int8_t user_defined_proto;
/* guess protocol */
flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_str, flow, flow->l4_proto, ntohs(flow->c_port), ntohs(flow->s_port), &user_defined_proto);
flow->guessed_protocol_id = (int16_t) ndpi_guess_protocol_id(ndpi_str, flow, flow->l4_proto,
ntohs(flow->c_port), ntohs(flow->s_port),
&user_defined_proto);
flow->guessed_protocol_id_by_ip = ndpi_guess_host_protocol_id(ndpi_str, flow);
ret->protocol_by_ip = flow->guessed_protocol_id_by_ip;
@@ -6554,10 +6646,10 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n
/* ********************************************************************************* */
ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow, const unsigned char *packet_data,
const unsigned short packetlen, const u_int64_t current_time_ms,
const struct ndpi_flow_input_info *input_info) {
static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow, const unsigned char *packet_data,
const unsigned short packetlen, const u_int64_t current_time_ms,
const struct ndpi_flow_input_info *input_info) {
struct ndpi_packet_struct *packet;
NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_packet;
u_int32_t num_calls = 0;
@@ -6565,16 +6657,17 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
memset(&ret, 0, sizeof(ret));
if(!flow || !ndpi_str)
if((!flow) || (!ndpi_str))
return(ret);
packet = &ndpi_str->packet;
NDPI_LOG_DBG(ndpi_str, "[%d/%d] START packet processing\n",
flow->detected_protocol_stack[0], flow->detected_protocol_stack[1]);
flow->detected_protocol_stack[0],
flow->detected_protocol_stack[1]);
ret.master_protocol = flow->detected_protocol_stack[1],
ret.app_protocol = flow->detected_protocol_stack[0];
ret.app_protocol = flow->detected_protocol_stack[0];
ret.protocol_by_ip = flow->guessed_protocol_id_by_ip;
ret.category = flow->category;
@@ -6600,16 +6693,16 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
ret.app_protocol = flow->detected_protocol_stack[0],
ret.category = flow->category;
return ret;
return(ret);
} else if(flow->detected_protocol_stack[0] != NDPI_PROTOCOL_UNKNOWN) {
if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen, input_info) != 0)
return ret;
return(ret);
goto ret_protocols;
}
if(ndpi_init_packet(ndpi_str, flow, current_time_ms, packet_data, packetlen, input_info) != 0)
return ret;
return(ret);
#ifdef HAVE_NBPF
if((flow->num_processed_pkts == 1) /* first packet of this flow to be analyzed */
@@ -6680,7 +6773,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
flow->protocol_id_already_guessed = 1;
if(ndpi_do_guess(ndpi_str, flow, &ret) == -1)
return ret;
return(ret);
}
num_calls = ndpi_check_flow_func(ndpi_str, flow, &ndpi_selection_packet);
@@ -6861,6 +6954,24 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
/* ********************************************************************************* */
ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow, const unsigned char *packet_data,
const unsigned short packetlen, const u_int64_t current_time_ms,
const struct ndpi_flow_input_info *input_info) {
ndpi_protocol p = ndpi_internal_detection_process_packet(ndpi_str, flow, packet_data,
packetlen, current_time_ms,
input_info);
p.master_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.master_protocol),
p.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.app_protocol);
return(p);
}
/* ********************************************************************************* */
u_int32_t ndpi_bytestream_to_number(const u_int8_t *str, u_int16_t max_chars_to_read, u_int16_t *bytes_read) {
u_int32_t val;
val = 0;
@@ -7718,17 +7829,17 @@ u_int16_t ndpi_get_upper_proto(ndpi_protocol proto) {
/* ****************************************************** */
ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow, u_int8_t proto,
u_int32_t shost /* host byte order */, u_int16_t sport,
u_int32_t dhost /* host byte order */, u_int16_t dport) {
static ndpi_protocol ndpi_internal_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow, u_int8_t proto,
u_int32_t shost /* host byte order */, u_int16_t sport,
u_int32_t dhost /* host byte order */, u_int16_t dport) {
u_int32_t rc;
struct in_addr addr;
ndpi_protocol ret = NDPI_PROTOCOL_NULL;
u_int8_t user_defined_proto;
if(!ndpi_str)
return ret;
return(ret);
#ifdef BITTORRENT_CACHE_DEBUG
printf("[%s:%u] ndpi_guess_undetected_protocol(%08X, %u, %08X, %u) [flow: %p]\n",
@@ -7816,6 +7927,21 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
/* ****************************************************** */
ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow, u_int8_t proto,
u_int32_t shost /* host byte order */, u_int16_t sport,
u_int32_t dhost /* host byte order */, u_int16_t dport) {
ndpi_protocol p = ndpi_internal_guess_undetected_protocol(ndpi_str, flow, proto,
shost, sport, dhost, dport);
p.master_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.master_protocol),
p.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, p.app_protocol);
return(p);
}
/* ****************************************************** */
char *ndpi_protocol2id(struct ndpi_detection_module_struct *ndpi_str,
ndpi_protocol proto, char *buf, u_int buf_len) {
if((proto.master_protocol != NDPI_PROTOCOL_UNKNOWN) && (proto.master_protocol != proto.app_protocol)) {
@@ -7987,9 +8113,15 @@ static int category_depends_on_master(int proto)
ndpi_protocol_category_t ndpi_get_proto_category(struct ndpi_detection_module_struct *ndpi_str,
ndpi_protocol proto) {
if(proto.category != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED)
return(proto.category);
#if 0
proto.master_protocol = ndpi_map_user_proto_id_to_ndpi_id(ndpi_str, proto.master_protocol),
proto.app_protocol = ndpi_map_user_proto_id_to_ndpi_id(ndpi_str, proto.app_protocol);
#endif
/* Simple rule: sub protocol first, master after, with some exceptions (i.e. mail) */
if(category_depends_on_master(proto.master_protocol)) {
@@ -8009,8 +8141,9 @@ ndpi_protocol_category_t ndpi_get_proto_category(struct ndpi_detection_module_st
char *ndpi_get_proto_name(struct ndpi_detection_module_struct *ndpi_str,
u_int16_t proto_id) {
if(!ndpi_str) return("Unknown");
proto_id = ndpi_map_user_proto_id_to_ndpi_id(ndpi_str, proto_id);
if((proto_id >= ndpi_str->ndpi_num_supported_protocols)
|| (!ndpi_is_valid_protoId(proto_id))
@@ -8026,6 +8159,8 @@ ndpi_protocol_breed_t ndpi_get_proto_breed(struct ndpi_detection_module_struct *
u_int16_t proto_id) {
if(!ndpi_str) return(NDPI_PROTOCOL_UNRATED);
proto_id = ndpi_map_user_proto_id_to_ndpi_id(ndpi_str, proto_id);
if((proto_id >= ndpi_str->ndpi_num_supported_protocols) ||
(!ndpi_is_valid_protoId(proto_id)) ||