Reducing the size of the ndpi_detection_module_struct structure. (#1490)
The ndpi_detection_module_struct structure contains 5 arrays "struct ndpi_call_function_struct" size 286*144=41 kB size, which are occupied by a small number of elements. At the moment we have callback_buffer_size = 172, tcp_with_payload=114, tcp_no_payload=8, udp=93, other 8. NDPI_MAX_SUPPORTED_PROTOCOLS = 285. Size of struct ndpi_detection_module_struct is 253136 bytes. Size of all structs ndpi_call_function_struct 5*286*144=205920 bytes. Real use memory size for struct ndpi_call_function_struct is (173+224)*144=57168 bytes.
This commit is contained in:
@@ -1011,20 +1011,17 @@ struct ndpi_detection_module_struct {
|
||||
void *user_data;
|
||||
#endif
|
||||
char custom_category_labels[NUM_CUSTOM_CATEGORIES][CUSTOM_CATEGORY_LABEL_LEN];
|
||||
|
||||
/* callback function buffer */
|
||||
struct ndpi_call_function_struct callback_buffer[NDPI_MAX_SUPPORTED_PROTOCOLS + 1];
|
||||
struct ndpi_call_function_struct *callback_buffer;
|
||||
struct ndpi_call_function_struct *callback_buffer_tcp_no_payload;
|
||||
struct ndpi_call_function_struct *callback_buffer_tcp_payload;
|
||||
struct ndpi_call_function_struct *callback_buffer_udp;
|
||||
struct ndpi_call_function_struct *callback_buffer_non_tcp_udp;
|
||||
u_int32_t callback_buffer_size;
|
||||
|
||||
struct ndpi_call_function_struct callback_buffer_tcp_no_payload[NDPI_MAX_SUPPORTED_PROTOCOLS + 1];
|
||||
u_int32_t callback_buffer_size_tcp_no_payload;
|
||||
|
||||
struct ndpi_call_function_struct callback_buffer_tcp_payload[NDPI_MAX_SUPPORTED_PROTOCOLS + 1];
|
||||
u_int32_t callback_buffer_size_tcp_payload;
|
||||
|
||||
struct ndpi_call_function_struct callback_buffer_udp[NDPI_MAX_SUPPORTED_PROTOCOLS + 1];
|
||||
u_int32_t callback_buffer_size_udp;
|
||||
|
||||
struct ndpi_call_function_struct callback_buffer_non_tcp_udp[NDPI_MAX_SUPPORTED_PROTOCOLS + 1];
|
||||
u_int32_t callback_buffer_size_non_tcp_udp;
|
||||
|
||||
ndpi_default_ports_tree_node_t *tcpRoot, *udpRoot;
|
||||
|
||||
+127
-39
@@ -157,6 +157,10 @@ static void ndpi_int_change_protocol(struct ndpi_detection_module_struct *ndpi_s
|
||||
u_int16_t upper_detected_protocol, u_int16_t lower_detected_protocol,
|
||||
ndpi_confidence_t confidence);
|
||||
|
||||
static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str);
|
||||
static void ndpi_enabled_callbacks_init(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const NDPI_PROTOCOL_BITMASK *dbm, int count_only);
|
||||
|
||||
/* ****************************************** */
|
||||
|
||||
ndpi_custom_dga_predict_fctn ndpi_dga_function = NULL;
|
||||
@@ -2563,6 +2567,11 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
|
||||
|
||||
ndpi_init_protocol_defaults(ndpi_str);
|
||||
|
||||
if(ndpi_callback_init(ndpi_str)) {
|
||||
NDPI_LOG_ERR(ndpi_str, "[NDPI] Error allocating callbacks\n");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++)
|
||||
snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u",
|
||||
(unsigned int) (i + 1));
|
||||
@@ -2956,6 +2965,10 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
|
||||
ndpi_free_geoip(ndpi_str);
|
||||
|
||||
if(ndpi_str->callback_buffer)
|
||||
ndpi_free(ndpi_str->callback_buffer);
|
||||
if(ndpi_str->callback_buffer_tcp_payload)
|
||||
ndpi_free(ndpi_str->callback_buffer_tcp_payload);
|
||||
ndpi_free(ndpi_str);
|
||||
}
|
||||
}
|
||||
@@ -3781,14 +3794,19 @@ void ndpi_set_bitmask_protocol_detection(char *label, struct ndpi_detection_modu
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const NDPI_PROTOCOL_BITMASK *dbm) {
|
||||
static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
|
||||
NDPI_PROTOCOL_BITMASK detection_bitmask_local;
|
||||
NDPI_PROTOCOL_BITMASK *detection_bitmask = &detection_bitmask_local;
|
||||
struct ndpi_call_function_struct *all_cb = NULL;
|
||||
u_int32_t a = 0;
|
||||
|
||||
NDPI_BITMASK_SET(detection_bitmask_local, *dbm);
|
||||
NDPI_BITMASK_SET(ndpi_str->detection_bitmask, *dbm);
|
||||
NDPI_ONE(detection_bitmask);
|
||||
|
||||
if(ndpi_str->callback_buffer) return 0;
|
||||
|
||||
ndpi_str->callback_buffer = ndpi_calloc(NDPI_MAX_SUPPORTED_PROTOCOLS+1,sizeof(struct ndpi_call_function_struct));
|
||||
if(!ndpi_str->callback_buffer) return 1;
|
||||
|
||||
/* set this here to zero to be interrupt safe */
|
||||
ndpi_str->callback_buffer_size = 0;
|
||||
@@ -4291,68 +4309,132 @@ void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *n
|
||||
/* ----------------------------------------------------------------- */
|
||||
|
||||
ndpi_str->callback_buffer_size = a;
|
||||
NDPI_BITMASK_SET(ndpi_str->detection_bitmask, detection_bitmask_local);
|
||||
|
||||
/* Resize callback_buffer */
|
||||
all_cb = ndpi_calloc(a+1,sizeof(struct ndpi_call_function_struct));
|
||||
if(all_cb) {
|
||||
memcpy((char *)all_cb,(char *)ndpi_str->callback_buffer, (a+1) * sizeof(struct ndpi_call_function_struct));
|
||||
ndpi_free(ndpi_str->callback_buffer);
|
||||
ndpi_str->callback_buffer = all_cb;
|
||||
}
|
||||
|
||||
NDPI_LOG_DBG2(ndpi_str, "callback_buffer_size is %u\n", ndpi_str->callback_buffer_size);
|
||||
/* Calculating the size of an array for callback functions */
|
||||
ndpi_enabled_callbacks_init(ndpi_str,detection_bitmask,1);
|
||||
all_cb = ndpi_calloc(ndpi_str->callback_buffer_size_tcp_payload +
|
||||
ndpi_str->callback_buffer_size_tcp_no_payload +
|
||||
ndpi_str->callback_buffer_size_udp +
|
||||
ndpi_str->callback_buffer_size_non_tcp_udp,
|
||||
sizeof(struct ndpi_call_function_struct));
|
||||
if(!all_cb) return 1;
|
||||
ndpi_str->callback_buffer_tcp_payload = all_cb;
|
||||
all_cb += ndpi_str->callback_buffer_size_tcp_payload;
|
||||
ndpi_str->callback_buffer_tcp_no_payload = all_cb;
|
||||
all_cb += ndpi_str->callback_buffer_size_tcp_no_payload;
|
||||
ndpi_str->callback_buffer_udp = all_cb;
|
||||
all_cb += ndpi_str->callback_buffer_size_udp;
|
||||
ndpi_str->callback_buffer_non_tcp_udp = all_cb;
|
||||
|
||||
ndpi_enabled_callbacks_init(ndpi_str,detection_bitmask,0);
|
||||
|
||||
/* When the module ends, it is necessary to free the memory ndpi_str->callback_buffer and
|
||||
ndpi_str->callback_buffer_tcp_payload */
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline int ndpi_proto_cb_tcp_payload(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
}
|
||||
|
||||
static inline int ndpi_proto_cb_tcp_nopayload(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0
|
||||
&& (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0;
|
||||
}
|
||||
|
||||
static inline int ndpi_proto_cb_udp(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
}
|
||||
|
||||
static inline int ndpi_proto_cb_other(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP)) == 0
|
||||
||
|
||||
(ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) != 0;
|
||||
}
|
||||
|
||||
static void ndpi_enabled_callbacks_init(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const NDPI_PROTOCOL_BITMASK *dbm, int count_only) {
|
||||
uint32_t a;
|
||||
|
||||
/* now build the specific buffer for tcp, udp and non_tcp_udp */
|
||||
ndpi_str->callback_buffer_size_tcp_payload = 0;
|
||||
ndpi_str->callback_buffer_size_tcp_no_payload = 0;
|
||||
for(a = 0; a < ndpi_str->callback_buffer_size; a++) {
|
||||
if((ndpi_str->callback_buffer[a].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP | NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0) {
|
||||
if(!NDPI_ISSET(dbm,ndpi_str->callback_buffer[a].ndpi_protocol_id)) continue;
|
||||
if(!ndpi_proto_cb_tcp_payload(ndpi_str,a)) continue;
|
||||
if(!count_only) {
|
||||
if(_ndpi_debug_callbacks)
|
||||
NDPI_LOG_DBG2(ndpi_str, "callback_buffer_tcp_payload, adding buffer %u as entry %u\n", a,
|
||||
ndpi_str->callback_buffer_size_tcp_payload);
|
||||
|
||||
memcpy(&ndpi_str->callback_buffer_tcp_payload[ndpi_str->callback_buffer_size_tcp_payload],
|
||||
&ndpi_str->callback_buffer[a], sizeof(struct ndpi_call_function_struct));
|
||||
ndpi_str->callback_buffer_size_tcp_payload++;
|
||||
|
||||
if((ndpi_str->callback_buffer[a].ndpi_selection_bitmask & NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) ==
|
||||
0) {
|
||||
if(_ndpi_debug_callbacks)
|
||||
NDPI_LOG_DBG2(
|
||||
ndpi_str,
|
||||
"\tcallback_buffer_tcp_no_payload, additional adding buffer %u to no_payload process\n", a);
|
||||
|
||||
memcpy(&ndpi_str->callback_buffer_tcp_no_payload[ndpi_str->callback_buffer_size_tcp_no_payload],
|
||||
&ndpi_str->callback_buffer[a], sizeof(struct ndpi_call_function_struct));
|
||||
ndpi_str->callback_buffer_size_tcp_no_payload++;
|
||||
}
|
||||
NDPI_LOG_DBG2(ndpi_str, "callback_buffer_tcp_payload, adding buffer %u as entry %u\n", a,
|
||||
ndpi_str->callback_buffer_size_tcp_payload);
|
||||
memcpy(&ndpi_str->callback_buffer_tcp_payload[ndpi_str->callback_buffer_size_tcp_payload],
|
||||
&ndpi_str->callback_buffer[a], sizeof(struct ndpi_call_function_struct));
|
||||
}
|
||||
ndpi_str->callback_buffer_size_tcp_payload++;
|
||||
}
|
||||
for(a = 0; a < ndpi_str->callback_buffer_size; a++) {
|
||||
if(!NDPI_ISSET(dbm,ndpi_str->callback_buffer[a].ndpi_protocol_id)) continue;
|
||||
if(!ndpi_proto_cb_tcp_nopayload(ndpi_str,a)) continue;
|
||||
if(!count_only) {
|
||||
if(_ndpi_debug_callbacks)
|
||||
NDPI_LOG_DBG2( ndpi_str,
|
||||
"\tcallback_buffer_tcp_no_payload, additional adding buffer %u to no_payload process\n", a);
|
||||
memcpy(&ndpi_str->callback_buffer_tcp_no_payload[ndpi_str->callback_buffer_size_tcp_no_payload],
|
||||
&ndpi_str->callback_buffer[a], sizeof(struct ndpi_call_function_struct));
|
||||
}
|
||||
ndpi_str->callback_buffer_size_tcp_no_payload++;
|
||||
}
|
||||
|
||||
ndpi_str->callback_buffer_size_udp = 0;
|
||||
for(a = 0; a < ndpi_str->callback_buffer_size; a++) {
|
||||
if((ndpi_str->callback_buffer[a].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP | NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0) {
|
||||
if(!NDPI_ISSET(dbm,ndpi_str->callback_buffer[a].ndpi_protocol_id)) continue;
|
||||
if(!ndpi_proto_cb_udp(ndpi_str,a)) continue;
|
||||
if(!count_only) {
|
||||
if(_ndpi_debug_callbacks)
|
||||
NDPI_LOG_DBG2(ndpi_str, "callback_buffer_size_udp: adding buffer : %u as entry %u\n", a,
|
||||
ndpi_str->callback_buffer_size_udp);
|
||||
NDPI_LOG_DBG2(ndpi_str, "callback_buffer_size_udp: adding buffer : %u\n", a);
|
||||
|
||||
memcpy(&ndpi_str->callback_buffer_udp[ndpi_str->callback_buffer_size_udp], &ndpi_str->callback_buffer[a],
|
||||
sizeof(struct ndpi_call_function_struct));
|
||||
ndpi_str->callback_buffer_size_udp++;
|
||||
}
|
||||
ndpi_str->callback_buffer_size_udp++;
|
||||
}
|
||||
|
||||
ndpi_str->callback_buffer_size_non_tcp_udp = 0;
|
||||
for(a = 0; a < ndpi_str->callback_buffer_size; a++) {
|
||||
if((ndpi_str->callback_buffer[a].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP | NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP)) == 0 ||
|
||||
(ndpi_str->callback_buffer[a].ndpi_selection_bitmask & NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) !=
|
||||
0) {
|
||||
if(!NDPI_ISSET(dbm,ndpi_str->callback_buffer[a].ndpi_protocol_id)) continue;
|
||||
if(!ndpi_proto_cb_other(ndpi_str,a)) continue;
|
||||
if(!count_only) {
|
||||
if(_ndpi_debug_callbacks)
|
||||
NDPI_LOG_DBG2(ndpi_str, "callback_buffer_non_tcp_udp: adding buffer : %u as entry %u\n", a,
|
||||
ndpi_str->callback_buffer_size_non_tcp_udp);
|
||||
NDPI_LOG_DBG2(ndpi_str, "callback_buffer_non_tcp_udp: adding buffer : %u\n", a);
|
||||
|
||||
memcpy(&ndpi_str->callback_buffer_non_tcp_udp[ndpi_str->callback_buffer_size_non_tcp_udp],
|
||||
&ndpi_str->callback_buffer[a], sizeof(struct ndpi_call_function_struct));
|
||||
ndpi_str->callback_buffer_size_non_tcp_udp++;
|
||||
}
|
||||
ndpi_str->callback_buffer_size_non_tcp_udp++;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4596,6 +4678,12 @@ void ndpi_free_flow_data(struct ndpi_flow_struct* flow) {
|
||||
}
|
||||
}
|
||||
|
||||
void ndpi_set_protocol_detection_bitmask2(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const NDPI_PROTOCOL_BITMASK *dbm) {
|
||||
ndpi_enabled_callbacks_init(ndpi_str,dbm,0);
|
||||
NDPI_BITMASK_SET(ndpi_str->detection_bitmask, *dbm);
|
||||
}
|
||||
|
||||
/* ************************************************ */
|
||||
|
||||
static int ndpi_init_packet(struct ndpi_detection_module_struct *ndpi_str,
|
||||
|
||||
Reference in New Issue
Block a user