Add basic profiling of memory allocations on data-path (#1789)
The goal is to have an idea of the memory allocation sizes performed in the **library data-path**, i.e. excluding init/deinit phases and all the allocations made by the application itself. In other words, how much memory is needed per-flow, by nDPI, other than `struct ndpi_flow_struct`? It works only on single-thread configurations. It is not enabled by default (in the unit tests) since different canfiguration options (example: `--enable-pcre`) lead to diffferent results. See: #1781
This commit is contained in:
+32
-1
@@ -127,6 +127,11 @@ extern u_int16_t min_pattern_len, max_pattern_len;
|
||||
extern void ndpi_self_check_host_match(); /* Self check function */
|
||||
u_int8_t dump_internal_stats;
|
||||
|
||||
struct ndpi_bin malloc_bins;
|
||||
int enable_malloc_bins = 0;
|
||||
int max_malloc_bins = 14;
|
||||
int malloc_size_stats = 0;
|
||||
|
||||
struct flow_info {
|
||||
struct ndpi_flow_info *flow;
|
||||
u_int16_t thread_id;
|
||||
@@ -506,6 +511,7 @@ static void help(u_int long_help) {
|
||||
" -I | Ignore VLAN id for flow hash calculation\n"
|
||||
" -z | Enable JA3+\n"
|
||||
" -A | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n"
|
||||
" -M | Memory allocation stats on data-path (only by the library). It works only on single-thread configuration\n"
|
||||
,
|
||||
human_readeable_string_len,
|
||||
min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection,
|
||||
@@ -808,7 +814,7 @@ static void parseOptions(int argc, char **argv) {
|
||||
}
|
||||
#endif
|
||||
|
||||
while((opt = getopt_long(argc, argv, "a:Ab:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zq0123:456:7:89:m:T:U:",
|
||||
while((opt = getopt_long(argc, argv, "a:Ab:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zq0123:456:7:89:m:MT:U:",
|
||||
longopts, &option_idx)) != EOF) {
|
||||
#ifdef DEBUG_TRACE
|
||||
if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : "");
|
||||
@@ -974,6 +980,11 @@ static void parseOptions(int argc, char **argv) {
|
||||
}
|
||||
break;
|
||||
|
||||
case 'M':
|
||||
enable_malloc_bins = 1;
|
||||
ndpi_init_bin(&malloc_bins, ndpi_bin_family64, max_malloc_bins);
|
||||
break;
|
||||
|
||||
case 'k':
|
||||
errno = 0;
|
||||
if((serialization_fp = fopen(optarg, "w")) == NULL)
|
||||
@@ -1110,6 +1121,12 @@ static void parseOptions(int argc, char **argv) {
|
||||
for(thread_id = 1; thread_id < num_threads; thread_id++)
|
||||
_pcap_file[thread_id] = _pcap_file[0];
|
||||
}
|
||||
|
||||
if(num_threads > 1 && enable_malloc_bins == 1)
|
||||
{
|
||||
printf("Memory profiling ('-M') is incompatible with multi-thread enviroment");
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef __linux__
|
||||
@@ -1881,8 +1898,10 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept
|
||||
if((!flow->detection_completed) && flow->ndpi_flow) {
|
||||
u_int8_t proto_guessed;
|
||||
|
||||
malloc_size_stats = 1;
|
||||
flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[0].workflow->ndpi_struct,
|
||||
flow->ndpi_flow, enable_protocol_guess, &proto_guessed);
|
||||
malloc_size_stats = 0;
|
||||
|
||||
if(enable_protocol_guess) ndpi_thread_info[thread_id].workflow->stats.guessed_flow_protocols++;
|
||||
}
|
||||
@@ -3528,6 +3547,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
|
||||
}
|
||||
|
||||
if(dump_internal_stats) {
|
||||
char buf[1024];
|
||||
|
||||
if(cumulative_stats.ndpi_flow_count)
|
||||
printf("\tNum dissector calls: %-13llu (%.2f diss/flow)\n",
|
||||
(long long unsigned int)cumulative_stats.num_dissector_calls,
|
||||
@@ -3587,6 +3608,9 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
|
||||
printf("\tPatricia protocols: %llu/%llu (search/found)\n",
|
||||
(long long unsigned int)cumulative_stats.patricia_stats[NDPI_PTREE_PROTOCOLS].n_search,
|
||||
(long long unsigned int)cumulative_stats.patricia_stats[NDPI_PTREE_PROTOCOLS].n_found);
|
||||
|
||||
if(enable_malloc_bins)
|
||||
printf("\tData-path malloc histogram: %s\n", ndpi_print_bin(&malloc_bins, 0, buf, sizeof(buf)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3615,6 +3639,8 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
|
||||
}
|
||||
|
||||
if(dump_internal_stats) {
|
||||
char buf[1024];
|
||||
|
||||
if(cumulative_stats.ndpi_flow_count)
|
||||
fprintf(results_file, "Num dissector calls: %llu (%.2f diss/flow)\n",
|
||||
(long long unsigned int)cumulative_stats.num_dissector_calls,
|
||||
@@ -3674,6 +3700,9 @@ static void printResults(u_int64_t processing_time_usec, u_int64_t setup_time_us
|
||||
fprintf(results_file, "Patricia protocols: %llu/%llu (search/found)\n",
|
||||
(long long unsigned int)cumulative_stats.patricia_stats[NDPI_PTREE_PROTOCOLS].n_search,
|
||||
(long long unsigned int)cumulative_stats.patricia_stats[NDPI_PTREE_PROTOCOLS].n_found);
|
||||
|
||||
if(enable_malloc_bins)
|
||||
fprintf(results_file, "Data-path malloc histogram: %s\n", ndpi_print_bin(&malloc_bins, 0, buf, sizeof(buf)));
|
||||
}
|
||||
|
||||
fprintf(results_file, "\n");
|
||||
@@ -5149,6 +5178,8 @@ void zscoreUnitTest() {
|
||||
if(extcap_dumper) pcap_dump_close(extcap_dumper);
|
||||
if(extcap_fifo_h) pcap_close(extcap_fifo_h);
|
||||
if(ndpi_info_mod) ndpi_exit_detection_module(ndpi_info_mod);
|
||||
if(enable_malloc_bins)
|
||||
ndpi_free_bin(&malloc_bins);
|
||||
if(csv_fp) fclose(csv_fp);
|
||||
ndpi_free(_debug_protocols);
|
||||
|
||||
|
||||
@@ -80,6 +80,11 @@ static u_int32_t flow_id = 0;
|
||||
u_int8_t enable_doh_dot_detection = 0;
|
||||
extern ndpi_init_prefs init_prefs;
|
||||
|
||||
extern int malloc_size_stats;
|
||||
extern struct ndpi_bin malloc_bins;
|
||||
extern int max_malloc_bins;
|
||||
extern int enable_malloc_bins;
|
||||
|
||||
/* ****************************************************** */
|
||||
|
||||
struct flow_id_stats {
|
||||
@@ -309,6 +314,17 @@ void ndpi_free_flow_info_half(struct ndpi_flow_info *flow) {
|
||||
|
||||
extern u_int32_t current_ndpi_memory, max_ndpi_memory;
|
||||
|
||||
static u_int32_t __slot_malloc_bins(u_int64_t v)
|
||||
{
|
||||
int i;
|
||||
|
||||
/* 0-2,3-4,5-8,9-16,17-32,33-64,65-128,129-256,257-512,513-1024,1025-2048,2049-4096,4097-8192,8193- */
|
||||
for(i=0; i < max_malloc_bins - 1; i++)
|
||||
if((1ULL << (i + 1)) >= v)
|
||||
return i;
|
||||
return i;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief ndpi_malloc wrapper function
|
||||
*/
|
||||
@@ -318,6 +334,9 @@ static void *ndpi_malloc_wrapper(size_t size) {
|
||||
if(current_ndpi_memory > max_ndpi_memory)
|
||||
max_ndpi_memory = current_ndpi_memory;
|
||||
|
||||
if(enable_malloc_bins && malloc_size_stats)
|
||||
ndpi_inc_bin(&malloc_bins, __slot_malloc_bins(size), 1);
|
||||
|
||||
return(malloc(size)); /* Don't change to ndpi_malloc !!!!! */
|
||||
}
|
||||
|
||||
@@ -1620,6 +1639,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
/* Set here any information (easily) available; in this trivial example we don't have any */
|
||||
input_info.in_pkt_dir = NDPI_IN_PKT_DIR_UNKNOWN;
|
||||
input_info.seen_flow_beginning = NDPI_FLOW_BEGINNING_UNKNOWN;
|
||||
malloc_size_stats = 1;
|
||||
flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
|
||||
iph ? (uint8_t *)iph : (uint8_t *)iph6,
|
||||
ipsize, time_ms, &input_info);
|
||||
@@ -1650,6 +1670,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
process_ndpi_collected_info(workflow, flow);
|
||||
}
|
||||
}
|
||||
malloc_size_stats = 0;
|
||||
}
|
||||
|
||||
#if 0
|
||||
|
||||
@@ -18,6 +18,10 @@ u_int8_t enable_flow_stats = 0;
|
||||
u_int8_t human_readeable_string_len = 5;
|
||||
u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */;
|
||||
ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus;
|
||||
int enable_malloc_bins = 0;
|
||||
int malloc_size_stats = 0;
|
||||
int max_malloc_bins = 0;
|
||||
struct ndpi_bin malloc_bins; /* unused */
|
||||
|
||||
int bufferToFile(const char * name, const uint8_t *Data, size_t Size) {
|
||||
FILE * fd;
|
||||
|
||||
Reference in New Issue
Block a user