Added -C to generate CSV analysis files
Improved IAT and byte distribution
This commit is contained in:
+92
-13
@@ -73,6 +73,7 @@ static char *_statsFilePath = NULL; /**< Top stats file path */
|
||||
static char *_diagnoseFilePath = NULL; /**< Top stats file path */
|
||||
static char *_jsonFilePath = NULL; /**< JSON file path */
|
||||
static FILE *stats_fp = NULL; /**< for Top Stats JSON file */
|
||||
static FILE *csv_fp = NULL; /**< for CSV export */
|
||||
#endif
|
||||
#ifdef HAVE_JSON_C
|
||||
static json_object *jArray_known_flows = NULL, *jArray_unknown_flows = NULL;
|
||||
@@ -347,7 +348,7 @@ static void help(u_int long_help) {
|
||||
#endif
|
||||
"[-f <filter>][-s <duration>][-m <duration>]\n"
|
||||
" [-p <protos>][-l <loops> [-q][-d][-J][-h][-e <len>][-t][-v <level>]\n"
|
||||
" [-n <threads>][-w <file>][-c <file>][-j <file>][-x <file>]\n"
|
||||
" [-n <threads>][-w <file>][-c <file>][-C <file>][-j <file>][-x <file>]\n"
|
||||
" [-T <num>][-U <num>]\n\n"
|
||||
"Usage:\n"
|
||||
" -i <file.pcap|device> | Specify a pcap file/playlist to read packets from or a\n"
|
||||
@@ -378,6 +379,7 @@ static void help(u_int long_help) {
|
||||
" | Default: %u:%u:%u:%u:%u\n"
|
||||
" -r | Print nDPI version and git revision\n"
|
||||
" -c <path> | Load custom categories from the specified file\n"
|
||||
" -C <path> | Write output in CSV format on the specified file\n"
|
||||
" -w <path> | Write test output on the specified file. This is useful for\n"
|
||||
" | testing purposes in order to compare results across runs\n"
|
||||
" -h | This help\n"
|
||||
@@ -438,6 +440,8 @@ static struct option longopts[] = {
|
||||
|
||||
/* ndpiReader options */
|
||||
{ "enable-protocol-guess", no_argument, NULL, 'd'},
|
||||
{ "categories", required_argument, NULL, 'c'},
|
||||
{ "csv-dump", required_argument, NULL, 'C'},
|
||||
{ "interface", required_argument, NULL, 'i'},
|
||||
{ "filter", required_argument, NULL, 'f'},
|
||||
{ "cpu-bind", required_argument, NULL, 'g'},
|
||||
@@ -576,6 +580,26 @@ void extcap_capture() {
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
void printCSVHeader() {
|
||||
if(!csv_fp) return;
|
||||
|
||||
fprintf(csv_fp, "#flow_id,protocol,first_seen,last_seen,src_ip,src_port,dst_ip,dst_port,ndpi_proto_num,ndpi_proto,");
|
||||
fprintf(csv_fp, "src2dst_packets,src2dst_bytes,dst2src_packets,dst2src_bytes,");
|
||||
fprintf(csv_fp, "data_ratio,str_data_ratio,");
|
||||
|
||||
/* IAT (Inter Arrival Time) */
|
||||
fprintf(csv_fp, "iat_flow_min,iat_flow_avg,iat_flow_max,iat_flow_stddev,");
|
||||
fprintf(csv_fp, "iat_c_to_s_min,iat_c_to_s_avg,iat_c_to_s_max,iat_c_to_s_stddev,");
|
||||
fprintf(csv_fp, "iat_s_to_c_min,iat_s_to_c_avg,iat_s_to_c_max,iat_s_to_c_stddev,");
|
||||
|
||||
/* Packet Length */
|
||||
fprintf(csv_fp, "pktlen_c_to_s_min,pktlen_c_to_s_avg,pktlen_c_to_s_max,pktlen_c_to_s_stddev");
|
||||
fprintf(csv_fp, "pktlen_s_to_c_min,pktlen_s_to_c_avg,pktlen_s_to_c_max,pktlen_s_to_c_stddev");
|
||||
fprintf(csv_fp, "\n");
|
||||
}
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
/**
|
||||
* @brief Option parser
|
||||
*/
|
||||
@@ -604,7 +628,7 @@ static void parseOptions(int argc, char **argv) {
|
||||
}
|
||||
#endif
|
||||
|
||||
while((opt = getopt_long(argc, argv, "e:c:df:g:i:hp:P:l:s:tv:V:n:j:Jrp:w:q0123:456:7:89:m:b:x:T:U:",
|
||||
while((opt = getopt_long(argc, argv, "e:c:C:df:g:i:hp:P:l:s:tv:V:n:j:Jrp:w:q0123:456:7:89:m:b:x:T:U:",
|
||||
longopts, &option_idx)) != EOF) {
|
||||
#ifdef DEBUG_TRACE
|
||||
if(trace) fprintf(trace, " #### -%c [%s] #### \n", opt, optarg ? optarg : "");
|
||||
@@ -671,6 +695,13 @@ static void parseOptions(int argc, char **argv) {
|
||||
_customCategoryFilePath = optarg;
|
||||
break;
|
||||
|
||||
case 'C':
|
||||
if((csv_fp = fopen(optarg, "w")) == NULL)
|
||||
printf("Unable to write on CSV file %s\n", optarg);
|
||||
else
|
||||
printCSVHeader();
|
||||
break;
|
||||
|
||||
case 's':
|
||||
capture_for = atoi(optarg);
|
||||
capture_until = capture_for + time(NULL);
|
||||
@@ -949,7 +980,48 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
json_object *jObj;
|
||||
#endif
|
||||
FILE *out = results_file ? results_file : stdout;
|
||||
|
||||
|
||||
if(csv_fp != NULL) {
|
||||
char buf[32];
|
||||
float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes);
|
||||
float f = (float)flow->first_seen, l = (float)flow->last_seen;
|
||||
|
||||
/* PLEASE KEEP IN SYNC WITH printCSVHeader() */
|
||||
|
||||
fprintf(csv_fp, "%u,%u,%.3f,%.3f,%s,%u,%s,%u,",
|
||||
flow->flow_id,
|
||||
flow->protocol,
|
||||
f/1000.0, l/1000.0,
|
||||
flow->src_name, ntohs(flow->src_port),
|
||||
flow->dst_name, ntohs(flow->dst_port)
|
||||
);
|
||||
|
||||
fprintf(csv_fp, "%u.%u,%s,",
|
||||
flow->detected_protocol.master_protocol, flow->detected_protocol.app_protocol,
|
||||
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
|
||||
flow->detected_protocol, buf, sizeof(buf)));
|
||||
|
||||
fprintf(csv_fp, "%u,%llu,", flow->src2dst_packets, (long long unsigned int) flow->src2dst_bytes);
|
||||
fprintf(csv_fp, "%u,%llu,", flow->dst2src_packets, (long long unsigned int) flow->dst2src_bytes);
|
||||
|
||||
fprintf(csv_fp, "%.3f,%s,", data_ratio, ndpi_data_ratio2str(data_ratio));
|
||||
|
||||
/* IAT (Inter Arrival Time) */
|
||||
fprintf(csv_fp, "%u,%.1f,%u,%.1f",
|
||||
ndpi_data_min(flow->iat_flow), ndpi_data_average(flow->iat_flow), ndpi_data_max(flow->iat_flow), ndpi_data_stddev(flow->iat_flow));
|
||||
|
||||
fprintf(csv_fp, "%u,%.1f,%u,%.1f,%u,%.1f,%u,%.1f",
|
||||
ndpi_data_min(flow->iat_c_to_s), ndpi_data_average(flow->iat_c_to_s), ndpi_data_max(flow->iat_c_to_s), ndpi_data_stddev(flow->iat_c_to_s),
|
||||
ndpi_data_min(flow->iat_s_to_c), ndpi_data_average(flow->iat_s_to_c), ndpi_data_max(flow->iat_s_to_c), ndpi_data_stddev(flow->iat_s_to_c));
|
||||
|
||||
/* Packet Length */
|
||||
fprintf(csv_fp, "%u,%.1f,%u,%.1f,%u,%.1f,%u,%.1f",
|
||||
ndpi_data_min(flow->pktlen_c_to_s), ndpi_data_average(flow->pktlen_c_to_s), ndpi_data_max(flow->pktlen_c_to_s), ndpi_data_stddev(flow->pktlen_c_to_s),
|
||||
ndpi_data_min(flow->pktlen_s_to_c), ndpi_data_average(flow->pktlen_s_to_c), ndpi_data_max(flow->pktlen_s_to_c), ndpi_data_stddev(flow->pktlen_s_to_c));
|
||||
|
||||
fprintf(csv_fp, "\n");
|
||||
}
|
||||
|
||||
if((verbose != 1) && (verbose != 2))
|
||||
return;
|
||||
|
||||
@@ -1006,13 +1078,22 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
if((flow->src2dst_packets+flow->dst2src_packets) > 5) {
|
||||
if(flow->iat_c_to_s && flow->iat_s_to_c) {
|
||||
float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes);
|
||||
|
||||
fprintf(out, "[bytes ratio: %.3f (%s)]", data_ratio, ndpi_data_ratio2str(data_ratio));
|
||||
|
||||
/* IAT (Inter Arrival Time) */
|
||||
fprintf(out, "[IAT c2s/s2c avg/stddev/entropy: %.1f/%.1f %.1f/%.1f %.1f/%.1f]",
|
||||
ndpi_data_average(flow->iat_c_to_s), ndpi_data_average(flow->iat_s_to_c),
|
||||
ndpi_data_stddev(flow->iat_c_to_s), ndpi_data_stddev(flow->iat_s_to_c),
|
||||
ndpi_data_entropy(flow->iat_c_to_s), ndpi_data_entropy(flow->iat_s_to_c));
|
||||
fprintf(out, "[IAT c2s/s2c min/avg/max/stddev: %u/%u %.1f/%.1f %u/%u %.1f/%.1f]",
|
||||
ndpi_data_min(flow->iat_c_to_s), ndpi_data_min(flow->iat_s_to_c),
|
||||
(float)ndpi_data_average(flow->iat_c_to_s), (float)ndpi_data_average(flow->iat_s_to_c),
|
||||
ndpi_data_max(flow->iat_c_to_s), ndpi_data_max(flow->iat_s_to_c),
|
||||
(float)ndpi_data_stddev(flow->iat_c_to_s), (float)ndpi_data_stddev(flow->iat_s_to_c));
|
||||
|
||||
/* Packet Length */
|
||||
fprintf(out, "[Pkt Len c2s/s2c min/avg/max/stddev: %u/%u %.1f/%.1f %u/%u %.1f/%.1f]",
|
||||
ndpi_data_min(flow->pktlen_c_to_s), ndpi_data_min(flow->pktlen_s_to_c),
|
||||
ndpi_data_average(flow->pktlen_c_to_s), ndpi_data_average(flow->pktlen_s_to_c),
|
||||
ndpi_data_max(flow->pktlen_c_to_s), ndpi_data_max(flow->pktlen_s_to_c),
|
||||
ndpi_data_stddev(flow->pktlen_c_to_s), ndpi_data_stddev(flow->pktlen_s_to_c));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2517,7 +2598,7 @@ static void printFlowsStats() {
|
||||
|
||||
if(verbose > 1) {
|
||||
for(i=0; i<num_flows; i++)
|
||||
printFlow(i+1, all_flows[i].flow, all_flows[i].thread_id);
|
||||
printFlow(i+1, all_flows[i].flow, all_flows[i].thread_id);
|
||||
}
|
||||
|
||||
for(thread_id = 0; thread_id < num_threads; thread_id++) {
|
||||
@@ -3998,7 +4079,7 @@ int orginal_main(int argc, char **argv) {
|
||||
int main(int argc, char **argv) {
|
||||
#endif
|
||||
int i;
|
||||
|
||||
|
||||
if(ndpi_get_api_version() != NDPI_API_VERSION) {
|
||||
printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
|
||||
return(-1);
|
||||
@@ -4041,14 +4122,12 @@ int orginal_main(int argc, char **argv) {
|
||||
for(i=0; i<num_loops; i++)
|
||||
test_lib();
|
||||
|
||||
|
||||
|
||||
if(results_path) free(results_path);
|
||||
if(results_file) fclose(results_file);
|
||||
if(extcap_dumper) pcap_dump_close(extcap_dumper);
|
||||
if(ndpi_info_mod) ndpi_exit_detection_module(ndpi_info_mod);
|
||||
|
||||
|
||||
if(csv_fp) fclose(csv_fp);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
+26
-4
@@ -451,12 +451,14 @@ void ndpi_flow_info_freer(void *node) {
|
||||
|
||||
ndpi_free_flow_info_half(flow);
|
||||
|
||||
if(flow->iat_c_to_s)
|
||||
ndpi_free_data_analysis(flow->iat_c_to_s);
|
||||
if(flow->iat_c_to_s) ndpi_free_data_analysis(flow->iat_c_to_s);
|
||||
if(flow->iat_s_to_c) ndpi_free_data_analysis(flow->iat_s_to_c);
|
||||
|
||||
if(flow->iat_s_to_c)
|
||||
ndpi_free_data_analysis(flow->iat_s_to_c);
|
||||
if(flow->pktlen_c_to_s) ndpi_free_data_analysis(flow->pktlen_c_to_s);
|
||||
if(flow->pktlen_s_to_c) ndpi_free_data_analysis(flow->pktlen_s_to_c);
|
||||
|
||||
if(flow->iat_flow) ndpi_free_data_analysis(flow->iat_flow);
|
||||
|
||||
ndpi_free(flow);
|
||||
}
|
||||
|
||||
@@ -762,6 +764,9 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
|
||||
newflow->ip_version = version;
|
||||
newflow->iat_c_to_s = ndpi_alloc_data_analysis(DATA_ANALUYSIS_SLIDING_WINDOW),
|
||||
newflow->iat_s_to_c = ndpi_alloc_data_analysis(DATA_ANALUYSIS_SLIDING_WINDOW);
|
||||
newflow->pktlen_c_to_s = ndpi_alloc_data_analysis(DATA_ANALUYSIS_SLIDING_WINDOW),
|
||||
newflow->pktlen_s_to_c = ndpi_alloc_data_analysis(DATA_ANALUYSIS_SLIDING_WINDOW),
|
||||
newflow->iat_flow = ndpi_alloc_data_analysis(DATA_ANALUYSIS_SLIDING_WINDOW);;
|
||||
|
||||
if(version == IPVERSION) {
|
||||
inet_ntop(AF_INET, &newflow->src_ip, newflow->src_name, sizeof(newflow->src_name));
|
||||
@@ -1045,6 +1050,18 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
workflow->stats.total_ip_bytes += rawsize;
|
||||
ndpi_flow = flow->ndpi_flow;
|
||||
|
||||
if(flow->flow_last_pkt_time.tv_sec) {
|
||||
ndpi_timer_sub(&when, &flow->flow_last_pkt_time, &tdiff);
|
||||
|
||||
if(flow->iat_flow) {
|
||||
u_int32_t ms = ndpi_timeval_to_milliseconds(tdiff);
|
||||
|
||||
if(ms > 0)
|
||||
ndpi_data_add_value(flow->iat_flow, ms);
|
||||
}
|
||||
}
|
||||
memcpy(&flow->flow_last_pkt_time, &when, sizeof(when));
|
||||
|
||||
if(src_to_dst_direction) {
|
||||
if(flow->src2dst_last_pkt_time.tv_sec) {
|
||||
ndpi_timer_sub(&when, &flow->src2dst_last_pkt_time, &tdiff);
|
||||
@@ -1056,6 +1073,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
}
|
||||
}
|
||||
|
||||
ndpi_data_add_value(flow->pktlen_c_to_s, rawsize);
|
||||
flow->src2dst_packets++, flow->src2dst_bytes += rawsize;
|
||||
flow->src2dst_l4_bytes += payload_len;
|
||||
memcpy(&flow->src2dst_last_pkt_time, &when, sizeof(when));
|
||||
@@ -1070,6 +1088,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
}
|
||||
}
|
||||
|
||||
ndpi_data_add_value(flow->pktlen_s_to_c, rawsize);
|
||||
flow->dst2src_packets++, flow->dst2src_bytes += rawsize;
|
||||
flow->dst2src_l4_bytes += payload_len;
|
||||
memcpy(&flow->dst2src_last_pkt_time, &when, sizeof(when));
|
||||
@@ -1086,6 +1105,9 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
||||
ndpi_flow_update_byte_dist_mean_var(flow, payload, payload_len, src_to_dst_direction);
|
||||
}
|
||||
|
||||
if(flow->first_seen == 0)
|
||||
flow->first_seen = time;
|
||||
|
||||
flow->last_seen = time;
|
||||
|
||||
if(!flow->has_human_readeable_strings) {
|
||||
|
||||
@@ -137,7 +137,7 @@ typedef struct ndpi_flow_info {
|
||||
struct ndpi_flow_struct *ndpi_flow;
|
||||
char src_name[48], dst_name[48];
|
||||
u_int8_t ip_version;
|
||||
u_int64_t last_seen;
|
||||
u_int64_t first_seen, last_seen;
|
||||
u_int64_t src2dst_bytes, dst2src_bytes;
|
||||
u_int32_t src2dst_packets, dst2src_packets;
|
||||
u_int32_t src2dst_opackets, dst2src_opackets;
|
||||
@@ -148,7 +148,8 @@ typedef struct ndpi_flow_info {
|
||||
ndpi_protocol detected_protocol;
|
||||
|
||||
// Flow data analysis
|
||||
struct ndpi_analyze_struct *iat_c_to_s, *iat_s_to_c;
|
||||
struct ndpi_analyze_struct *iat_c_to_s, *iat_s_to_c, *iat_flow,
|
||||
*pktlen_c_to_s, *pktlen_s_to_c;
|
||||
|
||||
char info[96];
|
||||
char host_server_name[256];
|
||||
@@ -167,7 +168,7 @@ typedef struct ndpi_flow_info {
|
||||
|
||||
void *src_id, *dst_id;
|
||||
|
||||
struct timeval src2dst_last_pkt_time, dst2src_last_pkt_time;
|
||||
struct timeval src2dst_last_pkt_time, dst2src_last_pkt_time, flow_last_pkt_time;
|
||||
|
||||
// Entropy fields
|
||||
u_int16_t src2dst_pkt_len[MAX_NUM_PKTS]; /*!< array of packet appdata lengths */
|
||||
|
||||
Reference in New Issue
Block a user