Added ndpi_serialize_raw_record() API call

This commit is contained in:
Luca Deri
2020-04-10 21:39:23 +02:00
parent db8497778d
commit d665df64e2
3 changed files with 101 additions and 54 deletions
+2 -1
View File
@@ -945,7 +945,8 @@ extern "C" {
const char *format /* e.f. "%.2f" */);
int ndpi_serialize_string_boolean(ndpi_serializer *serializer,
const char *key, u_int8_t value);
int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
u_char *record, u_int32_t record_len);
int ndpi_serialize_end_of_record(ndpi_serializer *serializer);
int ndpi_serialize_start_of_block(ndpi_serializer *serializer,
const char *key);
+16 -16
View File
@@ -40,7 +40,7 @@ void ndpi_init_data_analysis(struct ndpi_analyze_struct *ret, u_int16_t _max_ser
u_int32_t len;
memset(ret, 0, sizeof(struct ndpi_analyze_struct));
if(_max_series_len > MAX_SERIES_LEN) _max_series_len = MAX_SERIES_LEN;
ret->num_values_array_len = _max_series_len;
@@ -61,8 +61,8 @@ struct ndpi_analyze_struct* ndpi_alloc_data_analysis(u_int16_t _max_series_len)
struct ndpi_analyze_struct *ret = ndpi_malloc(sizeof(struct ndpi_analyze_struct));
if(ret != NULL)
ndpi_init_data_analysis(ret, _max_series_len);
ndpi_init_data_analysis(ret, _max_series_len);
return(ret);
}
@@ -89,18 +89,18 @@ void ndpi_data_add_value(struct ndpi_analyze_struct *s, const u_int32_t value) {
}
s->sum_total += value, s->num_data_entries++;
if(s->num_values_array_len) {
s->values[s->next_value_insert_index] = value;
if(++s->next_value_insert_index == s->num_values_array_len)
s->next_value_insert_index = 0;
}
/* Update stddev */
tmp_mu = s->stddev.mu;
s->stddev.mu = ((s->stddev.mu * (s->num_data_entries - 1)) + value) / s->num_data_entries;
s->stddev.q = s->stddev.q + (value - tmp_mu)*(value - s->stddev.mu);
s->stddev.q = s->stddev.q + (value - tmp_mu)*(value - s->stddev.mu);
}
/* ********************************************************************************* */
@@ -137,10 +137,10 @@ float ndpi_data_window_average(struct ndpi_analyze_struct *s) {
if(s->num_values_array_len) {
float sum = 0.0;
u_int16_t i, n = ndpi_min(s->num_data_entries, s->num_values_array_len);
for(i=0; i<n; i++)
sum += s->values[i];
return((float)sum / (float)n);
} else
return(0);
@@ -155,17 +155,17 @@ float ndpi_data_entropy(struct ndpi_analyze_struct *s) {
if(s->num_values_array_len) {
int i;
float sum = 0.0, total = 0.0;
for(i=0; i<s->num_values_array_len; i++)
total += s->values[i];
for (i=0; i<s->num_values_array_len; i++) {
float tmp = (float)s->values[i] / (float)total;
if(tmp > FLT_EPSILON)
sum -= tmp * logf(tmp);
sum -= tmp * logf(tmp);
}
return(sum / logf(2.0));
} else
return(0);
@@ -176,10 +176,10 @@ float ndpi_data_entropy(struct ndpi_analyze_struct *s) {
void ndpi_data_print_window_values(struct ndpi_analyze_struct *s) {
if(s->num_values_array_len) {
u_int16_t i, n = ndpi_min(s->num_data_entries, s->num_values_array_len);
for(i=0; i<n; i++)
printf("[%u: %u]", i, s->values[i]);
printf("\n");
}
}
@@ -196,7 +196,7 @@ void ndpi_data_print_window_values(struct ndpi_analyze_struct *s) {
float ndpi_data_ratio(u_int32_t sent, u_int32_t rcvd) {
float s = (float)((int64_t)sent + (int64_t)rcvd);
float d = (float)((int64_t)sent - (int64_t)rcvd);
return((s == 0) ? 0 : (d/s));
}
+83 -37
View File
@@ -3,9 +3,6 @@
*
* Copyright (C) 2011-20 - ntop.org
*
* This file is part of nDPI, an open source deep packet inspection
* library based on the OpenDPI and PACE technology by ipoque GmbH
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
@@ -170,7 +167,7 @@ int ndpi_init_serializer_ll(ndpi_serializer *_serializer,
memset(serializer, 0, sizeof(ndpi_private_serializer));
serializer->initial_buffer_size = serializer->buffer_size = buffer_size;
serializer->buffer = (u_int8_t *) malloc(serializer->buffer_size * sizeof(u_int8_t));
serializer->buffer = (u_int8_t *) calloc(serializer->buffer_size, sizeof(u_int8_t));
if(serializer->buffer == NULL)
return(-1);
@@ -197,6 +194,36 @@ int ndpi_init_serializer(ndpi_serializer *_serializer,
/* ********************************** */
static inline int ndpi_extend_serializer_buffer(ndpi_serializer *_serializer, u_int32_t min_len) {
u_int32_t new_size;
void *r;
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
if(min_len < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) {
if(serializer->initial_buffer_size < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) {
if(min_len < serializer->initial_buffer_size)
min_len = serializer->initial_buffer_size;
} else {
min_len = NDPI_SERIALIZER_DEFAULT_BUFFER_INCR;
}
}
new_size = serializer->buffer_size + min_len;
new_size = ((new_size / 4) + 1) * 4; /* required by zmq encryption */
r = realloc((void *) serializer->buffer, new_size);
if(r == NULL)
return(-1);
serializer->buffer = r;
serializer->buffer_size = new_size;
return(0);
}
/* ********************************** */
char* ndpi_serializer_get_buffer(ndpi_serializer *_serializer, u_int32_t *buffer_len) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
char *buf = (char*)serializer->buffer;
@@ -265,36 +292,6 @@ void ndpi_term_serializer(ndpi_serializer *_serializer) {
/* ********************************** */
static inline int ndpi_extend_serializer_buffer(ndpi_serializer *_serializer, u_int32_t min_len) {
u_int32_t new_size;
void *r;
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
if(min_len < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) {
if(serializer->initial_buffer_size < NDPI_SERIALIZER_DEFAULT_BUFFER_INCR) {
if(min_len < serializer->initial_buffer_size)
min_len = serializer->initial_buffer_size;
} else {
min_len = NDPI_SERIALIZER_DEFAULT_BUFFER_INCR;
}
}
new_size = serializer->buffer_size + min_len;
new_size = ((new_size / 4) + 1) * 4; /* required by zmq encryption */
r = realloc((void *) serializer->buffer, new_size);
if(r == NULL)
return(-1);
serializer->buffer = r;
serializer->buffer_size = new_size;
return(0);
}
/* ********************************** */
static inline void ndpi_serialize_single_uint8(ndpi_private_serializer *serializer,
u_int8_t s) {
u_int8_t v = s;
@@ -431,11 +428,60 @@ static inline void ndpi_deserialize_single_string(ndpi_private_deserializer *des
/* ********************************** */
/*
This function helps extending the existing serializer by adding a new
element in the array. This element is handled as raw without any
further check whatsoever
*/
int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
u_char *record, u_int32_t record_len) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used;
u_int16_t needed = record_len;
u_int8_t add_comma = 0;
if(serializer->fmt == ndpi_serialization_format_json) {
needed += 1;
if(serializer->status.size_used == 3) /* Empty buffer [{} */
serializer->status.size_used = 2; /* Remove {} */
else
needed += 2, add_comma = 1;
}
if(buff_diff < needed) {
if(ndpi_extend_serializer_buffer(_serializer, needed - buff_diff) < 0)
return(-1);
buff_diff = serializer->buffer_size - serializer->status.size_used;
}
if(serializer->fmt == ndpi_serialization_format_json) {
if(add_comma)
serializer->buffer[serializer->status.size_used-1] = ',';
else
serializer->status.size_used--;
}
memcpy(&serializer->buffer[serializer->status.size_used], record, record_len);
serializer->status.size_used += record_len;
if(serializer->fmt == ndpi_serialization_format_json) {
serializer->buffer[serializer->status.size_used] = ']';
if(add_comma) serializer->status.size_used++;
}
ndpi_serialize_end_of_record(_serializer);
return(0);
}
/* ********************************** */
int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
u_int32_t buff_diff = serializer->buffer_size - serializer->status.size_used;
u_int16_t needed =
sizeof(u_int8_t) /* type */;
u_int16_t needed = sizeof(u_int8_t) /* type */;
if(serializer->fmt == ndpi_serialization_format_json)
needed += 1;
@@ -452,7 +498,7 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
serializer->status.size_used += snprintf((char *) &serializer->buffer[serializer->status.size_used],
buff_diff, "]");
}
serializer->status.flags |= NDPI_SERIALIZER_STATUS_ARRAY | NDPI_SERIALIZER_STATUS_EOR;
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_COMMA;
} else {