Support serialization of double-precision floating-point numbers. Fixes #1702.
Signed-off-by: lns <matzeton@googlemail.com> Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
This commit is contained in:
@@ -1678,9 +1678,9 @@ static void printFlowSerialized(u_int16_t thread_id,
|
||||
float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes);
|
||||
|
||||
ndpi_serialize_string_uint32(serializer, "flow_id", flow->flow_id);
|
||||
ndpi_serialize_string_float(serializer, "first_seen_ms", f, "%.3f");
|
||||
ndpi_serialize_string_float(serializer, "last_seen_ms", l, "%.3f");
|
||||
ndpi_serialize_string_float(serializer, "duration_ms", (l-f)/1000.0, "%.3f");
|
||||
ndpi_serialize_string_double(serializer, "first_seen", f / 1000., "%.3f");
|
||||
ndpi_serialize_string_double(serializer, "last_seen", l / 1000., "%.3f");
|
||||
ndpi_serialize_string_double(serializer, "duration", (l-f)/1000.0, "%.3f");
|
||||
ndpi_serialize_string_uint32(serializer, "vlan_id", flow->vlan_id);
|
||||
ndpi_serialize_string_uint32(serializer, "bidirectional", flow->bidirectional);
|
||||
|
||||
|
||||
@@ -1229,6 +1229,16 @@ extern "C" {
|
||||
*/
|
||||
int ndpi_serialize_uint32_float(ndpi_serializer *serializer, u_int32_t key, float value, const char *format /* e.f. "%.2f" */);
|
||||
|
||||
/**
|
||||
* Serialize a 32-bit unsigned int key and a double value
|
||||
* @param serializer The serializer handle
|
||||
* @param key The field name or ID
|
||||
* @param value The field value
|
||||
* @param format The double value format
|
||||
* @return 0 on success, a negative number otherwise
|
||||
*/
|
||||
int ndpi_serialize_uint32_double(ndpi_serializer *serializer, u_int32_t key, double value, const char *format /* e.f. "%.2f" */);
|
||||
|
||||
/**
|
||||
* Serialize a 32-bit unsigned int key and a string value
|
||||
* @param serializer The serializer handle
|
||||
@@ -1395,6 +1405,17 @@ extern "C" {
|
||||
*/
|
||||
int ndpi_serialize_binary_float(ndpi_serializer *_serializer, const char *key, u_int16_t klen, float value, const char *format /* e.f. "%.2f" */);
|
||||
|
||||
/**
|
||||
* Serialize an unterminated string key and a double value
|
||||
* @param serializer The serializer handle
|
||||
* @param key The field name or ID
|
||||
* @param klen The key length
|
||||
* @param value The field value
|
||||
* @param format The double format
|
||||
* @return 0 on success, a negative number otherwise
|
||||
*/
|
||||
int ndpi_serialize_binary_double(ndpi_serializer *_serializer, const char *key, u_int16_t klen, double value, const char *format /* e.f. "%.2f" */);
|
||||
|
||||
/**
|
||||
* Serialize a string key and a a float value
|
||||
* @param serializer The serializer handle
|
||||
@@ -1405,6 +1426,16 @@ extern "C" {
|
||||
*/
|
||||
int ndpi_serialize_string_float(ndpi_serializer *serializer, const char *key, float value, const char *format /* e.f. "%.2f" */);
|
||||
|
||||
/**
|
||||
* Serialize a string key and a a double value
|
||||
* @param serializer The serializer handle
|
||||
* @param key The field name or ID
|
||||
* @param value The field value
|
||||
* @param format The double format
|
||||
* @return 0 on success, a negative number otherwise
|
||||
*/
|
||||
int ndpi_serialize_string_double(ndpi_serializer *serializer, const char *key, double value, const char *format /* e.f. "%.2f" */);
|
||||
|
||||
/**
|
||||
* Serialize an unterminated string key and a boolean value (JSON/CSV only, not supported by TLV)
|
||||
* @param serializer The serializer handle
|
||||
@@ -1586,6 +1617,7 @@ extern "C" {
|
||||
int ndpi_deserialize_value_int32(ndpi_deserializer *deserializer, int32_t *value);
|
||||
int ndpi_deserialize_value_int64(ndpi_deserializer *deserializer, int64_t *value);
|
||||
int ndpi_deserialize_value_float(ndpi_deserializer *deserializer, float *value);
|
||||
int ndpi_deserialize_value_double(ndpi_deserializer *deserializer, double *value);
|
||||
int ndpi_deserialize_value_string(ndpi_deserializer *deserializer, ndpi_string *value);
|
||||
|
||||
int ndpi_deserialize_clone_item(ndpi_deserializer *deserializer, ndpi_serializer *serializer);
|
||||
|
||||
@@ -1618,6 +1618,7 @@ typedef enum {
|
||||
ndpi_serialization_int32,
|
||||
ndpi_serialization_int64,
|
||||
ndpi_serialization_float,
|
||||
ndpi_serialization_double,
|
||||
ndpi_serialization_string,
|
||||
ndpi_serialization_start_of_block,
|
||||
ndpi_serialization_end_of_block,
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ $(NDPI_LIB_STATIC): $(OBJECTS)
|
||||
$(RANLIB) $@
|
||||
|
||||
$(NDPI_LIB_SHARED): $(OBJECTS)
|
||||
$(CC) -shared -fPIC $(SONAME_FLAG) -o $@ $(LDFLAGS) $(OBJECTS) $(LIBS)
|
||||
$(CC) -shared -fPIC $(CFLAGS) $(SONAME_FLAG) -o $@ $(LDFLAGS) $(OBJECTS) $(LIBS)
|
||||
ln -fs $(NDPI_LIB_SHARED) $(NDPI_LIB_SHARED_BASE)
|
||||
ln -fs $(NDPI_LIB_SHARED) $(NDPI_LIB_SHARED_BASE).$(NDPI_VERSION_MAJOR)
|
||||
|
||||
|
||||
+117
-4
@@ -492,6 +492,15 @@ static inline void ndpi_serialize_single_float(ndpi_private_serializer *serializ
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
/* TODO: fix portability across platforms */
|
||||
static inline void ndpi_serialize_single_double(ndpi_private_serializer *serializer,
|
||||
double s) {
|
||||
memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], &s, sizeof(s));
|
||||
serializer->status.buffer.size_used += sizeof(double);
|
||||
}
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
static inline void ndpi_serialize_single_string(ndpi_private_serializer *serializer,
|
||||
const char *s, u_int16_t slen) {
|
||||
u_int16_t l = htons(slen);
|
||||
@@ -571,6 +580,14 @@ static inline void ndpi_deserialize_single_float(ndpi_private_deserializer *dese
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
/* TODO: fix portability across platforms */
|
||||
static inline void ndpi_deserialize_single_double(ndpi_private_deserializer *deserializer,
|
||||
u_int32_t offset, double *s) {
|
||||
*s = *(double*)&deserializer->buffer.data[offset];
|
||||
}
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
static inline void ndpi_deserialize_single_string(ndpi_private_deserializer *deserializer,
|
||||
u_int32_t offset, ndpi_string *v) {
|
||||
v->str_len = ntohs(*((u_int16_t *) &deserializer->buffer.data[offset]));
|
||||
@@ -1633,6 +1650,62 @@ int ndpi_serialize_binary_float(ndpi_serializer *_serializer,
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
int ndpi_serialize_binary_double(ndpi_serializer *_serializer,
|
||||
const char *key,
|
||||
u_int16_t klen,
|
||||
double value,
|
||||
const char *format /* e.f. "%.2f" */) {
|
||||
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
|
||||
u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
|
||||
u_int32_t needed;
|
||||
|
||||
needed =
|
||||
sizeof(u_int8_t) /* type */ +
|
||||
sizeof(u_int16_t) /* key len */ +
|
||||
klen /* key */ +
|
||||
sizeof(double);
|
||||
|
||||
if(serializer->fmt == ndpi_serialization_format_json)
|
||||
needed += 32 + klen;
|
||||
|
||||
if(buff_diff < needed) {
|
||||
if(ndpi_extend_serializer_buffer(&serializer->buffer, needed - buff_diff) < 0)
|
||||
return(-1);
|
||||
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
|
||||
}
|
||||
|
||||
if(serializer->fmt == ndpi_serialization_format_json) {
|
||||
ndpi_serialize_json_pre(_serializer);
|
||||
|
||||
if (!(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST)) {
|
||||
serializer->status.buffer.size_used += ndpi_json_string_escape(key, klen,
|
||||
(char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff);
|
||||
serializer->buffer.data[serializer->status.buffer.size_used] = ':';
|
||||
serializer->status.buffer.size_used++;
|
||||
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
|
||||
}
|
||||
|
||||
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, format, value);
|
||||
|
||||
ndpi_serialize_json_post(_serializer);
|
||||
} else if(serializer->fmt == ndpi_serialization_format_csv) {
|
||||
if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1);
|
||||
ndpi_serialize_csv_pre(serializer);
|
||||
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
|
||||
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, format, value);
|
||||
} else {
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_double;
|
||||
|
||||
ndpi_serialize_single_string(serializer, key, klen);
|
||||
ndpi_serialize_single_double(serializer, value);
|
||||
}
|
||||
|
||||
serializer->status.flags |= NDPI_SERIALIZER_STATUS_NOT_EMPTY;
|
||||
return(0);
|
||||
}
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
int ndpi_serialize_string_float(ndpi_serializer *_serializer,
|
||||
const char *key,
|
||||
float value,
|
||||
@@ -1642,6 +1715,16 @@ int ndpi_serialize_string_float(ndpi_serializer *_serializer,
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
int ndpi_serialize_string_double(ndpi_serializer *_serializer,
|
||||
const char *key,
|
||||
double value,
|
||||
const char *format /* e.f. "%.2f" */)
|
||||
{
|
||||
return(ndpi_serialize_binary_double(_serializer, key, strlen(key), value, format));
|
||||
}
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
/* Key is a <string, len> pair, value is a raw value */
|
||||
static int ndpi_serialize_binary_raw(ndpi_serializer *_serializer,
|
||||
const char *key,
|
||||
@@ -1684,7 +1767,6 @@ static int ndpi_serialize_binary_raw(ndpi_serializer *_serializer,
|
||||
serializer->status.buffer.size_used += ndpi_json_string_escape(value, vlen,
|
||||
(char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff);
|
||||
else {
|
||||
//serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, value, vlen);
|
||||
memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], value, vlen);
|
||||
serializer->status.buffer.size_used += vlen;
|
||||
}
|
||||
@@ -1694,9 +1776,6 @@ static int ndpi_serialize_binary_raw(ndpi_serializer *_serializer,
|
||||
if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1);
|
||||
ndpi_serialize_csv_pre(serializer);
|
||||
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
|
||||
|
||||
//serializer->status.buffer.size_used += ndpi_snprintf((char *)
|
||||
// &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s", value);
|
||||
memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], value, vlen);
|
||||
serializer->status.buffer.size_used += vlen;
|
||||
|
||||
@@ -2210,6 +2289,9 @@ static inline int ndpi_deserialize_get_single_size(ndpi_private_deserializer *de
|
||||
case ndpi_serialization_float:
|
||||
size = sizeof(float);
|
||||
break;
|
||||
case ndpi_serialization_double:
|
||||
size = sizeof(double);
|
||||
break;
|
||||
case ndpi_serialization_string:
|
||||
case ndpi_serialization_start_of_block:
|
||||
case ndpi_serialization_start_of_list:
|
||||
@@ -2528,6 +2610,37 @@ int ndpi_deserialize_value_float(ndpi_deserializer *_deserializer,
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
int ndpi_deserialize_value_double(ndpi_deserializer *_deserializer,
|
||||
double *value) {
|
||||
ndpi_private_deserializer *deserializer = (ndpi_private_deserializer*)_deserializer;
|
||||
ndpi_serialization_type kt, et;
|
||||
u_int32_t buff_diff = deserializer->buffer.size - deserializer->status.buffer.size_used;
|
||||
u_int16_t expected;
|
||||
int size;
|
||||
|
||||
expected = sizeof(u_int8_t) /* type */;
|
||||
if(buff_diff < expected) return(-2);
|
||||
|
||||
kt = ndpi_deserialize_get_key_subtype(deserializer);
|
||||
size = ndpi_deserialize_get_single_size(deserializer, kt, deserializer->status.buffer.size_used + expected);
|
||||
if(size < 0) return(-2);
|
||||
|
||||
expected += size;
|
||||
|
||||
et = ndpi_deserialize_get_value_subtype(deserializer);
|
||||
size = ndpi_deserialize_get_single_size(deserializer, et, deserializer->status.buffer.size_used + expected);
|
||||
if(size < 0) return(-2);
|
||||
|
||||
if(et != ndpi_serialization_double)
|
||||
return(-1);
|
||||
|
||||
ndpi_deserialize_single_double(deserializer, deserializer->status.buffer.size_used + expected, value);
|
||||
|
||||
return(0);
|
||||
}
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
/* Return the string value for the current element */
|
||||
int ndpi_deserialize_value_string(ndpi_deserializer *_deserializer,
|
||||
ndpi_string *value) {
|
||||
|
||||
+37
-9
@@ -62,6 +62,7 @@ static int verbose = 0;
|
||||
|
||||
/* *********************************************** */
|
||||
|
||||
#define FLT_MAX 3.402823466e+38F
|
||||
int serializerUnitTest() {
|
||||
ndpi_serializer serializer, deserializer;
|
||||
int i, loop_id;
|
||||
@@ -99,6 +100,7 @@ int serializerUnitTest() {
|
||||
assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
|
||||
assert(ndpi_serialize_string_uint32(&serializer, kbuf, i*i) != -1);
|
||||
assert(ndpi_serialize_string_float(&serializer, kbuf, (float)(i*i), "%f") != -1);
|
||||
assert(ndpi_serialize_string_double(&serializer, kbuf, ((double)(FLT_MAX))*2, "%lf") != -1);
|
||||
assert(ndpi_serialize_string_int64(&serializer, kbuf, INT64_MAX) != -1);
|
||||
if ((i&0x3) == 0x3) ndpi_serialize_end_of_record(&serializer);
|
||||
}
|
||||
@@ -126,7 +128,8 @@ int serializerUnitTest() {
|
||||
jerr = json_tokener_success;
|
||||
j = json_tokener_parse_verbose(buffer, &jerr);
|
||||
if (j == NULL) {
|
||||
printf("%s: ERROR (json validation failed)\n", __FUNCTION__);
|
||||
printf("%s: ERROR (json validation failed: `%s')\n",
|
||||
__FUNCTION__, json_tokener_error_desc(jerr));
|
||||
return -1;
|
||||
} else {
|
||||
/* Validation ok */
|
||||
@@ -165,6 +168,7 @@ int serializerUnitTest() {
|
||||
int64_t v64;
|
||||
ndpi_string ks, vs;
|
||||
float vf;
|
||||
double vd;
|
||||
|
||||
switch(kt) {
|
||||
case ndpi_serialization_uint32:
|
||||
@@ -211,6 +215,11 @@ int serializerUnitTest() {
|
||||
if(verbose) printf("%f\n", vf);
|
||||
break;
|
||||
|
||||
case ndpi_serialization_double:
|
||||
assert(ndpi_deserialize_value_double(&deserializer, &vd) != -1);
|
||||
if(verbose) printf("%lf\n", vd);
|
||||
break;
|
||||
|
||||
default:
|
||||
if (verbose) printf("\n");
|
||||
printf("%s: ERROR (unsupported type %u detected)\n", __FUNCTION__, et);
|
||||
@@ -267,12 +276,15 @@ int serializeProtoUnitTest(void)
|
||||
NDPI_SET_BIT(risks, NDPI_TLS_OBSOLETE_VERSION);
|
||||
NDPI_SET_BIT(risks, NDPI_TLS_SELFSIGNED_CERTIFICATE);
|
||||
ndpi_serialize_proto(ndpi_info_mod, &serializer, risks, NDPI_CONFIDENCE_DPI, ndpi_proto);
|
||||
assert(ndpi_serialize_string_float(&serializer, "float", FLT_MAX, "%f") != -1);
|
||||
assert(ndpi_serialize_string_double(&serializer, "double", ((double)(FLT_MAX))*2, "%lf") != -1);
|
||||
|
||||
if (fmt == ndpi_serialization_format_json)
|
||||
{
|
||||
buffer_len = 0;
|
||||
buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
|
||||
char const * const expected_json_str = "{\"ndpi\": {\"flow_risk\": {\"6\": {\"risk\":\"Self-signed Cert\",\"severity\":\"High\",\"risk_score\": {\"total\":500,\"client\":450,\"server\":50}},\"7\": {\"risk\":\"Obsolete TLS (v1.1 or older)\",\"severity\":\"High\",\"risk_score\": {\"total\":510,\"client\":455,\"server\":55}},\"8\": {\"risk\":\"Weak TLS Cipher\",\"severity\":\"High\",\"risk_score\": {\"total\":250,\"client\":225,\"server\":25}},\"17\": {\"risk\":\"Malformed Packet\",\"severity\":\"Low\",\"risk_score\": {\"total\":260,\"client\":130,\"server\":130}}},\"confidence\": {\"6\":\"DPI\"},\"proto\":\"TLS.Facebook\",\"proto_id\":\"91.119\",\"encrypted\":1,\"breed\":\"Fun\",\"category_id\":6,\"category\":\"SocialNetwork\"}}";
|
||||
#ifndef WIN32
|
||||
char const * const expected_json_str = "{\"ndpi\": {\"flow_risk\": {\"6\": {\"risk\":\"Self-signed Cert\",\"severity\":\"High\",\"risk_score\": {\"total\":500,\"client\":450,\"server\":50}},\"7\": {\"risk\":\"Obsolete TLS (v1.1 or older)\",\"severity\":\"High\",\"risk_score\": {\"total\":510,\"client\":455,\"server\":55}},\"8\": {\"risk\":\"Weak TLS Cipher\",\"severity\":\"High\",\"risk_score\": {\"total\":250,\"client\":225,\"server\":25}},\"17\": {\"risk\":\"Malformed Packet\",\"severity\":\"Low\",\"risk_score\": {\"total\":260,\"client\":130,\"server\":130}}},\"confidence\": {\"6\":\"DPI\"},\"proto\":\"TLS.Facebook\",\"proto_id\":\"91.119\",\"encrypted\":1,\"breed\":\"Fun\",\"category_id\":6,\"category\":\"SocialNetwork\"},\"float\":340282346638528859811704183484516925440.000000,\"double\":680564693277057719623408366969033850880.000000}";
|
||||
|
||||
if (strncmp(buffer, expected_json_str, buffer_len) != 0)
|
||||
{
|
||||
@@ -280,6 +292,7 @@ int serializeProtoUnitTest(void)
|
||||
printf("%s: ERROR: got JSON str: \"%.*s\"\n", __FUNCTION__, (int)buffer_len, buffer);
|
||||
return -1;
|
||||
}
|
||||
#endif
|
||||
|
||||
if(verbose)
|
||||
printf("%s\n", buffer);
|
||||
@@ -288,7 +301,8 @@ int serializeProtoUnitTest(void)
|
||||
enum json_tokener_error jerr = json_tokener_success;
|
||||
json_object * const j = json_tokener_parse_verbose(buffer, &jerr);
|
||||
if (j == NULL) {
|
||||
printf("%s: ERROR (json validation failed)\n", __FUNCTION__);
|
||||
printf("%s: ERROR (json validation failed: `%s')\n",
|
||||
__FUNCTION__, json_tokener_error_desc(jerr));
|
||||
return -1;
|
||||
} else {
|
||||
/* Validation ok */
|
||||
@@ -296,14 +310,28 @@ int serializeProtoUnitTest(void)
|
||||
}
|
||||
} else if (fmt == ndpi_serialization_format_csv)
|
||||
{
|
||||
if(verbose) {
|
||||
buffer_len = 0;
|
||||
buffer = ndpi_serializer_get_header(&serializer, &buffer_len);
|
||||
char const * const expected_csv_hdr_str = "risk,severity,total,client,server,risk,severity,total,client,server,risk,severity,total,client,server,risk,severity,total,client,server,6,proto,proto_id,encrypted,breed,category_id,category,float,double";
|
||||
buffer_len = 0;
|
||||
buffer = ndpi_serializer_get_header(&serializer, &buffer_len);
|
||||
assert(buffer != NULL && buffer_len != 0);
|
||||
if (verbose)
|
||||
printf("%s\n", buffer);
|
||||
if (strncmp(buffer, expected_csv_hdr_str, buffer_len) != 0)
|
||||
{
|
||||
printf("%s: ERROR: expected CSV str: \"%s\"\n", __FUNCTION__, expected_csv_hdr_str);
|
||||
printf("%s: ERROR: got CSV str: \"%.*s\"\n", __FUNCTION__, (int)buffer_len, buffer);
|
||||
}
|
||||
|
||||
buffer_len = 0;
|
||||
buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
|
||||
printf("%s\n", buffer);
|
||||
char const * const expected_csv_buf_str = "Self-signed Cert,High,500,450,50,Obsolete TLS (v1.1 or older),High,510,455,55,Weak TLS Cipher,High,250,225,25,Malformed Packet,Low,260,130,130,DPI,TLS.Facebook,91.119,1,Fun,6,SocialNetwork,340282346638528859811704183484516925440.000000,680564693277057719623408366969033850880.000000";
|
||||
buffer_len = 0;
|
||||
buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
|
||||
assert(buffer != NULL && buffer_len != 0);
|
||||
if (verbose)
|
||||
printf("%s\n", buffer);
|
||||
if (strncmp(buffer, expected_csv_buf_str, buffer_len) != 0)
|
||||
{
|
||||
printf("%s: ERROR: expected CSV str: \"%s\"\n", __FUNCTION__, expected_csv_buf_str);
|
||||
printf("%s: ERROR: got CSV str: \"%.*s\"\n", __FUNCTION__, (int)buffer_len, buffer);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user