Support for multiple records in CSV serialization

This commit is contained in:
Alfredo Cardigliano
2020-06-05 17:28:27 +02:00
parent 597d6e5d60
commit 0da76fdac6
2 changed files with 82 additions and 45 deletions
+1
View File
@@ -3101,6 +3101,7 @@ void serializerUnitTest() {
assert(ndpi_serialize_string_string(&serializer, kbuf, vbuf) != -1);
assert(ndpi_serialize_string_uint32(&serializer, kbuf, i*i) != -1);
assert(ndpi_serialize_string_float(&serializer, kbuf, (float)(i*i), "%f") != -1);
if ((i&0x3) == 0x3) ndpi_serialize_end_of_record(&serializer);
}
if (fmt == ndpi_serialization_format_json) {
+81 -45
View File
@@ -40,14 +40,6 @@
/* ********************************** */
static u_int8_t ndpi_serialize_is_not_empty(ndpi_serializer *_serializer) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
return(((serializer->status.flags & NDPI_SERIALIZER_STATUS_NOT_EMPTY) == NDPI_SERIALIZER_STATUS_NOT_EMPTY) ? 1 : 0);
}
/* ********************************** */
static u_int64_t ndpi_htonll(u_int64_t v) {
union { u_int32_t lv[2]; u_int64_t llv; } u;
@@ -589,7 +581,8 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
u_int16_t needed = sizeof(u_int8_t) /* type */;
if(serializer->fmt == ndpi_serialization_format_json)
if(serializer->fmt == ndpi_serialization_format_json ||
serializer->fmt == ndpi_serialization_format_csv)
needed += 1;
if(buff_diff < needed) {
@@ -599,7 +592,11 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
}
if(serializer->fmt == ndpi_serialization_format_csv) {
serializer->buffer.data[serializer->status.buffer.size_used] = '\n';
serializer->status.buffer.size_used += 1;
serializer->buffer.data[serializer->status.buffer.size_used] = '\0';
serializer->status.flags |= NDPI_SERIALIZER_STATUS_HDR_DONE;
serializer->status.flags |= NDPI_SERIALIZER_STATUS_EOR;
} else if(serializer->fmt == ndpi_serialization_format_json) {
if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
serializer->buffer.data[0] = '[';
@@ -620,6 +617,21 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
/* ********************************** */
static inline void ndpi_serialize_csv_pre(ndpi_private_serializer *serializer) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_EOR) {
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_EOR;
} else if (serializer->status.buffer.size_used == 0) {
/* nothing to do */
} else {
u_int32_t buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s",
serializer->csv_separator);
}
}
/* ********************************** */
static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
@@ -721,8 +733,11 @@ int ndpi_serialize_uint32_uint32(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%u", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%u", value);
} else {
ndpi_serialization_type kt;
u_int8_t type = 0;
@@ -786,10 +801,11 @@ int ndpi_serialize_uint32_uint64(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%llu",
(serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "",
(unsigned long long)value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%llu", (unsigned long long)value);
} else {
if(value <= 0xffffffff) {
return(ndpi_serialize_uint32_uint32(_serializer, key, value));
@@ -849,8 +865,11 @@ int ndpi_serialize_uint32_int32(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%d", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%d", value);
} else {
ndpi_serialization_type kt;
u_int8_t type = 0;
@@ -914,11 +933,11 @@ int ndpi_serialize_uint32_int64(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%lld",
(serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "",
(long long int)value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%lld", (long long int)value);
}
else {
if((value & 0xFFFFFFFF) == value) {
@@ -976,8 +995,8 @@ int ndpi_serialize_uint32_float(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s",
(serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "");
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, format, value);
} else {
@@ -1033,8 +1052,11 @@ static int ndpi_serialize_uint32_binary(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%s", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s", value);
} else {
ndpi_serialization_type kt;
u_int8_t type = 0;
@@ -1096,9 +1118,11 @@ int ndpi_serialize_uint32_boolean(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_uint32(serializer, key) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%s", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "",
value ? "true" : "false");
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s", value ? "true" : "false");
}
serializer->status.flags |= NDPI_SERIALIZER_STATUS_NOT_EMPTY;
@@ -1150,8 +1174,11 @@ static int ndpi_serialize_binary_int32(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%d", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%d", value);
} else {
if(value <= 127 && value >= -128) {
serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_int8;
@@ -1227,9 +1254,10 @@ int ndpi_serialize_binary_int64(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%lld", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "",
(long long int)value);
"%lld", (long long int)value);
} else {
if ((value & 0xFFFFFFFF) == value) {
return(ndpi_serialize_string_int32(_serializer, key, value));
@@ -1298,8 +1326,11 @@ static int ndpi_serialize_binary_uint32(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%u", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "", value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%u", value);
} else {
if(value <= 0xff) {
serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_uint8;
@@ -1396,9 +1427,11 @@ static int ndpi_serialize_binary_uint64(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%llu", (serializer->status.buffer.size_used > 0) ? serializer->csv_separator : "",
(unsigned long long)value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%llu", (unsigned long long)value);
} else {
if(value <= 0xffffffff) {
return(ndpi_serialize_string_uint32(_serializer, key, value));
@@ -1465,8 +1498,8 @@ static int ndpi_serialize_binary_float(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1);
if(serializer->status.buffer.size_used > 0)
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "%s", serializer->csv_separator);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, format, value);
} else {
serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_float;
@@ -1538,9 +1571,11 @@ static int ndpi_serialize_binary_raw(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_string(serializer, key, klen) < 0) return(-1);
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%s", ndpi_serialize_is_not_empty(_serializer) ? serializer->csv_separator : "",
value);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *)
&serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s", value);
} else {
serializer->buffer.data[serializer->status.buffer.size_used++] = (ndpi_serialization_string << 4) | ndpi_serialization_string;
@@ -1637,9 +1672,10 @@ int ndpi_serialize_string_boolean(ndpi_serializer *_serializer,
ndpi_serialize_json_post(_serializer);
} else if(serializer->fmt == ndpi_serialization_format_csv) {
if (ndpi_serializer_header_string(serializer, key, strlen(key)) < 0) return(-1);
ndpi_serialize_csv_pre(serializer);
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
serializer->status.buffer.size_used += snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff,
"%s%s", ndpi_serialize_is_not_empty(_serializer) ? serializer->csv_separator : "",
value ? "true" : "false");
"%s", value ? "true" : "false");
}
serializer->status.flags |= NDPI_SERIALIZER_STATUS_NOT_EMPTY;