Add support for multiline json

This commit is contained in:
Alfredo Cardigliano
2023-07-12 10:47:49 +02:00
parent b1d25193d2
commit 669a02cd76
2 changed files with 80 additions and 37 deletions
+3 -1
View File
@@ -1756,7 +1756,8 @@ typedef enum {
ndpi_serialization_format_unknown = 0,
ndpi_serialization_format_tlv,
ndpi_serialization_format_json,
ndpi_serialization_format_csv
ndpi_serialization_format_csv,
ndpi_serialization_format_multiline_json
} ndpi_serialization_format;
/* Note:
@@ -1821,6 +1822,7 @@ typedef struct {
ndpi_serialization_format fmt;
char csv_separator[2];
u_int8_t has_snapshot;
u_int8_t multiline_json_array;
ndpi_private_serializer_status snapshot;
} ndpi_private_serializer;
+77 -36
View File
@@ -177,8 +177,12 @@ void ndpi_reset_serializer(ndpi_serializer *_serializer) {
serializer->status.buffer.size_used = 0;
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
/* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}");
if (serializer->multiline_json_array) {
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "{}");
} else {
/* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}");
}
} else if(serializer->fmt == ndpi_serialization_format_csv) {
serializer->status.header.size_used = 0;
serializer->status.buffer.size_used = 0;
@@ -215,6 +219,11 @@ int ndpi_init_serializer_ll(ndpi_serializer *_serializer,
memset(serializer, 0, sizeof(ndpi_private_serializer));
if (fmt == ndpi_serialization_format_multiline_json) {
fmt = ndpi_serialization_format_json;
serializer->multiline_json_array = 1;
}
serializer->fmt = fmt;
if (buffer_size < 3 ||
@@ -614,10 +623,18 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
if(serializer->fmt == ndpi_serialization_format_json) {
needed += 1;
if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */
serializer->status.buffer.size_used = 2; /* Remove {} */
else
needed += 2, add_comma = 1;
if (serializer->multiline_json_array) {
if(serializer->status.buffer.size_used == 2) /* Empty buffer {} */
serializer->status.buffer.size_used = 0; /* Remove {} */
else
needed += 2;
} else {
if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */
serializer->status.buffer.size_used = 2; /* Remove {} */
else
needed += 2, add_comma = 1;
}
}
if(buff_diff < needed) {
@@ -628,18 +645,22 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
}
if(serializer->fmt == ndpi_serialization_format_json) {
if(add_comma)
serializer->buffer.data[serializer->status.buffer.size_used-1] = ',';
else
serializer->status.buffer.size_used--;
if (!serializer->multiline_json_array) {
if (add_comma)
serializer->buffer.data[serializer->status.buffer.size_used-1] = ',';
else
serializer->status.buffer.size_used--;
}
}
memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], record, record_len);
serializer->status.buffer.size_used += record_len;
if(serializer->fmt == ndpi_serialization_format_json) {
serializer->buffer.data[serializer->status.buffer.size_used] = ']';
if(add_comma) serializer->status.buffer.size_used++;
if (!serializer->multiline_json_array) {
serializer->buffer.data[serializer->status.buffer.size_used] = ']';
if(add_comma) serializer->status.buffer.size_used++;
}
}
ndpi_serialize_end_of_record(_serializer);
@@ -665,20 +686,26 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
}
if(serializer->fmt == ndpi_serialization_format_csv) {
serializer->buffer.data[serializer->status.buffer.size_used] = '\n';
serializer->status.buffer.size_used += 1;
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
serializer->buffer.data[serializer->status.buffer.size_used] = '\0';
serializer->status.flags |= NDPI_SERIALIZER_STATUS_HDR_DONE;
serializer->status.flags |= NDPI_SERIALIZER_STATUS_EOR;
} else if(serializer->fmt == ndpi_serialization_format_json) {
if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
serializer->buffer.data[0] = '[';
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used],
buff_diff, "]");
}
} else if(serializer->fmt == ndpi_serialization_format_json) {
if(serializer->multiline_json_array) {
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
serializer->buffer.data[serializer->status.buffer.size_used] = '\0';
} else {
if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
serializer->buffer.data[0] = '[';
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "]");
}
}
serializer->status.flags |= NDPI_SERIALIZER_STATUS_ARRAY | NDPI_SERIALIZER_STATUS_EOR;
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_COMMA;
} else /* ndpi_serialization_format_tlv */ {
serializer->buffer.data[serializer->status.buffer.size_used++] = ndpi_serialization_end_of_record;
}
@@ -709,18 +736,26 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_EOR) {
serializer->status.buffer.size_used--; /* Remove ']' */
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_EOR;
serializer->buffer.data[serializer->status.buffer.size_used++] = ',';
serializer->buffer.data[serializer->status.buffer.size_used++] = '{';
} else {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)
if(serializer->multiline_json_array) {
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
} else {
serializer->status.buffer.size_used--; /* Remove ']' */
serializer->buffer.data[serializer->status.buffer.size_used++] = ',';
}
serializer->buffer.data[serializer->status.buffer.size_used++] = '{';
} else {
if(!serializer->multiline_json_array) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)
serializer->status.buffer.size_used--; /* Remove ']' */
}
serializer->status.buffer.size_used--; /* Remove '}' */
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
serializer->status.buffer.size_used--; /* Remove ']' */
if(!serializer->multiline_json_array)
serializer->status.buffer.size_used--; /* Remove ']' */
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOL)
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOL;
else
@@ -739,20 +774,24 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
static inline int ndpi_serialize_json_post(ndpi_serializer *_serializer) {
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
if (!serializer->multiline_json_array) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
}
}
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = '}';
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
if (!serializer->multiline_json_array) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
if(serializer->status.buffer.size_used >= serializer->buffer.size)
return -1;
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
}
}
serializer->status.flags |= NDPI_SERIALIZER_STATUS_COMMA;
@@ -2391,9 +2430,11 @@ void ndpi_serializer_rollback_snapshot(ndpi_serializer *_serializer) {
if(serializer->fmt == ndpi_serialization_format_json) {
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
serializer->buffer.data[serializer->status.buffer.size_used-1] = ']';
if (!serializer->multiline_json_array)
serializer->buffer.data[serializer->status.buffer.size_used-1] = ']';
} else {
serializer->buffer.data[0] = ' ';
if (!serializer->multiline_json_array)
serializer->buffer.data[0] = ' ';
serializer->buffer.data[serializer->status.buffer.size_used-1] = '}';
}
}