Add support for multiline json
This commit is contained in:
@@ -1756,7 +1756,8 @@ typedef enum {
|
||||
ndpi_serialization_format_unknown = 0,
|
||||
ndpi_serialization_format_tlv,
|
||||
ndpi_serialization_format_json,
|
||||
ndpi_serialization_format_csv
|
||||
ndpi_serialization_format_csv,
|
||||
ndpi_serialization_format_multiline_json
|
||||
} ndpi_serialization_format;
|
||||
|
||||
/* Note:
|
||||
@@ -1821,6 +1822,7 @@ typedef struct {
|
||||
ndpi_serialization_format fmt;
|
||||
char csv_separator[2];
|
||||
u_int8_t has_snapshot;
|
||||
u_int8_t multiline_json_array;
|
||||
ndpi_private_serializer_status snapshot;
|
||||
} ndpi_private_serializer;
|
||||
|
||||
|
||||
+77
-36
@@ -177,8 +177,12 @@ void ndpi_reset_serializer(ndpi_serializer *_serializer) {
|
||||
serializer->status.buffer.size_used = 0;
|
||||
buff_diff = serializer->buffer.size - serializer->status.buffer.size_used;
|
||||
|
||||
/* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */
|
||||
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}");
|
||||
if (serializer->multiline_json_array) {
|
||||
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "{}");
|
||||
} else {
|
||||
/* Note: please keep a space at the beginning as it is used for arrays when an end-of-record is used */
|
||||
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, " {}");
|
||||
}
|
||||
} else if(serializer->fmt == ndpi_serialization_format_csv) {
|
||||
serializer->status.header.size_used = 0;
|
||||
serializer->status.buffer.size_used = 0;
|
||||
@@ -215,6 +219,11 @@ int ndpi_init_serializer_ll(ndpi_serializer *_serializer,
|
||||
|
||||
memset(serializer, 0, sizeof(ndpi_private_serializer));
|
||||
|
||||
if (fmt == ndpi_serialization_format_multiline_json) {
|
||||
fmt = ndpi_serialization_format_json;
|
||||
serializer->multiline_json_array = 1;
|
||||
}
|
||||
|
||||
serializer->fmt = fmt;
|
||||
|
||||
if (buffer_size < 3 ||
|
||||
@@ -614,10 +623,18 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
|
||||
if(serializer->fmt == ndpi_serialization_format_json) {
|
||||
needed += 1;
|
||||
|
||||
if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */
|
||||
serializer->status.buffer.size_used = 2; /* Remove {} */
|
||||
else
|
||||
needed += 2, add_comma = 1;
|
||||
if (serializer->multiline_json_array) {
|
||||
if(serializer->status.buffer.size_used == 2) /* Empty buffer {} */
|
||||
serializer->status.buffer.size_used = 0; /* Remove {} */
|
||||
else
|
||||
needed += 2;
|
||||
|
||||
} else {
|
||||
if(serializer->status.buffer.size_used == 3) /* Empty buffer [{} */
|
||||
serializer->status.buffer.size_used = 2; /* Remove {} */
|
||||
else
|
||||
needed += 2, add_comma = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if(buff_diff < needed) {
|
||||
@@ -628,18 +645,22 @@ int ndpi_serialize_raw_record(ndpi_serializer *_serializer,
|
||||
}
|
||||
|
||||
if(serializer->fmt == ndpi_serialization_format_json) {
|
||||
if(add_comma)
|
||||
serializer->buffer.data[serializer->status.buffer.size_used-1] = ',';
|
||||
else
|
||||
serializer->status.buffer.size_used--;
|
||||
if (!serializer->multiline_json_array) {
|
||||
if (add_comma)
|
||||
serializer->buffer.data[serializer->status.buffer.size_used-1] = ',';
|
||||
else
|
||||
serializer->status.buffer.size_used--;
|
||||
}
|
||||
}
|
||||
|
||||
memcpy(&serializer->buffer.data[serializer->status.buffer.size_used], record, record_len);
|
||||
serializer->status.buffer.size_used += record_len;
|
||||
|
||||
if(serializer->fmt == ndpi_serialization_format_json) {
|
||||
serializer->buffer.data[serializer->status.buffer.size_used] = ']';
|
||||
if(add_comma) serializer->status.buffer.size_used++;
|
||||
if (!serializer->multiline_json_array) {
|
||||
serializer->buffer.data[serializer->status.buffer.size_used] = ']';
|
||||
if(add_comma) serializer->status.buffer.size_used++;
|
||||
}
|
||||
}
|
||||
|
||||
ndpi_serialize_end_of_record(_serializer);
|
||||
@@ -665,20 +686,26 @@ int ndpi_serialize_end_of_record(ndpi_serializer *_serializer) {
|
||||
}
|
||||
|
||||
if(serializer->fmt == ndpi_serialization_format_csv) {
|
||||
serializer->buffer.data[serializer->status.buffer.size_used] = '\n';
|
||||
serializer->status.buffer.size_used += 1;
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
|
||||
serializer->buffer.data[serializer->status.buffer.size_used] = '\0';
|
||||
serializer->status.flags |= NDPI_SERIALIZER_STATUS_HDR_DONE;
|
||||
serializer->status.flags |= NDPI_SERIALIZER_STATUS_EOR;
|
||||
} else if(serializer->fmt == ndpi_serialization_format_json) {
|
||||
if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
|
||||
serializer->buffer.data[0] = '[';
|
||||
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used],
|
||||
buff_diff, "]");
|
||||
}
|
||||
|
||||
} else if(serializer->fmt == ndpi_serialization_format_json) {
|
||||
|
||||
if(serializer->multiline_json_array) {
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
|
||||
serializer->buffer.data[serializer->status.buffer.size_used] = '\0';
|
||||
|
||||
} else {
|
||||
if(!(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)) {
|
||||
serializer->buffer.data[0] = '[';
|
||||
serializer->status.buffer.size_used += ndpi_snprintf((char *) &serializer->buffer.data[serializer->status.buffer.size_used], buff_diff, "]");
|
||||
}
|
||||
}
|
||||
serializer->status.flags |= NDPI_SERIALIZER_STATUS_ARRAY | NDPI_SERIALIZER_STATUS_EOR;
|
||||
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_COMMA;
|
||||
|
||||
} else /* ndpi_serialization_format_tlv */ {
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = ndpi_serialization_end_of_record;
|
||||
}
|
||||
@@ -709,18 +736,26 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
|
||||
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
|
||||
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_EOR) {
|
||||
serializer->status.buffer.size_used--; /* Remove ']' */
|
||||
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_EOR;
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = ',';
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = '{';
|
||||
} else {
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)
|
||||
if(serializer->multiline_json_array) {
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = '\n';
|
||||
} else {
|
||||
serializer->status.buffer.size_used--; /* Remove ']' */
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = ',';
|
||||
}
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = '{';
|
||||
|
||||
} else {
|
||||
if(!serializer->multiline_json_array) {
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY)
|
||||
serializer->status.buffer.size_used--; /* Remove ']' */
|
||||
}
|
||||
|
||||
serializer->status.buffer.size_used--; /* Remove '}' */
|
||||
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
|
||||
serializer->status.buffer.size_used--; /* Remove ']' */
|
||||
if(!serializer->multiline_json_array)
|
||||
serializer->status.buffer.size_used--; /* Remove ']' */
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_SOL)
|
||||
serializer->status.flags &= ~NDPI_SERIALIZER_STATUS_SOL;
|
||||
else
|
||||
@@ -739,20 +774,24 @@ static inline void ndpi_serialize_json_pre(ndpi_serializer *_serializer) {
|
||||
static inline int ndpi_serialize_json_post(ndpi_serializer *_serializer) {
|
||||
ndpi_private_serializer *serializer = (ndpi_private_serializer*)_serializer;
|
||||
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
|
||||
if(serializer->status.buffer.size_used >= serializer->buffer.size)
|
||||
return -1;
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
|
||||
if (!serializer->multiline_json_array) {
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_LIST) {
|
||||
if(serializer->status.buffer.size_used >= serializer->buffer.size)
|
||||
return -1;
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
|
||||
}
|
||||
}
|
||||
|
||||
if(serializer->status.buffer.size_used >= serializer->buffer.size)
|
||||
return -1;
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = '}';
|
||||
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
|
||||
if(serializer->status.buffer.size_used >= serializer->buffer.size)
|
||||
return -1;
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
|
||||
if (!serializer->multiline_json_array) {
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
|
||||
if(serializer->status.buffer.size_used >= serializer->buffer.size)
|
||||
return -1;
|
||||
serializer->buffer.data[serializer->status.buffer.size_used++] = ']';
|
||||
}
|
||||
}
|
||||
|
||||
serializer->status.flags |= NDPI_SERIALIZER_STATUS_COMMA;
|
||||
@@ -2391,9 +2430,11 @@ void ndpi_serializer_rollback_snapshot(ndpi_serializer *_serializer) {
|
||||
|
||||
if(serializer->fmt == ndpi_serialization_format_json) {
|
||||
if(serializer->status.flags & NDPI_SERIALIZER_STATUS_ARRAY) {
|
||||
serializer->buffer.data[serializer->status.buffer.size_used-1] = ']';
|
||||
if (!serializer->multiline_json_array)
|
||||
serializer->buffer.data[serializer->status.buffer.size_used-1] = ']';
|
||||
} else {
|
||||
serializer->buffer.data[0] = ' ';
|
||||
if (!serializer->multiline_json_array)
|
||||
serializer->buffer.data[0] = ' ';
|
||||
serializer->buffer.data[serializer->status.buffer.size_used-1] = '}';
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user