Improved nDPI JSON serialization. (#1689)

* fixed autoconf CFLAGS/LDFLAGS MSAN issue which could lead to build errors
 * introduced portable version of gmtime_r aka ndpi_gmtime_r
 * do as most as possible of the serialization work in ndpi_utils.c
 * use flow2json in ndpiReader

Signed-off-by: lns <matzeton@googlemail.com>
This commit is contained in:
Toni
2022-08-02 17:54:44 +02:00
committed by GitHub
parent 26aafd931c
commit b3e722e5a8
13 changed files with 247 additions and 325 deletions
+2 -2
View File
@@ -57,8 +57,8 @@ AS_IF([test "${with_thread_sanitizer+set}" = set],[
])
AS_IF([test "${with_memory_sanitizer+set}" = set],[
NDPI_CFLAGS="${CFLAGS} -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer"
NDPI_LDFLAGS="${LDFLAGS} -fsanitize=memory"
NDPI_CFLAGS="${NDPI_CFLAGS} -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer"
NDPI_LDFLAGS="${NDPI_LDFLAGS} -fsanitize=memory"
])
AS_IF([test "x${enable_code_coverage}" = "xyes"],[
+17 -191
View File
@@ -1060,12 +1060,15 @@ static void parseOptions(int argc, char **argv) {
}
}
if ((serialization_fp == NULL && serialization_format != ndpi_serialization_format_unknown) ||
(serialization_fp != NULL && serialization_format == ndpi_serialization_format_unknown))
if (serialization_fp == NULL && serialization_format != ndpi_serialization_format_unknown)
{
printf("Serializing detection results to a file requires command line arguments -k AND -K\n");
printf("Serializing detection results to a file requires command line arguments `-k'\n");
exit(1);
}
if (serialization_fp != NULL && serialization_format == ndpi_serialization_format_unknown)
{
serialization_format = ndpi_serialization_format_json;
}
if(extcap_exit)
exit(0);
@@ -1584,8 +1587,10 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
if(flow->ssh_tls.tls_subjectDN) fprintf(out, "[Subject: %s]", flow->ssh_tls.tls_subjectDN);
if(flow->ssh_tls.encrypted_sni.esni) {
char unknown_cipher[8];
fprintf(out, "[ESNI: %s]", flow->ssh_tls.encrypted_sni.esni);
fprintf(out, "[ESNI Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.encrypted_sni.cipher_suite));
fprintf(out, "[ESNI Cipher: %s]",
ndpi_cipher2str(flow->ssh_tls.encrypted_sni.cipher_suite, unknown_cipher));
}
if(flow->ssh_tls.sha1_cert_fingerprint_set) {
@@ -1605,8 +1610,8 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
if(flow->ssh_tls.notBefore && flow->ssh_tls.notAfter) {
char notBefore[32], notAfter[32];
struct tm a, b;
struct tm *before = gmtime_r(&flow->ssh_tls.notBefore, &a);
struct tm *after = gmtime_r(&flow->ssh_tls.notAfter, &b);
struct tm *before = ndpi_gmtime_r(&flow->ssh_tls.notBefore, &a);
struct tm *after = ndpi_gmtime_r(&flow->ssh_tls.notAfter, &b);
strftime(notBefore, sizeof(notBefore), "%Y-%m-%d %H:%M:%S", before);
strftime(notAfter, sizeof(notAfter), "%Y-%m-%d %H:%M:%S", after);
@@ -1614,7 +1619,11 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
fprintf(out, "[Validity: %s - %s]", notBefore, notAfter);
}
if(flow->ssh_tls.server_cipher != '\0') fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher));
char unknown_cipher[8];
if(flow->ssh_tls.server_cipher != '\0')
{
fprintf(out, "[Cipher: %s]", ndpi_cipher2str(flow->ssh_tls.server_cipher, unknown_cipher));
}
if(flow->bittorent_hash != NULL) fprintf(out, "[BT Hash: %s]", flow->bittorent_hash);
if(flow->dhcp_fingerprint != NULL) fprintf(out, "[DHCP Fingerprint: %s]", flow->dhcp_fingerprint);
if(flow->dhcp_class_ident) fprintf(out, "[DHCP Class Ident: %s]",
@@ -1639,69 +1648,17 @@ static void printFlowSerialized(u_int16_t thread_id,
{
char *json_str = NULL;
u_int32_t json_str_len = 0;
ndpi_serializer * const serializer = &ndpi_thread_info[thread_id].workflow->ndpi_serializer;
ndpi_serializer * const serializer = &flow->ndpi_flow_serializer;
//float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes);
double f = (double)flow->first_seen_ms, l = (double)flow->last_seen_ms;
u_int8_t known_tls;
char buf[64];
float data_ratio = ndpi_data_ratio(flow->src2dst_bytes, flow->dst2src_bytes);
ndpi_reset_serializer(serializer);
ndpi_serialize_string_uint32(serializer, "flow_id", flow->flow_id);
ndpi_serialize_string_uint32(serializer, "l4_protocol", flow->protocol);
ndpi_serialize_string_float(serializer, "first_seen_ms", f, "%.3f");
ndpi_serialize_string_float(serializer, "last_seen_ms", l, "%.3f");
ndpi_serialize_string_float(serializer, "duration_ms", (l-f)/1000.0, "%.3f");
ndpi_serialize_string_string(serializer, "src_name", flow->src_name);
ndpi_serialize_string_string(serializer, "dst_name", flow->dst_name);
ndpi_serialize_string_uint32(serializer, "src_port", ntohs(flow->src_port));
ndpi_serialize_string_uint32(serializer, "dst_port", ntohs(flow->dst_port));
ndpi_serialize_string_uint32(serializer, "ip_version", flow->ip_version);
ndpi_serialize_string_uint32(serializer, "vlan_id", flow->vlan_id);
ndpi_serialize_string_uint32(serializer, "bidirectional", flow->bidirectional);
ndpi_serialize_string_uint32(serializer, "encrypted",
ndpi_is_encrypted_proto(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol));
ndpi_serialize_string_string(serializer, "confidence",
ndpi_confidence_get_name(flow->confidence));
ndpi_serialize_string_uint32(serializer, "category_id",
flow->detected_protocol.category);
ndpi_serialize_string_string(
serializer, "category_name",
ndpi_category_get_name(
ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol.category
)
);
ndpi_serialize_string_string(serializer, "l7_protocol_id",
ndpi_protocol2id(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol, buf, sizeof(buf)));
ndpi_serialize_string_string(serializer, "l7_protocol_name",
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol, buf, sizeof(buf)));
ndpi_serialize_start_of_list(serializer, "risks");
if (flow->risk != NDPI_NO_RISK)
{
u_int32_t i;
for(i = 0; i < NDPI_MAX_RISK; ++i)
{
if (NDPI_ISSET_BIT(flow->risk, i) != 0)
{
ndpi_serialize_string_string(serializer, "str", ndpi_risk2str(i));
}
}
}
ndpi_serialize_end_of_list(serializer);
{
u_int16_t cli_score, srv_score;
ndpi_serialize_string_uint32(serializer, "risks_score",
ndpi_risk2score(flow->risk, &cli_score, &srv_score));
}
ndpi_serialize_string_string(serializer, "host_server_name",
flow->host_server_name);
/* XFER Packets/Bytes */
ndpi_serialize_start_of_block(serializer, "xfer");
@@ -1802,137 +1759,6 @@ static void printFlowSerialized(u_int16_t thread_id,
ndpi_serialize_string_uint32(serializer, "c_to_s_init_win", flow->c_to_s_init_win);
ndpi_serialize_string_uint32(serializer, "s_to_c_init_win", flow->s_to_c_init_win);
/* Protocol specific serialization */
ndpi_serialize_start_of_block(serializer, "l7_protocol_data");
if (flow->ssh_tls.server_info[0] != '\0')
{
ndpi_serialize_string_string(serializer, "server_info", flow->ssh_tls.server_info);
}
if (flow->ssh_tls.server_names != NULL)
{
ndpi_serialize_string_string(serializer, "server_names", flow->ssh_tls.server_names);
}
if (flow->ssh_tls.ssl_version != 0)
{
ndpi_ssl_version2str(buf, sizeof(buf), flow->ssh_tls.ssl_version, &known_tls);
ndpi_serialize_string_string(serializer, "version", buf);
}
if (flow->ssh_tls.ja3_client[0] != '\0')
{
ndpi_serialize_string_string(serializer, "ja3_client", flow->ssh_tls.ja3_client);
}
if (flow->ssh_tls.ja3_server[0] != '\0')
{
ndpi_serialize_string_string(serializer, "ja3_server", flow->ssh_tls.ja3_server);
}
if (flow->ssh_tls.tls_issuerDN != NULL)
{
ndpi_serialize_string_string(serializer, "issuerDN", flow->ssh_tls.tls_issuerDN);
}
if (flow->ssh_tls.tls_subjectDN != NULL)
{
ndpi_serialize_string_string(serializer, "subjectDN", flow->ssh_tls.tls_subjectDN);
}
if (flow->ssh_tls.client_hassh[0] != '\0')
{
ndpi_serialize_string_string(serializer, "client_hassh", flow->ssh_tls.client_hassh);
}
if (flow->ssh_tls.server_hassh[0] != '\0')
{
ndpi_serialize_string_string(serializer, "server_hassh", flow->ssh_tls.server_hassh);
}
if (flow->http.user_agent[0] != '\0')
{
ndpi_serialize_string_string(serializer, "user_agent", flow->http.user_agent);
}
if (flow->http.url[0] != '\0')
{
ndpi_risk_enum risk = ndpi_validate_url(flow->http.url);
if (risk != NDPI_NO_RISK)
{
NDPI_SET_BIT(flow->risk, risk);
}
ndpi_serialize_string_string(serializer, "url", flow->http.url);
ndpi_serialize_string_uint32(serializer, "code", flow->http.response_status_code);
if (flow->http.request_content_type[0] != '\0')
{
ndpi_serialize_string_string(serializer, "req_content_type",
flow->http.request_content_type);
}
if (flow->http.content_type[0] != '\0')
{
ndpi_serialize_string_string(serializer, "content_type",
flow->http.content_type);
}
}
switch (flow->info_type)
{
case INFO_INVALID:
break;
case INFO_GENERIC:
if (flow->info[0] != '\0')
{
ndpi_serialize_string_string(serializer, "info", flow->info);
}
break;
case INFO_KERBEROS:
if (flow->kerberos.domain[0] != '\0' ||
flow->kerberos.hostname[0] != '\0' ||
flow->kerberos.username[0] != '\0')
{
ndpi_serialize_string_string(serializer, "domain",
flow->kerberos.domain);
ndpi_serialize_string_string(serializer, "hostname",
flow->kerberos.hostname);
ndpi_serialize_string_string(serializer, "username",
flow->kerberos.username);
}
break;
case INFO_SOFTETHER:
ndpi_serialize_string_string(serializer, "client_ip", flow->softether.ip);
ndpi_serialize_string_string(serializer, "client_port", flow->softether.port);
ndpi_serialize_string_string(serializer, "hostname", flow->softether.hostname);
ndpi_serialize_string_string(serializer, "fqdn", flow->softether.fqdn);
break;
case INFO_FTP_IMAP_POP_SMTP:
ndpi_serialize_string_string(serializer, "username",
flow->ftp_imap_pop_smtp.username);
ndpi_serialize_string_string(serializer, "password",
flow->ftp_imap_pop_smtp.password);
ndpi_serialize_string_uint32(serializer, "auth_failed",
flow->ftp_imap_pop_smtp.auth_failed);
break;
case INFO_TLS_QUIC_ALPN_VERSION:
ndpi_serialize_string_string(serializer, "alpn", flow->tls_quic.alpn);
ndpi_serialize_string_string(serializer, "supported_versions",
flow->tls_quic.tls_supported_versions);
break;
case INFO_TLS_QUIC_ALPN_ONLY:
ndpi_serialize_string_string(serializer, "alpn", flow->tls_quic.alpn);
break;
}
ndpi_serialize_end_of_block(serializer);
json_str = ndpi_serializer_get_buffer(serializer, &json_str_len);
if (json_str == NULL || json_str_len == 0)
{
+22 -9
View File
@@ -434,13 +434,7 @@ struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * pref
if(do_init_flows_root)
workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *));
if (serialization_format != ndpi_serialization_format_unknown &&
ndpi_init_serializer(&workflow->ndpi_serializer,
serialization_format) != 0)
{
LOG(NDPI_LOG_ERROR, "serializer initialization failed\n");
exit(-1);
}
workflow->ndpi_serialization_format = serialization_format;
return workflow;
}
@@ -532,6 +526,7 @@ static void ndpi_free_flow_data_analysis(struct ndpi_flow_info *flow) {
void ndpi_flow_info_free_data(struct ndpi_flow_info *flow) {
ndpi_free_flow_info_half(flow);
ndpi_term_serializer(&flow->ndpi_flow_serializer);
ndpi_free_flow_data_analysis(flow);
ndpi_free_flow_tls_data(flow);
@@ -550,8 +545,6 @@ void ndpi_flow_info_free_data(struct ndpi_flow_info *flow) {
void ndpi_workflow_free(struct ndpi_workflow * workflow) {
u_int i;
ndpi_term_serializer(&workflow->ndpi_serializer);
for(i=0; i<workflow->prefs.num_roots; i++)
ndpi_tdestroy(workflow->ndpi_flows_root[i], ndpi_flow_info_freer);
@@ -1282,6 +1275,26 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
workflow->__flow_detected_callback(workflow, flow, workflow->__flow_detected_udata);
}
if (workflow->ndpi_serialization_format != ndpi_serialization_format_unknown)
{
if (ndpi_init_serializer(&flow->ndpi_flow_serializer,
workflow->ndpi_serialization_format) != 0)
{
LOG(NDPI_LOG_ERROR, "ndpi serializer init failed\n");
exit(-1);
}
if (ndpi_flow2json(workflow->ndpi_struct, flow->ndpi_flow,
flow->ip_version, flow->protocol,
flow->src_ip, flow->dst_ip,
&flow->src_ip6, &flow->dst_ip6,
flow->src_port, flow->dst_port,
flow->detected_protocol,
&flow->ndpi_flow_serializer) != 0)
{
LOG(NDPI_LOG_ERROR, "flow2json failed\n");
exit(-1);
}
}
ndpi_free_flow_info_half(flow);
}
}
+3 -1
View File
@@ -236,6 +236,8 @@ typedef struct ndpi_flow_info {
} softether;
};
ndpi_serializer ndpi_flow_serializer;
char flow_extra_info[16];
char host_server_name[80]; /* Hostname/SNI */
char *bittorent_hash;
@@ -348,7 +350,7 @@ typedef struct ndpi_workflow {
u_int32_t num_allocated_flows;
/* CSV,TLV,JSON serialization interface */
ndpi_serializer ndpi_serializer;
ndpi_serialization_format ndpi_serialization_format;
} ndpi_workflow_t;
-2
View File
@@ -79,7 +79,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
if (pkts == NULL) {
remove(pcap_path);
free(pcap_path);
ndpi_term_serializer(&workflow->ndpi_serializer);
return 0;
}
if (ndpi_is_datalink_supported(pcap_datalink(pkts)) == 0)
@@ -88,7 +87,6 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
pcap_close(pkts);
remove(pcap_path);
free(pcap_path);
ndpi_term_serializer(&workflow->ndpi_serializer);
return 0;
}
+1 -1
View File
@@ -1056,7 +1056,7 @@ extern "C" {
u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);
u_int8_t ndpi_is_safe_ssl_cipher(u_int32_t cipher);
const char* ndpi_cipher2str(u_int32_t cipher);
const char* ndpi_cipher2str(u_int32_t cipher, char unknown_cipher[8]);
const char* ndpi_tunnel2str(ndpi_packet_tunnel tt);
u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);
+1 -1
View File
@@ -654,7 +654,7 @@ struct ndpi_flow_tcp_struct {
/* NDPI_PROTOCOL_MAIL_SMTP */
/* NDPI_PROTOCOL_MAIL_POP */
/* NDPI_PROTOCOL_MAIL_IMAP */
/* NDPI_PROTOCOL_MAIL_FTP */
/* NDPI_PROTOCOL_FTP_CONTROL */
/* TODO: something clever to save memory */
struct {
u_int8_t auth_found:1, auth_failed:1, auth_tls:1, auth_done:1, _pad:4;
+2
View File
@@ -57,6 +57,8 @@ extern u_int8_t ndpi_ends_with(char *str, char *ends);
#ifndef NDPI_CFFI_PREPROCESSING
int ndpi_vsnprintf(char * str, size_t size, char const * format, va_list va_args);
int ndpi_snprintf(char * str, size_t size, char const * format, ...);
struct tm *ndpi_gmtime_r(const time_t *restrict timep,
struct tm *restrict result);
#endif
#endif
-2
View File
@@ -66,8 +66,6 @@ typedef uint u_int32_t;
typedef uint u_int;
typedef unsigned __int64 u_int64_t;
#define gmtime_r(a, b) memcpy(b, gmtime(a), sizeof(struct tm))
#define timegm _mkgmtime
#define sleep(a /* sec */) Sleep(1000*a /* ms */)
+191 -109
View File
@@ -381,7 +381,7 @@ u_int8_t ndpi_is_safe_ssl_cipher(u_int32_t cipher) {
/* ***************************************************** */
const char* ndpi_cipher2str(u_int32_t cipher) {
const char* ndpi_cipher2str(u_int32_t cipher, char unknown_cipher[8]) {
switch(cipher) {
case TLS_NULL_WITH_NULL_NULL: return("TLS_NULL_WITH_NULL_NULL");
case TLS_RSA_EXPORT_WITH_RC4_40_MD5: return("TLS_RSA_EXPORT_WITH_RC4_40_MD5");
@@ -748,10 +748,8 @@ const char* ndpi_cipher2str(u_int32_t cipher) {
default:
{
static char buf[8];
ndpi_snprintf(buf, sizeof(buf), "0X%04X", cipher);
return(buf);
ndpi_snprintf(unknown_cipher, 8, "0X%04X", cipher);
return(unknown_cipher);
}
}
}
@@ -875,15 +873,27 @@ int ndpi_has_human_readeable_string(struct ndpi_detection_module_struct *ndpi_st
static const char* ndpi_get_flow_info_by_proto_id(struct ndpi_flow_struct const * const flow,
u_int16_t proto_id) {
switch (proto_id) {
case NDPI_PROTOCOL_DNS:
case NDPI_PROTOCOL_HTTP:
return flow->host_server_name;
case NDPI_PROTOCOL_QUIC:
case NDPI_PROTOCOL_TLS:
if(flow->protos.tls_quic.hello_processed != 0)
return flow->host_server_name;
break;
case NDPI_PROTOCOL_WHOIS_DAS:
case NDPI_PROTOCOL_MAIL_SMTP:
case NDPI_PROTOCOL_NETBIOS:
case NDPI_PROTOCOL_SSDP:
case NDPI_PROTOCOL_MDNS:
case NDPI_PROTOCOL_STUN:
case NDPI_PROTOCOL_DNS:
case NDPI_PROTOCOL_DHCP:
case NDPI_PROTOCOL_XIAOMI:
case NDPI_PROTOCOL_SD_RTN:
case NDPI_PROTOCOL_COLLECTD:
case NDPI_PROTOCOL_HTTP:
case NDPI_PROTOCOL_HTTP_CONNECT:
case NDPI_PROTOCOL_HTTP_PROXY:
return flow->host_server_name;
case NDPI_PROTOCOL_QUIC:
case NDPI_PROTOCOL_TLS:
if(flow->protos.tls_quic.hello_processed != 0)
return flow->host_server_name;
break;
}
return NULL;
@@ -1192,28 +1202,126 @@ void ndpi_serialize_proto(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_serialize_risk(serializer, risk);
ndpi_serialize_confidence(serializer, confidence);
ndpi_serialize_string_string(serializer, "proto", ndpi_protocol2name(ndpi_struct, l7_protocol, buf, sizeof(buf)));
ndpi_serialize_string_string(serializer, "proto_id", ndpi_protocol2id(ndpi_struct, l7_protocol, buf, sizeof(buf)));
ndpi_serialize_string_uint32(serializer, "encrypted", ndpi_is_encrypted_proto(ndpi_struct, l7_protocol));
ndpi_protocol_breed_t breed =
ndpi_get_proto_breed(ndpi_struct,
(l7_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN ? l7_protocol.app_protocol : l7_protocol.master_protocol));
ndpi_serialize_string_string(serializer, "breed", ndpi_get_proto_breed_name(ndpi_struct, breed));
if(l7_protocol.category != NDPI_PROTOCOL_CATEGORY_UNSPECIFIED)
{
ndpi_serialize_string_uint32(serializer, "category_id", l7_protocol.category);
ndpi_serialize_string_string(serializer, "category", ndpi_category_get_name(ndpi_struct, l7_protocol.category));
}
ndpi_serialize_end_of_block(serializer);
}
/* ********************************** */
static void ndpi_tls2json(ndpi_serializer *serializer, struct ndpi_flow_struct *flow)
{
if(flow->protos.tls_quic.ssl_version)
{
char buf[64];
char notBefore[32], notAfter[32];
struct tm a, b, *before = NULL, *after = NULL;
u_int i, off;
u_int8_t unknown_tls_version;
char version[16], unknown_cipher[8];
ndpi_ssl_version2str(version, sizeof(version), flow->protos.tls_quic.ssl_version, &unknown_tls_version);
if(flow->protos.tls_quic.notBefore)
{
before = ndpi_gmtime_r((const time_t *)&flow->protos.tls_quic.notBefore, &a);
}
if(flow->protos.tls_quic.notAfter)
{
after = ndpi_gmtime_r((const time_t *)&flow->protos.tls_quic.notAfter, &b);
}
if(!unknown_tls_version)
{
ndpi_serialize_start_of_block(serializer, "tls");
ndpi_serialize_string_string(serializer, "version", version);
if(flow->protos.tls_quic.server_names)
{
ndpi_serialize_string_string(serializer, "server_names",
flow->protos.tls_quic.server_names);
}
if(before)
{
strftime(notBefore, sizeof(notBefore), "%Y-%m-%d %H:%M:%S", before);
ndpi_serialize_string_string(serializer, "notbefore", notBefore);
}
if(after)
{
strftime(notAfter, sizeof(notAfter), "%Y-%m-%d %H:%M:%S", after);
ndpi_serialize_string_string(serializer, "notafter", notAfter);
}
ndpi_serialize_string_string(serializer, "ja3", flow->protos.tls_quic.ja3_client);
ndpi_serialize_string_string(serializer, "ja3s", flow->protos.tls_quic.ja3_server);
ndpi_serialize_string_uint32(serializer, "unsafe_cipher", flow->protos.tls_quic.server_unsafe_cipher);
ndpi_serialize_string_string(serializer, "cipher",
ndpi_cipher2str(flow->protos.tls_quic.server_cipher, unknown_cipher));
if(flow->protos.tls_quic.issuerDN)
{
ndpi_serialize_string_string(serializer, "issuerDN", flow->protos.tls_quic.issuerDN);
}
if(flow->protos.tls_quic.subjectDN)
{
ndpi_serialize_string_string(serializer, "subjectDN", flow->protos.tls_quic.subjectDN);
}
if(flow->protos.tls_quic.alpn)
{
ndpi_serialize_string_string(serializer, "alpn", flow->protos.tls_quic.alpn);
}
if(flow->protos.tls_quic.tls_supported_versions)
{
ndpi_serialize_string_string(serializer, "tls_supported_versions", flow->protos.tls_quic.tls_supported_versions);
}
if(flow->protos.tls_quic.sha1_certificate_fingerprint[0] != '\0')
{
for(i=0, off=0; i<20; i++)
{
int rc = ndpi_snprintf(&buf[off], sizeof(buf)-off,"%s%02X", (i > 0) ? ":" : "",
flow->protos.tls_quic.sha1_certificate_fingerprint[i] & 0xFF);
if(rc <= 0) break; else off += rc;
}
ndpi_serialize_string_string(serializer, "fingerprint", buf);
}
ndpi_serialize_end_of_block(serializer);
}
}
}
/* NOTE: serializer must have been already initialized */
int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
ndpi_protocol l7_protocol,
ndpi_serializer *serializer) {
char buf[64];
char const *host_server_name;
if(flow == NULL) return(-1);
ndpi_serialize_proto(ndpi_struct, serializer, flow->risk, flow->confidence, l7_protocol);
host_server_name = ndpi_get_flow_info(flow, &l7_protocol);
if (host_server_name != NULL)
{
ndpi_serialize_string_string(serializer, "hostname", host_server_name);
}
switch(l7_protocol.master_protocol ? l7_protocol.master_protocol : l7_protocol.app_protocol) {
case NDPI_PROTOCOL_IP_ICMP:
if(flow->entropy > 0.0f) {
@@ -1223,7 +1331,6 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
case NDPI_PROTOCOL_DHCP:
ndpi_serialize_start_of_block(serializer, "dhcp");
ndpi_serialize_string_string(serializer, "hostname", flow->host_server_name);
ndpi_serialize_string_string(serializer, "fingerprint", flow->protos.dhcp.fingerprint);
ndpi_serialize_string_string(serializer, "class_ident", flow->protos.dhcp.class_ident);
ndpi_serialize_end_of_block(serializer);
@@ -1251,8 +1358,6 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
case NDPI_PROTOCOL_DNS:
ndpi_serialize_start_of_block(serializer, "dns");
if(flow->host_server_name[0] != '\0')
ndpi_serialize_string_string(serializer, "query", flow->host_server_name);
ndpi_serialize_string_uint32(serializer, "num_queries", flow->protos.dns.num_queries);
ndpi_serialize_string_uint32(serializer, "num_answers", flow->protos.dns.num_answers);
ndpi_serialize_string_uint32(serializer, "reply_code", flow->protos.dns.reply_code);
@@ -1273,7 +1378,6 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
case NDPI_PROTOCOL_MDNS:
ndpi_serialize_start_of_block(serializer, "mdns");
ndpi_serialize_string_string(serializer, "answer", flow->host_server_name);
ndpi_serialize_end_of_block(serializer);
break;
@@ -1291,6 +1395,25 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_SOFTETHER:
ndpi_serialize_start_of_block(serializer, "softether");
ndpi_serialize_string_string(serializer, "client_ip", flow->protos.softether.ip);
ndpi_serialize_string_string(serializer, "client_port", flow->protos.softether.port);
ndpi_serialize_string_string(serializer, "hostname", flow->protos.softether.hostname);
ndpi_serialize_string_string(serializer, "fqdn", flow->protos.softether.fqdn);
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_STUN:
ndpi_serialize_start_of_block(serializer, "stun");
ndpi_serialize_string_uint32(serializer, "num_udp_pkts", flow->stun.num_udp_pkts);
ndpi_serialize_string_uint32(serializer, "num_binding_requests",
flow->stun.num_binding_requests);
ndpi_serialize_string_uint32(serializer, "num_processed_pkts",
flow->stun.num_processed_pkts);
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_TELNET:
ndpi_serialize_start_of_block(serializer, "telnet");
ndpi_serialize_string_string(serializer, "username", flow->protos.telnet.username);
@@ -1302,40 +1425,42 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
case NDPI_PROTOCOL_HTTP_CONNECT:
case NDPI_PROTOCOL_HTTP_PROXY:
ndpi_serialize_start_of_block(serializer, "http");
if(flow->host_server_name[0] != '\0')
ndpi_serialize_string_string(serializer, "hostname", flow->host_server_name);
if(flow->http.url != NULL) {
ndpi_serialize_string_string(serializer, "url", flow->http.url);
ndpi_serialize_string_uint32(serializer, "code", flow->http.response_status_code);
ndpi_serialize_string_string(serializer, "content_type", flow->http.content_type);
ndpi_serialize_string_string(serializer, "user_agent", flow->http.user_agent);
ndpi_risk_enum risk = ndpi_validate_url(flow->http.url);
if (risk != NDPI_NO_RISK)
{
NDPI_SET_BIT(flow->risk, risk);
}
ndpi_serialize_string_string(serializer, "url", flow->http.url);
ndpi_serialize_string_uint32(serializer, "code", flow->http.response_status_code);
ndpi_serialize_string_string(serializer, "content_type", flow->http.content_type);
ndpi_serialize_string_string(serializer, "user_agent", flow->http.user_agent);
}
if (flow->http.request_content_type != NULL)
{
ndpi_serialize_string_string(serializer, "request_content_type",
flow->http.request_content_type);
}
if (flow->http.detected_os != NULL)
{
ndpi_serialize_string_string(serializer, "detected_os",
flow->http.detected_os);
}
if (flow->http.nat_ip != NULL)
{
ndpi_serialize_string_string(serializer, "nat_ip",
flow->http.nat_ip);
}
ndpi_serialize_end_of_block(serializer);
break;
case NDPI_PROTOCOL_QUIC:
ndpi_serialize_start_of_block(serializer, "quic");
if(flow->host_server_name[0] != '\0')
ndpi_serialize_string_string(serializer, "client_requested_server_name",
flow->host_server_name);
if(flow->protos.tls_quic.server_names)
ndpi_serialize_string_string(serializer, "server_names", flow->protos.tls_quic.server_names);
if(flow->http.user_agent)
ndpi_serialize_string_string(serializer, "user_agent", flow->http.user_agent);
if(flow->protos.tls_quic.ssl_version) {
u_int8_t unknown_tls_version;
char version[16];
ndpi_ssl_version2str(version, sizeof(version), flow->protos.tls_quic.ssl_version, &unknown_tls_version);
ndpi_tls2json(serializer, flow);
if(!unknown_tls_version)
ndpi_serialize_string_string(serializer, "version", version);
if(flow->protos.tls_quic.alpn)
ndpi_serialize_string_string(serializer, "alpn", flow->protos.tls_quic.alpn);
ndpi_serialize_string_string(serializer, "ja3", flow->protos.tls_quic.ja3_client);
if(flow->protos.tls_quic.tls_supported_versions)
ndpi_serialize_string_string(serializer, "tls_supported_versions", flow->protos.tls_quic.tls_supported_versions);
}
ndpi_serialize_end_of_block(serializer);
break;
@@ -1343,6 +1468,8 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_serialize_start_of_block(serializer, "imap");
ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username);
ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password);
ndpi_serialize_string_uint32(serializer, "auth_failed",
flow->l4.tcp.ftp_imap_pop_smtp.auth_failed);
ndpi_serialize_end_of_block(serializer);
break;
@@ -1350,6 +1477,8 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_serialize_start_of_block(serializer, "pop");
ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username);
ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password);
ndpi_serialize_string_uint32(serializer, "auth_failed",
flow->l4.tcp.ftp_imap_pop_smtp.auth_failed);
ndpi_serialize_end_of_block(serializer);
break;
@@ -1357,6 +1486,8 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_serialize_start_of_block(serializer, "smtp");
ndpi_serialize_string_string(serializer, "user", flow->l4.tcp.ftp_imap_pop_smtp.username);
ndpi_serialize_string_string(serializer, "password", flow->l4.tcp.ftp_imap_pop_smtp.password);
ndpi_serialize_string_uint32(serializer, "auth_failed",
flow->l4.tcp.ftp_imap_pop_smtp.auth_failed);
ndpi_serialize_end_of_block(serializer);
break;
@@ -1379,68 +1510,7 @@ int ndpi_dpi2json(struct ndpi_detection_module_struct *ndpi_struct,
case NDPI_PROTOCOL_TLS:
case NDPI_PROTOCOL_DTLS:
if(flow->protos.tls_quic.ssl_version) {
char notBefore[32], notAfter[32];
struct tm a, b, *before = NULL, *after = NULL;
u_int i, off;
u_int8_t unknown_tls_version;
char version[16];
ndpi_ssl_version2str(version, sizeof(version), flow->protos.tls_quic.ssl_version, &unknown_tls_version);
if(flow->protos.tls_quic.notBefore)
before = gmtime_r((const time_t *)&flow->protos.tls_quic.notBefore, &a);
if(flow->protos.tls_quic.notAfter)
after = gmtime_r((const time_t *)&flow->protos.tls_quic.notAfter, &b);
if(!unknown_tls_version) {
ndpi_serialize_start_of_block(serializer, "tls");
ndpi_serialize_string_string(serializer, "version", version);
ndpi_serialize_string_string(serializer, "client_requested_server_name",
flow->host_server_name);
if(flow->protos.tls_quic.server_names)
ndpi_serialize_string_string(serializer, "server_names", flow->protos.tls_quic.server_names);
if(before) {
strftime(notBefore, sizeof(notBefore), "%Y-%m-%d %H:%M:%S", before);
ndpi_serialize_string_string(serializer, "notbefore", notBefore);
}
if(after) {
strftime(notAfter, sizeof(notAfter), "%Y-%m-%d %H:%M:%S", after);
ndpi_serialize_string_string(serializer, "notafter", notAfter);
}
ndpi_serialize_string_string(serializer, "ja3", flow->protos.tls_quic.ja3_client);
ndpi_serialize_string_string(serializer, "ja3s", flow->protos.tls_quic.ja3_server);
ndpi_serialize_string_uint32(serializer, "unsafe_cipher", flow->protos.tls_quic.server_unsafe_cipher);
ndpi_serialize_string_string(serializer, "cipher", ndpi_cipher2str(flow->protos.tls_quic.server_cipher));
if(flow->protos.tls_quic.issuerDN)
ndpi_serialize_string_string(serializer, "issuerDN", flow->protos.tls_quic.issuerDN);
if(flow->protos.tls_quic.subjectDN)
ndpi_serialize_string_string(serializer, "subjectDN", flow->protos.tls_quic.subjectDN);
if(flow->protos.tls_quic.alpn)
ndpi_serialize_string_string(serializer, "alpn", flow->protos.tls_quic.alpn);
if(flow->protos.tls_quic.tls_supported_versions)
ndpi_serialize_string_string(serializer, "tls_supported_versions", flow->protos.tls_quic.tls_supported_versions);
if(flow->protos.tls_quic.sha1_certificate_fingerprint[0] != '\0') {
for(i=0, off=0; i<20; i++) {
int rc = ndpi_snprintf(&buf[off], sizeof(buf)-off,"%s%02X", (i > 0) ? ":" : "",
flow->protos.tls_quic.sha1_certificate_fingerprint[i] & 0xFF);
if(rc <= 0) break; else off += rc;
}
ndpi_serialize_string_string(serializer, "fingerprint", buf);
}
ndpi_serialize_end_of_block(serializer);
}
}
ndpi_tls2json(serializer, flow);
break;
} /* switch */
@@ -1459,10 +1529,7 @@ int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t src_port, u_int16_t dst_port,
ndpi_protocol l7_protocol,
ndpi_serializer *serializer) {
char src_name[32], dst_name[32];
if(ndpi_init_serializer(serializer, ndpi_serialization_format_json) == -1)
return(-1);
char src_name[32] = {}, dst_name[32] = {};
if(ip_version == 4) {
inet_ntop(AF_INET, &src_v4, src_name, sizeof(src_name));
@@ -1476,8 +1543,10 @@ int ndpi_flow2json(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_serialize_string_string(serializer, "src_ip", src_name);
ndpi_serialize_string_string(serializer, "dest_ip", dst_name);
if(src_port) ndpi_serialize_string_uint32(serializer, "src_port", src_port);
if(dst_port) ndpi_serialize_string_uint32(serializer, "dst_port", dst_port);
if(src_port) ndpi_serialize_string_uint32(serializer, "src_port", ntohs(src_port));
if(dst_port) ndpi_serialize_string_uint32(serializer, "dst_port", ntohs(dst_port));
ndpi_serialize_string_uint32(serializer, "ip", ip_version);
switch(l4_protocol) {
case IPPROTO_TCP:
@@ -2601,6 +2670,19 @@ int ndpi_vsnprintf(char * str, size_t size, char const * format, va_list va_args
/* ******************************************* */
struct tm *ndpi_gmtime_r(const time_t *restrict timep,
struct tm *restrict result)
{
#ifdef WIN32
gmtime_s(result, timep);
return result;
#else
return gmtime_r(timep, result);
#endif
}
/* ******************************************* */
int ndpi_snprintf(char * str, size_t size, char const * format, ...) {
va_list va_args;
+6 -5
View File
@@ -496,10 +496,10 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
time_t theTime;
theTime = flow->protos.tls_quic.notBefore;
strftime(b, sizeof(b), "%d/%b/%Y %H:%M:%S", gmtime_r(&theTime, &result));
strftime(b, sizeof(b), "%d/%b/%Y %H:%M:%S", ndpi_gmtime_r(&theTime, &result));
theTime = flow->protos.tls_quic.notAfter;
strftime(e, sizeof(e), "%d/%b/%Y %H:%M:%S", gmtime_r(&theTime, &result));
strftime(e, sizeof(e), "%d/%b/%Y %H:%M:%S", ndpi_gmtime_r(&theTime, &result));
snprintf(str, sizeof(str), "%s - %s", b, e);
ndpi_set_risk(ndpi_struct, flow, NDPI_TLS_CERTIFICATE_EXPIRED, str); /* Certificate expired */
@@ -510,10 +510,10 @@ static void processCertificateElements(struct ndpi_detection_module_struct *ndpi
time_t theTime;
theTime = flow->protos.tls_quic.notBefore;
strftime(b, sizeof(b), "%d/%b/%Y %H:%M:%S", gmtime_r(&theTime, &result));
strftime(b, sizeof(b), "%d/%b/%Y %H:%M:%S", ndpi_gmtime_r(&theTime, &result));
theTime = flow->protos.tls_quic.notAfter;
strftime(e, sizeof(e), "%d/%b/%Y %H:%M:%S", gmtime_r(&theTime, &result));
strftime(e, sizeof(e), "%d/%b/%Y %H:%M:%S", ndpi_gmtime_r(&theTime, &result));
snprintf(str, sizeof(str), "%s - %s", b, e);
ndpi_set_risk(ndpi_struct, flow, NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE, str); /* Certificate almost expired */
@@ -1446,8 +1446,9 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
ja3.server.num_cipher = 1, ja3.server.cipher[0] = ntohs(*((u_int16_t*)&packet->payload[offset]));
if((flow->protos.tls_quic.server_unsafe_cipher = ndpi_is_safe_ssl_cipher(ja3.server.cipher[0])) == 1) {
char str[64];
char unknown_cipher[8];
snprintf(str, sizeof(str), "Cipher %s", ndpi_cipher2str(ja3.server.cipher[0]));
snprintf(str, sizeof(str), "Cipher %s", ndpi_cipher2str(ja3.server.cipher[0], unknown_cipher));
ndpi_set_risk(ndpi_struct, flow, NDPI_TLS_WEAK_CIPHER, str);
}
+1 -1
View File
@@ -58,7 +58,7 @@ build_results() {
#echo $f
# create result files if not present
if [ ! -f result/$f.out ]; then
CMD="$READER -q -K JSON -k /dev/null -t -i pcap/$f -w result/$f.out -v 2"
CMD="$READER -q -k /dev/null -t -i pcap/$f -w result/$f.out -v 2"
$CMD
fi
done
+1 -1
View File
@@ -272,7 +272,7 @@ int serializeProtoUnitTest(void)
{
buffer_len = 0;
buffer = ndpi_serializer_get_buffer(&serializer, &buffer_len);
char const * const expected_json_str = "{\"ndpi\": {\"flow_risk\": {\"6\": {\"risk\":\"Self-signed Cert\",\"severity\":\"High\",\"risk_score\": {\"total\":500,\"client\":450,\"server\":50}},\"7\": {\"risk\":\"Obsolete TLS (v1.1 or older)\",\"severity\":\"High\",\"risk_score\": {\"total\":510,\"client\":455,\"server\":55}},\"8\": {\"risk\":\"Weak TLS Cipher\",\"severity\":\"High\",\"risk_score\": {\"total\":250,\"client\":225,\"server\":25}},\"17\": {\"risk\":\"Malformed Packet\",\"severity\":\"Low\",\"risk_score\": {\"total\":260,\"client\":130,\"server\":130}}},\"confidence\": {\"6\":\"DPI\"},\"proto\":\"TLS.Facebook\",\"breed\":\"Fun\",\"category\":\"SocialNetwork\"}}";
char const * const expected_json_str = "{\"ndpi\": {\"flow_risk\": {\"6\": {\"risk\":\"Self-signed Cert\",\"severity\":\"High\",\"risk_score\": {\"total\":500,\"client\":450,\"server\":50}},\"7\": {\"risk\":\"Obsolete TLS (v1.1 or older)\",\"severity\":\"High\",\"risk_score\": {\"total\":510,\"client\":455,\"server\":55}},\"8\": {\"risk\":\"Weak TLS Cipher\",\"severity\":\"High\",\"risk_score\": {\"total\":250,\"client\":225,\"server\":25}},\"17\": {\"risk\":\"Malformed Packet\",\"severity\":\"Low\",\"risk_score\": {\"total\":260,\"client\":130,\"server\":130}}},\"confidence\": {\"6\":\"DPI\"},\"proto\":\"TLS.Facebook\",\"proto_id\":\"91.119\",\"encrypted\":1,\"breed\":\"Fun\",\"category_id\":6,\"category\":\"SocialNetwork\"}}";
if (strncmp(buffer, expected_json_str, buffer_len) != 0)
{