Rework how hostname/SNI info is saved (#1330)
Looking at `struct ndpi_flow_struct` the two bigger fields are `host_server_name[240]` (mainly for HTTP hostnames and DNS domains) and `protos.tls_quic.client_requested_server_name[256]` (for TLS/QUIC SNIs). This commit aims to reduce `struct ndpi_flow_struct` size, according to two simple observations: 1) maximum one of these two fields is used for each flow. So it seems safe to merge them; 2) even if hostnames/SNIs might be very long, in practice they are rarely longer than a fews tens of bytes. So, using a (single) large buffer is a waste of memory for all kinds of flows. If we need to truncate the name, we keep the *last* characters, easing domain matching. Analyzing some real traffic, it seems safe to assume that the vast majority of hostnames/SNIs is shorter than 80 bytes. Hostnames/SNIs are always converted to lowercase. Attention was given so as to be sure that unit-tests outputs are not affected by this change. Because of a bug, TLS/QUIC SNI were always truncated to 64 bytes (the *first* 64 ones): as a consequence, there were some "Suspicious DGA domain name" and "TLS Certificate Mismatch" false positives.
This commit is contained in:
+27
-11
@@ -742,7 +742,7 @@ void extcap_capture() {
|
||||
void printCSVHeader() {
|
||||
if(!csv_fp) return;
|
||||
|
||||
fprintf(csv_fp, "#flow_id,protocol,first_seen,last_seen,duration,src_ip,src_port,dst_ip,dst_port,ndpi_proto_num,ndpi_proto,server_name,");
|
||||
fprintf(csv_fp, "#flow_id,protocol,first_seen,last_seen,duration,src_ip,src_port,dst_ip,dst_port,ndpi_proto_num,ndpi_proto,server_name_sni,");
|
||||
fprintf(csv_fp, "benign_score,dos_slow_score,dos_goldeneye_score,dos_hulk_score,ddos_score,hearthbleed_score,ftp_patator_score,ssh_patator_score,infiltration_score,");
|
||||
fprintf(csv_fp, "c_to_s_pkts,c_to_s_bytes,c_to_s_goodput_bytes,s_to_c_pkts,s_to_c_bytes,s_to_c_goodput_bytes,");
|
||||
fprintf(csv_fp, "data_ratio,str_data_ratio,c_to_s_goodput_ratio,s_to_c_goodput_ratio,");
|
||||
@@ -767,7 +767,7 @@ void printCSVHeader() {
|
||||
fprintf(csv_fp, "c_to_s_init_win,s_to_c_init_win,");
|
||||
|
||||
/* Flow info */
|
||||
fprintf(csv_fp, "client_info,server_info,");
|
||||
fprintf(csv_fp, "server_info,");
|
||||
fprintf(csv_fp, "tls_version,ja3c,tls_client_unsafe,");
|
||||
fprintf(csv_fp, "ja3s,tls_server_unsafe,");
|
||||
fprintf(csv_fp, "tls_alpn,tls_supported_versions,");
|
||||
@@ -1311,8 +1311,7 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
/* TCP window */
|
||||
fprintf(csv_fp, "%u,%u,", flow->c_to_s_init_win, flow->s_to_c_init_win);
|
||||
|
||||
fprintf(csv_fp, "%s,%s,",
|
||||
(flow->ssh_tls.client_requested_server_name[0] != '\0') ? flow->ssh_tls.client_requested_server_name : "",
|
||||
fprintf(csv_fp, "%s,",
|
||||
(flow->ssh_tls.server_info[0] != '\0') ? flow->ssh_tls.server_info : "");
|
||||
|
||||
fprintf(csv_fp, "%s,%s,%s,%s,%s,",
|
||||
@@ -1421,7 +1420,15 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
|
||||
if(flow->telnet.username) fprintf(out, "[Username: %s]", flow->telnet.username);
|
||||
if(flow->telnet.password) fprintf(out, "[Password: %s]", flow->telnet.password);
|
||||
if(flow->host_server_name[0] != '\0') fprintf(out, "[Host: %s]", flow->host_server_name);
|
||||
|
||||
if((flow->detected_protocol.master_protocol != NDPI_PROTOCOL_TLS)
|
||||
&& (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_TLS)
|
||||
&& (flow->detected_protocol.master_protocol != NDPI_PROTOCOL_QUIC)
|
||||
&& (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_QUIC)
|
||||
&& (flow->detected_protocol.app_protocol != NDPI_PROTOCOL_SSH)
|
||||
&& (flow->detected_protocol.master_protocol != NDPI_PROTOCOL_SSH)) {
|
||||
if(flow->host_server_name[0] != '\0') fprintf(out, "[Host: %s]", flow->host_server_name);
|
||||
}
|
||||
|
||||
if(flow->info[0] != '\0') fprintf(out, "[%s]", flow->info);
|
||||
if(flow->flow_extra_info[0] != '\0') fprintf(out, "[%s]", flow->flow_extra_info);
|
||||
@@ -1482,7 +1489,16 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
|
||||
}
|
||||
|
||||
if(flow->ssh_tls.ssl_version != 0) fprintf(out, "[%s]", ndpi_ssl_version2str(flow->ndpi_flow, flow->ssh_tls.ssl_version, &known_tls));
|
||||
if(flow->ssh_tls.client_requested_server_name[0] != '\0') fprintf(out, "[Client: %s]", flow->ssh_tls.client_requested_server_name);
|
||||
|
||||
if((flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS)
|
||||
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_TLS)
|
||||
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_QUIC)
|
||||
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_QUIC)
|
||||
|| (flow->detected_protocol.master_protocol == NDPI_PROTOCOL_SSH)
|
||||
|| (flow->detected_protocol.app_protocol == NDPI_PROTOCOL_SSH)) {
|
||||
if(flow->host_server_name[0] != '\0') fprintf(out, "[Client: %s]", flow->host_server_name);
|
||||
}
|
||||
|
||||
if(flow->ssh_tls.client_hassh[0] != '\0') fprintf(out, "[HASSH-C: %s]", flow->ssh_tls.client_hassh);
|
||||
|
||||
if(flow->ssh_tls.ja3_client[0] != '\0') fprintf(out, "[JA3C: %s%s]", flow->ssh_tls.ja3_client,
|
||||
@@ -2424,7 +2440,7 @@ static void printFlowsStats() {
|
||||
newHost->host_server_info_hasht = NULL;
|
||||
newHost->ip_string = all_flows[i].flow->src_name;
|
||||
newHost->ip = all_flows[i].flow->src_ip;
|
||||
newHost->dns_name = all_flows[i].flow->ssh_tls.client_requested_server_name;
|
||||
newHost->dns_name = all_flows[i].flow->host_server_name;
|
||||
|
||||
ndpi_ja3_info *newJA3 = ndpi_malloc(sizeof(ndpi_ja3_info));
|
||||
newJA3->ja3 = all_flows[i].flow->ssh_tls.ja3_client;
|
||||
@@ -2457,7 +2473,7 @@ static void printFlowsStats() {
|
||||
|
||||
newHost->ip = all_flows[i].flow->src_ip;
|
||||
newHost->ip_string = all_flows[i].flow->src_name;
|
||||
newHost->dns_name = all_flows[i].flow->ssh_tls.client_requested_server_name;;
|
||||
newHost->dns_name = all_flows[i].flow->host_server_name;
|
||||
|
||||
ndpi_ja3_fingerprints_host *newElement = ndpi_malloc(sizeof(ndpi_ja3_fingerprints_host));
|
||||
newElement->ja3 = all_flows[i].flow->ssh_tls.ja3_client;
|
||||
@@ -2474,7 +2490,7 @@ static void printFlowsStats() {
|
||||
ndpi_ip_dns *newInnerElement = ndpi_malloc(sizeof(ndpi_ip_dns));
|
||||
newInnerElement->ip = all_flows[i].flow->src_ip;
|
||||
newInnerElement->ip_string = all_flows[i].flow->src_name;
|
||||
newInnerElement->dns_name = all_flows[i].flow->ssh_tls.client_requested_server_name;
|
||||
newInnerElement->dns_name = all_flows[i].flow->host_server_name;
|
||||
HASH_ADD_INT(hostByJA3Found->ipToDNS_ht, ip, newInnerElement);
|
||||
}
|
||||
}
|
||||
@@ -2829,8 +2845,8 @@ static void printFlowsStats() {
|
||||
printf("][similarity: %f]",
|
||||
(similarity = ndpi_bin_similarity(¢roids[j], &bins[i], 0)));
|
||||
|
||||
if(all_flows[i].flow->ssh_tls.client_requested_server_name[0] != '\0')
|
||||
fprintf(out, "[%s]", all_flows[i].flow->ssh_tls.client_requested_server_name);
|
||||
if(all_flows[i].flow->host_server_name[0] != '\0')
|
||||
fprintf(out, "[%s]", all_flows[i].flow->host_server_name);
|
||||
|
||||
if(enable_doh_dot_detection) {
|
||||
if(((all_flows[i].flow->detected_protocol.master_protocol == NDPI_PROTOCOL_TLS)
|
||||
|
||||
@@ -924,7 +924,7 @@ static void ndpi_process_packet(uint8_t * const args,
|
||||
ndpi_ssl_version2str(flow_to_process->ndpi_flow,
|
||||
flow_to_process->ndpi_flow->protos.tls_quic.ssl_version,
|
||||
&unknown_tls_version),
|
||||
flow_to_process->ndpi_flow->protos.tls_quic.client_requested_server_name,
|
||||
flow_to_process->ndpi_flow->host_server_name,
|
||||
(flow_to_process->ndpi_flow->protos.tls_quic.alpn != NULL ?
|
||||
flow_to_process->ndpi_flow->protos.tls_quic.alpn : "-"));
|
||||
flow_to_process->tls_client_hello_seen = 1;
|
||||
|
||||
@@ -1105,8 +1105,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
|
||||
}
|
||||
/* MDNS */
|
||||
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_MDNS)) {
|
||||
char *name = (char*)flow->ndpi_flow->host_server_name; /* Trick to avoid warning(s) */
|
||||
snprintf(flow->info, sizeof(flow->info), "%s", name);
|
||||
snprintf(flow->info, sizeof(flow->info), "%s", flow->ndpi_flow->host_server_name);
|
||||
}
|
||||
/* UBNTAC2 */
|
||||
else if(is_ndpi_proto(flow, NDPI_PROTOCOL_UBNTAC2)) {
|
||||
@@ -1161,8 +1160,8 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
|
||||
if(flow->ndpi_flow->protos.telnet.password[0] != '\0')
|
||||
flow->telnet.password = ndpi_strdup(flow->ndpi_flow->protos.telnet.password);
|
||||
} else if(is_ndpi_proto(flow, NDPI_PROTOCOL_SSH)) {
|
||||
snprintf(flow->ssh_tls.client_requested_server_name,
|
||||
sizeof(flow->ssh_tls.client_requested_server_name), "%s",
|
||||
snprintf(flow->host_server_name,
|
||||
sizeof(flow->host_server_name), "%s",
|
||||
flow->ndpi_flow->protos.ssh.client_signature);
|
||||
snprintf(flow->ssh_tls.server_info, sizeof(flow->ssh_tls.server_info), "%s",
|
||||
flow->ndpi_flow->protos.ssh.server_signature);
|
||||
@@ -1178,9 +1177,6 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
|
||||
|| (flow->ndpi_flow->protos.tls_quic.ja3_client[0] != '\0')
|
||||
) {
|
||||
flow->ssh_tls.ssl_version = flow->ndpi_flow->protos.tls_quic.ssl_version;
|
||||
snprintf(flow->ssh_tls.client_requested_server_name,
|
||||
sizeof(flow->ssh_tls.client_requested_server_name), "%s",
|
||||
flow->ndpi_flow->protos.tls_quic.client_requested_server_name);
|
||||
|
||||
snprintf(flow->http.user_agent, sizeof(flow->http.user_agent), "%s", flow->ndpi_flow->http.user_agent ? flow->ndpi_flow->http.user_agent : "");
|
||||
|
||||
|
||||
@@ -197,7 +197,7 @@ typedef struct ndpi_flow_info {
|
||||
|
||||
char info[255];
|
||||
char flow_extra_info[16];
|
||||
char host_server_name[240];
|
||||
char host_server_name[80]; /* Hostname/SNI */
|
||||
char *bittorent_hash;
|
||||
char *dhcp_fingerprint;
|
||||
char *dhcp_class_ident;
|
||||
@@ -205,7 +205,7 @@ typedef struct ndpi_flow_info {
|
||||
|
||||
struct {
|
||||
u_int16_t ssl_version;
|
||||
char client_requested_server_name[256], server_info[64],
|
||||
char server_info[64],
|
||||
client_hassh[33], server_hassh[33], *server_names,
|
||||
*tls_alpn, *tls_supported_versions,
|
||||
*tls_issuerDN, *tls_subjectDN,
|
||||
|
||||
Reference in New Issue
Block a user