Add extra entropy checks and more precise(?) analysis. (#2383)

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
This commit is contained in:
Toni
2024-05-09 15:24:11 +02:00
committed by GitHub
parent b65a755e85
commit 18e03a26ca
88 changed files with 337 additions and 266 deletions
+9 -2
View File
@@ -113,8 +113,15 @@ extern "C" {
int ndpi_is_printable_buffer(u_int8_t const * const buf, size_t len);
int ndpi_normalize_printable_string(char * const str, size_t len);
int ndpi_is_valid_hostname(char * const str, size_t len);
#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy > 7.0f)
float ndpi_entropy(u_int8_t const * const buf, size_t len);
#define NDPI_ENTROPY_PLAINTEXT(entropy) (entropy < 4.941f)
#define NDPI_ENTROPY_EXECUTABLE(entropy) (entropy >= 4.941f)
#define NDPI_ENTROPY_EXECUTABLE_PACKED(entropy) (entropy >= 6.677f)
#define NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy) (entropy >= 7.174f)
#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy >= 7.312f)
float ndpi_entropy(u_int8_t const * const buf, size_t len);
char *ndpi_entropy2str(float entropy, char *buf, size_t len);
void ndpi_entropy2risk(struct ndpi_flow_struct *flow);
#ifdef __cplusplus
}
+1 -1
View File
@@ -1202,7 +1202,7 @@ struct ndpi_flow_struct {
/* init parameter, internal used to set up timestamp,... */
u_int16_t guessed_protocol_id, guessed_protocol_id_by_ip, guessed_category, guessed_header_category;
u_int8_t l4_proto, protocol_id_already_guessed:1, fail_with_unknown:1,
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, _pad1: 1;
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, skip_entropy_check: 1;
u_int16_t num_dissector_calls;
ndpi_confidence_t confidence; /* ndpi_confidence_t */
+13 -26
View File
@@ -175,7 +175,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
{ NDPI_TLS_CERT_VALIDITY_TOO_LONG, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_SERVER_ACCOUNTABLE },
{ NDPI_TLS_SUSPICIOUS_EXTENSION, NDPI_RISK_HIGH, CLIENT_HIGH_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
{ NDPI_TLS_FATAL_ALERT, NDPI_RISK_LOW, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
{ NDPI_SUSPICIOUS_ENTROPY, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
{ NDPI_SUSPICIOUS_ENTROPY, NDPI_RISK_LOW, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
{ NDPI_CLEAR_TEXT_CREDENTIALS, NDPI_RISK_HIGH, CLIENT_HIGH_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
{ NDPI_DNS_LARGE_PACKET, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
{ NDPI_DNS_FRAGMENTED, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
@@ -4396,14 +4396,10 @@ static u_int16_t guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);
if(packet->payload_packet_len > sizeof(struct ndpi_icmphdr)) {
flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
packet->payload_packet_len - sizeof(struct ndpi_icmphdr));
if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
char str[32];
snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
if (flow->skip_entropy_check == 0) {
flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
packet->payload_packet_len - sizeof(struct ndpi_icmphdr));
ndpi_entropy2risk(flow);
}
u_int16_t chksm = icmp4_checksum(packet->payload, packet->payload_packet_len);
@@ -8583,25 +8579,16 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
ndpi_search_shellscript(ndpi_str, flow);
}
if(flow->first_pkt_fully_encrypted == 0 &&
ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) == 0 &&
flow->packet_counter < 3)
if(flow->skip_entropy_check == 0 &&
flow->first_pkt_fully_encrypted == 0 &&
flow->packet_counter < 5 &&
/* The following protocols do their own entropy calculation/classification. */
ret.app_protocol != NDPI_PROTOCOL_IP_ICMP)
{
flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
char str[32];
snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
if (ret.app_protocol != NDPI_PROTOCOL_HTTP) {
flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
}
}
if(ret.app_protocol != NDPI_PROTOCOL_UNKNOWN &&
ret.app_protocol != NDPI_PROTOCOL_IP_ICMP &&
flow->entropy > 0.0f)
{
flow->entropy = 0.0f;
ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
ndpi_entropy2risk(flow);
}
return(ret);
+73
View File
@@ -2704,6 +2704,79 @@ float ndpi_entropy(u_int8_t const * const buf, size_t len) {
return entropy;
}
/* ******************************************************************** */
/* Losely implemented by: https://redirect.cs.umbc.edu/courses/graduate/CMSC691am/student%20talks/CMSC%20691%20Malware%20-%20Entropy%20Analysis%20Presentation.pdf */
char *ndpi_entropy2str(float entropy, char *buf, size_t len) {
if (buf == NULL) {
return NULL;
}
static const char entropy_fmtstr[] = "Entropy: %.3f (%s?)";
if (NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted or Random");
} else if (NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted Executable");
} else if (NDPI_ENTROPY_EXECUTABLE_PACKED(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Compressed Executable");
} else if (NDPI_ENTROPY_EXECUTABLE(entropy)) {
snprintf(buf, len, entropy_fmtstr, entropy, "Executable");
} else {
snprintf(buf, len, entropy_fmtstr, entropy, "Unknown");
}
return buf;
}
/* ******************************************************************** */
void ndpi_entropy2risk(struct ndpi_flow_struct *flow) {
char str[64];
if (NDPI_ENTROPY_PLAINTEXT(flow->entropy))
goto reset_risk;
if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_TLS ||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_QUIC ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_QUIC ||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_DTLS ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_DTLS)
{
flow->skip_entropy_check = 1;
goto reset_risk;
}
if (flow->confidence != NDPI_CONFIDENCE_DPI &&
flow->confidence != NDPI_CONFIDENCE_DPI_CACHE) {
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
ndpi_entropy2str(flow->entropy, str, sizeof(str)));
return;
}
if (ndpi_isset_risk(flow, NDPI_MALWARE_HOST_CONTACTED) ||
ndpi_isset_risk(flow, NDPI_BINARY_DATA_TRANSFER) ||
ndpi_isset_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER) ||
ndpi_isset_risk(flow, NDPI_POSSIBLE_EXPLOIT) ||
ndpi_isset_risk(flow, NDPI_HTTP_SUSPICIOUS_CONTENT) ||
ndpi_isset_risk(flow, NDPI_DNS_SUSPICIOUS_TRAFFIC) ||
ndpi_isset_risk(flow, NDPI_MALFORMED_PACKET) ||
(flow->category == NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT &&
(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP ||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_HTTP)) ||
flow->category == NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER ||
flow->category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED ||
flow->category == NDPI_PROTOCOL_CATEGORY_WEB)
{
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
ndpi_entropy2str(flow->entropy, str, sizeof(str)));
return;
}
reset_risk:
ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
}
/* ******************************************************************** */
static inline uint16_t get_n16bit(uint8_t const * cbuf) {
uint16_t r = ((uint16_t)cbuf[0]) | (((uint16_t)cbuf[1]) << 8);
+4
View File
@@ -222,8 +222,12 @@ static void ndpi_validate_http_content(struct ndpi_detection_module_struct *ndpi
if(len >= 8 /* 4 chars for \r\n\r\n and at least 4 charts for content guess */) {
double_ret += 4;
len -= 4;
ndpi_http_check_human_redeable_content(ndpi_struct, flow, double_ret, len);
if (flow->skip_entropy_check == 0) {
flow->entropy = ndpi_entropy(double_ret, len);
}
}
}