Add extra entropy checks and more precise(?) analysis. (#2383)
Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
This commit is contained in:
@@ -113,8 +113,15 @@ extern "C" {
|
||||
int ndpi_is_printable_buffer(u_int8_t const * const buf, size_t len);
|
||||
int ndpi_normalize_printable_string(char * const str, size_t len);
|
||||
int ndpi_is_valid_hostname(char * const str, size_t len);
|
||||
#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy > 7.0f)
|
||||
float ndpi_entropy(u_int8_t const * const buf, size_t len);
|
||||
|
||||
#define NDPI_ENTROPY_PLAINTEXT(entropy) (entropy < 4.941f)
|
||||
#define NDPI_ENTROPY_EXECUTABLE(entropy) (entropy >= 4.941f)
|
||||
#define NDPI_ENTROPY_EXECUTABLE_PACKED(entropy) (entropy >= 6.677f)
|
||||
#define NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy) (entropy >= 7.174f)
|
||||
#define NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy) (entropy >= 7.312f)
|
||||
float ndpi_entropy(u_int8_t const * const buf, size_t len);
|
||||
char *ndpi_entropy2str(float entropy, char *buf, size_t len);
|
||||
void ndpi_entropy2risk(struct ndpi_flow_struct *flow);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
||||
@@ -1202,7 +1202,7 @@ struct ndpi_flow_struct {
|
||||
/* init parameter, internal used to set up timestamp,... */
|
||||
u_int16_t guessed_protocol_id, guessed_protocol_id_by_ip, guessed_category, guessed_header_category;
|
||||
u_int8_t l4_proto, protocol_id_already_guessed:1, fail_with_unknown:1,
|
||||
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, _pad1: 1;
|
||||
init_finished:1, client_packet_direction:1, packet_direction:1, is_ipv6:1, first_pkt_fully_encrypted:1, skip_entropy_check: 1;
|
||||
|
||||
u_int16_t num_dissector_calls;
|
||||
ndpi_confidence_t confidence; /* ndpi_confidence_t */
|
||||
|
||||
+13
-26
@@ -175,7 +175,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
|
||||
{ NDPI_TLS_CERT_VALIDITY_TOO_LONG, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_SERVER_ACCOUNTABLE },
|
||||
{ NDPI_TLS_SUSPICIOUS_EXTENSION, NDPI_RISK_HIGH, CLIENT_HIGH_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
|
||||
{ NDPI_TLS_FATAL_ALERT, NDPI_RISK_LOW, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
|
||||
{ NDPI_SUSPICIOUS_ENTROPY, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
|
||||
{ NDPI_SUSPICIOUS_ENTROPY, NDPI_RISK_LOW, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_BOTH_ACCOUNTABLE },
|
||||
{ NDPI_CLEAR_TEXT_CREDENTIALS, NDPI_RISK_HIGH, CLIENT_HIGH_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
|
||||
{ NDPI_DNS_LARGE_PACKET, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
|
||||
{ NDPI_DNS_FRAGMENTED, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE, NDPI_CLIENT_ACCOUNTABLE },
|
||||
@@ -4396,14 +4396,10 @@ static u_int16_t guess_protocol_id(struct ndpi_detection_module_struct *ndpi_str
|
||||
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);
|
||||
|
||||
if(packet->payload_packet_len > sizeof(struct ndpi_icmphdr)) {
|
||||
flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
|
||||
packet->payload_packet_len - sizeof(struct ndpi_icmphdr));
|
||||
|
||||
if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
|
||||
char str[32];
|
||||
|
||||
snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
|
||||
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
|
||||
if (flow->skip_entropy_check == 0) {
|
||||
flow->entropy = ndpi_entropy(packet->payload + sizeof(struct ndpi_icmphdr),
|
||||
packet->payload_packet_len - sizeof(struct ndpi_icmphdr));
|
||||
ndpi_entropy2risk(flow);
|
||||
}
|
||||
|
||||
u_int16_t chksm = icmp4_checksum(packet->payload, packet->payload_packet_len);
|
||||
@@ -8583,25 +8579,16 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio
|
||||
ndpi_search_shellscript(ndpi_str, flow);
|
||||
}
|
||||
|
||||
if(flow->first_pkt_fully_encrypted == 0 &&
|
||||
ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
|
||||
NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) == 0 &&
|
||||
flow->packet_counter < 3)
|
||||
if(flow->skip_entropy_check == 0 &&
|
||||
flow->first_pkt_fully_encrypted == 0 &&
|
||||
flow->packet_counter < 5 &&
|
||||
/* The following protocols do their own entropy calculation/classification. */
|
||||
ret.app_protocol != NDPI_PROTOCOL_IP_ICMP)
|
||||
{
|
||||
flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
|
||||
if(NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(flow->entropy) != 0) {
|
||||
char str[32];
|
||||
|
||||
snprintf(str, sizeof(str), "Entropy %.2f", flow->entropy);
|
||||
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY, str);
|
||||
if (ret.app_protocol != NDPI_PROTOCOL_HTTP) {
|
||||
flow->entropy = ndpi_entropy(packet->payload, packet->payload_packet_len);
|
||||
}
|
||||
}
|
||||
if(ret.app_protocol != NDPI_PROTOCOL_UNKNOWN &&
|
||||
ret.app_protocol != NDPI_PROTOCOL_IP_ICMP &&
|
||||
flow->entropy > 0.0f)
|
||||
{
|
||||
flow->entropy = 0.0f;
|
||||
ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
|
||||
ndpi_entropy2risk(flow);
|
||||
}
|
||||
|
||||
return(ret);
|
||||
|
||||
@@ -2704,6 +2704,79 @@ float ndpi_entropy(u_int8_t const * const buf, size_t len) {
|
||||
return entropy;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
/* Losely implemented by: https://redirect.cs.umbc.edu/courses/graduate/CMSC691am/student%20talks/CMSC%20691%20Malware%20-%20Entropy%20Analysis%20Presentation.pdf */
|
||||
char *ndpi_entropy2str(float entropy, char *buf, size_t len) {
|
||||
if (buf == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static const char entropy_fmtstr[] = "Entropy: %.3f (%s?)";
|
||||
if (NDPI_ENTROPY_ENCRYPTED_OR_RANDOM(entropy)) {
|
||||
snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted or Random");
|
||||
} else if (NDPI_ENTROPY_EXECUTABLE_ENCRYPTED(entropy)) {
|
||||
snprintf(buf, len, entropy_fmtstr, entropy, "Encrypted Executable");
|
||||
} else if (NDPI_ENTROPY_EXECUTABLE_PACKED(entropy)) {
|
||||
snprintf(buf, len, entropy_fmtstr, entropy, "Compressed Executable");
|
||||
} else if (NDPI_ENTROPY_EXECUTABLE(entropy)) {
|
||||
snprintf(buf, len, entropy_fmtstr, entropy, "Executable");
|
||||
} else {
|
||||
snprintf(buf, len, entropy_fmtstr, entropy, "Unknown");
|
||||
}
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
void ndpi_entropy2risk(struct ndpi_flow_struct *flow) {
|
||||
char str[64];
|
||||
|
||||
if (NDPI_ENTROPY_PLAINTEXT(flow->entropy))
|
||||
goto reset_risk;
|
||||
|
||||
if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS ||
|
||||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_TLS ||
|
||||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_QUIC ||
|
||||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_QUIC ||
|
||||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_DTLS ||
|
||||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_DTLS)
|
||||
{
|
||||
flow->skip_entropy_check = 1;
|
||||
goto reset_risk;
|
||||
}
|
||||
|
||||
if (flow->confidence != NDPI_CONFIDENCE_DPI &&
|
||||
flow->confidence != NDPI_CONFIDENCE_DPI_CACHE) {
|
||||
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
|
||||
ndpi_entropy2str(flow->entropy, str, sizeof(str)));
|
||||
return;
|
||||
}
|
||||
|
||||
if (ndpi_isset_risk(flow, NDPI_MALWARE_HOST_CONTACTED) ||
|
||||
ndpi_isset_risk(flow, NDPI_BINARY_DATA_TRANSFER) ||
|
||||
ndpi_isset_risk(flow, NDPI_BINARY_APPLICATION_TRANSFER) ||
|
||||
ndpi_isset_risk(flow, NDPI_POSSIBLE_EXPLOIT) ||
|
||||
ndpi_isset_risk(flow, NDPI_HTTP_SUSPICIOUS_CONTENT) ||
|
||||
ndpi_isset_risk(flow, NDPI_DNS_SUSPICIOUS_TRAFFIC) ||
|
||||
ndpi_isset_risk(flow, NDPI_MALFORMED_PACKET) ||
|
||||
(flow->category == NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT &&
|
||||
(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP ||
|
||||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_HTTP)) ||
|
||||
flow->category == NDPI_PROTOCOL_CATEGORY_DATA_TRANSFER ||
|
||||
flow->category == NDPI_PROTOCOL_CATEGORY_UNSPECIFIED ||
|
||||
flow->category == NDPI_PROTOCOL_CATEGORY_WEB)
|
||||
{
|
||||
ndpi_set_risk(flow, NDPI_SUSPICIOUS_ENTROPY,
|
||||
ndpi_entropy2str(flow->entropy, str, sizeof(str)));
|
||||
return;
|
||||
}
|
||||
|
||||
reset_risk:
|
||||
ndpi_unset_risk(flow, NDPI_SUSPICIOUS_ENTROPY);
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
static inline uint16_t get_n16bit(uint8_t const * cbuf) {
|
||||
uint16_t r = ((uint16_t)cbuf[0]) | (((uint16_t)cbuf[1]) << 8);
|
||||
|
||||
@@ -222,8 +222,12 @@ static void ndpi_validate_http_content(struct ndpi_detection_module_struct *ndpi
|
||||
|
||||
if(len >= 8 /* 4 chars for \r\n\r\n and at least 4 charts for content guess */) {
|
||||
double_ret += 4;
|
||||
len -= 4;
|
||||
|
||||
ndpi_http_check_human_redeable_content(ndpi_struct, flow, double_ret, len);
|
||||
if (flow->skip_entropy_check == 0) {
|
||||
flow->entropy = ndpi_entropy(double_ret, len);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user