HTTP: improve extraction of metadata and of flow risks (#1959)

This commit is contained in:
Ivan Nardi
2023-05-05 13:35:20 +02:00
committed by GitHub
parent 02a2c80453
commit 0223d3c4f5
40 changed files with 1412 additions and 1299 deletions
-1
View File
@@ -742,7 +742,6 @@ struct ndpi_flow_tcp_struct {
/* NDPI_PROTOCOL_HTTP */
u_int32_t http_stage:2;
u_int32_t http_empty_line_seen:1;
/* NDPI_PROTOCOL_GNUTELLA */
u_int32_t gnutella_stage:2; // 0 - 2
+264 -256
View File
@@ -51,6 +51,8 @@ extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);
static void ndpi_check_http_header(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);
/* *********************************************** */
@@ -101,6 +103,9 @@ static void ndpi_analyze_content_signature(struct ndpi_detection_module_struct *
static int ndpi_search_http_tcp_again(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow) {
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
if(packet->payload_packet_len == 0 || packet->tcp_retransmission)
return 1;
ndpi_search_http_tcp(ndpi_struct, flow);
@@ -339,8 +344,7 @@ static ndpi_protocol_category_t ndpi_http_check_content(struct ndpi_detection_mo
static void ndpi_int_http_add_connection(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int16_t http_protocol,
ndpi_protocol_category_t category) {
u_int16_t http_protocol) {
u_int16_t master_protocol;
#ifdef HTTP_DEBUG
@@ -401,53 +405,124 @@ static void setHttpUserAgent(struct ndpi_detection_module_struct *ndpi_struct,
/* ************************************************************* */
static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow) {
struct ndpi_flow_struct *flow,
int hostname_just_set) {
u_int16_t master_protocol;
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
if((flow->l4.tcp.http_stage == 0) || (flow->http.url && flow->http_detected)) {
char *double_col = strchr((char*)flow->host_server_name, ':');
int a, b, c, d;
if(double_col) double_col[0] = '\0';
master_protocol = NDPI_PROTOCOL_HTTP;
if(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN)
master_protocol = flow->detected_protocol_stack[1];
else if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_CONNECT ||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_PROXY)
master_protocol = flow->detected_protocol_stack[0];
if(ndpi_struct->packet.iph
&& (sscanf(flow->host_server_name, "%d.%d.%d.%d", &a, &b, &c, &d) == 4)) {
/* IPv4 */
if(packet->server_line.len > 7 &&
strncmp((const char *)packet->server_line.ptr, "ntopng ", 7) == 0) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_NTOP, NDPI_PROTOCOL_HTTP, NDPI_CONFIDENCE_DPI);
ndpi_unset_risk(ndpi_struct, flow, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
}
if(ndpi_struct->packet.iph->daddr != inet_addr(flow->host_server_name)) {
char buf[64], msg[128];
snprintf(msg, sizeof(msg), "Expected %s, found %s",
ndpi_intoav4(ntohl(ndpi_struct->packet.iph->daddr), buf, sizeof(buf)), flow->host_server_name);
ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER, msg);
/* Matching on Content-Type.
OCSP: application/ocsp-request, application/ocsp-response
*/
/* We overwrite any previous sub-classification (example: via hostname) */
if(packet->content_line.len > 17 &&
strncmp((const char *)packet->content_line.ptr, "application/ocsp-", 17) == 0) {
NDPI_LOG_DBG2(ndpi_struct, "Found OCSP\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OCSP, master_protocol, NDPI_CONFIDENCE_DPI);
}
if(flow->http.method == NDPI_HTTP_METHOD_RPC_IN_DATA ||
flow->http.method == NDPI_HTTP_METHOD_RPC_OUT_DATA) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RPC, master_protocol, NDPI_CONFIDENCE_DPI);
}
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
hostname_just_set && flow->host_server_name[0] != '\0') {
ndpi_match_hostname_protocol(ndpi_struct, flow,
master_protocol,
flow->host_server_name,
strlen(flow->host_server_name));
}
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
packet->http_origin.len > 0) {
ndpi_protocol_match_result ret_match;
char *ptr, *origin_hostname;
size_t origin_hostname_len;
/* Origin syntax:
Origin: null
Origin: <scheme>://<hostname>
Origin: <scheme>://<hostname>:<port>
Try extracting hostname */
ptr = ndpi_strnstr((const char *)packet->http_origin.ptr, "://", packet->http_origin.len);
if(ptr) {
origin_hostname = ptr + 3;
origin_hostname_len = packet->http_origin.len - (ptr - (char *)packet->http_origin.ptr) - 3;
ptr = ndpi_strnstr(origin_hostname, ":", origin_hostname_len);
if(ptr) {
origin_hostname_len = ptr - origin_hostname;
}
NDPI_LOG_DBG2(ndpi_struct, "Origin: [%.*s] -> [%.*s]\n", packet->http_origin.len, packet->http_origin.ptr,
(int)origin_hostname_len, origin_hostname);
/* We already checked hostname...*/
if(strncmp(origin_hostname, flow->host_server_name, origin_hostname_len) != 0) {
ndpi_match_host_subprotocol(ndpi_struct, flow,
origin_hostname,
origin_hostname_len,
&ret_match,
master_protocol);
}
}
master_protocol = NDPI_PROTOCOL_HTTP;
if(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN)
master_protocol = flow->detected_protocol_stack[1];
else if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_CONNECT ||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP_PROXY)
master_protocol = flow->detected_protocol_stack[0];
}
if(ndpi_match_hostname_protocol(ndpi_struct, flow,
master_protocol,
flow->host_server_name,
strlen(flow->host_server_name)) == 0) {
if(flow->http.url &&
((strstr(flow->http.url, ":8080/downloading?n=0.") != NULL)
|| (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
/* This looks like Ookla speedtest */
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
ookla_add_to_cache(ndpi_struct, flow);
}
}
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
flow->http.url &&
((strstr(flow->http.url, ":8080/downloading?n=0.") != NULL) ||
(strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
/* This looks like Ookla speedtest */
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
ookla_add_to_cache(ndpi_struct, flow);
}
if (flow->http.url != NULL &&
strstr(flow->http.url, "micloud.xiaomi.net") != NULL)
{
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_XIAOMI, master_protocol, NDPI_CONFIDENCE_DPI);
}
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
flow->http.url != NULL &&
strstr(flow->http.url, "micloud.xiaomi.net") != NULL) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_XIAOMI, master_protocol, NDPI_CONFIDENCE_DPI);
}
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
packet->referer_line.len > 0 &&
ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
ookla_add_to_cache(ndpi_struct, flow);
}
/* WindowsUpdate over some kind of CDN */
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
flow->http.user_agent && flow->http.url &&
(strstr(flow->http.url, "delivery.mp.microsoft.com/") ||
strstr(flow->http.url, "download.windowsupdate.com/")) &&
strstr(flow->http.user_agent, "Microsoft-Delivery-Optimization/") &&
ndpi_isset_risk(ndpi_struct, flow, NDPI_NUMERIC_IP_HOST)) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WINDOWS_UPDATE, master_protocol, NDPI_CONFIDENCE_DPI);
}
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
packet->payload_packet_len >= 23 &&
memcmp(packet->payload, "<policy-file-request/>", 23) == 0) {
/*
<policy-file-request/>
<cross-domain-policy>
<allow-access-from domain="*.ookla.com" to-ports="8080"/>
<allow-access-from domain="*.speedtest.net" to-ports="8080"/>
</cross-domain-policy>
*/
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
ookla_add_to_cache(ndpi_struct, flow);
}
}
@@ -547,9 +622,9 @@ static void ndpi_check_user_agent(struct ndpi_detection_module_struct *ndpi_stru
/* ************************************************************* */
int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
const u_int8_t *ua_ptr, u_int16_t ua_ptr_len) {
void http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
const u_int8_t *ua_ptr, u_int16_t ua_ptr_len) {
/**
Format examples:
Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) ....
@@ -612,11 +687,10 @@ int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
if(ndpi_user_agent_set(flow, ua_ptr, ua_ptr_len) != NULL)
ndpi_check_user_agent(ndpi_struct, flow, flow->http.user_agent, ua_ptr_len);
else
NDPI_LOG_DBG2(ndpi_struct, "Could not set HTTP user agent\n");
NDPI_LOG_DBG2(ndpi_struct, "Could not set HTTP user agent (already set?)\n");
NDPI_LOG_DBG2(ndpi_struct, "User Agent Type line found %.*s\n",
ua_ptr_len, ua_ptr);
return 0;
}
/* ************************************************************* */
@@ -666,11 +740,8 @@ static void ndpi_check_http_server(struct ndpi_detection_module_struct *ndpi_str
if(server_len > 7) {
u_int off;
if(strncmp((const char *)server, "ntopng ", 7) == 0) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_NTOP, NDPI_PROTOCOL_HTTP, NDPI_CONFIDENCE_DPI);
NDPI_CLR_BIT(flow->risk, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
} else if((strncasecmp(server, "Apache/", off = 7) == 0) /* X.X.X */
|| (strncasecmp(server, "nginx/", off = 6) == 0) /* X.X.X */) {
if((strncasecmp(server, "Apache/", off = 7) == 0) /* X.X.X */
|| (strncasecmp(server, "nginx/", off = 6) == 0) /* X.X.X */) {
u_int i, j, a, b, c;
char buf[16] = { '\0' };
@@ -703,10 +774,8 @@ static void ndpi_check_http_server(struct ndpi_detection_module_struct *ndpi_str
static void check_content_type_and_change_protocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow) {
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
int ret;
if(flow->http_detected && (flow->http.response_status_code != 0))
return;
u_int len;
int hostname_just_set = 0;
if((flow->http.url == NULL)
&& (packet->http_url_name.len > 0)
@@ -748,101 +817,42 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
ndpi_check_http_url(ndpi_struct, flow, &flow->http.url[host_end]);
}
}
if(packet->http_method.ptr != NULL)
flow->http.method = ndpi_http_str2method((const char*)packet->http_method.ptr,
(u_int16_t)packet->http_method.len);
if((flow->http.method == NDPI_HTTP_METHOD_RPC_IN_DATA)
|| (flow->http.method == NDPI_HTTP_METHOD_RPC_OUT_DATA)) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RPC, flow->detected_protocol_stack[0], NDPI_CONFIDENCE_DPI);
check_content_type_and_change_protocol(ndpi_struct, flow);
}
}
if(packet->server_line.ptr != NULL)
ndpi_check_http_server(ndpi_struct, flow, (const char *)packet->server_line.ptr, packet->server_line.len);
if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0) {
ret = http_process_user_agent(ndpi_struct, flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
/* TODO: Is it correct to avoid setting ua, host_name,... if we have a (Netflix) subclassification? */
if(ret != 0)
return;
if(packet->user_agent_line.ptr != NULL) {
http_process_user_agent(ndpi_struct, flow, packet->user_agent_line.ptr, packet->user_agent_line.len);
}
/* check for host line */
if(packet->host_line.ptr != NULL) {
u_int len;
NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n",
packet->host_line.len, packet->host_line.ptr);
/* Copy result for nDPI apps */
ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len);
if(strlen(flow->host_server_name) > 0) {
if(ndpi_is_valid_hostname(flow->host_server_name,
strlen(flow->host_server_name)) == 0) {
char str[128];
snprintf(str, sizeof(str), "Invalid host %s", flow->host_server_name);
ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS, str);
/* This looks like an attack */
ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT, NULL);
}
}
if(packet->forwarded_line.ptr) {
if(flow->http.nat_ip == NULL) {
len = packet->forwarded_line.len;
flow->http.nat_ip = ndpi_malloc(len + 1);
if(flow->http.nat_ip != NULL) {
strncpy(flow->http.nat_ip, (char*)packet->forwarded_line.ptr, len);
flow->http.nat_ip[len] = '\0';
}
}
}
ndpi_http_parse_subprotocol(ndpi_struct, flow);
if(strlen(flow->host_server_name) > 0) {
ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1, 0);
}
if((flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
&& (flow->http_detected)
&& (packet->http_origin.len > 0)) {
ndpi_protocol_match_result ret_match;
ndpi_match_host_subprotocol(ndpi_struct, flow,
(char *)packet->http_origin.ptr,
packet->http_origin.len,
&ret_match,
NDPI_PROTOCOL_HTTP);
}
if(flow->detected_protocol_stack[1] != NDPI_PROTOCOL_UNKNOWN) {
if((flow->detected_protocol_stack[1] != NDPI_PROTOCOL_HTTP)
&& (flow->detected_protocol_stack[1] != NDPI_PROTOCOL_HTTP_CONNECT)
&& (flow->detected_protocol_stack[1] != NDPI_PROTOCOL_HTTP_PROXY)) {
NDPI_LOG_INFO(ndpi_struct, "found HTTP/%s\n",
ndpi_get_proto_name(ndpi_struct, flow->detected_protocol_stack[0]));
ndpi_int_http_add_connection(ndpi_struct, flow, flow->detected_protocol_stack[0], NDPI_PROTOCOL_CATEGORY_WEB);
return; /* We have identified a sub-protocol so we're done */
if(packet->forwarded_line.ptr != NULL) {
if(flow->http.nat_ip == NULL) {
len = packet->forwarded_line.len;
flow->http.nat_ip = ndpi_malloc(len + 1);
if(flow->http.nat_ip != NULL) {
strncpy(flow->http.nat_ip, (char*)packet->forwarded_line.ptr, len);
flow->http.nat_ip[len] = '\0';
}
}
}
if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP;
/* check for accept line */
if(packet->accept_line.ptr != NULL) {
NDPI_LOG_DBG2(ndpi_struct, "Accept line found %.*s\n",
packet->accept_line.len, packet->accept_line.ptr);
if(packet->server_line.ptr != NULL) {
if(flow->http.server == NULL) {
len = packet->server_line.len + 1;
flow->http.server = ndpi_malloc(len);
if(flow->http.server) {
strncpy(flow->http.server, (char*)packet->server_line.ptr,
packet->server_line.len);
flow->http.server[packet->server_line.len] = '\0';
}
}
}
/* check for authorization line */
if(packet->authorization_line.ptr != NULL) {
NDPI_LOG_DBG2(ndpi_struct, "Authorization line found %.*s\n",
packet->authorization_line.len, packet->authorization_line.ptr);
@@ -856,7 +866,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
}
}
if(packet->content_line.ptr != NULL && packet->content_line.len != 0) {
if(packet->content_line.ptr != NULL) {
NDPI_LOG_DBG2(ndpi_struct, "Content Type line found %.*s\n",
packet->content_line.len, packet->content_line.ptr);
@@ -886,54 +896,67 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
flow->guessed_category = flow->category = ndpi_http_check_content(ndpi_struct, flow);
}
}
if((flow->http.server == NULL) && (packet->server_line.len > 0)) {
int len = packet->server_line.len + 1;
flow->http.server = ndpi_malloc(len);
if(flow->http.server) {
strncpy(flow->http.server, (char*)packet->server_line.ptr,
packet->server_line.len);
flow->http.server[packet->server_line.len] = '\0';
}
}
}
if(flow->http_detected && packet->content_line.ptr && *(char*)packet->content_line.ptr) {
/* Matching on Content-Type.
OCSP: application/ocsp-request, application/ocsp-response
*/
if(packet->content_line.len > 17 &&
strncmp((const char *)packet->content_line.ptr, "application/ocsp-", 17) == 0) {
NDPI_LOG_DBG2(ndpi_struct, "Found OCSP\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OCSP, NDPI_PROTOCOL_HTTP, NDPI_CONFIDENCE_DPI);
}
}
}
if(packet->user_agent_line.ptr != NULL && packet->user_agent_line.len != 0 &&
flow->http.url) {
/* WindowsUpdate over some kind of CDN */
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN &&
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP &&
(strstr(flow->http.url, "delivery.mp.microsoft.com/") ||
strstr(flow->http.url, "download.windowsupdate.com/")) &&
ndpi_strnstr((const char *)packet->user_agent_line.ptr, "Microsoft-Delivery-Optimization/",
packet->user_agent_line.len) &&
ndpi_isset_risk(ndpi_struct, flow, NDPI_NUMERIC_IP_HOST)) {
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_WINDOWS_UPDATE, NDPI_PROTOCOL_HTTP, NDPI_CONFIDENCE_DPI);
/* check for host line (only if we don't already have an hostname) */
if(packet->host_line.ptr != NULL && flow->host_server_name[0] == '\0') {
NDPI_LOG_DBG2(ndpi_struct, "HOST line found %.*s\n",
packet->host_line.len, packet->host_line.ptr);
/* Copy result for nDPI apps */
ndpi_hostname_sni_set(flow, packet->host_line.ptr, packet->host_line.len);
if(strlen(flow->host_server_name) > 0) {
char *double_col;
int a, b, c, d;
hostname_just_set = 1;
if(ndpi_is_valid_hostname(flow->host_server_name,
strlen(flow->host_server_name)) == 0) {
char str[128];
snprintf(str, sizeof(str), "Invalid host %s", flow->host_server_name);
ndpi_set_risk(ndpi_struct, flow, NDPI_INVALID_CHARACTERS, str);
/* This looks like an attack */
ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT, NULL);
}
double_col = strchr((char*)flow->host_server_name, ':');
if(double_col) double_col[0] = '\0';
if(ndpi_struct->packet.iph
&& (sscanf(flow->host_server_name, "%d.%d.%d.%d", &a, &b, &c, &d) == 4)) {
/* IPv4 */
if(ndpi_struct->packet.iph->daddr != inet_addr(flow->host_server_name)) {
char buf[64], msg[128];
snprintf(msg, sizeof(msg), "Expected %s, found %s",
ndpi_intoav4(ntohl(ndpi_struct->packet.iph->daddr), buf, sizeof(buf)), flow->host_server_name);
ndpi_set_risk(ndpi_struct, flow, NDPI_HTTP_SUSPICIOUS_HEADER, msg);
}
}
}
}
ndpi_http_parse_subprotocol(ndpi_struct, flow, hostname_just_set);
if(hostname_just_set && strlen(flow->host_server_name) > 0) {
ndpi_check_dga_name(ndpi_struct, flow, flow->host_server_name, 1, 0);
}
if(flow->guessed_protocol_id == NDPI_PROTOCOL_UNKNOWN)
flow->guessed_protocol_id = NDPI_PROTOCOL_HTTP;
if(ndpi_get_http_method(ndpi_struct, flow) != NDPI_HTTP_METHOD_UNKNOWN) {
ndpi_int_http_add_connection(ndpi_struct, flow, flow->detected_protocol_stack[0], NDPI_PROTOCOL_CATEGORY_WEB);
ndpi_int_http_add_connection(ndpi_struct, flow, flow->detected_protocol_stack[0]);
}
}
/* ************************************************************* */
static void check_http_payload(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) {
/* Add here your payload code check */
ndpi_check_http_header(ndpi_struct, flow);
}
/* ************************************************************* */
@@ -1133,6 +1156,45 @@ static void ndpi_check_http_header(struct ndpi_detection_module_struct *ndpi_str
}
}
static void parse_response_code(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
char buf[4];
char ec[48];
if(packet->payload_packet_len >= 12) {
/* Set server HTTP response code */
strncpy(buf, (char*)&packet->payload[9], 3);
buf[3] = '\0';
flow->http.response_status_code = atoi(buf);
NDPI_LOG_DBG2(ndpi_struct, "Response code %d\n", flow->http.response_status_code);
/* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */
if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509))
flow->http.response_status_code = 0; /* Out of range */
if(flow->http.response_status_code >= 400) {
snprintf(ec, sizeof(ec), "HTTP Error Code %u", flow->http.response_status_code);
ndpi_set_risk(ndpi_struct, flow, NDPI_ERROR_CODE_DETECTED, ec);
if(flow->http.url != NULL) {
/* Let's check for Wordpress */
char *slash = strchr(flow->http.url, '/');
if(slash != NULL &&
(((flow->http.method == NDPI_HTTP_METHOD_POST) && (strncmp(slash, "/wp-admin/", 10) == 0))
|| ((flow->http.method == NDPI_HTTP_METHOD_GET) && (strncmp(slash, "/wp-content/uploads/", 20) == 0))
)) {
/* Example of popular exploits https://www.wordfence.com/blog/2022/05/millions-of-attacks-target-tatsu-builder-plugin/ */
ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT, "Possible Wordpress Exploit");
}
}
}
}
}
/*************************************************************************************************/
static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
@@ -1148,8 +1210,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
}
/* Check if we so far detected the protocol in the request or not. */
if((packet->payload_packet_len > 0) /* Needed in case of extra packet processing */
&& (flow->l4.tcp.http_stage == 0)) {
if(flow->l4.tcp.http_stage == 0) {
/* Expected a request */
flow->http_detected = 0;
@@ -1163,70 +1224,25 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
if(packet->payload_packet_len >= 7 && strncasecmp((const char *)packet->payload, "HTTP/1.", 7) == 0) {
NDPI_LOG_INFO(ndpi_struct, "found HTTP response\n");
if(packet->payload_packet_len >= 12) {
char buf[4];
parse_response_code(ndpi_struct, flow);
/* Set server HTTP response code */
strncpy(buf, (char*)&packet->payload[9], 3);
buf[3] = '\0';
flow->http.response_status_code = atoi(buf);
/* https://en.wikipedia.org/wiki/List_of_HTTP_status_codes */
if((flow->http.response_status_code < 100) || (flow->http.response_status_code > 509))
flow->http.response_status_code = 0; /* Out of range */
else if(flow->http.response_status_code >= 400) {
char ec[48];
if(flow->flow_payload) {
char *endl;
if(flow->http.url != NULL) {
/* Let's check for Wordpress */
char *slash = strchr(flow->http.url, '/');
flow->flow_payload[flow->flow_payload_len] = '\0';
if((endl = strrchr(flow->flow_payload, '\r')) == NULL)
endl = strrchr(flow->flow_payload, '\n');
if(slash != NULL &&
(((flow->http.method == NDPI_HTTP_METHOD_POST) && (strncmp(slash, "/wp-admin/", 10) == 0))
|| ((flow->http.method == NDPI_HTTP_METHOD_GET) && (strncmp(slash, "/wp-content/uploads/", 20) == 0))
)) {
/* Example of popular exploits https://www.wordfence.com/blog/2022/05/millions-of-attacks-target-tatsu-builder-plugin/ */
ndpi_set_risk(ndpi_struct, flow, NDPI_POSSIBLE_EXPLOIT, "Possible Wordpress Exploit");
}
}
snprintf(ec, sizeof(ec), "HTTP Error Code %u", flow->http.response_status_code);
ndpi_set_risk(ndpi_struct, flow, NDPI_ERROR_CODE_DETECTED, ec);
}
if(flow->flow_payload) {
char *endl;
flow->flow_payload[flow->flow_payload_len] = '\0';
if((endl = strrchr(flow->flow_payload, '\r')) == NULL)
endl = strrchr(flow->flow_payload, '\n');
if(endl != NULL) {
endl[0] = '\0';
flow->flow_payload_len = endl - flow->flow_payload;
}
if(endl != NULL) {
endl[0] = '\0';
flow->flow_payload_len = endl - flow->flow_payload;
}
}
ndpi_parse_packet_line_info(ndpi_struct, flow);
check_content_type_and_change_protocol(ndpi_struct, flow);
ndpi_validate_http_content(ndpi_struct, flow);
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
return;
}
if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "<policy-file-request/>", 23) == 0)) {
/*
<policy-file-request/>
<cross-domain-policy>
<allow-access-from domain="*.ookla.com" to-ports="8080"/>
<allow-access-from domain="*.speedtest.net" to-ports="8080"/>
</cross-domain-policy>
*/
ookla_found:
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_WEB);
ookla_add_to_cache(ndpi_struct, flow);
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
return;
}
@@ -1247,7 +1263,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
return;
} else {
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
}
}
@@ -1255,15 +1271,11 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
"Filename HTTP found: %d, we look for line info..\n", filename_start);
ndpi_parse_packet_line_info(ndpi_struct, flow);
ndpi_check_http_header(ndpi_struct, flow);
if(packet->parsed_lines <= 1) {
NDPI_LOG_DBG2(ndpi_struct,
"Found just one line, we will look further for the next packet...\n");
packet->http_method.ptr = packet->line[0].ptr;
packet->http_method.len = filename_start - 1;
/* Encode the direction of the packet in the stage, so we will know when we need to look for the response packet. */
flow->l4.tcp.http_stage = packet->packet_direction + 1; // packet_direction 0: stage 1, packet_direction 1: stage 2
return;
@@ -1288,12 +1300,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
else
flow->http.request_version = 0;
/* Check for Ookla */
if((packet->referer_line.len > 0)
&& ndpi_strnstr((const char *)packet->referer_line.ptr, "www.speedtest.net", packet->referer_line.len)) {
goto ookla_found;
}
if((packet->http_url_name.len > 7)
&& (!strncasecmp((const char*) packet->http_url_name.ptr, "http://", 7))) {
NDPI_LOG_INFO(ndpi_struct, "found HTTP_PROXY\n");
@@ -1328,7 +1334,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
u_int proto = flow->detected_protocol_stack[1] ? flow->detected_protocol_stack[1] : flow->detected_protocol_stack[0];
if(proto == NDPI_PROTOCOL_UNKNOWN) proto = NDPI_PROTOCOL_HTTP;
ndpi_int_http_add_connection(ndpi_struct, flow, proto, NDPI_PROTOCOL_CATEGORY_WEB);
ndpi_int_http_add_connection(ndpi_struct, flow, proto);
flow->http_detected = 1;
NDPI_LOG_DBG2(ndpi_struct,
"HTTP START Found, we will look further for the response...\n");
@@ -1350,10 +1356,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
otherwise check the second packet for the HTTP request
*/
if((flow->l4.tcp.http_stage - packet->packet_direction) == 1) { /* Expected a response package */
if(flow->http_detected)
return;
NDPI_LOG_DBG2(ndpi_struct,
" SECOND PAYLOAD TRAFFIC FROM CLIENT, FIRST PACKET MIGHT HAVE BEEN HTTP...UNKNOWN TRAFFIC, HERE FOR HTTP again.. \n");
@@ -1371,12 +1373,20 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
return;
}
}
/* The previous pkt was a valid HTTP request and this one is in the same direction:
splitted request? Try to extract more metadata */
if(flow->http_detected) {
check_content_type_and_change_protocol(ndpi_struct, flow);
}
// http://www.slideshare.net/DSPIP/rtsp-analysis-wireshark
if(packet->line[0].len >= 9
if(!flow->http_detected
&& packet->line[0].len >= 9
&& strncasecmp((const char *)&packet->line[0].ptr[packet->line[0].len - 9], " HTTP/1.", 8) == 0) {
NDPI_LOG_INFO(ndpi_struct, "found HTTP\n");
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
check_content_type_and_change_protocol(ndpi_struct, flow);
NDPI_LOG_DBG2(ndpi_struct,
@@ -1398,19 +1408,17 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
if((packet->parsed_lines == 1) && (packet->packet_direction == 1 /* server -> client */)) {
/* In Apache if you do "GET /\n\n" the response comes without any header */
NDPI_LOG_INFO(ndpi_struct, "found HTTP. (apache)\n");
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_CATEGORY_WEB);
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_HTTP);
check_content_type_and_change_protocol(ndpi_struct, flow);
return;
}
parse_response_code(ndpi_struct, flow);
/* Parse packet line and we look for the subprotocols */
ndpi_parse_packet_line_info(ndpi_struct, flow);
check_content_type_and_change_protocol(ndpi_struct, flow);
if(packet->empty_line_position_set != 0 || flow->l4.tcp.http_empty_line_seen == 1) {
NDPI_LOG_DBG2(ndpi_struct, "empty line. check_http_payload\n");
check_http_payload(ndpi_struct, flow);
}
ndpi_validate_http_content(ndpi_struct, flow);
flow->l4.tcp.http_stage = 0;
return;