config: HTTP: enable/disable processing of HTTP responses

This commit is contained in:
Nardi Ivan
2024-01-18 10:21:24 +01:00
committed by Ivan Nardi
parent c669044a44
commit 950f209a17
12 changed files with 111 additions and 14 deletions
+1
View File
@@ -26,5 +26,6 @@ TODO
| "pop" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext POP flows (because of opportunistic TLS, via STARTTLS msg) |
| "ftp" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext FTP flows (because of opportunistic TLS, via AUTH TLS msg) |
| "stun" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets multiplexed into STUN flows |
| "http" | "process_response.enable" | 1 | NULL | NULL | Enable/disable processing of HTTP responses. By default, HTTP flows are usually fully classified after the first request/response pair. If this parameter is disabled, the flows are fully classified after the first request (or after the first response, if the request is missing); in that case, some flow risks are not checked and some metadata are not exported |
| "ookla" | "aggressiveness", | 0x01 | 0x00 | 0x01 | Detection aggressiveness for Ookla. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable heuristic for detection over TLS (via Ookla LRU cache) |
| $PROTO_NAME | "ip_list.load" | 1 | NULL | NULL | Enable/disable loading of internal list of IP addresses (used for (sub)classification) specific to that protocol. Use "any" as protocol name if you want to easily enable/disable all lists. This knob is valid only for the following protocols: Alibaba, Amazon AWS, Apple, Avast, Bloomberg, Cachefly, Cloudflare, Discord, Disney+, Dropbox, Edgecast, EpicGames, Ethereum, Facebook, Github, Google, Google Cloud, GoTo, Hotspot Shield, Hulu, Line, Microsoft 365, Microsoft Azure, Microsoft One Drive, Microsoft Outlook, Mullvad, Netflix, Nvidia, OpenDNS, ProtonVPN, RiotGames, Roblox, Skype/Teams, Starcraft, Steam, Teamviewer, Telegram, Tencent, Threema, TOR, Twitch, Twitter, UbuntuONE, VK, Yandex, Yandex Cloud, Webex, Whatsapp, Zoom |
+5
View File
@@ -117,6 +117,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, "stun", "tls_dissection.enable", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 1 + 1);
sprintf(cfg_value, "%d", value);
ndpi_set_config(ndpi_info_mod, "http", "process_response.enable", cfg_value);
}
if(fuzzed_data.ConsumeBool()) {
value = fuzzed_data.ConsumeIntegralInRange(0, 0x01 + 1);
sprintf(cfg_value, "%d", value);
+2 -5
View File
@@ -205,6 +205,8 @@ struct ndpi_detection_module_config_struct {
int stun_opportunistic_tls_enabled;
int http_parse_response_enabled;
int ookla_aggressiveness;
NDPI_PROTOCOL_BITMASK ip_list_bitmask;
@@ -327,14 +329,9 @@ struct ndpi_detection_module_struct {
/* *** If you add a new LRU cache, please update lru_cache_type above! *** */
int tcp_ack_paylod_heuristic;
int fully_encrypted_based_on_first_pkt_heuristic;
u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */
ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
u_int8_t direction_detect_disable:1, /* disable internal detection of packet direction */ _pad:7;
#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_typedefs.h"
#endif
+2
View File
@@ -10790,6 +10790,8 @@ static const struct cfg_param {
{ "stun", "tls_dissection.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(stun_opportunistic_tls_enabled) },
{ "http", "process_response.enable", "1", NULL, NULL, CFG_PARAM_ENABLE_DISABLE, __OFF(http_parse_response_enabled) },
{ "ookla", "aggressiveness", "0x01", "0", "1", CFG_PARAM_INT, __OFF(ookla_aggressiveness) },
{ "$PROTO_NAME_OR_ID", "ip_list.load", "1", NULL, NULL, CFG_PARAM_PROTOCOL_ENABLE_DISABLE, __OFF(ip_list_bitmask)},
+13 -9
View File
@@ -1493,16 +1493,20 @@ static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struc
NDPI_LOG_DBG(ndpi_struct, "search HTTP\n");
ndpi_check_http_tcp(ndpi_struct, flow);
if((flow->host_server_name[0] != '\0'&&
if((ndpi_struct->cfg.http_parse_response_enabled &&
flow->host_server_name[0] != '\0'&&
flow->http.response_status_code != 0) ||
/* We have found 3 consecutive requests (without the reply) or 3
consecutive replies (without the request). If the traffic is really
asymmetric, stop here, because we will never find the metadata from
both the request and the reply. We wait for 3 events (instead of 2)
to avoid false positives triggered by missing/dropped packets */
(flow->l4.tcp.http_asymmetric_stage == 2 &&
(flow->packet_direction_complete_counter[0] == 0 ||
flow->packet_direction_complete_counter[1] == 0))) {
(!ndpi_struct->cfg.http_parse_response_enabled &&
(flow->host_server_name[0] != '\0' ||
flow->http.response_status_code != 0)) ||
/* We have found 3 consecutive requests (without the reply) or 3
consecutive replies (without the request). If the traffic is really
asymmetric, stop here, because we will never find the metadata from
both the request and the reply. We wait for 3 events (instead of 2)
to avoid false positives triggered by missing/dropped packets */
(flow->l4.tcp.http_asymmetric_stage == 2 &&
(flow->packet_direction_complete_counter[0] == 0 ||
flow->packet_direction_complete_counter[1] == 0))) {
flow->extra_packets_func = NULL; /* We're good now */
if(flow->initial_binary_bytes_len) ndpi_analyze_content_signature(ndpi_struct, flow);
Binary file not shown.
+28
View File
@@ -0,0 +1,28 @@
DPI Packets (TCP): 6 (6.00 pkts/flow)
Confidence DPI : 1 (flows)
Num dissector calls: 15 (15.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)
LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache stun_zoom: 0/0/0 (insert/search/found)
Automa host: 1/1 (search/found)
Automa domain: 1/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 0/0 (search/found)
Automa common alpns: 0/0 (search/found)
Patricia risk mask: 0/0 (search/found)
Patricia risk mask IPv6: 0/0 (search/found)
Patricia risk: 0/0 (search/found)
Patricia risk IPv6: 0/0 (search/found)
Patricia protocols: 1/1 (search/found)
Patricia protocols IPv6: 0/0 (search/found)
Google 10 1278 1
Acceptable 10 1278 1
1 TCP 192.168.1.128:42170 <-> 216.58.208.142:80 [proto: 7.126/HTTP.Google][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 6][cat: Web/5][6 pkts/478 bytes <-> 4 pkts/800 bytes][Goodput ratio: 15/66][0.04 sec][Hostname/SNI: google.com][bytes ratio: -0.252 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/7 8/14 28/20 10/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 80/200 140/594 27/227][URL: google.com/][StatusCode: 301][Content-Type: text/html][Server: gws][User-Agent: curl/7.68.0][PLAIN TEXT (GET / HTTP/1.1)][Plen Bins: 0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
@@ -0,0 +1 @@
--cfg=http,process_response.enable,0
@@ -0,0 +1 @@
../../default/pcap/http.pcapng
@@ -0,0 +1 @@
../../default/pcap/http_asymmetric.pcapng
@@ -0,0 +1,28 @@
DPI Packets (TCP): 4 (4.00 pkts/flow)
Confidence DPI : 1 (flows)
Num dissector calls: 15 (15.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)
LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache stun_zoom: 0/0/0 (insert/search/found)
Automa host: 1/1 (search/found)
Automa domain: 1/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 0/0 (search/found)
Automa common alpns: 0/0 (search/found)
Patricia risk mask: 0/0 (search/found)
Patricia risk mask IPv6: 0/0 (search/found)
Patricia risk: 0/0 (search/found)
Patricia risk IPv6: 0/0 (search/found)
Patricia protocols: 1/1 (search/found)
Patricia protocols IPv6: 0/0 (search/found)
Google 10 1278 1
Acceptable 10 1278 1
1 TCP 192.168.1.128:42170 <-> 216.58.208.142:80 [proto: 7.126/HTTP.Google][IP: 126/Google][ClearText][Confidence: DPI][DPI packets: 4][cat: Web/5][6 pkts/478 bytes <-> 4 pkts/800 bytes][Goodput ratio: 15/66][0.04 sec][Hostname/SNI: google.com][bytes ratio: -0.252 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/7 8/14 28/20 10/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 80/200 140/594 27/227][URL: google.com/][User-Agent: curl/7.68.0][PLAIN TEXT (GET / HTTP/1.1)][Plen Bins: 0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
@@ -0,0 +1,29 @@
DPI Packets (TCP): 6 (3.00 pkts/flow)
Confidence DPI : 2 (flows)
Num dissector calls: 30 (15.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
LRU cache zoom: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)
LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
LRU cache stun_zoom: 0/0/0 (insert/search/found)
Automa host: 1/0 (search/found)
Automa domain: 1/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 0/0 (search/found)
Automa common alpns: 0/0 (search/found)
Patricia risk mask: 4/0 (search/found)
Patricia risk mask IPv6: 0/0 (search/found)
Patricia risk: 0/0 (search/found)
Patricia risk IPv6: 0/0 (search/found)
Patricia protocols: 4/0 (search/found)
Patricia protocols IPv6: 0/0 (search/found)
HTTP 23 9961 2
Acceptable 23 9961 2
1 TCP 192.168.1.146:80 -> 192.168.1.103:1044 [proto: 7/HTTP][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 3][cat: Web/5][13 pkts/8357 bytes -> 0 pkts/0 bytes][Goodput ratio: 91/0][5.11 sec][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 464/0 5000/0 1435/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/0 643/0 1514/0 626/0][StatusCode: 200][Content-Type: text/html][Server: Apache/2.4.41 (Ubuntu)][Risk: ** HTTP Susp User-Agent **** Unidirectional Traffic **][Risk Score: 110][Risk Info: No client to server traffic / Empty or missing User-Agent][PLAIN TEXT (HTTP/1.1 200 OK)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,14,14,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,57,0,0]
2 TCP 192.168.0.1:1044 -> 10.10.10.1:80 [proto: 7/HTTP][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 3][cat: Web/5][10 pkts/1604 bytes -> 0 pkts/0 bytes][Goodput ratio: 66/0][5.11 sec][Hostname/SNI: proxy.wiresharkfest.acropolis.local][bytes ratio: 1.000 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 567/0 4951/0 1550/0][Pkt Len c2s/s2c min/avg/max/stddev: 54/0 160/0 418/0 160/0][URL: proxy.wiresharkfest.acropolis.local/][User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101 Firefox/91.0][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][PLAIN TEXT (GET / HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,66,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]