Ookla: rework detection (#1922)
The logic of the LRU cache has been changed: once we know an ip has connected to an Ookla server, all the following (unknown) flows (for a short time interval) from the same ip to the port 8080 are treated as Ookla ones. Most of the changes in this commit are about introducing the concept of "aggressive detection". In some cases, to properly detect a protocol we might use some statistical/behavior logic that, from one side, let us to identify the protocol more often but, from the other side, might lead to some false positives. To allow the user/application to easily detect when such logic has been triggered, the new confidence value `NDPI_CONFIDENCE_DPI_AGGRESSIVE` has been added. It is always possible to disable/configure this kind of logic via the API. Detection of Ookla flows using plain TLS over port 8080 is the first example of aggressive detection in nDPI. Tested with: * Android 9.0 with app 4.8.3 * Ubuntu 20.04 with Firefox 110 * Win 10 with app 1.15 and 1.16 * Win 10 with Chrome 108, Edge 108 and Firefox 106
This commit is contained in:
+43
-2
@@ -96,6 +96,7 @@ u_int8_t verbose = 0, enable_flow_stats = 0;
|
||||
int nDPI_LogLevel = 0;
|
||||
char *_debug_protocols = NULL;
|
||||
char *_disabled_protocols = NULL;
|
||||
int aggressiveness[NDPI_MAX_SUPPORTED_PROTOCOLS];
|
||||
static u_int8_t stats_flag = 0;
|
||||
ndpi_init_prefs init_prefs = ndpi_no_prefs;
|
||||
u_int8_t human_readeable_string_len = 5;
|
||||
@@ -515,6 +516,7 @@ static void help(u_int long_help) {
|
||||
" -z | Enable JA3+\n"
|
||||
" -A | Dump internal statistics (LRU caches / Patricia trees / Ahocarasick automas / ...\n"
|
||||
" -M | Memory allocation stats on data-path (only by the library). It works only on single-thread configuration\n"
|
||||
" -Z proto:value | Set this value of aggressiveness for this protocol (0 to disable it). This flag can be used multiple times\n"
|
||||
,
|
||||
human_readeable_string_len,
|
||||
min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection,
|
||||
@@ -797,7 +799,7 @@ void printCSVHeader() {
|
||||
*/
|
||||
static void parseOptions(int argc, char **argv) {
|
||||
int option_idx = 0;
|
||||
int opt;
|
||||
int opt, i;
|
||||
#ifndef USE_DPDK
|
||||
char *__pcap_file = NULL;
|
||||
int thread_id, do_capture = 0;
|
||||
@@ -818,7 +820,10 @@ static void parseOptions(int argc, char **argv) {
|
||||
}
|
||||
#endif
|
||||
|
||||
while((opt = getopt_long(argc, argv, "a:Ab:B:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zq0123:456:7:89:m:MT:U:",
|
||||
for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++)
|
||||
aggressiveness[i] = -1; /* Use the default value */
|
||||
|
||||
while((opt = getopt_long(argc, argv, "a:Ab:B:e:Ec:C:dDf:g:i:Ij:k:K:S:hHp:pP:l:r:s:tu:v:V:n:rp:x:w:zZ:q0123:456:7:89:m:MT:U:",
|
||||
longopts, &option_idx)) != EOF) {
|
||||
#ifdef DEBUG_TRACE
|
||||
if(trace) fprintf(trace, " #### Handling option -%c [%s] #### \n", opt, optarg ? optarg : "");
|
||||
@@ -950,6 +955,35 @@ static void parseOptions(int argc, char **argv) {
|
||||
_disabled_protocols = ndpi_strdup(optarg);
|
||||
break;
|
||||
|
||||
case 'Z': /* proto_name:aggr_value */
|
||||
{
|
||||
struct ndpi_detection_module_struct *module_tmp;
|
||||
NDPI_PROTOCOL_BITMASK all;
|
||||
char *saveptr, *tmp_str, *proto_str, *aggr_str;
|
||||
|
||||
/* Use a temporary module with all protocols enabled */
|
||||
module_tmp = ndpi_init_detection_module(0);
|
||||
if(!module_tmp)
|
||||
break;
|
||||
NDPI_BITMASK_SET_ALL(all);
|
||||
ndpi_set_protocol_detection_bitmask2(module_tmp, &all);
|
||||
ndpi_finalize_initialization(module_tmp);
|
||||
|
||||
tmp_str = ndpi_strdup(optarg);
|
||||
if(tmp_str) {
|
||||
proto_str = strtok_r(tmp_str, ":", &saveptr);
|
||||
if(proto_str) {
|
||||
aggr_str = strtok_r(NULL, ":", &saveptr);
|
||||
if(aggr_str) {
|
||||
aggressiveness[ndpi_get_protocol_id(module_tmp, proto_str)] = atoi(aggr_str);
|
||||
}
|
||||
}
|
||||
}
|
||||
ndpi_free(tmp_str);
|
||||
ndpi_exit_detection_module(module_tmp);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'h':
|
||||
help(0);
|
||||
break;
|
||||
@@ -2413,6 +2447,7 @@ static void debug_printf(u_int32_t protocol, void *id_struct,
|
||||
static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
|
||||
NDPI_PROTOCOL_BITMASK enabled_bitmask;
|
||||
struct ndpi_workflow_prefs prefs;
|
||||
int i;
|
||||
|
||||
memset(&prefs, 0, sizeof(prefs));
|
||||
prefs.decode_tunnels = decode_tunnels;
|
||||
@@ -2472,6 +2507,12 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
|
||||
NDPI_LRUCACHE_BITTORRENT, 32768);
|
||||
/* Enable/disable LRU caches TTL here */
|
||||
|
||||
/* Set aggressiviness here */
|
||||
for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
|
||||
if(aggressiveness[i] != -1)
|
||||
ndpi_set_protocol_aggressiveness(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, aggressiveness[i]);
|
||||
}
|
||||
|
||||
ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct);
|
||||
|
||||
if(enable_doh_dot_detection)
|
||||
|
||||
@@ -108,6 +108,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
||||
ndpi_set_opportunistic_tls(ndpi_info_mod, random_proto, random_value);
|
||||
ndpi_get_opportunistic_tls(ndpi_info_mod, random_proto);
|
||||
|
||||
for(i = 0; i < NDPI_MAX_SUPPORTED_PROTOCOLS; i++) {
|
||||
ndpi_set_protocol_aggressiveness(ndpi_info_mod, i, random_value);
|
||||
ndpi_get_protocol_aggressiveness(ndpi_info_mod, i);
|
||||
}
|
||||
|
||||
ndpi_finalize_initialization(ndpi_info_mod);
|
||||
|
||||
/* Random protocol configuration */
|
||||
|
||||
@@ -1031,6 +1031,11 @@ extern "C" {
|
||||
int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
u_int16_t proto);
|
||||
|
||||
int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
u_int16_t proto, u_int32_t value);
|
||||
u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
u_int16_t proto);
|
||||
|
||||
/**
|
||||
* Find a protocol id associated with a string automata
|
||||
*
|
||||
|
||||
@@ -659,6 +659,14 @@ struct ndpi_lru_cache {
|
||||
struct ndpi_lru_cache_entry *entries;
|
||||
};
|
||||
|
||||
|
||||
/* Aggressiveness values */
|
||||
|
||||
#define NDPI_AGGRESSIVENESS_DISABLED 0x00 /* For all protocols */
|
||||
|
||||
/* Ookla */
|
||||
#define NDPI_AGGRESSIVENESS_OOKLA_TLS 0x01 /* Enable detection over TLS (using ookla cache) */
|
||||
|
||||
/* ************************************************** */
|
||||
|
||||
struct ndpi_flow_tcp_struct {
|
||||
@@ -920,7 +928,8 @@ typedef enum {
|
||||
NDPI_CONFIDENCE_DPI_CACHE, /* Classification results based on some LRU cache (i.e. correlation among sessions) */
|
||||
NDPI_CONFIDENCE_DPI, /* Deep packet inspection */
|
||||
NDPI_CONFIDENCE_MATCH_BY_IP, /* Classification obtained looking only at the IP addresses */
|
||||
|
||||
NDPI_CONFIDENCE_DPI_AGGRESSIVE, /* Aggressive DPI: it might be a false positive */
|
||||
|
||||
/*
|
||||
IMPORTANT
|
||||
|
||||
@@ -1229,6 +1238,8 @@ struct ndpi_detection_module_struct {
|
||||
int opportunistic_tls_pop_enabled;
|
||||
int opportunistic_tls_ftp_enabled;
|
||||
|
||||
u_int32_t aggressiveness_ookla;
|
||||
|
||||
u_int16_t ndpi_to_user_proto_id[NDPI_MAX_NUM_CUSTOM_PROTOCOLS]; /* custom protocolId mapping */
|
||||
ndpi_proto_defaults_t proto_defaults[NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS];
|
||||
|
||||
@@ -1563,6 +1574,10 @@ struct ndpi_flow_struct {
|
||||
/* NDPI_PROTOCOL_Z3950 */
|
||||
u_int8_t z3950_stage : 2; // 0-3
|
||||
|
||||
/* NDPI_PROTOCOL_OOKLA */
|
||||
u_int8_t ookla_stage : 1;
|
||||
|
||||
|
||||
/* NDPI_PROTOCOL_OPENVPN */
|
||||
u_int8_t ovpn_session_id[8];
|
||||
u_int8_t ovpn_counter;
|
||||
|
||||
+55
-1
@@ -198,6 +198,10 @@ extern void ndpi_unset_risk(struct ndpi_detection_module_struct *ndpi_str,
|
||||
struct ndpi_flow_struct *flow, ndpi_risk_enum r);
|
||||
extern u_int32_t make_mining_key(struct ndpi_flow_struct *flow);
|
||||
extern int stun_search_into_zoom_cache(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow);
|
||||
extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow);
|
||||
extern int ookla_search_into_cache(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow);
|
||||
|
||||
/* Forward */
|
||||
static int addDefaultPort(struct ndpi_detection_module_struct *ndpi_str,
|
||||
@@ -2932,7 +2936,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
|
||||
ndpi_str->msteams_cache_num_entries = 1024;
|
||||
ndpi_str->stun_zoom_cache_num_entries = 1024;
|
||||
|
||||
ndpi_str->ookla_cache_ttl = 0;
|
||||
ndpi_str->ookla_cache_ttl = 120; /* sec */
|
||||
ndpi_str->bittorrent_cache_ttl = 0;
|
||||
ndpi_str->zoom_cache_ttl = 0;
|
||||
ndpi_str->stun_cache_ttl = 0;
|
||||
@@ -2946,6 +2950,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
|
||||
ndpi_str->opportunistic_tls_pop_enabled = 1;
|
||||
ndpi_str->opportunistic_tls_ftp_enabled = 1;
|
||||
|
||||
ndpi_str->aggressiveness_ookla = NDPI_AGGRESSIVENESS_OOKLA_TLS;
|
||||
|
||||
for(i = 0; i < NUM_CUSTOM_CATEGORIES; i++)
|
||||
ndpi_snprintf(ndpi_str->custom_category_labels[i], CUSTOM_CATEGORY_LABEL_LEN, "User custom category %u",
|
||||
(unsigned int) (i + 1));
|
||||
@@ -6254,6 +6260,13 @@ ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_st
|
||||
ret.app_protocol = flow->detected_protocol_stack[0];
|
||||
}
|
||||
|
||||
/* Does it looks like Ookla? */
|
||||
if(ret.app_protocol == NDPI_PROTOCOL_UNKNOWN &&
|
||||
ntohs(flow->s_port) == 8080 && ookla_search_into_cache(ndpi_str, flow)) {
|
||||
ndpi_set_detected_protocol(ndpi_str, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_PARTIAL_CACHE);
|
||||
ret.app_protocol = flow->detected_protocol_stack[0];
|
||||
}
|
||||
|
||||
/* Classification by-port is the last resort */
|
||||
if(enable_guess && ret.app_protocol == NDPI_PROTOCOL_UNKNOWN) {
|
||||
|
||||
@@ -8052,6 +8065,9 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)
|
||||
case NDPI_CONFIDENCE_MATCH_BY_IP:
|
||||
return "Match by IP";
|
||||
|
||||
case NDPI_CONFIDENCE_DPI_AGGRESSIVE:
|
||||
return "DPI (aggressive)";
|
||||
|
||||
default:
|
||||
return NULL;
|
||||
}
|
||||
@@ -8572,6 +8588,11 @@ int ndpi_match_hostname_protocol(struct ndpi_detection_module_struct *ndpi_struc
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, subproto, master_protocol, NDPI_CONFIDENCE_DPI);
|
||||
if(!category_depends_on_master(master_protocol))
|
||||
ndpi_int_change_category(ndpi_struct, flow, ret_match.protocol_category);
|
||||
|
||||
if(subproto == NDPI_PROTOCOL_OOKLA) {
|
||||
ookla_add_to_cache(ndpi_struct, flow);
|
||||
}
|
||||
|
||||
return(1);
|
||||
} else
|
||||
return(0);
|
||||
@@ -9643,3 +9664,36 @@ int ndpi_get_opportunistic_tls(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
int ndpi_set_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
u_int16_t proto, u_int32_t value)
|
||||
{
|
||||
if(!ndpi_struct)
|
||||
return -1;
|
||||
|
||||
switch(proto) {
|
||||
case NDPI_PROTOCOL_OOKLA:
|
||||
ndpi_struct->aggressiveness_ookla = value;
|
||||
return 0;
|
||||
default:
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
u_int32_t ndpi_get_protocol_aggressiveness(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
u_int16_t proto)
|
||||
{
|
||||
if(!ndpi_struct)
|
||||
return -1;
|
||||
|
||||
switch(proto) {
|
||||
case NDPI_PROTOCOL_OOKLA:
|
||||
return ndpi_struct->aggressiveness_ookla;
|
||||
default:
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +46,9 @@ static const char* binary_file_ext[] = {
|
||||
NULL
|
||||
};
|
||||
|
||||
extern void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow);
|
||||
|
||||
static void ndpi_search_http_tcp(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow);
|
||||
|
||||
@@ -436,6 +439,7 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp
|
||||
|| (strstr(flow->http.url, ":8080/upload?n=0.") != NULL))) {
|
||||
/* This looks like Ookla speedtest */
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, master_protocol, NDPI_CONFIDENCE_DPI);
|
||||
ookla_add_to_cache(ndpi_struct, flow);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1217,30 +1221,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
|
||||
return;
|
||||
}
|
||||
|
||||
if((packet->payload_packet_len == 3) && memcmp(packet->payload, "HI\n", 3) == 0) {
|
||||
/* This looks like Ookla: we don't give up with HTTP yet */
|
||||
flow->l4.tcp.http_stage = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if((packet->payload_packet_len == 40) && (flow->l4.tcp.http_stage == 0)) {
|
||||
/*
|
||||
-> QR O06L0072-6L91-4O43-857J-K8OO172L6L51
|
||||
<- QNUUX 2.5 2017-08-15.1314.4jn12m5
|
||||
-> MXFWUXJM 31625365
|
||||
*/
|
||||
|
||||
if((packet->payload[2] == ' ')
|
||||
&& (packet->payload[11] == '-')
|
||||
&& (packet->payload[16] == '-')
|
||||
&& (packet->payload[21] == '-')
|
||||
&& (packet->payload[26] == '-')
|
||||
&& (packet->payload[39] == 0x0A)
|
||||
)
|
||||
flow->l4.tcp.http_stage = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if((packet->payload_packet_len == 23) && (memcmp(packet->payload, "<policy-file-request/>", 23) == 0)) {
|
||||
/*
|
||||
<policy-file-request/>
|
||||
@@ -1251,25 +1231,7 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
|
||||
*/
|
||||
ookla_found:
|
||||
ndpi_int_http_add_connection(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_CATEGORY_WEB);
|
||||
|
||||
if(ndpi_struct->ookla_cache != NULL) {
|
||||
if(packet->iph != NULL) {
|
||||
if(packet->tcp->source == htons(8080))
|
||||
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->saddr, 1 /* dummy */, ndpi_get_current_time(flow));
|
||||
else
|
||||
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, packet->iph->daddr, 1 /* dummy */, ndpi_get_current_time(flow));
|
||||
} else if(packet->iphv6 != NULL) {
|
||||
u_int32_t h;
|
||||
|
||||
if(packet->tcp->source == htons(8080))
|
||||
h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_src, sizeof(packet->iphv6->ip6_src));
|
||||
else
|
||||
h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
|
||||
|
||||
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
|
||||
}
|
||||
}
|
||||
|
||||
ookla_add_to_cache(ndpi_struct, flow);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1389,18 +1351,6 @@ static void ndpi_check_http_tcp(struct ndpi_detection_module_struct *ndpi_struct
|
||||
} else if((flow->l4.tcp.http_stage == 1) || (flow->l4.tcp.http_stage == 2)) {
|
||||
NDPI_LOG_DBG2(ndpi_struct, "HTTP stage %u: \n", flow->l4.tcp.http_stage);
|
||||
|
||||
if((packet->payload_packet_len == 34) && (flow->l4.tcp.http_stage == 1)) {
|
||||
if((packet->payload[5] == ' ') && (packet->payload[9] == ' ')) {
|
||||
goto ookla_found;
|
||||
}
|
||||
}
|
||||
|
||||
if((packet->payload_packet_len > 6) && memcmp(packet->payload, "HELLO ", 6) == 0) {
|
||||
/* This looks like Ookla */
|
||||
goto ookla_found;
|
||||
} else
|
||||
NDPI_ADD_PROTOCOL_TO_BITMASK(flow->excluded_protocol_bitmask, NDPI_PROTOCOL_OOKLA);
|
||||
|
||||
/**
|
||||
At first check, if this is for sure a response packet
|
||||
(in another direction. If not, if HTTP is detected do nothing now and return,
|
||||
|
||||
+77
-96
@@ -23,111 +23,92 @@
|
||||
|
||||
#include "ndpi_api.h"
|
||||
|
||||
/* #define DEBUG_OOKLA_LRU */
|
||||
|
||||
const u_int16_t ookla_port = 8080;
|
||||
|
||||
/* ************************************************************* */
|
||||
|
||||
static void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) {
|
||||
struct ndpi_packet_struct* packet = &ndpi_struct->packet;
|
||||
u_int32_t addr = 0;
|
||||
u_int16_t sport, dport;
|
||||
|
||||
static u_int32_t get_ookla_key(struct ndpi_flow_struct *flow)
|
||||
{
|
||||
if(flow->is_ipv6)
|
||||
return ndpi_quick_hash(flow->c_address.v6, 16);
|
||||
else
|
||||
return ntohl(flow->c_address.v4);
|
||||
}
|
||||
|
||||
/* ************************************************************* */
|
||||
|
||||
int ookla_search_into_cache(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow)
|
||||
{
|
||||
u_int32_t key;
|
||||
u_int16_t dummy;
|
||||
|
||||
if(ndpi_struct->ookla_cache) {
|
||||
key = get_ookla_key(flow);
|
||||
#ifdef DEBUG_OOKLA_LRU
|
||||
printf("[LRU OOKLA] Search %u\n", key);
|
||||
#endif
|
||||
|
||||
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, key,
|
||||
&dummy, 0 /* Don't remove it as it can be used for other connections */,
|
||||
ndpi_get_current_time(flow))) {
|
||||
#ifdef DEBUG_OOKLA_LRU
|
||||
printf("[LRU OOKLA] Found\n");
|
||||
#endif
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ************************************************************* */
|
||||
|
||||
void ookla_add_to_cache(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow)
|
||||
{
|
||||
u_int32_t key;
|
||||
|
||||
if(ndpi_struct->ookla_cache) {
|
||||
key = get_ookla_key(flow);
|
||||
#ifdef DEBUG_OOKLA_LRU
|
||||
printf("[LRU OOKLA] ADDING %u\n", key);
|
||||
#endif
|
||||
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, key, 1 /* dummy */,
|
||||
ndpi_get_current_time(flow));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/* ************************************************************* */
|
||||
|
||||
void ndpi_search_ookla(struct ndpi_detection_module_struct* ndpi_struct, struct ndpi_flow_struct* flow) {
|
||||
struct ndpi_packet_struct *packet = &ndpi_struct->packet;
|
||||
|
||||
NDPI_LOG_DBG(ndpi_struct, "Ookla detection\n");
|
||||
|
||||
if(packet->tcp)
|
||||
sport = ntohs(packet->tcp->source), dport = htons(packet->tcp->dest);
|
||||
else
|
||||
sport = ntohs(packet->udp->source), dport = htons(packet->udp->dest);
|
||||
if(ntohs(flow->s_port) != ookla_port && ntohs(flow->c_port) != ookla_port) {
|
||||
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
|
||||
return;
|
||||
}
|
||||
|
||||
if((sport != ookla_port) && (dport != ookla_port)) {
|
||||
#ifdef OOKLA_DEBUG
|
||||
printf("=>>>>>>>> [OOKLA IPv6] Skipping flow [%u -> %u]\n", sport, dport);
|
||||
#endif
|
||||
goto ookla_exclude;
|
||||
if(flow->packet_counter == 1 &&
|
||||
packet->payload_packet_len >= NDPI_STATICSTRING_LEN("HI") &&
|
||||
memcmp(packet->payload, "HI", NDPI_STATICSTRING_LEN("HI")) == 0) {
|
||||
flow->ookla_stage = 1;
|
||||
return;
|
||||
}
|
||||
if(flow->packet_counter == 2 &&
|
||||
flow->ookla_stage == 1 &&
|
||||
packet->payload_packet_len >= NDPI_STATICSTRING_LEN("HELLO") &&
|
||||
memcmp(packet->payload, "HELLO", NDPI_STATICSTRING_LEN("HELLO")) == 0) {
|
||||
NDPI_LOG_INFO(ndpi_struct, "found ookla (Hi + Hello)\n");
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
|
||||
ookla_add_to_cache(ndpi_struct, flow);
|
||||
return;
|
||||
}
|
||||
|
||||
if(packet->iphv6 != NULL) {
|
||||
if((dport == ookla_port) && (packet->payload_packet_len >= 3)) {
|
||||
u_int32_t h;
|
||||
|
||||
if((packet->payload_packet_len == 3)
|
||||
&& (packet->payload[0] == 0x48) /* HI\n */
|
||||
&& (packet->payload[1] == 0x49)
|
||||
&& (packet->payload[2] == 0x0A)) {
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
|
||||
|
||||
if(ndpi_struct->ookla_cache != NULL) {
|
||||
/* In order to avoid creating an IPv6 LRU we hash the IPv6 address */
|
||||
h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
|
||||
|
||||
#ifdef OOKLA_DEBUG
|
||||
printf("=>>>>>>>> [OOKLA IPv6] Adding %u\n", h);
|
||||
#endif
|
||||
ndpi_lru_add_to_cache(ndpi_struct->ookla_cache, h, 1 /* dummy */, ndpi_get_current_time(flow));
|
||||
}
|
||||
return;
|
||||
} else {
|
||||
if(sport == ookla_port)
|
||||
h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_src, sizeof(packet->iphv6->ip6_src));
|
||||
else
|
||||
h = ndpi_quick_hash((unsigned char *)&packet->iphv6->ip6_dst, sizeof(packet->iphv6->ip6_dst));
|
||||
|
||||
if(ndpi_struct->ookla_cache != NULL) {
|
||||
u_int16_t dummy;
|
||||
|
||||
#ifdef OOKLA_DEBUG
|
||||
printf("=>>>>>>>> [OOKLA IPv6] Searching %u\n", h);
|
||||
#endif
|
||||
|
||||
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, h, &dummy, 0 /* Don't remove it as it can be used for other connections */,
|
||||
ndpi_get_current_time(flow))) {
|
||||
NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
|
||||
#ifdef OOKLA_DEBUG
|
||||
printf("=>>>>> Found %u\n", h);
|
||||
#endif
|
||||
return;
|
||||
} else {
|
||||
#ifdef OOKLA_DEBUG
|
||||
printf("=>>>>> NOT Found %u\n", h);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
|
||||
goto ookla_exclude;
|
||||
}
|
||||
} else {
|
||||
if(sport == ookla_port)
|
||||
addr = packet->iph->saddr;
|
||||
else
|
||||
addr = packet->iph->daddr;
|
||||
|
||||
#ifdef OOKLA_DEBUG
|
||||
printf("=>>>>>>>> [OOKLA IPv4] Searching %u\n", addr);
|
||||
#endif
|
||||
|
||||
if(ndpi_struct->ookla_cache != NULL) {
|
||||
u_int16_t dummy;
|
||||
|
||||
if(ndpi_lru_find_cache(ndpi_struct->ookla_cache, addr, &dummy, 0 /* Don't remove it as it can be used for other connections */,
|
||||
ndpi_get_current_time(flow))) {
|
||||
NDPI_LOG_INFO(ndpi_struct, "found ookla tcp connection\n");
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI_CACHE);
|
||||
#ifdef OOKLA_DEBUG
|
||||
printf("=>>>>> Found %u\n", addr);
|
||||
#endif
|
||||
return;
|
||||
} else {
|
||||
#ifdef OOKLA_DEBUG
|
||||
printf("=>>>>> NOT Found %u\n", addr);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ookla_exclude:
|
||||
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
|
||||
}
|
||||
|
||||
|
||||
+23
-2
@@ -33,6 +33,8 @@ extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_st
|
||||
extern int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow,
|
||||
const u_int8_t *ua_ptr, u_int16_t ua_ptr_len);
|
||||
extern int ookla_search_into_cache(struct ndpi_detection_module_struct* ndpi_struct,
|
||||
struct ndpi_flow_struct* flow);
|
||||
/* QUIC/GQUIC stuff */
|
||||
extern int quic_len(const uint8_t *buf, uint64_t *value);
|
||||
extern int quic_len_buffer_still_required(uint8_t value);
|
||||
@@ -1153,8 +1155,27 @@ static int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
#ifdef DEBUG_TLS_BLOCKS
|
||||
printf("*** [TLS Block] No more blocks\n");
|
||||
#endif
|
||||
flow->extra_packets_func = NULL;
|
||||
return(0); /* That's all */
|
||||
/* An ookla flow? */
|
||||
if((ndpi_struct->aggressiveness_ookla & NDPI_AGGRESSIVENESS_OOKLA_TLS) && /* Feature enabled */
|
||||
(!something_went_wrong &&
|
||||
flow->tls_quic.certificate_processed == 1 &&
|
||||
flow->protos.tls_quic.hello_processed == 1) && /* TLS handshake found without errors */
|
||||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_TLS && /* No IMAPS/FTPS/... */
|
||||
flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN && /* No sub-classification */
|
||||
ntohs(flow->s_port) == 8080 && /* Ookla port */
|
||||
ookla_search_into_cache(ndpi_struct, flow)) {
|
||||
NDPI_LOG_INFO(ndpi_struct, "found ookla (cache over TLS)\n");
|
||||
/* Even if a LRU cache is involved, NDPI_CONFIDENCE_DPI_AGGRESSIVE seems more
|
||||
suited than NDPI_CONFIDENCE_DPI_CACHE */
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OOKLA, NDPI_PROTOCOL_TLS, NDPI_CONFIDENCE_DPI_AGGRESSIVE);
|
||||
/* TLS over port 8080 usually triggers that risk; clear it */
|
||||
ndpi_unset_risk(ndpi_struct, flow, NDPI_KNOWN_PROTOCOL_ON_NON_STANDARD_PORT);
|
||||
flow->extra_packets_func = NULL;
|
||||
return(0); /* That's all */
|
||||
} else {
|
||||
flow->extra_packets_func = NULL;
|
||||
return(0); /* That's all */
|
||||
}
|
||||
} else
|
||||
return(1);
|
||||
}
|
||||
|
||||
Binary file not shown.
@@ -7,7 +7,7 @@ Confidence Match by port : 5 (flows)
|
||||
Confidence DPI (partial) : 9 (flows)
|
||||
Confidence DPI : 146 (flows)
|
||||
Num dissector calls: 473 (2.96 diss/flow)
|
||||
LRU cache ookla: 0/0/0 (insert/search/found)
|
||||
LRU cache ookla: 0/5/0 (insert/search/found)
|
||||
LRU cache bittorrent: 0/42/0 (insert/search/found)
|
||||
LRU cache zoom: 0/0/0 (insert/search/found)
|
||||
LRU cache stun: 0/0/0 (insert/search/found)
|
||||
|
||||
+28
-18
@@ -1,27 +1,37 @@
|
||||
Guessed flow protos: 0
|
||||
Guessed flow protos: 1
|
||||
|
||||
DPI Packets (TCP): 16 (8.00 pkts/flow)
|
||||
Confidence DPI (cache) : 1 (flows)
|
||||
Confidence DPI : 1 (flows)
|
||||
Num dissector calls: 97 (48.50 diss/flow)
|
||||
LRU cache ookla: 3/1/1 (insert/search/found)
|
||||
LRU cache bittorrent: 0/0/0 (insert/search/found)
|
||||
DPI Packets (TCP): 46 (7.67 pkts/flow)
|
||||
Confidence DPI (partial cache): 1 (flows)
|
||||
Confidence DPI : 4 (flows)
|
||||
Confidence DPI (aggressive) : 1 (flows)
|
||||
Num dissector calls: 488 (81.33 diss/flow)
|
||||
LRU cache ookla: 6/2/2 (insert/search/found)
|
||||
LRU cache bittorrent: 0/3/0 (insert/search/found)
|
||||
LRU cache zoom: 0/0/0 (insert/search/found)
|
||||
LRU cache stun: 0/0/0 (insert/search/found)
|
||||
LRU cache tls_cert: 0/0/0 (insert/search/found)
|
||||
LRU cache mining: 0/0/0 (insert/search/found)
|
||||
LRU cache tls_cert: 0/2/0 (insert/search/found)
|
||||
LRU cache mining: 0/1/0 (insert/search/found)
|
||||
LRU cache msteams: 0/0/0 (insert/search/found)
|
||||
LRU cache stun_zoom: 0/0/0 (insert/search/found)
|
||||
Automa host: 0/0 (search/found)
|
||||
Automa domain: 0/0 (search/found)
|
||||
Automa host: 2/1 (search/found)
|
||||
Automa domain: 2/0 (search/found)
|
||||
Automa tls cert: 0/0 (search/found)
|
||||
Automa risk mask: 0/0 (search/found)
|
||||
Automa common alpns: 0/0 (search/found)
|
||||
Patricia risk mask: 4/0 (search/found)
|
||||
Automa risk mask: 1/0 (search/found)
|
||||
Automa common alpns: 4/4 (search/found)
|
||||
Patricia risk mask: 12/0 (search/found)
|
||||
Patricia risk: 0/0 (search/found)
|
||||
Patricia protocols: 4/0 (search/found)
|
||||
Patricia protocols: 11/1 (search/found)
|
||||
|
||||
Ookla 50 6661 2
|
||||
Ookla 113 38411 6
|
||||
|
||||
1 TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 12][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][Risk: ** HTTP Susp User-Agent **** HTTP Obsolete Server **][Risk Score: 150][Risk Info: Obsolete Apache server 2.2.22 / Empty or missing User-Agent][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
||||
2 TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI (cache)][DPI packets: 4][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
||||
JA3 Host Stats:
|
||||
IP Address # JA3C
|
||||
1 192.168.1.128 2
|
||||
|
||||
|
||||
1 TCP 192.168.1.128:35830 <-> 89.96.108.170:8080 [proto: 91.191/TLS.Ookla][IP: 0/Unknown][Encrypted][Confidence: DPI (aggressive)][DPI packets: 6][cat: Web/5][21 pkts/21216 bytes <-> 8 pkts/1950 bytes][Goodput ratio: 93/72][0.32 sec][Hostname/SNI: spd-pub-mi-01-01.fastwebnet.it][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: 0.832 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/61 274/280 62/109][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1010/244 1514/387 612/138][TLSv1.3][JA3C: c279b0189edb9269da7bc43dea5e0c36][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,4,0,0,0,0,4,9,0,9,0,0,0,0,0,4,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0]
|
||||
2 TCP 192.168.1.128:48854 <-> 104.16.209.12:443 [proto: 91.191/TLS.Ookla][IP: 220/Cloudflare][Encrypted][Confidence: DPI][DPI packets: 6][cat: Network/14][8 pkts/1620 bytes <-> 6 pkts/3818 bytes][Goodput ratio: 67/89][0.06 sec][Hostname/SNI: www.speedtest.net][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.404 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/5 18/15 7/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 202/636 583/1514 181/646][TLSv1.3][JA3C: 579ccef312d18482fc42e2b822ca2430][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Firefox][Cipher: TLS_AES_128_GCM_SHA256][PLAIN TEXT (@oTAgOeedtest.net)][Plen Bins: 0,0,14,0,0,14,0,0,0,0,14,0,0,0,0,0,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,0,0]
|
||||
3 TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 12][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][Risk: ** HTTP Susp User-Agent **** HTTP Obsolete Server **][Risk Score: 150][Risk Info: Obsolete Apache server 2.2.22 / Empty or missing User-Agent][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
||||
4 TCP 192.168.1.192:51156 <-> 89.96.108.170:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI (partial cache)][DPI packets: 10][cat: Network/14][6 pkts/591 bytes <-> 4 pkts/1784 bytes][Goodput ratio: 32/85][0.05 sec][bytes ratio: -0.502 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/10 15/20 6/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/446 143/1514 31/617][PLAIN TEXT (gKRZvA)][Plen Bins: 0,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0]
|
||||
5 TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
||||
6 TCP 192.168.1.192:37790 <-> 185.157.229.246:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][6 pkts/454 bytes <-> 4 pkts/317 bytes][Goodput ratio: 11/14][0.06 sec][bytes ratio: 0.178 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/5 46/9 17/4][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 76/79 106/108 14/17][PLAIN TEXT (HELLO 2.9 )][Plen Bins: 50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
||||
|
||||
@@ -7,7 +7,7 @@ Confidence Unknown : 59 (flows)
|
||||
Confidence Match by port : 27 (flows)
|
||||
Confidence DPI (partial) : 1 (flows)
|
||||
Confidence DPI : 206 (flows)
|
||||
Num dissector calls: 26899 (91.81 diss/flow)
|
||||
Num dissector calls: 26897 (91.80 diss/flow)
|
||||
LRU cache ookla: 0/0/0 (insert/search/found)
|
||||
LRU cache bittorrent: 0/261/0 (insert/search/found)
|
||||
LRU cache zoom: 0/0/0 (insert/search/found)
|
||||
|
||||
@@ -6,7 +6,7 @@ DPI Packets (other): 5 (1.00 pkts/flow)
|
||||
Confidence Unknown : 44 (flows)
|
||||
Confidence Match by port : 22 (flows)
|
||||
Confidence DPI : 201 (flows)
|
||||
Num dissector calls: 22405 (83.91 diss/flow)
|
||||
Num dissector calls: 22404 (83.91 diss/flow)
|
||||
LRU cache ookla: 0/0/0 (insert/search/found)
|
||||
LRU cache bittorrent: 0/198/0 (insert/search/found)
|
||||
LRU cache zoom: 0/0/0 (insert/search/found)
|
||||
|
||||
@@ -4,7 +4,7 @@ DPI Packets (TCP): 2011 (1.01 pkts/flow)
|
||||
Confidence Unknown : 1862 (flows)
|
||||
Confidence Match by port : 132 (flows)
|
||||
Num dissector calls: 0 (0.00 diss/flow)
|
||||
LRU cache ookla: 0/0/0 (insert/search/found)
|
||||
LRU cache ookla: 0/2/0 (insert/search/found)
|
||||
LRU cache bittorrent: 0/5976/0 (insert/search/found)
|
||||
LRU cache zoom: 0/0/0 (insert/search/found)
|
||||
LRU cache stun: 0/0/0 (insert/search/found)
|
||||
|
||||
Reference in New Issue
Block a user