Add a new flow risk NDPI_ANONYMOUS_SUBSCRIBER (#1462)
The main goal of a DPI engine is usually to determine "what", i.e. which types of traffic flow on the network. However the applications using DPI are often interested also in "who", i.e. which "user/subscriber" generated that traffic. The association between a flow and a subscriber is usually done via some kind of DHCP/GTP/RADIUS/NAT mappings. In all these cases the key element of the flow used to identify the user is the source ip address. That usually happens for the vast majority of the traffic. However, depending on the protocols involved and on the position on the net where the traffic is captured, the source ip address might have been changed/anonymized. In that case, that address is useless for any flow-username association. Example: iCloud Private Relay traffic captured between the exit relay and the server. See the picture at page 5 on: https://www.apple.com/privacy/docs/iCloud_Private_Relay_Overview_Dec2021.PDF This commit adds new generic flow risk `NDPI_ANONYMOUS_SUBSCRIBER` hinting that the ip addresses shouldn't be used to identify the user associated with the flow. As a first example of this new feature, the entire list of the relay ip addresses used by Private Relay is added. A key point to note is that list is NOT used for flow classification (unlike all the other ip lists present in nDPI) but only for setting this new flow risk. TODO: IPv6
This commit is contained in:
@@ -272,4 +272,9 @@ NDPI_HTTP_CRAWLER_BOT
|
||||
===================================
|
||||
The risk is set whenever a crawler/bot/robot has been detected
|
||||
|
||||
.. _Risk 045:
|
||||
|
||||
NDPI_ANONYMOUS_SUBSCRIBER
|
||||
===================================
|
||||
The risk is set whenever the (source) ip address has been anonymized and it can't be used to identify the subscriber.
|
||||
Example: the flow is generated by an iCloud-private-relay exit node.
|
||||
|
||||
@@ -336,6 +336,7 @@ typedef enum {
|
||||
NDPI_PUNYCODE_IDN,
|
||||
NDPI_ERROR_CODE_DETECTED,
|
||||
NDPI_HTTP_CRAWLER_BOT,
|
||||
NDPI_ANONYMOUS_SUBSCRIBER,
|
||||
|
||||
/* Leave this as last member */
|
||||
NDPI_MAX_RISK
|
||||
|
||||
@@ -117,7 +117,8 @@ typedef enum {
|
||||
NDPI_PUNYCODE_IDN, /* https://en.wikipedia.org/wiki/Punycode */
|
||||
NDPI_ERROR_CODE_DETECTED,
|
||||
NDPI_HTTP_CRAWLER_BOT,
|
||||
|
||||
NDPI_ANONYMOUS_SUBSCRIBER,
|
||||
|
||||
/* Leave this as last member */
|
||||
NDPI_MAX_RISK /* must be <= 63 due to (**) */
|
||||
} ndpi_risk_enum;
|
||||
@@ -1040,6 +1041,7 @@ struct ndpi_detection_module_struct {
|
||||
ndpi_list *trusted_issuer_dn;
|
||||
|
||||
void *ip_risk_mask_ptree;
|
||||
void *ip_risk_ptree;
|
||||
|
||||
struct {
|
||||
ndpi_automa hostnames, hostnames_shadow;
|
||||
@@ -1175,7 +1177,7 @@ struct ndpi_flow_struct {
|
||||
char host_server_name[80];
|
||||
|
||||
u_int8_t initial_binary_bytes[8], initial_binary_bytes_len;
|
||||
u_int8_t risk_checked:1, ip_risk_mask_evaluated:1, host_risk_mask_evaluated:1, _notused:5;
|
||||
u_int8_t risk_checked:1, ip_risk_mask_evaluated:1, host_risk_mask_evaluated:1, tree_risk_checked:1, _notused:4;
|
||||
ndpi_risk risk_mask; /* Stores the flow risk mask for flow peers */
|
||||
ndpi_risk risk; /* Issues found with this flow [bitmask of ndpi_risk] */
|
||||
|
||||
@@ -1392,6 +1394,8 @@ typedef enum
|
||||
ndpi_dont_load_google_list = (1 << 10),
|
||||
ndpi_dont_load_google_cloud_list = (1 << 11),
|
||||
ndpi_dont_load_asn_lists = (1 << 12),
|
||||
ndpi_dont_load_icloud_private_relay_list = (1 << 13),
|
||||
ndpi_dont_init_risk_ptree = (1 << 14),
|
||||
} ndpi_prefs;
|
||||
|
||||
typedef struct {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -63,6 +63,7 @@
|
||||
#include "ndpi_ms_skype_teams_match.c.inc"
|
||||
#include "ndpi_google_match.c.inc"
|
||||
#include "ndpi_google_cloud_match.c.inc"
|
||||
#include "ndpi_icloud_private_relay_match.c.inc"
|
||||
#include "ndpi_asn_telegram.c.inc"
|
||||
#include "ndpi_asn_apple.c.inc"
|
||||
#include "ndpi_asn_twitter.c.inc"
|
||||
@@ -138,6 +139,7 @@ static ndpi_risk_info ndpi_known_risks[] = {
|
||||
{ NDPI_PUNYCODE_IDN, NDPI_RISK_LOW, CLIENT_LOW_RISK_PERCENTAGE },
|
||||
{ NDPI_ERROR_CODE_DETECTED, NDPI_RISK_LOW, CLIENT_LOW_RISK_PERCENTAGE },
|
||||
{ NDPI_HTTP_CRAWLER_BOT, NDPI_RISK_LOW, CLIENT_LOW_RISK_PERCENTAGE },
|
||||
{ NDPI_ANONYMOUS_SUBSCRIBER, NDPI_RISK_MEDIUM, CLIENT_FAIR_RISK_PERCENTAGE },
|
||||
|
||||
/* Leave this as last member */
|
||||
{ NDPI_MAX_RISK, NDPI_RISK_LOW, CLIENT_FAIR_RISK_PERCENTAGE }
|
||||
@@ -2051,6 +2053,23 @@ u_int16_t ndpi_network_port_ptree_match(struct ndpi_detection_module_struct *ndp
|
||||
|
||||
/* ******************************************* */
|
||||
|
||||
ndpi_risk_enum ndpi_network_risk_ptree_match(struct ndpi_detection_module_struct *ndpi_str,
|
||||
struct in_addr *pin /* network byte order */) {
|
||||
ndpi_prefix_t prefix;
|
||||
ndpi_patricia_node_t *node;
|
||||
|
||||
/* Make sure all in network byte order otherwise compares wont work */
|
||||
ndpi_fill_prefix_v4(&prefix, pin, 32, ((ndpi_patricia_tree_t *) ndpi_str->ip_risk_ptree)->maxbits);
|
||||
node = ndpi_patricia_search_best(ndpi_str->ip_risk_ptree, &prefix);
|
||||
|
||||
if(node)
|
||||
return((ndpi_risk_enum)node->value.u.uv32.user_value);
|
||||
|
||||
return(NDPI_NO_RISK);
|
||||
}
|
||||
|
||||
/* ******************************************* */
|
||||
|
||||
#if 0
|
||||
static u_int8_t tor_ptree_match(struct ndpi_detection_module_struct *ndpi_str, struct in_addr *pin) {
|
||||
return((ndpi_network_ptree_match(ndpi_str, pin) == NDPI_PROTOCOL_TOR) ? 1 : 0);
|
||||
@@ -2151,6 +2170,10 @@ static void ndpi_init_ptree_ipv4(struct ndpi_detection_module_struct *ndpi_str,
|
||||
|
||||
pin.s_addr = htonl(host_list[i].network);
|
||||
if((node = add_to_ptree(ptree, AF_INET, &pin, host_list[i].cidr /* bits */)) != NULL) {
|
||||
/* Two main cases:
|
||||
1) ip -> protocol: uv32.user_value = protocol; uv32.additional_user_value = 0;
|
||||
2) ip -> risk: uv32.user_value = risk; uv32.additional_user_value = 0;
|
||||
*/
|
||||
node->value.u.uv32.user_value = host_list[i].value, node->value.u.uv32.additional_user_value = 0;
|
||||
}
|
||||
}
|
||||
@@ -2447,6 +2470,14 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
|
||||
|
||||
ndpi_str->ip_risk_mask_ptree = ndpi_patricia_new(32 /* IPv4 */);
|
||||
|
||||
if(!(prefs & ndpi_dont_init_risk_ptree)) {
|
||||
if((ndpi_str->ip_risk_ptree = ndpi_patricia_new(32 /* IPv4 */)) != NULL) {
|
||||
if(!(prefs & ndpi_dont_load_icloud_private_relay_list)) {
|
||||
ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_anonymous_subscriber_protocol_list);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
NDPI_BITMASK_RESET(ndpi_str->detection_bitmask);
|
||||
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
|
||||
ndpi_str->user_data = NULL;
|
||||
@@ -2850,6 +2881,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
if(ndpi_str->ip_risk_mask_ptree)
|
||||
ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->ip_risk_mask_ptree, free_ptree_data);
|
||||
|
||||
if(ndpi_str->ip_risk_ptree)
|
||||
ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->ip_risk_ptree, free_ptree_data);
|
||||
|
||||
if(ndpi_str->udpRoot != NULL)
|
||||
ndpi_tdestroy(ndpi_str->udpRoot, ndpi_free);
|
||||
if(ndpi_str->tcpRoot != NULL)
|
||||
@@ -5892,6 +5926,27 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
|
||||
|
||||
flow->risk_checked = 1;
|
||||
}
|
||||
if(!flow->tree_risk_checked) {
|
||||
if(ndpi_str->ip_risk_ptree) {
|
||||
/* TODO: ipv6 */
|
||||
if(packet->iph &&
|
||||
ndpi_is_public_ipv4(ntohl(packet->iph->saddr)) &&
|
||||
ndpi_is_public_ipv4(ntohl(packet->iph->daddr))) {
|
||||
struct in_addr addr;
|
||||
ndpi_risk_enum net_risk;
|
||||
|
||||
addr.s_addr = packet->iph->saddr;
|
||||
net_risk = ndpi_network_risk_ptree_match(ndpi_str, &addr);
|
||||
if(net_risk == NDPI_NO_RISK) {
|
||||
addr.s_addr = packet->iph->daddr;
|
||||
net_risk = ndpi_network_risk_ptree_match(ndpi_str, &addr);
|
||||
}
|
||||
if(net_risk != NDPI_NO_RISK)
|
||||
ndpi_set_risk(ndpi_str, flow, net_risk);
|
||||
}
|
||||
}
|
||||
flow->tree_risk_checked = 1;
|
||||
}
|
||||
|
||||
ndpi_reconcile_protocols(ndpi_str, flow, &ret);
|
||||
|
||||
|
||||
@@ -1895,11 +1895,15 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
|
||||
case NDPI_ERROR_CODE_DETECTED:
|
||||
return("Error Code Detected");
|
||||
break;
|
||||
|
||||
|
||||
case NDPI_HTTP_CRAWLER_BOT:
|
||||
return("Crawler/Bot Detected");
|
||||
break;
|
||||
|
||||
|
||||
case NDPI_ANONYMOUS_SUBSCRIBER:
|
||||
return("Anonymous Subscriber");
|
||||
break;
|
||||
|
||||
default:
|
||||
snprintf(buf, sizeof(buf), "%d", (int)risk);
|
||||
return(buf);
|
||||
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")" || return
|
||||
|
||||
DEST=../src/lib/ndpi_icloud_private_relay_match.c.inc
|
||||
TMP=/tmp/icloud.csv
|
||||
LIST=/tmp/icloud.list
|
||||
LIST_MERGED=/tmp/icloud.list_m
|
||||
ORIGIN="https://mask-api.icloud.com/egress-ip-ranges.csv"
|
||||
|
||||
|
||||
echo "(1) Downloading file..."
|
||||
http_response=$(curl -s -o "$TMP" -w "%{http_code}" ${ORIGIN})
|
||||
if [ "$http_response" != "200" ]; then
|
||||
echo "Error $http_response: you probably need to update the list url!"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "(2) Processing IP addresses..."
|
||||
|
||||
# Note: the "grep -v :" is used to skip IPv6 addresses
|
||||
cut -d ',' -f 1 $TMP | grep -v ':' > $LIST
|
||||
./mergeipaddrlist.py $LIST > $LIST_MERGED
|
||||
./ipaddr2list.py $LIST_MERGED NDPI_ANONYMOUS_SUBSCRIBER > $DEST
|
||||
rm -f $TMP $LIST $LIST_MERGED
|
||||
|
||||
echo "(3) iCloud Private Relay IPs are available in $DEST"
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
import sys
|
||||
import socket, struct
|
||||
|
||||
# This scripts is mainly used to create "ip -> protocols" lists.
|
||||
# However it is also used to create "ip -> risk" lists
|
||||
proto = "NDPI_PROTOCOL_XYX"
|
||||
if len (sys.argv) < 2 :
|
||||
print("Usage: ipaddr2list.py <file> <protocol>")
|
||||
|
||||
+2
-1
@@ -83,9 +83,10 @@ flow_risks[41] = ProtoField.bool("ndpi.flow_risk.cert_about_to_expire", "TLS cer
|
||||
flow_risks[42] = ProtoField.bool("ndpi.flow_risk.punycode_idn", "IDN Domain Name", num_bits_flow_risks, nil, bit(10), "nDPI Flow Risk: IDN Domain Name")
|
||||
flow_risks[43] = ProtoField.bool("ndpi.flow_risk.error_code_detected", "Error Code Detected", num_bits_flow_risks, nil, bit(11), "nDPI Flow Risk: Error Code Detected")
|
||||
flow_risks[44] = ProtoField.bool("ndpi.flow_risk.crawler_bot", "Crawler/Bot Detected", num_bits_flow_risks, nil, bit(12), "nDPI Flow Risk: Crawler/Bot Detected")
|
||||
flow_risks[45] = ProtoField.bool("ndpi.flow_risk.anonymous_subscriber", "Anonymous Subscriber", num_bits_flow_risks, nil, bit(13), "nDPI Flow Risk: Anonymous Subscriber")
|
||||
|
||||
-- Last one: keep in sync the bitmask when adding new risks!!
|
||||
flow_risks[64] = ProtoField.new("Unused", "ndpi.flow_risk.unused", ftypes.UINT32, nil, base.HEX, bit(32) - bit(10))
|
||||
flow_risks[64] = ProtoField.new("Unused", "ndpi.flow_risk.unused", ftypes.UINT32, nil, base.HEX, bit(32) - bit(13))
|
||||
|
||||
for _,v in pairs(flow_risks) do
|
||||
ndpi_fds[#ndpi_fds + 1] = v
|
||||
|
||||
Reference in New Issue
Block a user