Changes for supporinng more efficient sub-string matching

This commit is contained in:
Luca Deri
2023-08-26 17:55:50 +02:00
parent 4ca94369e1
commit eeeee46b1e
4 changed files with 67 additions and 33866 deletions
+12 -7
View File
@@ -31,6 +31,8 @@
/* Used by both nDPI core and patricia code under third-party */
#include "ndpi_patricia_typedefs.h"
#define USE_LEGACY_AHO_CORASICK
#ifndef NDPI_CFFI_PREPROCESSING
#ifndef u_char
typedef unsigned char u_char;
@@ -1170,6 +1172,8 @@ typedef struct ndpi_proto {
#define NUM_CUSTOM_CATEGORIES 5
#define CUSTOM_CATEGORY_LABEL_LEN 32
typedef void ndpi_domain_classify;
#ifdef NDPI_LIB_COMPILATION
/* Needed to have access to HAVE_* defines */
@@ -1200,7 +1204,7 @@ struct ndpi_detection_module_struct {
u_int64_t current_ts;
u_int16_t max_packets_to_process;
u_int16_t num_tls_blocks_to_follow;
u_int8_t skip_tls_blocks_until_change_cipher:1, enable_ja3_plus:1, enable_load_gambling_list:1, _notused:5;
u_int8_t skip_tls_blocks_until_change_cipher:1, enable_ja3_plus:1, _notused:6;
u_int8_t tls_certificate_expire_in_x_days;
void *user_data;
@@ -1262,7 +1266,11 @@ struct ndpi_detection_module_struct {
/* *** If you add a new Patricia tree, please update ptree_type above! *** */
struct {
#ifdef USE_LEGACY_AHO_CORASICK
ndpi_automa hostnames, hostnames_shadow;
#else
ndpi_domain_classify *sc_hostnames, *sc_hostnames_shadow;
#endif
void *ipAddresses, *ipAddresses_shadow; /* Patricia */
u_int8_t categories_loaded;
} custom_categories;
@@ -1761,14 +1769,13 @@ typedef enum {
ndpi_enable_tcp_ack_payload_heuristic = (1 << 17),
ndpi_dont_load_crawlers_list = (1 << 18),
ndpi_dont_load_protonvpn_list = (1 << 19),
ndpi_dont_load_gambling_list = (1 << 20),
/* Heuristic to detect fully encrypted sessions, i.e. flows where every bytes of
the payload is encrypted in an attempt to “look like nothing”.
This heuristic only analyzes the first packet of the flow.
See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf */
ndpi_disable_fully_encrypted_heuristic = (1 << 21),
ndpi_dont_load_protonvpn_exit_nodes_list = (1 << 22),
ndpi_dont_load_mullvad_list = (1 << 23),
ndpi_disable_fully_encrypted_heuristic = (1 << 20),
ndpi_dont_load_protonvpn_exit_nodes_list = (1 << 21),
ndpi_dont_load_mullvad_list = (1 << 22),
} ndpi_prefs;
typedef struct {
@@ -2028,8 +2035,6 @@ typedef struct {
#define MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS 6
typedef void ndpi_domain_classify;
/* **************************************** */
#endif /* __NDPI_TYPEDEFS_H__ */
File diff suppressed because it is too large Load Diff
+52 -34
View File
@@ -71,7 +71,6 @@
#include "inc_generated/ndpi_ms_onedrive_match.c.inc"
#include "inc_generated/ndpi_ms_outlook_match.c.inc"
#include "inc_generated/ndpi_ms_skype_teams_match.c.inc"
#include "inc_generated/ndpi_gambling_match.c.inc"
#include "inc_generated/ndpi_google_match.c.inc"
#include "inc_generated/ndpi_google_cloud_match.c.inc"
#include "inc_generated/ndpi_crawlers_match.c.inc"
@@ -969,22 +968,9 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
for(i = 0; host_match[i].string_to_match != NULL; i++)
ndpi_init_protocol_match(ndpi_str, &host_match[i]);
if(ndpi_str->enable_load_gambling_list)
for(i = 0; ndpi_protocol_gambling_hostname_list[i].string_to_match != NULL; i++)
ndpi_init_protocol_match(ndpi_str, &ndpi_protocol_gambling_hostname_list[i]);
else {
ndpi_protocol_match gambling_match;
if (ndpi_init_empty_app_protocol(ndpi_protocol_gambling_hostname_list, &gambling_match) != 0 ||
ndpi_init_app_protocol(ndpi_str, &gambling_match) != 0) {
NDPI_LOG_ERR(ndpi_str,
"[NDPI] INTERNAL ERROR could not initialize empty gambling app protocol\n");
}
}
/* ************************ */
for(i = 0; tls_certificate_match[i].string_to_match != NULL; i++) {
#if 0
printf("%s() %s / %u\n", __FUNCTION__,
tls_certificate_match[i].string_to_match,
@@ -2136,6 +2122,10 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
"BITCOIN", NDPI_PROTOCOL_CATEGORY_CRYPTO_CURRENCY,
ndpi_build_default_ports(ports_a, 8333, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 1 /* app proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_GAMBLING,
"Gambling", NDPI_PROTOCOL_CATEGORY_WEB,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 1 /* app proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_PROTONVPN,
"ProtonVPN", NDPI_PROTOCOL_CATEGORY_VPN,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
@@ -2820,16 +2810,12 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
set_ndpi_debug_function(ndpi_str, (ndpi_debug_function_ptr) ndpi_debug_printf);
NDPI_BITMASK_RESET(ndpi_str->debug_bitmask);
NDPI_BITMASK_RESET(ndpi_str->debug_!bitmask);
#endif /* NDPI_ENABLE_DEBUG_MESSAGES */
if(prefs & ndpi_enable_ja3_plus)
ndpi_str->enable_ja3_plus = 1;
ndpi_str->enable_load_gambling_list = 1;
if(prefs & ndpi_dont_load_gambling_list)
ndpi_str->enable_load_gambling_list = 0;
if(!(prefs & ndpi_dont_init_libgcrypt)) {
if(!gcry_control (GCRYCTL_INITIALIZATION_FINISHED_P)) {
const char *gcrypt_ver = gcry_check_version(NULL);
@@ -2988,29 +2974,41 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->risky_domain_automa.ac_automa = NULL; /* Initialized on demand */
ndpi_str->trusted_issuer_dn = NULL;
#ifdef USE_LEGACY_AHO_CORASICK
ndpi_str->custom_categories.hostnames.ac_automa = ac_automata_init(ac_domain_match_handler);
if(!ndpi_str->custom_categories.hostnames.ac_automa) {
ndpi_exit_detection_module(ndpi_str);
return(NULL);
}
ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_domain_match_handler);
if(!ndpi_str->custom_categories.hostnames_shadow.ac_automa) {
ndpi_exit_detection_module(ndpi_str);
return(NULL);
}
if(ndpi_str->custom_categories.hostnames.ac_automa)
ac_automata_feature(ndpi_str->custom_categories.hostnames.ac_automa, AC_FEATURE_LC);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa, AC_FEATURE_LC);
if(ndpi_str->custom_categories.hostnames.ac_automa)
ac_automata_name(ndpi_str->custom_categories.hostnames.ac_automa, "ccat", 0);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
ac_automata_name(ndpi_str->custom_categories.hostnames_shadow.ac_automa, "ccat_sh", 0);
#else
ndpi_str->custom_categories.sc_hostnames = ndpi_domain_classify_alloc();
ndpi_str->custom_categories.sc_hostnames_shadow = ndpi_domain_classify_alloc();
#endif
ndpi_str->custom_categories.ipAddresses = ndpi_patricia_new(32 /* IPv4 */);
ndpi_str->custom_categories.ipAddresses_shadow = ndpi_patricia_new(32 /* IPv4 */);
if(ndpi_str->host_automa.ac_automa)
ac_automata_feature(ndpi_str->host_automa.ac_automa,AC_FEATURE_LC);
if(ndpi_str->custom_categories.hostnames.ac_automa)
ac_automata_feature(ndpi_str->custom_categories.hostnames.ac_automa,AC_FEATURE_LC);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa,AC_FEATURE_LC);
if(ndpi_str->tls_cert_subject_automa.ac_automa)
ac_automata_feature(ndpi_str->tls_cert_subject_automa.ac_automa,AC_FEATURE_LC);
@@ -3025,12 +3023,6 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
if(ndpi_str->host_automa.ac_automa)
ac_automata_name(ndpi_str->host_automa.ac_automa,"host",AC_FEATURE_DEBUG);
if(ndpi_str->custom_categories.hostnames.ac_automa)
ac_automata_name(ndpi_str->custom_categories.hostnames.ac_automa,"ccat",0);
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa)
ac_automata_name(ndpi_str->custom_categories.hostnames_shadow.ac_automa,"ccat_sh",0);
if(ndpi_str->tls_cert_subject_automa.ac_automa)
ac_automata_name(ndpi_str->tls_cert_subject_automa.ac_automa,"tls_cert",AC_FEATURE_DEBUG);
@@ -3401,11 +3393,22 @@ int ndpi_match_string_value(void *automa, char *string_to_match,
int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_str,
char *name, u_int name_len,
ndpi_protocol_category_t *category) {
#ifdef USE_LEGACY_AHO_CORASICK
u_int32_t id;
int rc = ndpi_match_string_common(ndpi_str->custom_categories.hostnames.ac_automa,
name, name_len, &id, category, NULL);
if(rc < 0) return rc;
return(id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1);
#else
u_int16_t rc = ndpi_domain_classify_contains(ndpi_str->custom_categories.sc_hostnames, name);
if(rc == 0)
return(-1); /* Not found */
else {
*category = (ndpi_protocol_category_t)rc;
return(0);
}
#endif
}
/* *********************************************** */
@@ -3541,6 +3544,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
if(ndpi_str->malicious_sha1_hashmap != NULL)
ndpi_hash_free(&ndpi_str->malicious_sha1_hashmap, NULL);
#ifdef USE_LEGACY_AHO_CORASICK
if(ndpi_str->custom_categories.hostnames.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->custom_categories.hostnames.ac_automa,
1 /* free patterns strings memory */);
@@ -3548,7 +3552,11 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->custom_categories.hostnames_shadow.ac_automa,
1 /* free patterns strings memory */);
#else
ndpi_domain_classify_free(ndpi_str->custom_categories.sc_hostnames_shadow);
ndpi_domain_classify_free(ndpi_str->custom_categories.sc_hostnames);
#endif
if(ndpi_str->custom_categories.ipAddresses != NULL)
ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses, free_ptree_data);
@@ -6746,7 +6754,7 @@ int ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_str,
int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_str,
const char *name_to_add,
ndpi_protocol_category_t category) {
#ifdef USE_LEGACY_AHO_CORASICK
if(ndpi_str->custom_categories.hostnames_shadow.ac_automa == NULL)
return(-1);
@@ -6756,6 +6764,10 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_str,
return ndpi_string_to_automa(ndpi_str,
(AC_AUTOMATA_t *)ndpi_str->custom_categories.hostnames_shadow.ac_automa,
name_to_add,category,category, 0, 0, 1); /* at_end */
#else
return(ndpi_domain_classify_add(ndpi_str->custom_categories.sc_hostnames_shadow,
(u_int16_t)category, (char*)name_to_add) ? 0 : -1);
#endif
}
/* ********************************************************************************* */
@@ -6793,6 +6805,7 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
ndpi_load_category(ndpi_str, category_match[i].string_to_match,
category_match[i].protocol_category, built_in);
#ifdef USE_LEGACY_AHO_CORASICK
/* Free */
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->custom_categories.hostnames.ac_automa,
1 /* free patterns strings memory */);
@@ -6810,7 +6823,12 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
ac_automata_feature(ndpi_str->custom_categories.hostnames_shadow.ac_automa,AC_FEATURE_LC);
ac_automata_name(ndpi_str->custom_categories.hostnames_shadow.ac_automa,"ccat_sh",0);
}
#else
ndpi_domain_classify_free(ndpi_str->custom_categories.sc_hostnames);
ndpi_str->custom_categories.sc_hostnames = ndpi_str->custom_categories.sc_hostnames_shadow;
ndpi_str->custom_categories.sc_hostnames_shadow = ndpi_domain_classify_alloc();
#endif
if(ndpi_str->custom_categories.ipAddresses != NULL)
ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses, free_ptree_data);
+3 -7
View File
@@ -5,21 +5,17 @@ set -e
cd "$(dirname "${0}")" || exit 1
. ./common.sh || exit 1
DEST=../src/lib/inc_generated/ndpi_gambling_match.c.inc
LIST=/tmp/gambling.list
LIST=../lists/gambling.list
printf '(1) %s\n' "Scraping Illegal Gambling Sites (Belgium)"
DOMAINS="$(curl -s 'https://www.gamingcommission.be/en/gaming-commission/illegal-games-of-chance/list-of-illegal-gambling-sites' | sed -n 's/^<td[^>]\+>\(.\+\.[a-zA-Z0-9]\+\)\(\|\/.*[^<]*\)<\/td>/\1/gp' || exit 1)"
is_str_empty "${DOMAINS}" "Please check gambling sites URL and sed REGEX."
printf '(1) %s\n' "Downloading Gambling Sites (Poland)"
printf '(2) %s\n' "Downloading Gambling Sites (Poland)"
DOMAINS_PL="$(curl -s https://hazard.mf.gov.pl/api/Register | xmllint --xpath "/*[local-name(.)='Rejestr']/*[local-name(.)='PozycjaRejestru']/*[local-name(.)='AdresDomeny']/text()" -)"
is_str_empty "${DOMAINS_PL}" "Please check gambling sites URL and XPath."
printf '(2) %s\n' "Processing IP addresses..."
echo "${DOMAINS}" "${DOMAINS_PL}" | sort | uniq >${LIST}
./hostname2list.py "${LIST}" "Gambling" NDPI_PROTOCOL_GAMBLING NDPI_PROTOCOL_CATEGORY_WEB NDPI_PROTOCOL_UNSAFE >${DEST}
rm -f "${LIST}"
is_file_empty "${DEST}"
printf '(3) %s\n' "List ${LIST} is now ready"
exit 0