Added new API calls

- ndpi_load_domain_suffixes()
- ndpi_get_host_domain_suffix()

whose goal is to find the domain name of a hostname. Example:

www.bbc.co.uk   -> co.uk
mail.apple.com  -> com
This commit is contained in:
Luca
2024-01-15 19:03:46 +01:00
parent 61a18623e7
commit 162c38f18f
9 changed files with 15639 additions and 40 deletions
+24 -1
View File
@@ -5633,6 +5633,29 @@ void outlierUnitTest() {
/* *********************************************** */ /* *********************************************** */
void domainsUnitTest() {
NDPI_PROTOCOL_BITMASK all;
struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module(init_prefs);
const char *lists_path = "../lists/public_suffix_list.dat";
struct stat st;
if(stat(lists_path, &st) == 0) {
NDPI_BITMASK_SET_ALL(all);
ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all);
assert(ndpi_load_domain_suffixes(ndpi_info_mod, "../lists/public_suffix_list.dat") == 0);
assert(strcmp(ndpi_get_host_domain_suffix(ndpi_info_mod, "www.chosei.chiba.jp"), "chosei.chiba.jp") == 0);
assert(strcmp(ndpi_get_host_domain_suffix(ndpi_info_mod, "www.unipi.it"), "it") == 0);
assert(strcmp(ndpi_get_host_domain_suffix(ndpi_info_mod, "mail.apple.com"), "com") == 0);
assert(strcmp(ndpi_get_host_domain_suffix(ndpi_info_mod, "www.bbc.co.uk"), "co.uk") == 0);
ndpi_exit_detection_module(ndpi_info_mod);
}
}
/* *********************************************** */
void domainSearchUnitTest() { void domainSearchUnitTest() {
ndpi_domain_classify *sc = ndpi_domain_classify_alloc(); ndpi_domain_classify *sc = ndpi_domain_classify_alloc();
char *domain = "ntop.org"; char *domain = "ntop.org";
@@ -5702,7 +5725,6 @@ int main(int argc, char **argv) {
} }
#endif #endif
if(ndpi_get_api_version() != NDPI_API_VERSION) { if(ndpi_get_api_version() != NDPI_API_VERSION) {
printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n"); printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
return(-1); return(-1);
@@ -5721,6 +5743,7 @@ int main(int argc, char **argv) {
exit(0); exit(0);
#endif #endif
domainsUnitTest();
outlierUnitTest(); outlierUnitTest();
pearsonUnitTest(); pearsonUnitTest();
binaryBitmapUnitTest(); binaryBitmapUnitTest();
+10 -1
View File
@@ -8,4 +8,13 @@ File Name Format
---------------- ----------------
The file name format is <category id>_<string>.list The file name format is <category id>_<string>.list
The category id list can be found in src/include/ndpi_typedefs.h The category id list can be found in src/include/ndpi_typedefs.h
Domain Suffix List
------------------
public_suffix_list.dat can be downloaded from
https://publicsuffix.org/list/public_suffix_list.dat and
it contains the list of domain suffixes
File diff suppressed because it is too large Load Diff
+35
View File
@@ -2147,6 +2147,9 @@ extern "C" {
u_int8_t class_id, u_int8_t class_id,
char *file_path); char *file_path);
bool ndpi_domain_classify_finalize(ndpi_domain_classify *s); bool ndpi_domain_classify_finalize(ndpi_domain_classify *s);
const char* ndpi_domain_classify_longest_prefix(ndpi_domain_classify *s,
u_int8_t *class_id /* out */,
const char *hostnname);
bool ndpi_domain_classify_contains(ndpi_domain_classify *s, bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
u_int8_t *class_id /* out */, u_int8_t *class_id /* out */,
const char *domain); const char *domain);
@@ -2188,6 +2191,8 @@ extern "C" {
/** /**
* Get user data which was previously set with `ndpi_set_user_data()`. * Get user data which was previously set with `ndpi_set_user_data()`.
* *
* @par ndpi_str = the struct created for the protocol detection
*
* @return the user data pointer * @return the user data pointer
* *
*/ */
@@ -2195,6 +2200,36 @@ extern "C" {
/* ******************************* */ /* ******************************* */
/**
* Loads the domain suffixes from the specified path. You need to
* perform this action once
*
* @par ndpi_str = the struct created for the protocol detection
* @par public_suffix_list_path = path of the public_suffix_list path
*
* @return 0 = no error, -1 otherwise
*
*/
int ndpi_load_domain_suffixes(struct ndpi_detection_module_struct *ndpi_str,
char *public_suffix_list_path);
/**
* Returns the domain suffix out of the specified hostname.
* The returned pointer is an offset of the original hostname.
* Note that you need to call ndpi_load_domain_suffixes() before
* calling this function.
*
* @par ndpi_str = the struct created for the protocol detection
* @par hostname = the hostname from which the domain name has to be extracted
*
* @return The host domain name or the hostitself if not found.
*
*/
const char* ndpi_get_host_domain_suffix(struct ndpi_detection_module_struct *ndpi_str,
const char *hostname);
/* ******************************* */
/* Can't call libc functions from kernel space, define some stub instead */ /* Can't call libc functions from kernel space, define some stub instead */
#define ndpi_isalpha(ch) (((ch) >= 'a' && (ch) <= 'z') || ((ch) >= 'A' && (ch) <= 'Z')) #define ndpi_isalpha(ch) (((ch) >= 'a' && (ch) <= 'z') || ((ch) >= 'A' && (ch) <= 'Z'))
+7 -5
View File
@@ -152,7 +152,7 @@ struct ndpi_detection_module_struct {
u_int16_t num_tls_blocks_to_follow; u_int16_t num_tls_blocks_to_follow;
u_int8_t skip_tls_blocks_until_change_cipher:1, _notused:7; u_int8_t skip_tls_blocks_until_change_cipher:1, _notused:7;
u_int8_t tls_certificate_expire_in_x_days; u_int8_t tls_certificate_expire_in_x_days;
void *user_data; void *user_data;
char custom_category_labels[NUM_CUSTOM_CATEGORIES][CUSTOM_CATEGORY_LABEL_LEN]; char custom_category_labels[NUM_CUSTOM_CATEGORIES][CUSTOM_CATEGORY_LABEL_LEN];
@@ -206,11 +206,11 @@ struct ndpi_detection_module_struct {
/* Patricia trees */ /* Patricia trees */
ndpi_patricia_tree_t *ip_risk_mask_ptree; ndpi_patricia_tree_t *ip_risk_mask_ptree;
ndpi_patricia_tree_t *ip_risk_mask_ptree6; ndpi_patricia_tree_t *ip_risk_mask_ptree6;
ndpi_patricia_tree_t *ip_risk_ptree; ndpi_patricia_tree_t *ip_risk_ptree;
ndpi_patricia_tree_t *ip_risk_ptree6; ndpi_patricia_tree_t *ip_risk_ptree6;
ndpi_patricia_tree_t *protocols_ptree; /* IP-based protocol detection */ ndpi_patricia_tree_t *protocols_ptree; /* IP-based protocol detection */
ndpi_patricia_tree_t *protocols_ptree6; ndpi_patricia_tree_t *protocols_ptree6;
/* *** If you add a new Patricia tree, please update ptree_type above! *** */ /* *** If you add a new Patricia tree, please update ptree_type above! *** */
struct { struct {
@@ -256,7 +256,7 @@ struct ndpi_detection_module_struct {
struct ndpi_lru_cache *tls_cert_cache; struct ndpi_lru_cache *tls_cert_cache;
u_int32_t tls_cert_cache_num_entries; u_int32_t tls_cert_cache_num_entries;
int32_t tls_cert_cache_ttl; int32_t tls_cert_cache_ttl;
/* NDPI_PROTOCOL_MINING and subprotocols */ /* NDPI_PROTOCOL_MINING and subprotocols */
struct ndpi_lru_cache *mining_cache; struct ndpi_lru_cache *mining_cache;
u_int32_t mining_cache_num_entries; u_int32_t mining_cache_num_entries;
@@ -302,7 +302,9 @@ struct ndpi_detection_module_struct {
nbpf_filter nbpf_custom_proto[MAX_NBPF_CUSTOM_PROTO]; nbpf_filter nbpf_custom_proto[MAX_NBPF_CUSTOM_PROTO];
#endif #endif
u_int16_t max_payload_track_len; u_int16_t max_payload_track_len;
ndpi_domain_classify *public_domain_suffixes;
}; };
+43 -31
View File
@@ -89,7 +89,7 @@ bool ndpi_domain_classify_add(ndpi_domain_classify *s,
u_int32_t i; u_int32_t i;
char *dot; char *dot;
if(!s || !domain) if((!s) || (!domain))
return(false); return(false);
/* Skip initial string . in domain names */ /* Skip initial string . in domain names */
@@ -97,18 +97,21 @@ bool ndpi_domain_classify_add(ndpi_domain_classify *s,
dot = strrchr(domain, '.'); dot = strrchr(domain, '.');
if(!dot) return(false); if(dot) {
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local"))) if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local")))
return(false); return(false);
}
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) { for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
if(s->classes[i].class_id == class_id) { if(s->classes[i].class_id == class_id) {
break; break;
} else if(s->classes[i].class_id == 0) { } else if(s->classes[i].class_id == 0) {
s->classes[i].class_id = class_id; s->classes[i].class_id = class_id;
s->classes[i].domains = ndpi_bitmap64_alloc(); s->classes[i].domains = ndpi_bitmap64_alloc();
if(!s->classes[i].domains) if(!s->classes[i].domains)
s->classes[i].class_id = 0; s->classes[i].class_id = 0;
break; break;
} }
} }
@@ -130,7 +133,7 @@ u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
FILE *fd; FILE *fd;
char *line; char *line;
if(!s || !file_path) if((!s) || (!file_path))
return(false); return(false);
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) { for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
@@ -199,9 +202,9 @@ bool ndpi_domain_classify_finalize(ndpi_domain_classify *s) {
/* ********************************************************** */ /* ********************************************************** */
static bool is_valid_domain_char(u_char c) { static bool is_valid_domain_char(u_char c) {
if(((c >= 'A')&& (c <= 'Z')) if(((c >= 'A') && (c <= 'Z'))
|| ((c >= 'a')&& (c <= 'z')) || ((c >= 'a') && (c <= 'z'))
|| ((c >= '0')&& (c <= '9')) || ((c >= '0') && (c <= '9'))
|| (c == '_') || (c == '_')
|| (c == '-') || (c == '-')
|| (c == '.')) || (c == '.'))
@@ -212,35 +215,37 @@ static bool is_valid_domain_char(u_char c) {
/* ********************************************************** */ /* ********************************************************** */
bool ndpi_domain_classify_contains(ndpi_domain_classify *s, const char* ndpi_domain_classify_longest_prefix(ndpi_domain_classify *s,
u_int8_t *class_id /* out */, u_int8_t *class_id /* out */,
const char *domain) { const char *hostname) {
u_int32_t i, len; u_int32_t i, len;
const char *dot, *elem; const char *dot, *elem;
if(!domain || !s) return(false); *class_id = 0; /* Unknown class_id */
if((len = strlen(domain)) == 0) return(false);
if((dot = strrchr(domain, '.')) == NULL) return(false); if(!hostname || !s) return(hostname);
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local"))) return(false); if((len = strlen(hostname)) == 0) return(hostname);
if((dot = strrchr(hostname, '.')) == NULL) return(hostname);
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local"))) return(hostname);
/* This is a number or a numeric IP or similar */ /* This is a number or a numeric IP or similar */
if(ndpi_isdigit(domain[len-1]) && isdigit(domain[0])) { if(ndpi_isdigit(hostname[len-1]) && isdigit(hostname[0])) {
#ifdef DEBUG_CONTAINS #ifdef DEBUG_CONTAINS
printf("[contains] %s INVALID\n", domain); printf("[contains] %s INVALID\n", hostname);
#endif #endif
return(false); return(hostname);
} }
if(!is_valid_domain_char(domain[0])) { if(!is_valid_domain_char(hostname[0])) {
#ifdef DEBUG_CONTAINS #ifdef DEBUG_CONTAINS
printf("[contains] %s INVALID\n", domain); printf("[contains] %s INVALID\n", hostname);
#endif #endif
return(false); return(hostname);
} }
elem = domain; elem = hostname;
while(elem != NULL) { while(elem != NULL) {
u_int64_t hash = ndpi_quick_hash64(elem, strlen(elem)); u_int64_t hash = ndpi_quick_hash64(elem, strlen(elem));
@@ -249,10 +254,10 @@ bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
if(s->classes[i].class_id != 0) { if(s->classes[i].class_id != 0) {
if(ndpi_bitmap64_isset(s->classes[i].domains, hash)) { if(ndpi_bitmap64_isset(s->classes[i].domains, hash)) {
#ifdef DEBUG_CONTAINS #ifdef DEBUG_CONTAINS
printf("[contains] %s = %d\n", domain, s->classes[i].class_id); printf("[contains] %s = %d\n", hostname, s->classes[i].class_id);
#endif #endif
*class_id = s->classes[i].class_id; *class_id = s->classes[i].class_id;
return(true); return(elem);
} }
} else } else
break; break;
@@ -260,16 +265,23 @@ bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
elem = strchr(elem, '.'); elem = strchr(elem, '.');
if((elem == NULL) || (elem == dot)) if((elem == NULL) /* || (elem == dot) */)
break; break;
else else
elem = &elem[1]; elem = &elem[1];
} /* while */ } /* while */
#ifdef DEBUG_CONTAINS
printf("[contains] %s NOT FOUND\n", domain);
#endif
return(false); /* Not found */
return(hostname);
}
/* ********************************************************** */
bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
u_int8_t *class_id /* out */,
const char *domain) {
(void)ndpi_domain_classify_longest_prefix(s, class_id, domain); /* UNUSED */
return((*class_id == 0) ? false : true);
} }
+93
View File
@@ -0,0 +1,93 @@
/*
* ndpi_domains.c
*
* Copyright (C) 2011-24 - ntop.org and contributors
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "ndpi_config.h"
#include "ndpi_api.h"
#include "ndpi_includes.h"
#include "ndpi_private.h"
/* ******************************* */
int ndpi_load_domain_suffixes(struct ndpi_detection_module_struct *ndpi_str,
char *public_suffix_list_path) {
char buf[256], *line;
FILE *fd;
bool do_trace = false;
u_int num_domains = 0;
if(public_suffix_list_path == NULL)
return(-1);
if((fd = fopen(public_suffix_list_path, "r")) == NULL)
return(-2);
if(ndpi_str->public_domain_suffixes != NULL) {
/* An existing license was aleady loaded: free it and start over */
ndpi_domain_classify_free(ndpi_str->public_domain_suffixes);
}
if((ndpi_str->public_domain_suffixes = ndpi_domain_classify_alloc()) == NULL)
return(-3);
while((line = fgets(buf, sizeof(buf), fd)) != NULL) {
u_int offset, len;
/* Skip empty lines or comments */
if((line[0] == '\0') || (line[0] == '/') || (line[0] == '\n') || (line[0] == '\r'))
continue;
if((line[0] == '*') && (line[1] == '.') && (line[2] != '\0'))
offset = 2;
else
offset = 0;
len = strlen(line) - 1;
while((len > 0) && (line[len] == '\n'))
line[len--] = '\0';
if(!ndpi_domain_classify_add(ndpi_str->public_domain_suffixes,
1 /* dummy */, &line[offset])) {
if(do_trace) printf("Error while processing domain %s\n", &line[offset]);
} else
num_domains++;
}
if(!ndpi_domain_classify_finalize(ndpi_str->public_domain_suffixes)) {
if(do_trace) printf("Error while finalizing domain processing\n");
}
if(do_trace) printf("Loaded %u domains\n", num_domains);
return(0);
}
/* ******************************* */
const char* ndpi_get_host_domain_suffix(struct ndpi_detection_module_struct *ndpi_str,
const char *hostname) {
if(ndpi_str->public_domain_suffixes == NULL)
return(hostname);
else {
u_int8_t class_id;
return(ndpi_domain_classify_longest_prefix(ndpi_str->public_domain_suffixes,
&class_id, hostname));
}
}
+7 -2
View File
@@ -4000,9 +4000,14 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
ndpi_free_geoip(ndpi_str); ndpi_free_geoip(ndpi_str);
if(ndpi_str->callback_buffer) if(ndpi_str->callback_buffer)
ndpi_free(ndpi_str->callback_buffer); ndpi_free(ndpi_str->callback_buffer);
if(ndpi_str->callback_buffer_tcp_payload) if(ndpi_str->callback_buffer_tcp_payload)
ndpi_free(ndpi_str->callback_buffer_tcp_payload); ndpi_free(ndpi_str->callback_buffer_tcp_payload);
if(ndpi_str->public_domain_suffixes)
ndpi_domain_classify_free(ndpi_str->public_domain_suffixes);
ndpi_free(ndpi_str); ndpi_free(ndpi_str);
} }
+1
View File
@@ -132,6 +132,7 @@
<ClCompile Include="..\src\lib\ndpi_memory.c" /> <ClCompile Include="..\src\lib\ndpi_memory.c" />
<ClCompile Include="..\src\lib\ndpi_serializer.c" /> <ClCompile Include="..\src\lib\ndpi_serializer.c" />
<ClCompile Include="..\src\lib\ndpi_utils.c" /> <ClCompile Include="..\src\lib\ndpi_utils.c" />
<ClCompile Include="..\src\lib\ndpi_domains.c" />
<ClCompile Include="..\src\lib\ndpi_binary_bitmap.c" /> <ClCompile Include="..\src\lib\ndpi_binary_bitmap.c" />
<ClCompile Include="..\src\lib\ndpi_hash.c" /> <ClCompile Include="..\src\lib\ndpi_hash.c" />
<ClCompile Include="..\src\lib\ndpi_domain_classify.c" /> <ClCompile Include="..\src\lib\ndpi_domain_classify.c" />