Added new API calls
- ndpi_load_domain_suffixes() - ndpi_get_host_domain_suffix() whose goal is to find the domain name of a hostname. Example: www.bbc.co.uk -> co.uk mail.apple.com -> com
This commit is contained in:
+24
-1
@@ -5633,6 +5633,29 @@ void outlierUnitTest() {
|
||||
|
||||
/* *********************************************** */
|
||||
|
||||
void domainsUnitTest() {
|
||||
NDPI_PROTOCOL_BITMASK all;
|
||||
struct ndpi_detection_module_struct *ndpi_info_mod = ndpi_init_detection_module(init_prefs);
|
||||
const char *lists_path = "../lists/public_suffix_list.dat";
|
||||
struct stat st;
|
||||
|
||||
if(stat(lists_path, &st) == 0) {
|
||||
NDPI_BITMASK_SET_ALL(all);
|
||||
ndpi_set_protocol_detection_bitmask2(ndpi_info_mod, &all);
|
||||
|
||||
assert(ndpi_load_domain_suffixes(ndpi_info_mod, "../lists/public_suffix_list.dat") == 0);
|
||||
|
||||
assert(strcmp(ndpi_get_host_domain_suffix(ndpi_info_mod, "www.chosei.chiba.jp"), "chosei.chiba.jp") == 0);
|
||||
assert(strcmp(ndpi_get_host_domain_suffix(ndpi_info_mod, "www.unipi.it"), "it") == 0);
|
||||
assert(strcmp(ndpi_get_host_domain_suffix(ndpi_info_mod, "mail.apple.com"), "com") == 0);
|
||||
assert(strcmp(ndpi_get_host_domain_suffix(ndpi_info_mod, "www.bbc.co.uk"), "co.uk") == 0);
|
||||
|
||||
ndpi_exit_detection_module(ndpi_info_mod);
|
||||
}
|
||||
}
|
||||
|
||||
/* *********************************************** */
|
||||
|
||||
void domainSearchUnitTest() {
|
||||
ndpi_domain_classify *sc = ndpi_domain_classify_alloc();
|
||||
char *domain = "ntop.org";
|
||||
@@ -5702,7 +5725,6 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
if(ndpi_get_api_version() != NDPI_API_VERSION) {
|
||||
printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
|
||||
return(-1);
|
||||
@@ -5721,6 +5743,7 @@ int main(int argc, char **argv) {
|
||||
exit(0);
|
||||
#endif
|
||||
|
||||
domainsUnitTest();
|
||||
outlierUnitTest();
|
||||
pearsonUnitTest();
|
||||
binaryBitmapUnitTest();
|
||||
|
||||
@@ -9,3 +9,12 @@ File Name Format
|
||||
|
||||
The file name format is <category id>_<string>.list
|
||||
The category id list can be found in src/include/ndpi_typedefs.h
|
||||
|
||||
|
||||
Domain Suffix List
|
||||
------------------
|
||||
|
||||
public_suffix_list.dat can be downloaded from
|
||||
https://publicsuffix.org/list/public_suffix_list.dat and
|
||||
it contains the list of domain suffixes
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2147,6 +2147,9 @@ extern "C" {
|
||||
u_int8_t class_id,
|
||||
char *file_path);
|
||||
bool ndpi_domain_classify_finalize(ndpi_domain_classify *s);
|
||||
const char* ndpi_domain_classify_longest_prefix(ndpi_domain_classify *s,
|
||||
u_int8_t *class_id /* out */,
|
||||
const char *hostnname);
|
||||
bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
|
||||
u_int8_t *class_id /* out */,
|
||||
const char *domain);
|
||||
@@ -2188,6 +2191,8 @@ extern "C" {
|
||||
/**
|
||||
* Get user data which was previously set with `ndpi_set_user_data()`.
|
||||
*
|
||||
* @par ndpi_str = the struct created for the protocol detection
|
||||
*
|
||||
* @return the user data pointer
|
||||
*
|
||||
*/
|
||||
@@ -2195,6 +2200,36 @@ extern "C" {
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
/**
|
||||
* Loads the domain suffixes from the specified path. You need to
|
||||
* perform this action once
|
||||
*
|
||||
* @par ndpi_str = the struct created for the protocol detection
|
||||
* @par public_suffix_list_path = path of the public_suffix_list path
|
||||
*
|
||||
* @return 0 = no error, -1 otherwise
|
||||
*
|
||||
*/
|
||||
int ndpi_load_domain_suffixes(struct ndpi_detection_module_struct *ndpi_str,
|
||||
char *public_suffix_list_path);
|
||||
|
||||
/**
|
||||
* Returns the domain suffix out of the specified hostname.
|
||||
* The returned pointer is an offset of the original hostname.
|
||||
* Note that you need to call ndpi_load_domain_suffixes() before
|
||||
* calling this function.
|
||||
*
|
||||
* @par ndpi_str = the struct created for the protocol detection
|
||||
* @par hostname = the hostname from which the domain name has to be extracted
|
||||
*
|
||||
* @return The host domain name or the hostitself if not found.
|
||||
*
|
||||
*/
|
||||
const char* ndpi_get_host_domain_suffix(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const char *hostname);
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
/* Can't call libc functions from kernel space, define some stub instead */
|
||||
|
||||
#define ndpi_isalpha(ch) (((ch) >= 'a' && (ch) <= 'z') || ((ch) >= 'A' && (ch) <= 'Z'))
|
||||
|
||||
@@ -303,6 +303,8 @@ struct ndpi_detection_module_struct {
|
||||
#endif
|
||||
|
||||
u_int16_t max_payload_track_len;
|
||||
|
||||
ndpi_domain_classify *public_domain_suffixes;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -89,7 +89,7 @@ bool ndpi_domain_classify_add(ndpi_domain_classify *s,
|
||||
u_int32_t i;
|
||||
char *dot;
|
||||
|
||||
if(!s || !domain)
|
||||
if((!s) || (!domain))
|
||||
return(false);
|
||||
|
||||
/* Skip initial string . in domain names */
|
||||
@@ -97,9 +97,10 @@ bool ndpi_domain_classify_add(ndpi_domain_classify *s,
|
||||
|
||||
dot = strrchr(domain, '.');
|
||||
|
||||
if(!dot) return(false);
|
||||
if(dot) {
|
||||
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local")))
|
||||
return(false);
|
||||
}
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
|
||||
if(s->classes[i].class_id == class_id) {
|
||||
@@ -107,8 +108,10 @@ bool ndpi_domain_classify_add(ndpi_domain_classify *s,
|
||||
} else if(s->classes[i].class_id == 0) {
|
||||
s->classes[i].class_id = class_id;
|
||||
s->classes[i].domains = ndpi_bitmap64_alloc();
|
||||
|
||||
if(!s->classes[i].domains)
|
||||
s->classes[i].class_id = 0;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -130,7 +133,7 @@ u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
|
||||
FILE *fd;
|
||||
char *line;
|
||||
|
||||
if(!s || !file_path)
|
||||
if((!s) || (!file_path))
|
||||
return(false);
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
|
||||
@@ -199,9 +202,9 @@ bool ndpi_domain_classify_finalize(ndpi_domain_classify *s) {
|
||||
/* ********************************************************** */
|
||||
|
||||
static bool is_valid_domain_char(u_char c) {
|
||||
if(((c >= 'A')&& (c <= 'Z'))
|
||||
|| ((c >= 'a')&& (c <= 'z'))
|
||||
|| ((c >= '0')&& (c <= '9'))
|
||||
if(((c >= 'A') && (c <= 'Z'))
|
||||
|| ((c >= 'a') && (c <= 'z'))
|
||||
|| ((c >= '0') && (c <= '9'))
|
||||
|| (c == '_')
|
||||
|| (c == '-')
|
||||
|| (c == '.'))
|
||||
@@ -212,35 +215,37 @@ static bool is_valid_domain_char(u_char c) {
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
|
||||
const char* ndpi_domain_classify_longest_prefix(ndpi_domain_classify *s,
|
||||
u_int8_t *class_id /* out */,
|
||||
const char *domain) {
|
||||
const char *hostname) {
|
||||
u_int32_t i, len;
|
||||
const char *dot, *elem;
|
||||
|
||||
if(!domain || !s) return(false);
|
||||
if((len = strlen(domain)) == 0) return(false);
|
||||
if((dot = strrchr(domain, '.')) == NULL) return(false);
|
||||
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local"))) return(false);
|
||||
*class_id = 0; /* Unknown class_id */
|
||||
|
||||
if(!hostname || !s) return(hostname);
|
||||
if((len = strlen(hostname)) == 0) return(hostname);
|
||||
if((dot = strrchr(hostname, '.')) == NULL) return(hostname);
|
||||
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local"))) return(hostname);
|
||||
|
||||
/* This is a number or a numeric IP or similar */
|
||||
if(ndpi_isdigit(domain[len-1]) && isdigit(domain[0])) {
|
||||
if(ndpi_isdigit(hostname[len-1]) && isdigit(hostname[0])) {
|
||||
#ifdef DEBUG_CONTAINS
|
||||
printf("[contains] %s INVALID\n", domain);
|
||||
printf("[contains] %s INVALID\n", hostname);
|
||||
#endif
|
||||
|
||||
return(false);
|
||||
return(hostname);
|
||||
}
|
||||
|
||||
if(!is_valid_domain_char(domain[0])) {
|
||||
if(!is_valid_domain_char(hostname[0])) {
|
||||
#ifdef DEBUG_CONTAINS
|
||||
printf("[contains] %s INVALID\n", domain);
|
||||
printf("[contains] %s INVALID\n", hostname);
|
||||
#endif
|
||||
|
||||
return(false);
|
||||
return(hostname);
|
||||
}
|
||||
|
||||
elem = domain;
|
||||
elem = hostname;
|
||||
|
||||
while(elem != NULL) {
|
||||
u_int64_t hash = ndpi_quick_hash64(elem, strlen(elem));
|
||||
@@ -249,10 +254,10 @@ bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
|
||||
if(s->classes[i].class_id != 0) {
|
||||
if(ndpi_bitmap64_isset(s->classes[i].domains, hash)) {
|
||||
#ifdef DEBUG_CONTAINS
|
||||
printf("[contains] %s = %d\n", domain, s->classes[i].class_id);
|
||||
printf("[contains] %s = %d\n", hostname, s->classes[i].class_id);
|
||||
#endif
|
||||
*class_id = s->classes[i].class_id;
|
||||
return(true);
|
||||
return(elem);
|
||||
}
|
||||
} else
|
||||
break;
|
||||
@@ -260,16 +265,23 @@ bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
|
||||
|
||||
elem = strchr(elem, '.');
|
||||
|
||||
if((elem == NULL) || (elem == dot))
|
||||
if((elem == NULL) /* || (elem == dot) */)
|
||||
break;
|
||||
else
|
||||
elem = &elem[1];
|
||||
} /* while */
|
||||
|
||||
#ifdef DEBUG_CONTAINS
|
||||
printf("[contains] %s NOT FOUND\n", domain);
|
||||
#endif
|
||||
/* Not found */
|
||||
return(hostname);
|
||||
}
|
||||
|
||||
return(false);
|
||||
/* ********************************************************** */
|
||||
|
||||
bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
|
||||
u_int8_t *class_id /* out */,
|
||||
const char *domain) {
|
||||
(void)ndpi_domain_classify_longest_prefix(s, class_id, domain); /* UNUSED */
|
||||
|
||||
return((*class_id == 0) ? false : true);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
* ndpi_domains.c
|
||||
*
|
||||
* Copyright (C) 2011-24 - ntop.org and contributors
|
||||
*
|
||||
* nDPI is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* nDPI is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
#include "ndpi_config.h"
|
||||
#include "ndpi_api.h"
|
||||
#include "ndpi_includes.h"
|
||||
#include "ndpi_private.h"
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
int ndpi_load_domain_suffixes(struct ndpi_detection_module_struct *ndpi_str,
|
||||
char *public_suffix_list_path) {
|
||||
char buf[256], *line;
|
||||
FILE *fd;
|
||||
bool do_trace = false;
|
||||
u_int num_domains = 0;
|
||||
|
||||
if(public_suffix_list_path == NULL)
|
||||
return(-1);
|
||||
|
||||
if((fd = fopen(public_suffix_list_path, "r")) == NULL)
|
||||
return(-2);
|
||||
|
||||
if(ndpi_str->public_domain_suffixes != NULL) {
|
||||
/* An existing license was aleady loaded: free it and start over */
|
||||
ndpi_domain_classify_free(ndpi_str->public_domain_suffixes);
|
||||
}
|
||||
|
||||
if((ndpi_str->public_domain_suffixes = ndpi_domain_classify_alloc()) == NULL)
|
||||
return(-3);
|
||||
|
||||
while((line = fgets(buf, sizeof(buf), fd)) != NULL) {
|
||||
u_int offset, len;
|
||||
|
||||
/* Skip empty lines or comments */
|
||||
if((line[0] == '\0') || (line[0] == '/') || (line[0] == '\n') || (line[0] == '\r'))
|
||||
continue;
|
||||
|
||||
if((line[0] == '*') && (line[1] == '.') && (line[2] != '\0'))
|
||||
offset = 2;
|
||||
else
|
||||
offset = 0;
|
||||
|
||||
len = strlen(line) - 1;
|
||||
while((len > 0) && (line[len] == '\n'))
|
||||
line[len--] = '\0';
|
||||
|
||||
if(!ndpi_domain_classify_add(ndpi_str->public_domain_suffixes,
|
||||
1 /* dummy */, &line[offset])) {
|
||||
if(do_trace) printf("Error while processing domain %s\n", &line[offset]);
|
||||
} else
|
||||
num_domains++;
|
||||
}
|
||||
|
||||
if(!ndpi_domain_classify_finalize(ndpi_str->public_domain_suffixes)) {
|
||||
if(do_trace) printf("Error while finalizing domain processing\n");
|
||||
}
|
||||
|
||||
if(do_trace) printf("Loaded %u domains\n", num_domains);
|
||||
|
||||
return(0);
|
||||
}
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
const char* ndpi_get_host_domain_suffix(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const char *hostname) {
|
||||
if(ndpi_str->public_domain_suffixes == NULL)
|
||||
return(hostname);
|
||||
else {
|
||||
u_int8_t class_id;
|
||||
|
||||
return(ndpi_domain_classify_longest_prefix(ndpi_str->public_domain_suffixes,
|
||||
&class_id, hostname));
|
||||
}
|
||||
}
|
||||
@@ -4001,8 +4001,13 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
|
||||
if(ndpi_str->callback_buffer)
|
||||
ndpi_free(ndpi_str->callback_buffer);
|
||||
|
||||
if(ndpi_str->callback_buffer_tcp_payload)
|
||||
ndpi_free(ndpi_str->callback_buffer_tcp_payload);
|
||||
|
||||
if(ndpi_str->public_domain_suffixes)
|
||||
ndpi_domain_classify_free(ndpi_str->public_domain_suffixes);
|
||||
|
||||
ndpi_free(ndpi_str);
|
||||
}
|
||||
|
||||
|
||||
@@ -132,6 +132,7 @@
|
||||
<ClCompile Include="..\src\lib\ndpi_memory.c" />
|
||||
<ClCompile Include="..\src\lib\ndpi_serializer.c" />
|
||||
<ClCompile Include="..\src\lib\ndpi_utils.c" />
|
||||
<ClCompile Include="..\src\lib\ndpi_domains.c" />
|
||||
<ClCompile Include="..\src\lib\ndpi_binary_bitmap.c" />
|
||||
<ClCompile Include="..\src\lib\ndpi_hash.c" />
|
||||
<ClCompile Include="..\src\lib\ndpi_domain_classify.c" />
|
||||
|
||||
Reference in New Issue
Block a user