Domain Classification Improvements (#2396)
* Added size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize); size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize); used to compress short strings such as domain names. This code is based on https://github.com/Ed-von-Schleck/shoco * Major code rewrite for ndpi_hash and ndpi_domain_classify * Improvements to make sure custom categories are loaded and enabled * Fixed string encoding * Extended SalesForce/Cloudflare domains list
This commit is contained in:
+36
-24
@@ -1931,11 +1931,9 @@ extern "C" {
|
||||
* Free the hashmap.
|
||||
*
|
||||
* @par h = pointer to the hash map [in, out]
|
||||
* @par cleanup_func = pointer to a optional callback function
|
||||
* called for each element in the hashmap [in]
|
||||
*
|
||||
*/
|
||||
void ndpi_hash_free(ndpi_str_hash **h, void (*cleanup_func)(ndpi_str_hash *h));
|
||||
void ndpi_hash_free(ndpi_str_hash **h);
|
||||
|
||||
/**
|
||||
* Search for an entry in the hashmap.
|
||||
@@ -1949,7 +1947,7 @@ extern "C" {
|
||||
* @return 0 if an entry with that key was found, 1 otherwise
|
||||
*
|
||||
*/
|
||||
int ndpi_hash_find_entry(ndpi_str_hash *h, char *key, u_int key_len, void **value);
|
||||
int ndpi_hash_find_entry(ndpi_str_hash *h, char *key, u_int key_len, u_int16_t *value);
|
||||
|
||||
/**
|
||||
* Add an entry to the hashmap.
|
||||
@@ -1957,12 +1955,12 @@ extern "C" {
|
||||
* @par h = pointer to the hash map [in, out]
|
||||
* @par key = character string (no '\0' required) [in]
|
||||
* @par key_len = length of the character string @key [in]
|
||||
* @par value = pointer to the value to add [in]
|
||||
* @par value = value to add [in]
|
||||
*
|
||||
* @return 0 if the entry was added, 1 otherwise
|
||||
*
|
||||
*/
|
||||
int ndpi_hash_add_entry(ndpi_str_hash **h, char *key, u_int8_t key_len, void *value);
|
||||
int ndpi_hash_add_entry(ndpi_str_hash **h, char *key, u_int8_t key_len, u_int16_t value);
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
@@ -2076,23 +2074,21 @@ extern "C" {
|
||||
for substring domain matching and classification
|
||||
*/
|
||||
|
||||
ndpi_domain_classify* ndpi_domain_classify_alloc(void);
|
||||
void ndpi_domain_classify_free(ndpi_domain_classify *s);
|
||||
u_int32_t ndpi_domain_classify_size(ndpi_domain_classify *s);
|
||||
bool ndpi_domain_classify_add(ndpi_domain_classify *s,
|
||||
u_int8_t class_id, const char *domain);
|
||||
u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
|
||||
u_int8_t class_id,
|
||||
char *file_path);
|
||||
bool ndpi_domain_classify_finalize(ndpi_domain_classify *s);
|
||||
const char* ndpi_domain_classify_longest_prefix(ndpi_domain_classify *s,
|
||||
u_int8_t *class_id /* out */,
|
||||
const char *hostnname,
|
||||
bool return_subprefix);
|
||||
bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
|
||||
u_int8_t *class_id /* out */,
|
||||
const char *domain);
|
||||
|
||||
ndpi_domain_classify* ndpi_domain_classify_alloc();
|
||||
void ndpi_domain_classify_free(ndpi_domain_classify *s);
|
||||
u_int32_t ndpi_domain_classify_size(ndpi_domain_classify *s);
|
||||
bool ndpi_domain_classify_add(struct ndpi_detection_module_struct *ndpi_mod,
|
||||
ndpi_domain_classify *s,
|
||||
u_int16_t class_id, char *domain);
|
||||
u_int32_t ndpi_domain_classify_add_domains(struct ndpi_detection_module_struct *ndpi_mod,
|
||||
ndpi_domain_classify *s,
|
||||
u_int16_t class_id,
|
||||
char *file_path);
|
||||
bool ndpi_domain_classify_hostname(struct ndpi_detection_module_struct *ndpi_mod,
|
||||
ndpi_domain_classify *s,
|
||||
u_int16_t *class_id /* out */,
|
||||
char *hostname);
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
/*
|
||||
@@ -2160,12 +2156,14 @@ extern "C" {
|
||||
*
|
||||
* @par ndpi_str = the struct created for the protocol detection
|
||||
* @par hostname = the hostname from which the domain name has to be extracted
|
||||
* @par suffix_id = the id of the returned domain
|
||||
*
|
||||
* @return The host domain name suffic or the host itself if not found.
|
||||
*
|
||||
*/
|
||||
const char* ndpi_get_host_domain_suffix(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const char *hostname);
|
||||
const char *hostname,
|
||||
u_int16_t *suffix_id /* out */);
|
||||
|
||||
/**
|
||||
* Returns the domain (including the TLS) suffix out of the specified hostname.
|
||||
@@ -2217,6 +2215,20 @@ extern "C" {
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize);
|
||||
size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize);
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
/* NOTE
|
||||
this function works best if yout have loaded in memory domain
|
||||
suffixes using ndpi_load_domain_suffixes()
|
||||
*/
|
||||
u_int ndpi_encode_domain(struct ndpi_detection_module_struct *ndpi_str,
|
||||
char *domain, char *out, u_int out_len);
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
const char *ndpi_lru_cache_idx_to_name(lru_cache_type idx);
|
||||
|
||||
/**
|
||||
|
||||
@@ -400,7 +400,7 @@ struct ndpi_detection_module_struct {
|
||||
|
||||
u_int16_t max_payload_track_len;
|
||||
|
||||
ndpi_domain_classify *public_domain_suffixes;
|
||||
ndpi_str_hash *public_domain_suffixes;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@@ -1127,11 +1127,7 @@ typedef struct _ndpi_automa {
|
||||
struct ndpi_automa_stats stats;
|
||||
} ndpi_automa;
|
||||
|
||||
typedef struct ndpi_str_hash {
|
||||
unsigned int hash;
|
||||
void *value;
|
||||
// u_int8_t private_data[1]; /* Avoid error C2466 and do not initiate private data with 0 */
|
||||
} ndpi_str_hash;
|
||||
typedef void ndpi_str_hash;
|
||||
|
||||
typedef struct ndpi_proto {
|
||||
/*
|
||||
@@ -1164,10 +1160,7 @@ typedef struct {
|
||||
#define MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS 16
|
||||
|
||||
typedef struct {
|
||||
struct {
|
||||
u_int16_t class_id;
|
||||
ndpi_bitmap64_fuse *domains;
|
||||
} classes[MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS];
|
||||
ndpi_str_hash *domains;
|
||||
} ndpi_domain_classify;
|
||||
|
||||
typedef enum {
|
||||
|
||||
@@ -385,6 +385,7 @@ static ndpi_protocol_match host_match[] =
|
||||
{ "bloombergvault.com", "Bloomberg", NDPI_PROTOCOL_BLOOMBERG, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "bloomberg.com", "Bloomberg", NDPI_PROTOCOL_BLOOMBERG, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "salesforce.com", "Salesforce", NDPI_PROTOCOL_SALESFORCE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "force.com", "Salesforce", NDPI_PROTOCOL_SALESFORCE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "salesforceliveagent.com", "Salesforce", NDPI_PROTOCOL_SALESFORCE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "apple-dns.net", "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "origin-apple.com.akadns.net", "Apple", NDPI_PROTOCOL_APPLE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
@@ -920,8 +921,9 @@ static ndpi_protocol_match host_match[] =
|
||||
{ "whiteboard.microsoft.com", "Microsoft365", NDPI_PROTOCOL_MICROSOFT_365, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "events.data.microsoft.com", "Microsoft365", NDPI_PROTOCOL_MICROSOFT_365, NDPI_PROTOCOL_CATEGORY_COLLABORATIVE, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
|
||||
{ "cloudflare.com", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "cdnjs.cloudflare.com", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "cloudflare.com", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "cdnjs.cloudflare.com", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "cf-ipfs.com", "Cloudflare", NDPI_PROTOCOL_CLOUDFLARE, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
|
||||
{ "d295hzzivaok4k.cloudfront.net","OpenDNS", NDPI_PROTOCOL_OPENDNS, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
{ "opendns.com", "OpenDNS", NDPI_PROTOCOL_OPENDNS, NDPI_PROTOCOL_CATEGORY_NETWORK, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
|
||||
|
||||
+65
-163
@@ -1,7 +1,7 @@
|
||||
/*
|
||||
* ndpi_domain_classify.c
|
||||
*
|
||||
* Copyright (C) 2011-23 - ntop.org and contributors
|
||||
* Copyright (C) 2011-24 - ntop.org and contributors
|
||||
*
|
||||
* nDPI is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
@@ -27,40 +27,31 @@
|
||||
#include "ndpi_config.h"
|
||||
#include "ndpi_api.h"
|
||||
|
||||
#if 0
|
||||
#define DEBUG_ADD
|
||||
#define DEBUG_CONTAINS
|
||||
#endif
|
||||
#define ENCODE_DATA
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
ndpi_domain_classify* ndpi_domain_classify_alloc() {
|
||||
int i;
|
||||
ndpi_domain_classify *cat = (ndpi_domain_classify*)ndpi_malloc(sizeof(ndpi_domain_classify));
|
||||
ndpi_domain_classify *s = (ndpi_domain_classify*)ndpi_malloc(sizeof(ndpi_domain_classify));
|
||||
|
||||
if(!cat)
|
||||
if(!s)
|
||||
return NULL;
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++)
|
||||
cat->classes[i].class_id = 0, cat->classes[i].domains = NULL;
|
||||
if(ndpi_hash_init(&s->domains) != 0) {
|
||||
ndpi_free(s);
|
||||
return(NULL);
|
||||
}
|
||||
|
||||
return((ndpi_domain_classify*)cat);
|
||||
return((ndpi_domain_classify*)s);
|
||||
}
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
void ndpi_domain_classify_free(ndpi_domain_classify *s) {
|
||||
u_int32_t i;
|
||||
|
||||
if(!s)
|
||||
return;
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
|
||||
if(s->classes[i].domains != NULL) {
|
||||
ndpi_bitmap64_fuse_free(s->classes[i].domains);
|
||||
} else
|
||||
break;
|
||||
}
|
||||
ndpi_hash_free(&s->domains);
|
||||
|
||||
ndpi_free(s);
|
||||
}
|
||||
@@ -68,28 +59,26 @@ void ndpi_domain_classify_free(ndpi_domain_classify *s) {
|
||||
/* ********************************************************** */
|
||||
|
||||
u_int32_t ndpi_domain_classify_size(ndpi_domain_classify *s) {
|
||||
u_int32_t i, tot_len = sizeof(ndpi_domain_classify);
|
||||
u_int32_t tot_len = sizeof(ndpi_domain_classify);
|
||||
|
||||
if(!s)
|
||||
return(0);
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
|
||||
if(s->classes[i].domains != NULL) {
|
||||
tot_len += ndpi_bitmap64_fuse_size(s->classes[i].domains);
|
||||
} else
|
||||
break;
|
||||
}
|
||||
/* TODO */
|
||||
|
||||
return(tot_len);
|
||||
}
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
bool ndpi_domain_classify_add(ndpi_domain_classify *s,
|
||||
u_int8_t class_id,
|
||||
const char *domain) {
|
||||
u_int32_t i;
|
||||
u_int64_t hash;
|
||||
bool ndpi_domain_classify_add(struct ndpi_detection_module_struct *ndpi_str,
|
||||
ndpi_domain_classify *s,
|
||||
u_int16_t class_id,
|
||||
char *domain) {
|
||||
#ifdef ENCODE_DATA
|
||||
u_int32_t out_len;
|
||||
char out[256];
|
||||
#endif
|
||||
|
||||
if((!s) || (!domain))
|
||||
return(false);
|
||||
@@ -97,43 +86,27 @@ bool ndpi_domain_classify_add(ndpi_domain_classify *s,
|
||||
/* Skip initial string . in domain names */
|
||||
while(domain[0] == '.') domain++;
|
||||
|
||||
#if 0
|
||||
char *dot = strrchr(domain, '.');
|
||||
//printf("%s\n", domain);
|
||||
// fprintf(stdout, "."); fflush(stdout);
|
||||
|
||||
if(dot) {
|
||||
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local")))
|
||||
return(false);
|
||||
}
|
||||
#endif
|
||||
#ifdef ENCODE_DATA
|
||||
out_len = ndpi_encode_domain(ndpi_str, domain, out, sizeof(out));
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
|
||||
if(s->classes[i].class_id == class_id) {
|
||||
break;
|
||||
} else if(s->classes[i].class_id == 0) {
|
||||
s->classes[i].class_id = class_id;
|
||||
s->classes[i].domains = ndpi_bitmap64_fuse_alloc();
|
||||
ndpi_hash_add_entry(&s->domains, out, out_len, class_id);
|
||||
#else
|
||||
ndpi_hash_add_entry(&s->domains, domain, strlen(domain), class_id);
|
||||
#endif
|
||||
|
||||
if(!s->classes[i].domains)
|
||||
s->classes[i].class_id = 0;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(i == MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS)
|
||||
return(false);
|
||||
|
||||
hash = ndpi_quick_hash64(domain, strlen(domain));
|
||||
|
||||
return(ndpi_bitmap64_fuse_set(s->classes[i].domains, hash));
|
||||
return(true);
|
||||
}
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
|
||||
u_int8_t class_id,
|
||||
u_int32_t ndpi_domain_classify_add_domains(struct ndpi_detection_module_struct *ndpi_mod,
|
||||
ndpi_domain_classify *s,
|
||||
u_int16_t class_id,
|
||||
char *file_path) {
|
||||
u_int32_t i, num_added = 0;
|
||||
u_int32_t num_added = 0;
|
||||
char buf[256];
|
||||
FILE *fd;
|
||||
char *line;
|
||||
@@ -141,30 +114,12 @@ u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
|
||||
if((!s) || (!file_path))
|
||||
return(false);
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
|
||||
if(s->classes[i].class_id == class_id) {
|
||||
break;
|
||||
} else if(s->classes[i].class_id == 0) {
|
||||
s->classes[i].class_id = class_id;
|
||||
s->classes[i].domains = ndpi_bitmap64_fuse_alloc();
|
||||
if(!s->classes[i].domains)
|
||||
s->classes[i].class_id = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(i == MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS)
|
||||
return(false);
|
||||
|
||||
/* *************************************** */
|
||||
|
||||
fd = fopen(file_path, "r");
|
||||
if(fd == NULL)
|
||||
return(false);
|
||||
|
||||
while((line = fgets(buf, sizeof(buf), fd)) != NULL) {
|
||||
u_int len;
|
||||
u_int64_t hash;
|
||||
|
||||
if((line[0] == '#') || (line[0] == '\0'))
|
||||
continue;
|
||||
@@ -177,9 +132,7 @@ u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
|
||||
line[len] = '\0';
|
||||
}
|
||||
|
||||
hash = ndpi_quick_hash64(line, strlen(line));
|
||||
|
||||
if(ndpi_bitmap64_fuse_set(s->classes[i].domains, hash))
|
||||
if(ndpi_domain_classify_add(ndpi_mod, s, class_id, line))
|
||||
num_added++;
|
||||
}
|
||||
|
||||
@@ -191,104 +144,53 @@ u_int32_t ndpi_domain_classify_add_domains(ndpi_domain_classify *s,
|
||||
/* ********************************************************** */
|
||||
|
||||
bool ndpi_domain_classify_finalize(ndpi_domain_classify *s) {
|
||||
u_int32_t i;
|
||||
|
||||
if(!s)
|
||||
return(false);
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
|
||||
if(s->classes[i].class_id != 0) {
|
||||
ndpi_bitmap64_fuse_compress(s->classes[i].domains);
|
||||
}
|
||||
}
|
||||
return(true);
|
||||
}
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
static bool is_valid_domain_char(u_char c) {
|
||||
if(((c >= 'A') && (c <= 'Z'))
|
||||
|| ((c >= 'a') && (c <= 'z'))
|
||||
|| ((c >= '0') && (c <= '9'))
|
||||
|| (c == '_')
|
||||
|| (c == '-')
|
||||
|| (c == '.'))
|
||||
return(true);
|
||||
else
|
||||
return(false);
|
||||
}
|
||||
bool ndpi_domain_classify_hostname(struct ndpi_detection_module_struct *ndpi_mod,
|
||||
ndpi_domain_classify *s,
|
||||
u_int16_t *class_id /* out */,
|
||||
char *hostname) {
|
||||
u_int32_t len;
|
||||
const char *dot;
|
||||
char *item;
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
const char* ndpi_domain_classify_longest_prefix(ndpi_domain_classify *s,
|
||||
u_int8_t *class_id /* out */,
|
||||
const char *hostname,
|
||||
bool return_subprefix) {
|
||||
u_int32_t i, len;
|
||||
const char *dot, *elem, *prev_elem;
|
||||
// ndpi_enable_loaded_categories(ndpi_mod); /* Make sure they have been enabled */
|
||||
|
||||
*class_id = 0; /* Unknown class_id */
|
||||
|
||||
if(!hostname || !s) return(hostname);
|
||||
if((len = strlen(hostname)) == 0) return(hostname);
|
||||
if((dot = strrchr(hostname, '.')) == NULL) return(hostname);
|
||||
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local"))) return(hostname);
|
||||
if(!hostname || !s) return(false);
|
||||
if((len = strlen(hostname)) == 0) return(false);
|
||||
if((dot = strrchr(hostname, '.')) == NULL) return(false);
|
||||
if((!strcmp(dot, ".arpa")) || (!strcmp(dot, ".local"))) return(false);
|
||||
|
||||
/* This is a number or a numeric IP or similar */
|
||||
if(ndpi_isdigit(hostname[len-1]) && isdigit(hostname[0])) {
|
||||
#ifdef DEBUG_CONTAINS
|
||||
printf("[contains] %s INVALID\n", hostname);
|
||||
item = hostname;
|
||||
|
||||
while(true) {
|
||||
char *next;
|
||||
|
||||
/* This looks like a match so let's check the hash now */
|
||||
#ifdef ENCODE_DATA
|
||||
char out[256];
|
||||
u_int32_t out_len = ndpi_encode_domain(ndpi_mod, item, out, sizeof(out));
|
||||
|
||||
if(ndpi_hash_find_entry(s->domains, out, out_len, class_id) == 0)
|
||||
return(true);
|
||||
#else
|
||||
if(ndpi_hash_find_entry(s->domains, item, strlen(item), class_id) == 0)
|
||||
return(true);
|
||||
#endif
|
||||
|
||||
return(hostname);
|
||||
next = strchr(item, '.');
|
||||
|
||||
if(!next) break; else item = &next[1];
|
||||
}
|
||||
|
||||
if(!is_valid_domain_char(hostname[0])) {
|
||||
#ifdef DEBUG_CONTAINS
|
||||
printf("[contains] %s INVALID\n", hostname);
|
||||
#endif
|
||||
|
||||
return(hostname);
|
||||
}
|
||||
|
||||
elem = prev_elem = hostname;
|
||||
|
||||
while(elem != NULL) {
|
||||
u_int64_t hash = ndpi_quick_hash64(elem, strlen(elem));
|
||||
|
||||
for(i=0; i<MAX_NUM_NDPI_DOMAIN_CLASSIFICATIONS; i++) {
|
||||
if(s->classes[i].class_id != 0) {
|
||||
if(ndpi_bitmap64_fuse_isset(s->classes[i].domains, hash)) {
|
||||
#ifdef DEBUG_CONTAINS
|
||||
printf("[contains] %s = %d [%llu]\n",
|
||||
hostname, s->classes[i].class_id, hash);
|
||||
#endif
|
||||
*class_id = s->classes[i].class_id;
|
||||
return(return_subprefix ? prev_elem : elem);
|
||||
}
|
||||
} else
|
||||
break;
|
||||
}
|
||||
|
||||
prev_elem = elem;
|
||||
elem = strchr(elem, '.');
|
||||
|
||||
if(elem == NULL) break;
|
||||
// if(elem == dot) break;
|
||||
|
||||
elem = &elem[1];
|
||||
} /* while */
|
||||
|
||||
/* Not found */
|
||||
return(hostname);
|
||||
}
|
||||
|
||||
/* ********************************************************** */
|
||||
|
||||
bool ndpi_domain_classify_contains(ndpi_domain_classify *s,
|
||||
u_int8_t *class_id /* out */,
|
||||
const char *domain) {
|
||||
(void)ndpi_domain_classify_longest_prefix(s, class_id, domain, false); /* UNUSED */
|
||||
|
||||
return((*class_id == 0) ? false : true);
|
||||
return(false);
|
||||
}
|
||||
|
||||
+73
-28
@@ -29,29 +29,29 @@ int ndpi_load_domain_suffixes(struct ndpi_detection_module_struct *ndpi_str,
|
||||
char *public_suffix_list_path) {
|
||||
char buf[256], *line;
|
||||
FILE *fd;
|
||||
u_int num_domains = 0;
|
||||
|
||||
u_int16_t domain_id = 1;
|
||||
|
||||
if(ndpi_str == NULL || public_suffix_list_path == NULL)
|
||||
return(-1);
|
||||
|
||||
if((fd = fopen(public_suffix_list_path, "r")) == NULL)
|
||||
return(-2);
|
||||
|
||||
|
||||
if(ndpi_str->public_domain_suffixes != NULL) {
|
||||
/* An existing license was aleady loaded: free it and start over */
|
||||
ndpi_domain_classify_free(ndpi_str->public_domain_suffixes);
|
||||
ndpi_hash_free(&ndpi_str->public_domain_suffixes);
|
||||
}
|
||||
|
||||
if((ndpi_str->public_domain_suffixes = ndpi_domain_classify_alloc()) == NULL)
|
||||
if(ndpi_hash_init(&ndpi_str->public_domain_suffixes) != 0)
|
||||
return(-3);
|
||||
|
||||
while((line = fgets(buf, sizeof(buf), fd)) != NULL) {
|
||||
u_int offset, len;
|
||||
|
||||
|
||||
/* Skip private domains */
|
||||
if(strstr(line, "// ===END ICANN DOMAINS==="))
|
||||
break;
|
||||
|
||||
|
||||
/* Skip empty lines or comments */
|
||||
if((line[0] == '\0') || (line[0] == '/') || (line[0] == '\n') || (line[0] == '\r'))
|
||||
continue;
|
||||
@@ -65,54 +65,99 @@ int ndpi_load_domain_suffixes(struct ndpi_detection_module_struct *ndpi_str,
|
||||
while((len > 0) && (line[len] == '\n'))
|
||||
line[len--] = '\0';
|
||||
|
||||
if(!ndpi_domain_classify_add(ndpi_str->public_domain_suffixes,
|
||||
1 /* dummy */, &line[offset])) {
|
||||
if(ndpi_hash_add_entry(&ndpi_str->public_domain_suffixes,
|
||||
&line[offset], strlen(&line[offset]), domain_id) != 0) {
|
||||
|
||||
NDPI_LOG_ERR(ndpi_str, "Error while processing domain %s\n", &line[offset]);
|
||||
} else
|
||||
num_domains++;
|
||||
domain_id++;
|
||||
}
|
||||
|
||||
fclose(fd);
|
||||
|
||||
if(!ndpi_domain_classify_finalize(ndpi_str->public_domain_suffixes)) {
|
||||
NDPI_LOG_ERR(ndpi_str, "Error while finalizing domain processing\n");
|
||||
}
|
||||
|
||||
if(num_domains > 0) {
|
||||
NDPI_LOG_DBG(ndpi_str, "Loaded %u domains\n", num_domains);
|
||||
}
|
||||
|
||||
if(domain_id > 0)
|
||||
NDPI_LOG_DBG(ndpi_str, "Loaded %u domains\n", domain_id-1);
|
||||
|
||||
return(0);
|
||||
}
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
/*
|
||||
Example
|
||||
- www.ntop.org -> org
|
||||
- www.bbc.co.uk -> co.uk
|
||||
*/
|
||||
|
||||
const char* ndpi_get_host_domain_suffix(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const char *hostname) {
|
||||
const char *hostname,
|
||||
u_int16_t *domain_id /* out */) {
|
||||
char *dot, *prev_dot;
|
||||
|
||||
*domain_id = 0;
|
||||
|
||||
if(!ndpi_str)
|
||||
return NULL;
|
||||
|
||||
if(ndpi_str->public_domain_suffixes == NULL)
|
||||
return(hostname);
|
||||
else {
|
||||
u_int8_t class_id;
|
||||
|
||||
prev_dot = dot = strrchr(hostname, '.');
|
||||
|
||||
while(dot != NULL) {
|
||||
while((dot != hostname) && (dot[0] != '.'))
|
||||
dot--;
|
||||
|
||||
return(ndpi_domain_classify_longest_prefix(ndpi_str->public_domain_suffixes,
|
||||
&class_id, hostname, false));
|
||||
if((dot == hostname)
|
||||
|| (ndpi_hash_find_entry(ndpi_str->public_domain_suffixes,
|
||||
&dot[1], strlen(&dot[1]), domain_id) != 0)) {
|
||||
/* Not found: end of search */
|
||||
return(&prev_dot[1]);
|
||||
}
|
||||
|
||||
prev_dot = dot;
|
||||
dot--;
|
||||
}
|
||||
|
||||
return(hostname);
|
||||
}
|
||||
|
||||
/* ******************************* */
|
||||
|
||||
/*
|
||||
Example
|
||||
- www.ntop.org -> ntop.org
|
||||
- www.bbc.co.uk -> bbc.co.uk
|
||||
*/
|
||||
const char* ndpi_get_host_domain(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const char *hostname) {
|
||||
const char *ret;
|
||||
char *dot;
|
||||
u_int16_t domain_id;
|
||||
|
||||
if(!ndpi_str)
|
||||
return NULL;
|
||||
|
||||
if(ndpi_str->public_domain_suffixes == NULL)
|
||||
return(hostname);
|
||||
else {
|
||||
u_int8_t class_id;
|
||||
|
||||
return(ndpi_domain_classify_longest_prefix(ndpi_str->public_domain_suffixes,
|
||||
&class_id, hostname, true));
|
||||
|
||||
ret = ndpi_get_host_domain_suffix(ndpi_str, hostname, &domain_id);
|
||||
|
||||
if((ret == NULL) || (ret == hostname))
|
||||
return(hostname);
|
||||
|
||||
dot = strstr(hostname, ret);
|
||||
|
||||
if(dot == NULL)
|
||||
return(hostname);
|
||||
|
||||
dot--;
|
||||
while(dot != hostname) {
|
||||
dot--;
|
||||
|
||||
if(dot[0] == '.')
|
||||
return(&dot[1]);
|
||||
}
|
||||
|
||||
return(hostname);
|
||||
}
|
||||
|
||||
+56
-37
@@ -954,7 +954,7 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
|
||||
|
||||
/* ************************ */
|
||||
|
||||
ndpi_enable_loaded_categories(ndpi_str);
|
||||
//ndpi_enable_loaded_categories(ndpi_str);
|
||||
|
||||
if(!ndpi_xgrams_inited) {
|
||||
ndpi_xgrams_inited = 1;
|
||||
@@ -3454,11 +3454,17 @@ static int is_ip_list_enabled(struct ndpi_detection_module_struct *ndpi_str, int
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* *********************************************** */
|
||||
|
||||
int ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
u_int i;
|
||||
|
||||
if(!ndpi_str)
|
||||
return -1;
|
||||
|
||||
if(!ndpi_str->custom_categories.categories_loaded)
|
||||
ndpi_enable_loaded_categories(ndpi_str);
|
||||
|
||||
if(ndpi_str->finalized) /* Already finalized */
|
||||
return 0;
|
||||
|
||||
@@ -4038,15 +4044,18 @@ int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_str,
|
||||
return(id != NDPI_PROTOCOL_UNKNOWN ? 0 : -1);
|
||||
#else
|
||||
char buf[128];
|
||||
u_int8_t class_id;
|
||||
u_int16_t class_id;
|
||||
u_int max_len = sizeof(buf)-1;
|
||||
|
||||
if(!ndpi_str->custom_categories.categories_loaded)
|
||||
ndpi_enable_loaded_categories(ndpi_str);
|
||||
|
||||
if(name_len > max_len) name_len = max_len;
|
||||
memcpy(buf, name, name_len);
|
||||
buf[name_len] = '\0';
|
||||
|
||||
if(ndpi_domain_classify_contains(ndpi_str->custom_categories.sc_hostnames,
|
||||
&class_id, buf)) {
|
||||
if(ndpi_domain_classify_hostname(ndpi_str, ndpi_str->custom_categories.sc_hostnames,
|
||||
&class_id, buf)) {
|
||||
*category = (ndpi_protocol_category_t)class_id;
|
||||
return(0);
|
||||
} else
|
||||
@@ -4066,9 +4075,11 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
|
||||
ndpi_patricia_node_t *node;
|
||||
u_int cp_len = ndpi_min(sizeof(ipbuf) - 1, name_len);
|
||||
|
||||
*id = 0;
|
||||
|
||||
if(!ndpi_str->custom_categories.categories_loaded)
|
||||
return(-1);
|
||||
|
||||
ndpi_enable_loaded_categories(ndpi_str);
|
||||
|
||||
if(cp_len > 0) {
|
||||
memcpy(ipbuf, name_or_ip, cp_len);
|
||||
ipbuf[cp_len] = '\0';
|
||||
@@ -4085,7 +4096,8 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
|
||||
/* Search IPv4 */
|
||||
|
||||
/* Make sure all in network byte order otherwise compares wont work */
|
||||
ndpi_fill_prefix_v4(&prefix, &pin, 32, ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses)->maxbits);
|
||||
ndpi_fill_prefix_v4(&prefix, &pin, 32,
|
||||
((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses)->maxbits);
|
||||
node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses, &prefix);
|
||||
|
||||
if(node) {
|
||||
@@ -4095,7 +4107,8 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str
|
||||
return(-1);
|
||||
} else if(inet_pton(AF_INET6, ipbuf, &pin6) == 1) {
|
||||
/* Search IPv6 */
|
||||
ndpi_fill_prefix_v6(&prefix, &pin6, 128, ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits);
|
||||
ndpi_fill_prefix_v6(&prefix, &pin6, 128,
|
||||
((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits);
|
||||
node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses6, &prefix);
|
||||
|
||||
if(node) {
|
||||
@@ -4196,10 +4209,10 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->tls_cert_subject_automa.ac_automa, 0);
|
||||
|
||||
if(ndpi_str->malicious_ja3_hashmap != NULL)
|
||||
ndpi_hash_free(&ndpi_str->malicious_ja3_hashmap, NULL);
|
||||
ndpi_hash_free(&ndpi_str->malicious_ja3_hashmap);
|
||||
|
||||
if(ndpi_str->malicious_sha1_hashmap != NULL)
|
||||
ndpi_hash_free(&ndpi_str->malicious_sha1_hashmap, NULL);
|
||||
ndpi_hash_free(&ndpi_str->malicious_sha1_hashmap);
|
||||
|
||||
#ifdef USE_LEGACY_AHO_CORASICK
|
||||
if(ndpi_str->custom_categories.hostnames.ac_automa != NULL)
|
||||
@@ -4260,7 +4273,7 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
ndpi_free(ndpi_str->callback_buffer_tcp_payload);
|
||||
|
||||
if(ndpi_str->public_domain_suffixes)
|
||||
ndpi_domain_classify_free(ndpi_str->public_domain_suffixes);
|
||||
ndpi_hash_free(&ndpi_str->public_domain_suffixes);
|
||||
|
||||
ndpi_free(ndpi_str);
|
||||
}
|
||||
@@ -4865,6 +4878,8 @@ int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str,
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
int load_categories_file_fd(struct ndpi_detection_module_struct *ndpi_str,
|
||||
FILE *fd, void *user_data) {
|
||||
char buffer[512], *line, *name, *category, *saveptr;
|
||||
@@ -5064,10 +5079,10 @@ int ndpi_load_categories_dir(struct ndpi_detection_module_struct *ndpi_str,
|
||||
|
||||
if(failed_files)
|
||||
return(-1 * failed_files);
|
||||
|
||||
return(num_loaded);
|
||||
}
|
||||
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static int ndpi_load_risky_domain(struct ndpi_detection_module_struct *ndpi_str,
|
||||
@@ -5209,7 +5224,7 @@ int load_malicious_ja3_file_fd(struct ndpi_detection_module_struct *ndpi_str, FI
|
||||
continue;
|
||||
}
|
||||
|
||||
if(ndpi_hash_add_entry(&ndpi_str->malicious_ja3_hashmap, line, len, NULL) == 0)
|
||||
if(ndpi_hash_add_entry(&ndpi_str->malicious_ja3_hashmap, line, len, 0) == 0)
|
||||
num++;
|
||||
}
|
||||
|
||||
@@ -5287,7 +5302,8 @@ int load_malicious_sha1_file_fd(struct ndpi_detection_module_struct *ndpi_str, F
|
||||
for (i = 0; i < 40; ++i)
|
||||
first_comma[i] = toupper(first_comma[i]);
|
||||
|
||||
if(ndpi_hash_add_entry(&ndpi_str->malicious_sha1_hashmap, first_comma, second_comma - first_comma, NULL) == 0)
|
||||
if(ndpi_hash_add_entry(&ndpi_str->malicious_sha1_hashmap, first_comma,
|
||||
second_comma - first_comma, 0) == 0)
|
||||
num++;
|
||||
}
|
||||
|
||||
@@ -6225,42 +6241,42 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
|
||||
/* ******************************************************************** */
|
||||
|
||||
static inline int ndpi_proto_cb_tcp_payload(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static inline int ndpi_proto_cb_tcp_nopayload(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0
|
||||
&& (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0;
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0
|
||||
&& (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_HAS_PAYLOAD) == 0;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static inline int ndpi_proto_cb_udp(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC)) != 0;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
static inline int ndpi_proto_cb_other(const struct ndpi_detection_module_struct *ndpi_str, uint32_t idx) {
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP)) == 0
|
||||
||
|
||||
(ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) != 0;
|
||||
return (ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
(NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_UDP |
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_INT_TCP_OR_UDP)) == 0
|
||||
||
|
||||
(ndpi_str->callback_buffer[idx].ndpi_selection_bitmask &
|
||||
NDPI_SELECTION_BITMASK_PROTOCOL_COMPLETE_TRAFFIC) != 0;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
@@ -7896,7 +7912,7 @@ int ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_str,
|
||||
if(ndpi_str->custom_categories.sc_hostnames_shadow == NULL)
|
||||
return(-1);
|
||||
|
||||
return(ndpi_domain_classify_add(ndpi_str->custom_categories.sc_hostnames_shadow,
|
||||
return(ndpi_domain_classify_add(ndpi_str, ndpi_str->custom_categories.sc_hostnames_shadow,
|
||||
(u_int16_t)category, (char*)name_to_add) ? 0 : -1);
|
||||
#endif
|
||||
}
|
||||
@@ -7931,6 +7947,9 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
|
||||
int i;
|
||||
static char *built_in = "built-in";
|
||||
|
||||
if(ndpi_str->custom_categories.categories_loaded)
|
||||
return(-1); /* Already loaded */
|
||||
|
||||
/* First add the nDPI known categories matches */
|
||||
for(i = 0; category_match[i].string_to_match != NULL; i++)
|
||||
ndpi_load_category(ndpi_str, category_match[i].string_to_match,
|
||||
@@ -7956,7 +7975,7 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str)
|
||||
}
|
||||
#else
|
||||
ndpi_domain_classify_free(ndpi_str->custom_categories.sc_hostnames);
|
||||
ndpi_domain_classify_finalize(ndpi_str->custom_categories.sc_hostnames_shadow);
|
||||
// ndpi_domain_classify_finalize(ndpi_str->custom_categories.sc_hostnames_shadow);
|
||||
ndpi_str->custom_categories.sc_hostnames = ndpi_str->custom_categories.sc_hostnames_shadow;
|
||||
ndpi_str->custom_categories.sc_hostnames_shadow = ndpi_domain_classify_alloc();
|
||||
#endif
|
||||
|
||||
+194
-76
@@ -1,7 +1,7 @@
|
||||
/*
|
||||
* ndpi_utils.c
|
||||
*
|
||||
* Copyright (C) 2011-23 - ntop.org and contributors
|
||||
* Copyright (C) 2011-24 - ntop.org and contributors
|
||||
*
|
||||
* This file is part of nDPI, an open source deep packet inspection
|
||||
* library based on the OpenDPI and PACE technology by ipoque GmbH
|
||||
@@ -38,6 +38,7 @@
|
||||
|
||||
#include "ahocorasick.h"
|
||||
#include "libcache.h"
|
||||
#include "shoco.h"
|
||||
|
||||
#include <time.h>
|
||||
#ifndef WIN32
|
||||
@@ -72,21 +73,11 @@ struct pcre2_struct {
|
||||
};
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Please keep this strcture in sync with
|
||||
* `struct ndpi_str_hash` in src/include/ndpi_typedefs.h
|
||||
*/
|
||||
|
||||
typedef struct ndpi_str_hash_private {
|
||||
unsigned int hash;
|
||||
void *value;
|
||||
// u_int8_t private_data[1]; /* Avoid error C2466 and do not initiate private data with 0 */
|
||||
typedef struct {
|
||||
char *key;
|
||||
u_int16_t value16;
|
||||
UT_hash_handle hh;
|
||||
} ndpi_str_hash_private;
|
||||
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
|
||||
_Static_assert(sizeof(struct ndpi_str_hash) == sizeof(struct ndpi_str_hash_private) - sizeof(UT_hash_handle),
|
||||
"Please keep `struct ndpi_str_hash` and `struct ndpi_str_hash_private` syncd.");
|
||||
#endif
|
||||
} ndpi_str_hash_priv;
|
||||
|
||||
/* ****************************************** */
|
||||
|
||||
@@ -1246,15 +1237,15 @@ static void ndpi_tls2json(ndpi_serializer *serializer, struct ndpi_flow_struct *
|
||||
|
||||
static char* print_ndpi_address_port(ndpi_address_port *ap, char *buf, u_int buf_len) {
|
||||
char ipbuf[INET6_ADDRSTRLEN];
|
||||
|
||||
|
||||
if(ap->is_ipv6) {
|
||||
inet_ntop(AF_INET6, &ap->address, ipbuf, sizeof(ipbuf));
|
||||
} else {
|
||||
inet_ntop(AF_INET, &ap->address, ipbuf, sizeof(ipbuf));
|
||||
}
|
||||
|
||||
|
||||
snprintf(buf, buf_len, "%s:%u", ipbuf, ap->port);
|
||||
|
||||
|
||||
return(buf);
|
||||
}
|
||||
|
||||
@@ -2280,12 +2271,9 @@ ndpi_http_method ndpi_http_str2method(const char* method, u_int16_t method_len)
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
int ndpi_hash_init(ndpi_str_hash **h)
|
||||
{
|
||||
int ndpi_hash_init(ndpi_str_hash **h) {
|
||||
if (h == NULL)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
*h = NULL;
|
||||
return 0;
|
||||
@@ -2293,77 +2281,69 @@ int ndpi_hash_init(ndpi_str_hash **h)
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
void ndpi_hash_free(ndpi_str_hash **h, void (*cleanup_func)(ndpi_str_hash *h))
|
||||
{
|
||||
struct ndpi_str_hash_private *h_priv;
|
||||
struct ndpi_str_hash_private *current, *tmp;
|
||||
|
||||
if (h == NULL)
|
||||
{
|
||||
return;
|
||||
}
|
||||
h_priv = *(struct ndpi_str_hash_private **)h;
|
||||
|
||||
HASH_ITER(hh, h_priv, current, tmp) {
|
||||
HASH_DEL(h_priv, current);
|
||||
if (cleanup_func != NULL)
|
||||
{
|
||||
cleanup_func((ndpi_str_hash *)current);
|
||||
void ndpi_hash_free(ndpi_str_hash **h) {
|
||||
if(h != NULL) {
|
||||
ndpi_str_hash_priv *h_priv = *((ndpi_str_hash_priv **)h);
|
||||
ndpi_str_hash_priv *current, *tmp;
|
||||
|
||||
HASH_ITER(hh, h_priv, current, tmp) {
|
||||
HASH_DEL(h_priv, current);
|
||||
ndpi_free(current->key);
|
||||
ndpi_free(current);
|
||||
}
|
||||
ndpi_free(current);
|
||||
|
||||
*h = NULL;
|
||||
}
|
||||
|
||||
*h = NULL;
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
int ndpi_hash_find_entry(ndpi_str_hash *h, char *key, u_int key_len, void **value)
|
||||
{
|
||||
struct ndpi_str_hash_private *h_priv = (struct ndpi_str_hash_private *)h;
|
||||
struct ndpi_str_hash_private *found;
|
||||
unsigned int hash_value;
|
||||
int ndpi_hash_find_entry(ndpi_str_hash *h, char *key, u_int key_len, u_int16_t *value) {
|
||||
ndpi_str_hash_priv *h_priv = (ndpi_str_hash_priv *)h;
|
||||
ndpi_str_hash_priv *item;
|
||||
|
||||
HASH_FIND(hh, h_priv, key, key_len, item);
|
||||
|
||||
if (item != NULL) {
|
||||
if(value != NULL)
|
||||
*value = item->value16;
|
||||
|
||||
HASH_VALUE(key, key_len, hash_value);
|
||||
HASH_FIND_INT(h_priv, &hash_value, found);
|
||||
if (found != NULL)
|
||||
{
|
||||
if (value != NULL)
|
||||
{
|
||||
*value = found->value;
|
||||
}
|
||||
return 0;
|
||||
} else {
|
||||
} else
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
|
||||
int ndpi_hash_add_entry(ndpi_str_hash **h, char *key, u_int8_t key_len, void *value)
|
||||
{
|
||||
struct ndpi_str_hash_private **h_priv = (struct ndpi_str_hash_private **)h;
|
||||
struct ndpi_str_hash_private *new = ndpi_calloc(1, sizeof(*new));
|
||||
struct ndpi_str_hash_private *found;
|
||||
unsigned int hash_value;
|
||||
int ndpi_hash_add_entry(ndpi_str_hash **h, char *key, u_int8_t key_len, u_int16_t value) {
|
||||
ndpi_str_hash_priv *h_priv = (ndpi_str_hash_priv *)*h;
|
||||
ndpi_str_hash_priv *item;
|
||||
|
||||
if (new == NULL)
|
||||
{
|
||||
return 1;
|
||||
HASH_FIND(hh, h_priv, key, key_len, item);
|
||||
|
||||
if(item != NULL) {
|
||||
item->value16 = value;
|
||||
return(1); /* Entry already present */
|
||||
}
|
||||
|
||||
HASH_VALUE(key, key_len, hash_value);
|
||||
new->hash = hash_value;
|
||||
new->value = value;
|
||||
HASH_ADD_INT(*h_priv, hash, new);
|
||||
item = ndpi_calloc(1, sizeof(ndpi_str_hash_priv));
|
||||
if(item == NULL)
|
||||
return(2);
|
||||
|
||||
HASH_FIND_INT(*h_priv, &hash_value, found);
|
||||
if (found == NULL) /* The insertion failed (because of a memory allocation error) */
|
||||
{
|
||||
ndpi_free(new);
|
||||
return 1;
|
||||
item->key = ndpi_malloc(key_len+1);
|
||||
|
||||
if(item->key == NULL) {
|
||||
ndpi_free(item);
|
||||
return(1);
|
||||
} else {
|
||||
memcpy(item->key, key, key_len);
|
||||
item->key[key_len] = '\0';
|
||||
}
|
||||
|
||||
item->value16 = value;
|
||||
|
||||
HASH_ADD(hh, *((ndpi_str_hash_priv **)h), key[0], key_len, item);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -2502,7 +2482,7 @@ void ndpi_handle_risk_exceptions(struct ndpi_detection_module_struct *ndpi_str,
|
||||
if(host && (host[0] != '\0')) {
|
||||
/* Check host exception */
|
||||
ndpi_check_hostname_risk_exception(ndpi_str, flow, host);
|
||||
|
||||
|
||||
if(flow->risk_mask == 0) {
|
||||
u_int i;
|
||||
|
||||
@@ -2555,7 +2535,7 @@ void ndpi_set_risk(struct ndpi_flow_struct *flow, ndpi_risk_enum r,
|
||||
/* In case there is an exception set, take it into account */
|
||||
if(flow->host_risk_mask_evaluated)
|
||||
v &= flow->risk_mask;
|
||||
|
||||
|
||||
// NDPI_SET_BIT(flow->risk, (u_int32_t)r);
|
||||
flow->risk |= v;
|
||||
|
||||
@@ -3149,3 +3129,141 @@ const char *ndpi_lru_cache_idx_to_name(lru_cache_type idx)
|
||||
return "unknown";
|
||||
return names[idx];
|
||||
}
|
||||
|
||||
/* ******************************************* */
|
||||
|
||||
size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize) {
|
||||
size_t ret = shoco_compress(in, len, out, bufsize);
|
||||
|
||||
if(ret > bufsize)
|
||||
return(0); /* Better not to compress data (it is longer than the uncompressed data) */
|
||||
|
||||
return(ret);
|
||||
}
|
||||
|
||||
/* ******************************************* */
|
||||
|
||||
size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize) {
|
||||
return(shoco_decompress(in, len, out, bufsize));
|
||||
|
||||
}
|
||||
|
||||
/* ******************************************* */
|
||||
|
||||
static u_char ndpi_domain_mapper[256];
|
||||
static bool ndpi_domain_mapper_initialized = false;
|
||||
|
||||
#define IGNORE_CHAR 0xFF
|
||||
#define NUM_BITS_NIBBLE 6 /* each 'nibble' is encoded with 6 bits */
|
||||
#define NIBBLE_ELEM_OFFSET 24
|
||||
|
||||
/* Used fo encoding domain names 8 bits -> 6 bits */
|
||||
static void ndpi_domain_mapper_init() {
|
||||
u_int i;
|
||||
u_char idx = 1 /* start from 1 to make sure 0 is no ambiguous */;
|
||||
|
||||
memset(ndpi_domain_mapper, IGNORE_CHAR, 256);
|
||||
|
||||
for(i='a'; i<= 'z'; i++)
|
||||
ndpi_domain_mapper[i] = idx++;
|
||||
|
||||
for(i='0'; i<= '9'; i++)
|
||||
ndpi_domain_mapper[i] = idx++;
|
||||
|
||||
ndpi_domain_mapper['-'] = idx++;
|
||||
ndpi_domain_mapper['_'] = idx++;
|
||||
ndpi_domain_mapper['.'] = idx++;
|
||||
}
|
||||
|
||||
/* ************************************************ */
|
||||
|
||||
u_int ndpi_encode_domain(struct ndpi_detection_module_struct *ndpi_str,
|
||||
char *domain, char *out, u_int out_len) {
|
||||
u_int out_idx = 0, i, buf_shift = 0, domain_buf_len, compressed_len, suffix_len, domain_len;
|
||||
u_int32_t value = 0;
|
||||
u_char domain_buf[256], compressed[128];
|
||||
u_int16_t domain_id = 0;
|
||||
const char *suffix;
|
||||
|
||||
if(!ndpi_domain_mapper_initialized) {
|
||||
ndpi_domain_mapper_init();
|
||||
ndpi_domain_mapper_initialized = true;
|
||||
}
|
||||
|
||||
domain_len = strlen(domain);
|
||||
|
||||
if(domain_len >= (out_len-3))
|
||||
return(0);
|
||||
|
||||
if(domain_len <= 4)
|
||||
return((u_int)snprintf(out, out_len, "%s", domain)); /* Too short */
|
||||
|
||||
/* [1] Encode the domain in 6 bits */
|
||||
suffix = ndpi_get_host_domain_suffix(ndpi_str, domain, &domain_id);
|
||||
|
||||
if(suffix == NULL)
|
||||
return((u_int)snprintf(out, out_len, "%s", domain)); /* Unknown suffix */
|
||||
|
||||
snprintf((char*)domain_buf, sizeof(domain_buf), "%s", domain);
|
||||
domain_buf_len = strlen((char*)domain_buf), suffix_len = strlen(suffix);
|
||||
|
||||
if(domain_buf_len > suffix_len) {
|
||||
snprintf((char*)domain_buf, sizeof(domain_buf), "%s", domain);
|
||||
domain_buf_len = domain_buf_len-suffix_len-1;
|
||||
domain_buf[domain_buf_len] = '\0';
|
||||
|
||||
for(i=0; domain_buf[i] != '\0'; i++) {
|
||||
u_int32_t mapped_idx = ndpi_domain_mapper[domain_buf[i]];
|
||||
|
||||
if(mapped_idx != IGNORE_CHAR) {
|
||||
mapped_idx <<= buf_shift;
|
||||
value |= mapped_idx, buf_shift += NUM_BITS_NIBBLE;
|
||||
|
||||
if(buf_shift == NIBBLE_ELEM_OFFSET) {
|
||||
memcpy(&out[out_idx], &value, 3);
|
||||
out_idx += 3;
|
||||
buf_shift = 0; /* Move to the next buffer */
|
||||
value = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(buf_shift != 0) {
|
||||
u_int bytes = buf_shift / NUM_BITS_NIBBLE;
|
||||
|
||||
memcpy(&out[out_idx], &value, bytes);
|
||||
out_idx += bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/* [2] Check if compressing the string is more efficient */
|
||||
compressed_len = ndpi_compress_str((char*)domain_buf, domain_buf_len,
|
||||
(char*)compressed, sizeof(compressed));
|
||||
|
||||
if((compressed_len > 0) && ((out_idx == 0) || (compressed_len < out_idx))) {
|
||||
if(compressed_len >= domain_len) {
|
||||
/* Compression creates a longer buffer */
|
||||
return((u_int)snprintf(out, out_len, "%s", domain));
|
||||
} else {
|
||||
compressed_len = ndpi_min(ndpi_min(compressed_len, sizeof(compressed)), out_len-3);
|
||||
memcpy(out, compressed, compressed_len);
|
||||
out_idx = compressed_len;
|
||||
}
|
||||
}
|
||||
|
||||
/* Add trailer domainId value */
|
||||
out[out_idx++] = (domain_id >> 8) & 0xFF;
|
||||
out[out_idx++] = domain_id & 0xFF;
|
||||
|
||||
#ifdef DEBUG
|
||||
{
|
||||
u_int i;
|
||||
|
||||
fprintf(stdout, "%s [len: %u][", domain, out_idx);
|
||||
for(i=0; i<out_idx; i++) fprintf(stdout, "%02X", out[i] & 0xFF);
|
||||
fprintf(stdout, "]\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
return(out_idx);
|
||||
}
|
||||
|
||||
@@ -766,7 +766,7 @@ static void ndpi_search_dns(struct ndpi_detection_module_struct *ndpi_struct, st
|
||||
ndpi_hostname_sni_set(flow, (const u_int8_t *)_hostname, len, is_mdns ? NDPI_HOSTNAME_NORM_LC : NDPI_HOSTNAME_NORM_ALL);
|
||||
|
||||
if (hostname_is_valid == 0)
|
||||
ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, NULL);
|
||||
ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, "Invalid chars detected in domain name");
|
||||
|
||||
dot = strchr(_hostname, '.');
|
||||
if(dot) {
|
||||
|
||||
@@ -221,7 +221,7 @@ static void ndpi_search_fastcgi(struct ndpi_detection_module_struct *ndpi_struct
|
||||
ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, str);
|
||||
|
||||
/* This looks like an attack */
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Suspicious hostname: attack ?");
|
||||
}
|
||||
ndpi_int_fastcgi_add_connection(ndpi_struct, flow, &ret_match);
|
||||
}
|
||||
|
||||
@@ -1007,7 +1007,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
|
||||
ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, str);
|
||||
|
||||
/* This looks like an attack */
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Suspicious hostname: attack ?");
|
||||
}
|
||||
|
||||
double_col = strchr((char*)flow->host_server_name, ':');
|
||||
|
||||
@@ -1475,7 +1475,7 @@ void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, str);
|
||||
|
||||
/* This looks like an attack */
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Suspicious hostname: attack ?");
|
||||
}
|
||||
|
||||
sni_found = 1;
|
||||
@@ -1503,7 +1503,7 @@ void process_chlo(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
/* Add check for missing SNI */
|
||||
if(flow->host_server_name[0] == '\0') {
|
||||
/* This is a bit suspicious */
|
||||
ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI, NULL);
|
||||
ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI, "SNI should be present all time: attack ?");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+12
-9
@@ -643,7 +643,7 @@ void processCertificateElements(struct ndpi_detection_module_struct *ndpi_struct
|
||||
ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, dNSName);
|
||||
|
||||
/* This looks like an attack */
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Invalid dNSName name");
|
||||
}
|
||||
|
||||
if(matched_name == 0) {
|
||||
@@ -695,10 +695,13 @@ void processCertificateElements(struct ndpi_detection_module_struct *ndpi_struct
|
||||
|
||||
i += len;
|
||||
} else {
|
||||
char buf[32];
|
||||
|
||||
snprintf(buf, sizeof(buf), "Unknown extension %02X", general_name_type);
|
||||
#if DEBUG_TLS
|
||||
printf("[TLS] Leftover %u bytes", packet->payload_packet_len - i);
|
||||
#endif
|
||||
ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_EXTENSION, NULL);
|
||||
ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_EXTENSION, buf);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
@@ -781,7 +784,7 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
|
||||
if((packet->payload_packet_len != (length + 4 + (is_dtls ? 8 : 0))) || (packet->payload[1] != 0x0) ||
|
||||
certificates_offset >= packet->payload_packet_len) {
|
||||
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);
|
||||
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, "Unvalid lenght");
|
||||
return(-1); /* Invalid length */
|
||||
}
|
||||
|
||||
@@ -790,7 +793,7 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
packet->payload[certificates_offset - 1];
|
||||
|
||||
if((packet->payload[certificates_offset - 3] != 0x0) || ((certificates_length+3) != length)) {
|
||||
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, NULL);
|
||||
ndpi_set_risk(flow, NDPI_MALFORMED_PACKET, "Invalid certificate offset");
|
||||
return(-2); /* Invalid length */
|
||||
}
|
||||
|
||||
@@ -1056,7 +1059,7 @@ static int ndpi_search_tls_tcp(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
u_int8_t alert_level = message->buffer[5];
|
||||
|
||||
if(alert_level == 2 /* Warning (1), Fatal (2) */)
|
||||
ndpi_set_risk(flow, NDPI_TLS_FATAL_ALERT, NULL);
|
||||
ndpi_set_risk(flow, NDPI_TLS_FATAL_ALERT, "Found fatal TLS alert");
|
||||
}
|
||||
|
||||
u_int16_t const alert_len = ntohs(*(u_int16_t const *)&message->buffer[3]);
|
||||
@@ -1516,7 +1519,7 @@ static void checkExtensions(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
printf("[TLS] extension length exceeds remaining packet length: %u > %u.\n",
|
||||
extension_len, packet->payload_packet_len - extension_payload_offset);
|
||||
#endif
|
||||
ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_EXTENSION, NULL);
|
||||
ndpi_set_risk(flow, NDPI_TLS_SUSPICIOUS_EXTENSION, "Invalid extension len");
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -2264,7 +2267,7 @@ int processClientServerHello(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
ndpi_set_risk(flow, NDPI_INVALID_CHARACTERS, sni);
|
||||
|
||||
/* This looks like an attack */
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, NULL);
|
||||
ndpi_set_risk(flow, NDPI_POSSIBLE_EXPLOIT, "Invalid chars found in SNI: exploit or misconfiguration?");
|
||||
}
|
||||
|
||||
if(!is_quic) {
|
||||
@@ -2847,7 +2850,7 @@ compute_ja3c:
|
||||
&& (flow->protos.tls_quic.encrypted_sni.esni == NULL) /* No ESNI */
|
||||
) {
|
||||
/* This is a bit suspicious */
|
||||
ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI, NULL);
|
||||
ndpi_set_risk(flow, NDPI_TLS_MISSING_SNI, "SNI should always be present");
|
||||
|
||||
if(flow->protos.tls_quic.advertised_alpns != NULL) {
|
||||
char buf[256], *tmp, *item;
|
||||
@@ -2859,7 +2862,7 @@ compute_ja3c:
|
||||
while(item != NULL) {
|
||||
if(item[0] == 'h') {
|
||||
/* Example 'h2' */
|
||||
ndpi_set_risk(flow, NDPI_TLS_ALPN_SNI_MISMATCH, NULL);
|
||||
ndpi_set_risk(flow, NDPI_TLS_ALPN_SNI_MISMATCH, item);
|
||||
break;
|
||||
} else
|
||||
item = strtok_r(NULL, ",", &tmp);
|
||||
|
||||
Vendored
+24
@@ -0,0 +1,24 @@
|
||||
#pragma once
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#define shoco_restrict __restrict
|
||||
#elif __GNUC__
|
||||
#define shoco_restrict __restrict__
|
||||
#else
|
||||
#define shoco_restrict restrict
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
size_t shoco_compress(const char * const shoco_restrict in, size_t len, char * const shoco_restrict out, size_t bufsize);
|
||||
size_t shoco_decompress(const char * const shoco_restrict in, size_t len, char * const shoco_restrict out, size_t bufsize);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
/*
|
||||
Note
|
||||
This file has been generated (by ntop) as indicated in https://github.com/Ed-von-Schleck/shoco
|
||||
using generate_compression_model.py and trained using domain names for obtaining optimal
|
||||
performance when used with Internet domain names
|
||||
*/
|
||||
|
||||
#ifndef _SHOCO_INTERNAL
|
||||
#error This header file is only to be included by 'shoco.c'.
|
||||
#endif
|
||||
#pragma once
|
||||
/*
|
||||
This file was generated by 'generate_compressor_model.py'
|
||||
so don't edit this by hand. Also, do not include this file
|
||||
anywhere. It is internal to 'shoco.c'. Include 'shoco.h'
|
||||
if you want to use shoco in your project.
|
||||
*/
|
||||
|
||||
#define MIN_CHR 45
|
||||
#define MAX_CHR 123
|
||||
|
||||
static const char chrs_by_chr_id[32] = {
|
||||
'o', '.', 'c', 'e', 'a', 'i', 'r', 'n', 's', 't', 'l', 'd', 'm', 'u', 'p', 'g', 'h', 'b', 'f', 'k', 'y', 'v', 'w', '-', 'x', 'j', 'z', 'q', '1', '2', '0', '3'
|
||||
};
|
||||
|
||||
static const int8_t chr_ids_by_chr[256] = {
|
||||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 1, -1, 30, 28, 29, 31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, 17, 2, 11, 3, 18, 15, 16, 5, 25, 19, 10, 12, 7, 0, 14, 27, 6, 8, 9, 13, 21, 22, 24, 20, 26, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
|
||||
};
|
||||
|
||||
static const int8_t successor_ids_by_chr_id_and_chr_id[32][32] = {
|
||||
{7, 3, 11, -1, -1, -1, 1, 2, 6, 8, 5, 12, 0, 4, 9, 15, -1, 13, -1, -1, -1, 10, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{2, -1, 0, 6, 8, 3, 10, 1, 15, 11, -1, 5, 13, 4, 12, 7, -1, 9, -1, -1, -1, -1, -1, -1, -1, 14, -1, -1, -1, -1, -1, -1},
|
||||
{0, 4, 14, 3, 1, 8, 10, 9, 12, 6, 5, 13, -1, 11, -1, -1, 2, -1, -1, 7, -1, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1},
|
||||
{-1, 1, 8, 10, 7, -1, 0, 4, 3, 2, 6, 5, 9, -1, 13, -1, -1, 12, -1, -1, -1, 15, 11, -1, 14, -1, -1, -1, -1, -1, -1, -1},
|
||||
{-1, 4, 6, -1, -1, 10, 1, 0, 5, 3, 2, 8, 7, 12, 9, 11, -1, 13, -1, -1, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{3, 10, 2, 8, 6, -1, 9, 0, 4, 1, 5, 7, 11, -1, 14, 12, -1, -1, 15, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{2, 5, 9, 0, 1, 4, -1, 12, 8, 6, -1, 10, 13, 7, -1, 3, -1, -1, -1, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{9, 2, 8, 0, 5, 7, -1, 13, 6, 1, 10, 4, -1, 14, -1, 3, -1, -1, 12, 11, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{6, 0, 8, 2, 4, 3, -1, -1, 7, 1, -1, 15, 11, 10, 9, -1, 5, -1, -1, 12, 14, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{4, 1, 11, 0, 3, 2, 5, -1, 7, 10, 14, -1, 15, 8, -1, -1, 6, -1, -1, -1, 9, 13, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{3, 4, 11, 1, 2, 0, -1, -1, 8, 7, 5, 10, 13, 6, 12, -1, -1, 15, 14, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{5, 2, 9, 0, 3, 1, 8, 7, 6, -1, 12, 14, 11, 4, -1, 10, -1, 13, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{3, 2, 11, 1, 0, 4, -1, -1, 7, 13, -1, 15, 10, 8, 5, 14, -1, 9, -1, -1, 6, -1, -1, -1, 12, -1, -1, -1, -1, -1, -1, -1},
|
||||
{-1, 5, 14, 12, 11, 13, 1, 2, 0, 4, 9, 6, 8, -1, 7, 15, -1, 10, -1, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{1, 5, 12, 3, 0, 6, 2, -1, 8, 9, 4, 14, 13, 11, 7, -1, 10, -1, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{0, 2, 13, 1, 3, 5, 4, 10, 9, 12, 6, -1, 15, 7, -1, 14, 8, -1, -1, -1, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{1, 4, 10, 0, 2, 3, 7, 9, 8, 6, 13, -1, 14, 5, 15, -1, -1, -1, -1, 12, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{4, 5, 9, 1, 0, 3, 2, 15, 8, 12, 6, -1, 14, 7, -1, 13, -1, 11, -1, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{0, 9, 10, 3, 4, 1, 2, -1, 11, 6, 5, 15, 13, 7, 14, -1, -1, -1, 8, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{6, 0, 12, 1, 3, 2, 5, 11, 4, 8, 9, -1, 14, 7, 13, -1, -1, -1, -1, -1, 10, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{3, 0, 4, 5, 2, 14, -1, 9, 1, 6, 10, -1, 7, 13, 8, -1, -1, 11, -1, -1, -1, -1, -1, 15, -1, -1, 12, -1, -1, -1, -1, -1},
|
||||
{4, 2, 6, 0, 3, 1, 8, 5, 7, 11, -1, 14, 12, 10, 9, -1, -1, -1, -1, -1, 15, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{2, 5, 8, 0, 1, 3, 9, 7, 4, 12, 11, 13, 14, -1, 10, -1, 6, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{13, -1, 1, 7, 2, 5, 14, 12, 0, 6, 10, 8, 3, -1, 4, 11, -1, 9, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{10, 0, 6, 7, 5, 1, -1, 12, 8, 3, 15, -1, 13, 11, 2, -1, -1, -1, -1, -1, 4, -1, -1, 14, 9, -1, -1, -1, -1, -1, -1, -1},
|
||||
{1, 6, 8, 3, 2, 5, -1, 11, 7, 12, -1, 10, 9, 4, 0, -1, 14, 13, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
|
||||
{4, 2, 11, 0, 1, 3, -1, 14, 10, 15, 9, 12, 13, 6, -1, -1, 7, -1, -1, -1, 8, -1, -1, -1, -1, -1, 5, -1, -1, -1, -1, -1},
|
||||
{7, 1, 4, 12, 3, 2, 8, -1, 6, 9, 5, 15, 10, 0, 14, -1, -1, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1, -1},
|
||||
{-1, 0, 14, -1, 11, -1, -1, -1, 12, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1, -1, 3, 1, 2, 6},
|
||||
{-1, 0, 7, -1, 15, -1, -1, -1, 8, -1, -1, 11, 14, -1, 10, 12, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, 5, 3, 2},
|
||||
{-1, 0, 13, -1, 10, -1, -1, -1, 14, -1, -1, -1, 15, -1, -1, -1, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 3, 1, 7},
|
||||
{-1, 0, 5, -1, 14, -1, -1, -1, 13, -1, -1, 2, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8, 6, 3, 4}
|
||||
};
|
||||
|
||||
static const int8_t chrs_by_chr_and_successor_id[MAX_CHR - MIN_CHR][16] = {
|
||||
{'s', 'c', 'a', 'm', 'p', 'i', 't', 'e', 'd', 'b', 'l', 'g', 'n', 'o', 'r', 'f'},
|
||||
{'c', 'n', 'o', 'i', 'u', 'd', 'e', 'g', 'a', 'b', 'r', 't', 'p', 'm', 'j', 's'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'.', '0', '1', '2', '7', '5', '4', '3', '8', '6', 'a', '9', 'b', 'c', 's', 'm'},
|
||||
{'.', '2', '0', '1', '8', '6', '3', '9', '7', '4', '5', 'a', 's', '-', 'c', 'p'},
|
||||
{'.', '4', '3', '0', '1', '2', 'b', 'c', 's', '5', 'p', 'd', 'g', '6', 'm', 'a'},
|
||||
{'.', '6', 'd', '0', '3', 'c', '2', '4', '1', '8', '5', '7', '9', 's', 'a', 'm'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
|
||||
{'n', 'r', 'l', 't', '.', 's', 'c', 'm', 'd', 'p', 'i', 'g', 'u', 'b', 'y', 'v'},
|
||||
{'a', 'e', 'r', 'i', 'o', '.', 'l', 'u', 's', 'c', 'y', 'b', 't', 'g', 'm', 'n'},
|
||||
{'o', 'a', 'h', 'e', '.', 'l', 't', 'k', 'i', 'n', 'r', 'u', 's', 'd', 'c', 'z'},
|
||||
{'e', 'i', '.', 'a', 'u', 'o', 's', 'n', 'r', 'c', 'g', 'm', 'l', 'b', 'd', 'y'},
|
||||
{'r', '.', 't', 's', 'n', 'd', 'l', 'a', 'c', 'm', 'e', 'w', 'b', 'p', 'x', 'v'},
|
||||
{'o', 'i', 'r', 'e', 'a', 'l', 't', 'u', 'f', '.', 'c', 's', 'y', 'm', 'p', 'd'},
|
||||
{'o', 'e', '.', 'a', 'r', 'i', 'l', 'u', 'h', 's', 'n', 'y', 't', 'c', 'g', 'm'},
|
||||
{'e', 'o', 'a', 'i', '.', 'u', 't', 'r', 's', 'n', 'c', 'y', 'k', 'l', 'm', 'p'},
|
||||
{'n', 't', 'c', 'o', 's', 'l', 'a', 'd', 'e', 'r', '.', 'm', 'g', 'v', 'p', 'f'},
|
||||
{'p', 'o', 'a', 'e', 'u', 'i', '.', 's', 'c', 'm', 'd', 'n', 't', 'b', 'h', 'k'},
|
||||
{'.', 'e', 'i', 'a', 's', 'r', 'o', 'u', 't', 'l', 'y', 'n', 'c', 'p', 'm', '-'},
|
||||
{'i', 'e', 'a', 'o', '.', 'l', 'u', 't', 's', 'y', 'd', 'c', 'p', 'm', 'f', 'b'},
|
||||
{'a', 'e', '.', 'o', 'i', 'p', 'y', 's', 'u', 'b', 'm', 'c', 'x', 't', 'g', 'd'},
|
||||
{'e', 't', '.', 'g', 'd', 'a', 's', 'i', 'c', 'o', 'l', 'k', 'f', 'n', 'u', '-'},
|
||||
{'m', 'r', 'n', '.', 'u', 'l', 's', 'o', 't', 'p', 'v', 'c', 'd', 'b', 'w', 'g'},
|
||||
{'a', 'o', 'r', 'e', 'l', '.', 'i', 'p', 's', 't', 'h', 'u', 'c', 'm', 'd', '-'},
|
||||
{'u', '.', 'i', 'a', 'c', 'l', 's', 'o', 'r', 't', 'm', 'b', 'e', 'q', 'p', 'd'},
|
||||
{'e', 'a', 'o', 'g', 'i', '.', 't', 'u', 's', 'c', 'd', 'k', 'n', 'm', 'y', 'v'},
|
||||
{'.', 't', 'e', 'i', 'a', 'h', 'o', 's', 'c', 'p', 'u', 'm', 'k', '-', 'y', 'd'},
|
||||
{'e', '.', 'i', 'a', 'o', 'r', 'h', 's', 'u', 'y', 't', 'c', 'w', 'v', 'l', 'm'},
|
||||
{'s', 'r', 'n', 'k', 't', '.', 'd', 'p', 'm', 'l', 'b', 'a', 'e', 'i', 'c', 'g'},
|
||||
{'e', 'i', '.', 'a', 'o', 'n', 'c', 's', 'r', 'p', 'u', 't', 'm', '-', 'd', 'y'},
|
||||
{'e', 'a', 'o', 'i', 's', '.', 'h', 'n', 'c', 'r', 'p', 'l', 't', 'd', 'm', 'w'},
|
||||
{'.', 'i', 'p', 't', 'y', 'a', 'c', 'e', 's', 'x', 'o', 'u', 'n', 'm', '-', 'l'},
|
||||
{'.', 's', 'a', 'o', 'c', 'e', 't', 'm', 'p', 'n', 'l', 'b', 'z', 'u', 'i', '-'},
|
||||
{'e', 'a', '.', 'i', 'o', 'z', 'u', 'h', 'y', 'l', 's', 'c', 'd', 'm', 'n', 't'}
|
||||
};
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(push)
|
||||
#pragma warning(disable: 4324) // structure was padded due to __declspec(align())
|
||||
#endif
|
||||
|
||||
typedef struct Pack {
|
||||
const uint32_t word;
|
||||
const unsigned int bytes_packed;
|
||||
const unsigned int bytes_unpacked;
|
||||
const unsigned int offsets[8];
|
||||
const int16_t _ALIGNED masks[8];
|
||||
const unsigned char header_mask;
|
||||
const unsigned char header;
|
||||
} Pack;
|
||||
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
||||
#define PACK_COUNT 3
|
||||
#define MAX_SUCCESSOR_N 7
|
||||
|
||||
static const Pack packs[PACK_COUNT] = {
|
||||
{ 0x80000000, 1, 2, { 26, 24, 24, 24, 24, 24, 24, 24 }, { 15, 3, 0, 0, 0, 0, 0, 0 }, 0xc0, 0x80 },
|
||||
{ 0xc0000000, 2, 4, { 25, 22, 19, 16, 16, 16, 16, 16 }, { 15, 7, 7, 7, 0, 0, 0, 0 }, 0xe0, 0xc0 },
|
||||
{ 0xe0000000, 4, 8, { 23, 19, 15, 11, 8, 5, 2, 0 }, { 31, 15, 15, 15, 7, 7, 7, 3 }, 0xf0, 0xe0 }
|
||||
};
|
||||
Vendored
+233
@@ -0,0 +1,233 @@
|
||||
/* https://github.com/Ed-von-Schleck/shoco */
|
||||
|
||||
#include <stdint.h>
|
||||
|
||||
#if (defined (__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || __BIG_ENDIAN__)
|
||||
#define swap(x) (x)
|
||||
#else
|
||||
#if defined(_MSC_VER)
|
||||
#include <stdlib.h>
|
||||
#define swap(x) _byteswap_ulong(x)
|
||||
#elif defined (__GNUC__)
|
||||
#if defined(__builtin_bswap32)
|
||||
#define swap(x) __builtin_bswap32(x)
|
||||
#else
|
||||
#define swap(x) ((x<<24) + ((x&0x0000FF00)<<8) + ((x&0x00FF0000)>>8) + (x>>24))
|
||||
#endif
|
||||
#else
|
||||
#include <byteswap.h>
|
||||
#define swap(x) bswap_32(x)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
#define _ALIGNED __declspec(align(16))
|
||||
#define inline __inline
|
||||
#elif defined(__GNUC__)
|
||||
#define _ALIGNED __attribute__ ((aligned(16)))
|
||||
#else
|
||||
#define _ALIGNED
|
||||
#endif
|
||||
|
||||
#if defined(_M_X64) || defined (_M_AMD64) || defined (__x86_64__)
|
||||
#include "emmintrin.h"
|
||||
#define HAVE_SSE2
|
||||
#endif
|
||||
|
||||
#include "shoco.h"
|
||||
#define _SHOCO_INTERNAL
|
||||
#include "shoco_domains_model.h" /* we have built a model trained on domain names */
|
||||
|
||||
static inline int decode_header(unsigned char val) {
|
||||
int i = -1;
|
||||
while ((signed char)val < 0) {
|
||||
val <<= 1;
|
||||
++i;
|
||||
}
|
||||
return i;
|
||||
}
|
||||
|
||||
union Code {
|
||||
uint32_t word;
|
||||
char bytes[4];
|
||||
};
|
||||
|
||||
#ifdef HAVE_SSE2
|
||||
static inline int check_indices(const int16_t * shoco_restrict indices, int pack_n) {
|
||||
__m128i zero = _mm_setzero_si128();
|
||||
__m128i indis = _mm_load_si128 ((__m128i *)indices);
|
||||
__m128i masks = _mm_load_si128 ((__m128i *)packs[pack_n].masks);
|
||||
__m128i cmp = _mm_cmpgt_epi16 (indis, masks);
|
||||
__m128i mmask = _mm_cmpgt_epi16 (masks, zero);
|
||||
cmp = _mm_and_si128 (cmp, mmask);
|
||||
int result = _mm_movemask_epi8 (cmp);
|
||||
return (result == 0);
|
||||
}
|
||||
#else
|
||||
static inline int check_indices(const int16_t * shoco_restrict indices, int pack_n) {
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < packs[pack_n].bytes_unpacked; ++i)
|
||||
if (indices[i] > packs[pack_n].masks[i])
|
||||
return 0;
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline int find_best_encoding(const int16_t * shoco_restrict indices, unsigned int n_consecutive) {
|
||||
int p;
|
||||
|
||||
for (p = PACK_COUNT - 1; p >= 0; --p)
|
||||
if ((n_consecutive >= packs[p].bytes_unpacked) && (check_indices(indices, p)))
|
||||
return p;
|
||||
return -1;
|
||||
}
|
||||
|
||||
size_t shoco_compress(const char * const shoco_restrict original, size_t strlen, char * const shoco_restrict out, size_t bufsize) {
|
||||
char *o = out;
|
||||
char * const out_end = out + bufsize;
|
||||
const char *in = original;
|
||||
int16_t _ALIGNED indices[MAX_SUCCESSOR_N + 1] = { 0 };
|
||||
int last_chr_index;
|
||||
int current_index;
|
||||
int successor_index;
|
||||
unsigned int n_consecutive;
|
||||
union Code code;
|
||||
int pack_n;
|
||||
unsigned int rest;
|
||||
const char * const in_end = original + strlen;
|
||||
|
||||
while ((*in != '\0')) {
|
||||
if (strlen && (in == in_end))
|
||||
break;
|
||||
|
||||
// find the longest string of known successors
|
||||
indices[0] = chr_ids_by_chr[(unsigned char)in[0]];
|
||||
last_chr_index = indices[0];
|
||||
if (last_chr_index < 0)
|
||||
goto last_resort;
|
||||
|
||||
rest = in_end - in;
|
||||
for (n_consecutive = 1; n_consecutive <= MAX_SUCCESSOR_N; ++n_consecutive) {
|
||||
if (strlen && (n_consecutive == rest))
|
||||
break;
|
||||
|
||||
current_index = chr_ids_by_chr[(unsigned char)in[n_consecutive]];
|
||||
if (current_index < 0) // '\0' is always -1
|
||||
break;
|
||||
|
||||
successor_index = successor_ids_by_chr_id_and_chr_id[last_chr_index][current_index];
|
||||
if (successor_index < 0)
|
||||
break;
|
||||
|
||||
indices[n_consecutive] = (int16_t)successor_index;
|
||||
last_chr_index = current_index;
|
||||
}
|
||||
if (n_consecutive < 2)
|
||||
goto last_resort;
|
||||
|
||||
pack_n = find_best_encoding(indices, n_consecutive);
|
||||
if (pack_n >= 0) {
|
||||
unsigned int i;
|
||||
|
||||
if (o + packs[pack_n].bytes_packed > out_end)
|
||||
return bufsize + 1;
|
||||
|
||||
code.word = packs[pack_n].word;
|
||||
for (i = 0; i < packs[pack_n].bytes_unpacked; ++i)
|
||||
code.word |= indices[i] << packs[pack_n].offsets[i];
|
||||
|
||||
// In the little-endian world, we need to swap what's
|
||||
// in the register to match the memory representation.
|
||||
// On big-endian systems, this is a dummy.
|
||||
code.word = swap(code.word);
|
||||
|
||||
// if we'd just copy the word, we might write over the end
|
||||
// of the output string
|
||||
for (i = 0; i < packs[pack_n].bytes_packed; ++i)
|
||||
o[i] = code.bytes[i];
|
||||
|
||||
o += packs[pack_n].bytes_packed;
|
||||
in += packs[pack_n].bytes_unpacked;
|
||||
} else {
|
||||
last_resort:
|
||||
if (*in & 0x80) {
|
||||
// non-ascii case
|
||||
if (o + 2 > out_end)
|
||||
return bufsize + 1;
|
||||
// put in a sentinel byte
|
||||
*o++ = 0x00;
|
||||
} else {
|
||||
// an ascii byte
|
||||
if (o + 1 > out_end)
|
||||
return bufsize + 1;
|
||||
}
|
||||
*o++ = *in++;
|
||||
}
|
||||
}
|
||||
|
||||
return o - out;
|
||||
}
|
||||
|
||||
size_t shoco_decompress(const char * const shoco_restrict original, size_t complen, char * const shoco_restrict out, size_t bufsize) {
|
||||
char *o = out;
|
||||
char * const out_end = out + bufsize;
|
||||
const char *in = original;
|
||||
char last_chr;
|
||||
union Code code = { 0 };
|
||||
int offset;
|
||||
int mask;
|
||||
int mark;
|
||||
const char * const in_end = original + complen;
|
||||
|
||||
while (in < in_end) {
|
||||
mark = decode_header(*in);
|
||||
if (mark < 0) {
|
||||
if (o >= out_end)
|
||||
return bufsize + 1;
|
||||
|
||||
// ignore the sentinel value for non-ascii chars
|
||||
if (*in == 0x00) {
|
||||
if (++in >= in_end)
|
||||
return SIZE_MAX;
|
||||
}
|
||||
|
||||
*o++ = *in++;
|
||||
} else {
|
||||
unsigned int i;
|
||||
|
||||
if (o + packs[mark].bytes_unpacked > out_end)
|
||||
return bufsize + 1;
|
||||
else if (in + packs[mark].bytes_packed > in_end)
|
||||
return SIZE_MAX;
|
||||
|
||||
// This should be OK as well, but it fails with emscripten.
|
||||
// Test this with new versions of emcc.
|
||||
//code.word = swap(*(uint32_t *)in);
|
||||
for (i = 0; i < packs[mark].bytes_packed; ++i)
|
||||
code.bytes[i] = in[i];
|
||||
code.word = swap(code.word);
|
||||
|
||||
// unpack the leading char
|
||||
offset = packs[mark].offsets[0];
|
||||
mask = packs[mark].masks[0];
|
||||
last_chr = o[0] = chrs_by_chr_id[(code.word >> offset) & mask];
|
||||
|
||||
// unpack the successor chars
|
||||
for (i = 1; i < packs[mark].bytes_unpacked; ++i) {
|
||||
offset = packs[mark].offsets[i];
|
||||
mask = packs[mark].masks[i];
|
||||
last_chr = o[i] = chrs_by_chr_and_successor_id[(unsigned char)last_chr - MIN_CHR][(code.word >> offset) & mask];
|
||||
}
|
||||
|
||||
o += packs[mark].bytes_unpacked;
|
||||
in += packs[mark].bytes_packed;
|
||||
}
|
||||
}
|
||||
|
||||
// append a 0-terminator if it fits
|
||||
if (o < out_end)
|
||||
*o = '\0';
|
||||
|
||||
return o - out;
|
||||
}
|
||||
Reference in New Issue
Block a user