Domain Classification Improvements (#2396)

* Added
size_t ndpi_compress_str(const char * in, size_t len, char * out, size_t bufsize);
size_t ndpi_decompress_str(const char * in, size_t len, char * out, size_t bufsize);

used to compress short strings such as domain names. This code is based on
https://github.com/Ed-von-Schleck/shoco

* Major code rewrite for ndpi_hash and ndpi_domain_classify

* Improvements to make sure custom categories are loaded and enabled

* Fixed string encoding

* Extended SalesForce/Cloudflare domains list
This commit is contained in:
Luca Deri
2024-04-18 23:21:40 +02:00
committed by GitHub
parent 108b8331d5
commit ad117bfaab
65 changed files with 1194 additions and 612 deletions
+24
View File
@@ -0,0 +1,24 @@
#pragma once
#include <stddef.h>
#if defined(_MSC_VER)
#define shoco_restrict __restrict
#elif __GNUC__
#define shoco_restrict __restrict__
#else
#define shoco_restrict restrict
#endif
#ifdef __cplusplus
extern "C" {
#endif
size_t shoco_compress(const char * const shoco_restrict in, size_t len, char * const shoco_restrict out, size_t bufsize);
size_t shoco_decompress(const char * const shoco_restrict in, size_t len, char * const shoco_restrict out, size_t bufsize);
#ifdef __cplusplus
}
#endif
+172
View File
@@ -0,0 +1,172 @@
/*
Note
This file has been generated (by ntop) as indicated in https://github.com/Ed-von-Schleck/shoco
using generate_compression_model.py and trained using domain names for obtaining optimal
performance when used with Internet domain names
*/
#ifndef _SHOCO_INTERNAL
#error This header file is only to be included by 'shoco.c'.
#endif
#pragma once
/*
This file was generated by 'generate_compressor_model.py'
so don't edit this by hand. Also, do not include this file
anywhere. It is internal to 'shoco.c'. Include 'shoco.h'
if you want to use shoco in your project.
*/
#define MIN_CHR 45
#define MAX_CHR 123
static const char chrs_by_chr_id[32] = {
'o', '.', 'c', 'e', 'a', 'i', 'r', 'n', 's', 't', 'l', 'd', 'm', 'u', 'p', 'g', 'h', 'b', 'f', 'k', 'y', 'v', 'w', '-', 'x', 'j', 'z', 'q', '1', '2', '0', '3'
};
static const int8_t chr_ids_by_chr[256] = {
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 23, 1, -1, 30, 28, 29, 31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, 17, 2, 11, 3, 18, 15, 16, 5, 25, 19, 10, 12, 7, 0, 14, 27, 6, 8, 9, 13, 21, 22, 24, 20, 26, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
static const int8_t successor_ids_by_chr_id_and_chr_id[32][32] = {
{7, 3, 11, -1, -1, -1, 1, 2, 6, 8, 5, 12, 0, 4, 9, 15, -1, 13, -1, -1, -1, 10, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{2, -1, 0, 6, 8, 3, 10, 1, 15, 11, -1, 5, 13, 4, 12, 7, -1, 9, -1, -1, -1, -1, -1, -1, -1, 14, -1, -1, -1, -1, -1, -1},
{0, 4, 14, 3, 1, 8, 10, 9, 12, 6, 5, 13, -1, 11, -1, -1, 2, -1, -1, 7, -1, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1},
{-1, 1, 8, 10, 7, -1, 0, 4, 3, 2, 6, 5, 9, -1, 13, -1, -1, 12, -1, -1, -1, 15, 11, -1, 14, -1, -1, -1, -1, -1, -1, -1},
{-1, 4, 6, -1, -1, 10, 1, 0, 5, 3, 2, 8, 7, 12, 9, 11, -1, 13, -1, -1, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{3, 10, 2, 8, 6, -1, 9, 0, 4, 1, 5, 7, 11, -1, 14, 12, -1, -1, 15, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{2, 5, 9, 0, 1, 4, -1, 12, 8, 6, -1, 10, 13, 7, -1, 3, -1, -1, -1, 11, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{9, 2, 8, 0, 5, 7, -1, 13, 6, 1, 10, 4, -1, 14, -1, 3, -1, -1, 12, 11, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1},
{6, 0, 8, 2, 4, 3, -1, -1, 7, 1, -1, 15, 11, 10, 9, -1, 5, -1, -1, 12, 14, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1},
{4, 1, 11, 0, 3, 2, 5, -1, 7, 10, 14, -1, 15, 8, -1, -1, 6, -1, -1, -1, 9, 13, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{3, 4, 11, 1, 2, 0, -1, -1, 8, 7, 5, 10, 13, 6, 12, -1, -1, 15, 14, -1, 9, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{5, 2, 9, 0, 3, 1, 8, 7, 6, -1, 12, 14, 11, 4, -1, 10, -1, 13, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{3, 2, 11, 1, 0, 4, -1, -1, 7, 13, -1, 15, 10, 8, 5, 14, -1, 9, -1, -1, 6, -1, -1, -1, 12, -1, -1, -1, -1, -1, -1, -1},
{-1, 5, 14, 12, 11, 13, 1, 2, 0, 4, 9, 6, 8, -1, 7, 15, -1, 10, -1, 3, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{1, 5, 12, 3, 0, 6, 2, -1, 8, 9, 4, 14, 13, 11, 7, -1, 10, -1, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1},
{0, 2, 13, 1, 3, 5, 4, 10, 9, 12, 6, -1, 15, 7, -1, 14, 8, -1, -1, -1, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{1, 4, 10, 0, 2, 3, 7, 9, 8, 6, 13, -1, 14, 5, 15, -1, -1, -1, -1, 12, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{4, 5, 9, 1, 0, 3, 2, 15, 8, 12, 6, -1, 14, 7, -1, 13, -1, 11, -1, -1, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{0, 9, 10, 3, 4, 1, 2, -1, 11, 6, 5, 15, 13, 7, 14, -1, -1, -1, 8, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{6, 0, 12, 1, 3, 2, 5, 11, 4, 8, 9, -1, 14, 7, 13, -1, -1, -1, -1, -1, 10, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1},
{3, 0, 4, 5, 2, 14, -1, 9, 1, 6, 10, -1, 7, 13, 8, -1, -1, 11, -1, -1, -1, -1, -1, 15, -1, -1, 12, -1, -1, -1, -1, -1},
{4, 2, 6, 0, 3, 1, 8, 5, 7, 11, -1, 14, 12, 10, 9, -1, -1, -1, -1, -1, 15, -1, -1, 13, -1, -1, -1, -1, -1, -1, -1, -1},
{2, 5, 8, 0, 1, 3, 9, 7, 4, 12, 11, 13, 14, -1, 10, -1, 6, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{13, -1, 1, 7, 2, 5, 14, 12, 0, 6, 10, 8, 3, -1, 4, 11, -1, 9, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{10, 0, 6, 7, 5, 1, -1, 12, 8, 3, 15, -1, 13, 11, 2, -1, -1, -1, -1, -1, 4, -1, -1, 14, 9, -1, -1, -1, -1, -1, -1, -1},
{1, 6, 8, 3, 2, 5, -1, 11, 7, 12, -1, 10, 9, 4, 0, -1, 14, 13, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1},
{4, 2, 11, 0, 1, 3, -1, 14, 10, 15, 9, 12, 13, 6, -1, -1, 7, -1, -1, -1, 8, -1, -1, -1, -1, -1, 5, -1, -1, -1, -1, -1},
{7, 1, 4, 12, 3, 2, 8, -1, 6, 9, 5, 15, 10, 0, 14, -1, -1, 11, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1, -1},
{-1, 0, 14, -1, 11, -1, -1, -1, 12, -1, -1, -1, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, 13, -1, -1, -1, -1, 3, 1, 2, 6},
{-1, 0, 7, -1, 15, -1, -1, -1, 8, -1, -1, 11, 14, -1, 10, 12, -1, 6, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 4, 5, 3, 2},
{-1, 0, 13, -1, 10, -1, -1, -1, 14, -1, -1, -1, 15, -1, -1, -1, -1, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 3, 1, 7},
{-1, 0, 5, -1, 14, -1, -1, -1, 13, -1, -1, 2, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8, 6, 3, 4}
};
static const int8_t chrs_by_chr_and_successor_id[MAX_CHR - MIN_CHR][16] = {
{'s', 'c', 'a', 'm', 'p', 'i', 't', 'e', 'd', 'b', 'l', 'g', 'n', 'o', 'r', 'f'},
{'c', 'n', 'o', 'i', 'u', 'd', 'e', 'g', 'a', 'b', 'r', 't', 'p', 'm', 'j', 's'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'.', '0', '1', '2', '7', '5', '4', '3', '8', '6', 'a', '9', 'b', 'c', 's', 'm'},
{'.', '2', '0', '1', '8', '6', '3', '9', '7', '4', '5', 'a', 's', '-', 'c', 'p'},
{'.', '4', '3', '0', '1', '2', 'b', 'c', 's', '5', 'p', 'd', 'g', '6', 'm', 'a'},
{'.', '6', 'd', '0', '3', 'c', '2', '4', '1', '8', '5', '7', '9', 's', 'a', 'm'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'},
{'n', 'r', 'l', 't', '.', 's', 'c', 'm', 'd', 'p', 'i', 'g', 'u', 'b', 'y', 'v'},
{'a', 'e', 'r', 'i', 'o', '.', 'l', 'u', 's', 'c', 'y', 'b', 't', 'g', 'm', 'n'},
{'o', 'a', 'h', 'e', '.', 'l', 't', 'k', 'i', 'n', 'r', 'u', 's', 'd', 'c', 'z'},
{'e', 'i', '.', 'a', 'u', 'o', 's', 'n', 'r', 'c', 'g', 'm', 'l', 'b', 'd', 'y'},
{'r', '.', 't', 's', 'n', 'd', 'l', 'a', 'c', 'm', 'e', 'w', 'b', 'p', 'x', 'v'},
{'o', 'i', 'r', 'e', 'a', 'l', 't', 'u', 'f', '.', 'c', 's', 'y', 'm', 'p', 'd'},
{'o', 'e', '.', 'a', 'r', 'i', 'l', 'u', 'h', 's', 'n', 'y', 't', 'c', 'g', 'm'},
{'e', 'o', 'a', 'i', '.', 'u', 't', 'r', 's', 'n', 'c', 'y', 'k', 'l', 'm', 'p'},
{'n', 't', 'c', 'o', 's', 'l', 'a', 'd', 'e', 'r', '.', 'm', 'g', 'v', 'p', 'f'},
{'p', 'o', 'a', 'e', 'u', 'i', '.', 's', 'c', 'm', 'd', 'n', 't', 'b', 'h', 'k'},
{'.', 'e', 'i', 'a', 's', 'r', 'o', 'u', 't', 'l', 'y', 'n', 'c', 'p', 'm', '-'},
{'i', 'e', 'a', 'o', '.', 'l', 'u', 't', 's', 'y', 'd', 'c', 'p', 'm', 'f', 'b'},
{'a', 'e', '.', 'o', 'i', 'p', 'y', 's', 'u', 'b', 'm', 'c', 'x', 't', 'g', 'd'},
{'e', 't', '.', 'g', 'd', 'a', 's', 'i', 'c', 'o', 'l', 'k', 'f', 'n', 'u', '-'},
{'m', 'r', 'n', '.', 'u', 'l', 's', 'o', 't', 'p', 'v', 'c', 'd', 'b', 'w', 'g'},
{'a', 'o', 'r', 'e', 'l', '.', 'i', 'p', 's', 't', 'h', 'u', 'c', 'm', 'd', '-'},
{'u', '.', 'i', 'a', 'c', 'l', 's', 'o', 'r', 't', 'm', 'b', 'e', 'q', 'p', 'd'},
{'e', 'a', 'o', 'g', 'i', '.', 't', 'u', 's', 'c', 'd', 'k', 'n', 'm', 'y', 'v'},
{'.', 't', 'e', 'i', 'a', 'h', 'o', 's', 'c', 'p', 'u', 'm', 'k', '-', 'y', 'd'},
{'e', '.', 'i', 'a', 'o', 'r', 'h', 's', 'u', 'y', 't', 'c', 'w', 'v', 'l', 'm'},
{'s', 'r', 'n', 'k', 't', '.', 'd', 'p', 'm', 'l', 'b', 'a', 'e', 'i', 'c', 'g'},
{'e', 'i', '.', 'a', 'o', 'n', 'c', 's', 'r', 'p', 'u', 't', 'm', '-', 'd', 'y'},
{'e', 'a', 'o', 'i', 's', '.', 'h', 'n', 'c', 'r', 'p', 'l', 't', 'd', 'm', 'w'},
{'.', 'i', 'p', 't', 'y', 'a', 'c', 'e', 's', 'x', 'o', 'u', 'n', 'm', '-', 'l'},
{'.', 's', 'a', 'o', 'c', 'e', 't', 'm', 'p', 'n', 'l', 'b', 'z', 'u', 'i', '-'},
{'e', 'a', '.', 'i', 'o', 'z', 'u', 'h', 'y', 'l', 's', 'c', 'd', 'm', 'n', 't'}
};
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable: 4324) // structure was padded due to __declspec(align())
#endif
typedef struct Pack {
const uint32_t word;
const unsigned int bytes_packed;
const unsigned int bytes_unpacked;
const unsigned int offsets[8];
const int16_t _ALIGNED masks[8];
const unsigned char header_mask;
const unsigned char header;
} Pack;
#ifdef _MSC_VER
#pragma warning(pop)
#endif
#define PACK_COUNT 3
#define MAX_SUCCESSOR_N 7
static const Pack packs[PACK_COUNT] = {
{ 0x80000000, 1, 2, { 26, 24, 24, 24, 24, 24, 24, 24 }, { 15, 3, 0, 0, 0, 0, 0, 0 }, 0xc0, 0x80 },
{ 0xc0000000, 2, 4, { 25, 22, 19, 16, 16, 16, 16, 16 }, { 15, 7, 7, 7, 0, 0, 0, 0 }, 0xe0, 0xc0 },
{ 0xe0000000, 4, 8, { 23, 19, 15, 11, 8, 5, 2, 0 }, { 31, 15, 15, 15, 7, 7, 7, 3 }, 0xf0, 0xe0 }
};
+233
View File
@@ -0,0 +1,233 @@
/* https://github.com/Ed-von-Schleck/shoco */
#include <stdint.h>
#if (defined (__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || __BIG_ENDIAN__)
#define swap(x) (x)
#else
#if defined(_MSC_VER)
#include <stdlib.h>
#define swap(x) _byteswap_ulong(x)
#elif defined (__GNUC__)
#if defined(__builtin_bswap32)
#define swap(x) __builtin_bswap32(x)
#else
#define swap(x) ((x<<24) + ((x&0x0000FF00)<<8) + ((x&0x00FF0000)>>8) + (x>>24))
#endif
#else
#include <byteswap.h>
#define swap(x) bswap_32(x)
#endif
#endif
#if defined(_MSC_VER)
#define _ALIGNED __declspec(align(16))
#define inline __inline
#elif defined(__GNUC__)
#define _ALIGNED __attribute__ ((aligned(16)))
#else
#define _ALIGNED
#endif
#if defined(_M_X64) || defined (_M_AMD64) || defined (__x86_64__)
#include "emmintrin.h"
#define HAVE_SSE2
#endif
#include "shoco.h"
#define _SHOCO_INTERNAL
#include "shoco_domains_model.h" /* we have built a model trained on domain names */
static inline int decode_header(unsigned char val) {
int i = -1;
while ((signed char)val < 0) {
val <<= 1;
++i;
}
return i;
}
union Code {
uint32_t word;
char bytes[4];
};
#ifdef HAVE_SSE2
static inline int check_indices(const int16_t * shoco_restrict indices, int pack_n) {
__m128i zero = _mm_setzero_si128();
__m128i indis = _mm_load_si128 ((__m128i *)indices);
__m128i masks = _mm_load_si128 ((__m128i *)packs[pack_n].masks);
__m128i cmp = _mm_cmpgt_epi16 (indis, masks);
__m128i mmask = _mm_cmpgt_epi16 (masks, zero);
cmp = _mm_and_si128 (cmp, mmask);
int result = _mm_movemask_epi8 (cmp);
return (result == 0);
}
#else
static inline int check_indices(const int16_t * shoco_restrict indices, int pack_n) {
unsigned int i;
for (i = 0; i < packs[pack_n].bytes_unpacked; ++i)
if (indices[i] > packs[pack_n].masks[i])
return 0;
return 1;
}
#endif
static inline int find_best_encoding(const int16_t * shoco_restrict indices, unsigned int n_consecutive) {
int p;
for (p = PACK_COUNT - 1; p >= 0; --p)
if ((n_consecutive >= packs[p].bytes_unpacked) && (check_indices(indices, p)))
return p;
return -1;
}
size_t shoco_compress(const char * const shoco_restrict original, size_t strlen, char * const shoco_restrict out, size_t bufsize) {
char *o = out;
char * const out_end = out + bufsize;
const char *in = original;
int16_t _ALIGNED indices[MAX_SUCCESSOR_N + 1] = { 0 };
int last_chr_index;
int current_index;
int successor_index;
unsigned int n_consecutive;
union Code code;
int pack_n;
unsigned int rest;
const char * const in_end = original + strlen;
while ((*in != '\0')) {
if (strlen && (in == in_end))
break;
// find the longest string of known successors
indices[0] = chr_ids_by_chr[(unsigned char)in[0]];
last_chr_index = indices[0];
if (last_chr_index < 0)
goto last_resort;
rest = in_end - in;
for (n_consecutive = 1; n_consecutive <= MAX_SUCCESSOR_N; ++n_consecutive) {
if (strlen && (n_consecutive == rest))
break;
current_index = chr_ids_by_chr[(unsigned char)in[n_consecutive]];
if (current_index < 0) // '\0' is always -1
break;
successor_index = successor_ids_by_chr_id_and_chr_id[last_chr_index][current_index];
if (successor_index < 0)
break;
indices[n_consecutive] = (int16_t)successor_index;
last_chr_index = current_index;
}
if (n_consecutive < 2)
goto last_resort;
pack_n = find_best_encoding(indices, n_consecutive);
if (pack_n >= 0) {
unsigned int i;
if (o + packs[pack_n].bytes_packed > out_end)
return bufsize + 1;
code.word = packs[pack_n].word;
for (i = 0; i < packs[pack_n].bytes_unpacked; ++i)
code.word |= indices[i] << packs[pack_n].offsets[i];
// In the little-endian world, we need to swap what's
// in the register to match the memory representation.
// On big-endian systems, this is a dummy.
code.word = swap(code.word);
// if we'd just copy the word, we might write over the end
// of the output string
for (i = 0; i < packs[pack_n].bytes_packed; ++i)
o[i] = code.bytes[i];
o += packs[pack_n].bytes_packed;
in += packs[pack_n].bytes_unpacked;
} else {
last_resort:
if (*in & 0x80) {
// non-ascii case
if (o + 2 > out_end)
return bufsize + 1;
// put in a sentinel byte
*o++ = 0x00;
} else {
// an ascii byte
if (o + 1 > out_end)
return bufsize + 1;
}
*o++ = *in++;
}
}
return o - out;
}
size_t shoco_decompress(const char * const shoco_restrict original, size_t complen, char * const shoco_restrict out, size_t bufsize) {
char *o = out;
char * const out_end = out + bufsize;
const char *in = original;
char last_chr;
union Code code = { 0 };
int offset;
int mask;
int mark;
const char * const in_end = original + complen;
while (in < in_end) {
mark = decode_header(*in);
if (mark < 0) {
if (o >= out_end)
return bufsize + 1;
// ignore the sentinel value for non-ascii chars
if (*in == 0x00) {
if (++in >= in_end)
return SIZE_MAX;
}
*o++ = *in++;
} else {
unsigned int i;
if (o + packs[mark].bytes_unpacked > out_end)
return bufsize + 1;
else if (in + packs[mark].bytes_packed > in_end)
return SIZE_MAX;
// This should be OK as well, but it fails with emscripten.
// Test this with new versions of emcc.
//code.word = swap(*(uint32_t *)in);
for (i = 0; i < packs[mark].bytes_packed; ++i)
code.bytes[i] = in[i];
code.word = swap(code.word);
// unpack the leading char
offset = packs[mark].offsets[0];
mask = packs[mark].masks[0];
last_chr = o[0] = chrs_by_chr_id[(code.word >> offset) & mask];
// unpack the successor chars
for (i = 1; i < packs[mark].bytes_unpacked; ++i) {
offset = packs[mark].offsets[i];
mask = packs[mark].masks[i];
last_chr = o[i] = chrs_by_chr_and_successor_id[(unsigned char)last_chr - MIN_CHR][(code.word >> offset) & mask];
}
o += packs[mark].bytes_unpacked;
in += packs[mark].bytes_packed;
}
}
// append a 0-terminator if it fits
if (o < out_end)
*o = '\0';
return o - out;
}