Reworked categories handling

Removed GenericProtocol and replaced with categories
Removed ndpi_pref_enable_category_substring_match option: substring matching is now default
This commit is contained in:
Luca Deri
2019-09-29 21:46:41 +02:00
parent 016d568c2b
commit f2a5bbef17
15 changed files with 1301 additions and 1314 deletions
+13 -14
View File
@@ -383,13 +383,12 @@ extern "C" {
* @return the ID of the matched subprotocol
*
*/
int ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
char *string_to_match,
u_int string_to_match_len,
ndpi_protocol_match_result *ret_match,
u_int16_t master_protocol_id);
u_int16_t ndpi_match_host_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
char *string_to_match,
u_int string_to_match_len,
ndpi_protocol_match_result *ret_match,
u_int16_t master_protocol_id);
/**
* Check if the string content passed match with a protocol
@@ -403,12 +402,12 @@ extern "C" {
* @return the ID of the matched subprotocol
*
*/
int ndpi_match_content_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
char *string_to_match,
u_int string_to_match_len,
ndpi_protocol_match_result *ret_match,
u_int16_t master_protocol_id);
u_int16_t ndpi_match_content_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
char *string_to_match,
u_int string_to_match_len,
ndpi_protocol_match_result *ret_match,
u_int16_t master_protocol_id);
/**
* Exclude protocol from search
*
@@ -755,7 +754,7 @@ extern "C" {
u_int32_t daddr,
ndpi_protocol *ret);
int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, unsigned long *id);
char *name, unsigned long *id);
void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
ndpi_protocol *ret);
+1 -1
View File
@@ -171,7 +171,7 @@ typedef enum {
NDPI_PROTOCOL_LASTFM = 134,
NDPI_PROTOCOL_WAZE = 135,
NDPI_PROTOCOL_YOUTUBE_UPLOAD = 136, /* Upload files to youtube */
NDPI_PROTOCOL_GENERIC = 137, /* Generic protocol used for category matching */
NDPI_PROTOCOL_FREE_137 = 137, /* Free */
NDPI_PROTOCOL_CHECKMK = 138,
NDPI_PROTOCOL_AJP = 139, /* Leonn Paiva <leonn.paiva@gmail.com> */
NDPI_PROTOCOL_APPLE = 140,
+10 -8
View File
@@ -919,11 +919,10 @@ typedef enum {
} ndpi_protocol_category_t;
typedef enum {
ndpi_pref_http_dont_dissect_response = 0,
ndpi_pref_dns_dont_dissect_response,
ndpi_pref_direction_detect_disable,
ndpi_pref_disable_metadata_export,
ndpi_pref_enable_category_substring_match
ndpi_pref_http_dont_dissect_response = 0,
ndpi_pref_dns_dont_dissect_response,
ndpi_pref_direction_detect_disable,
ndpi_pref_disable_metadata_export,
} ndpi_detection_preference;
/* ntop extensions */
@@ -1048,7 +1047,6 @@ struct ndpi_detection_module_struct {
#else
ndpi_automa hostnames, hostnames_shadow;
#endif
void *hostnames_hash;
void *ipAddresses, *ipAddresses_shadow; /* Patricia */
u_int8_t categories_loaded;
} custom_categories;
@@ -1102,8 +1100,7 @@ struct ndpi_detection_module_struct {
u_int8_t http_dont_dissect_response:1, dns_dont_dissect_response:1,
direction_detect_disable:1, /* disable internal detection of packet direction */
disable_metadata_export:1, /* No metadata is exported */
enable_category_substring_match:1 /* Default is perfect match */
disable_metadata_export:1 /* No metadata is exported */
;
void *hyperscan; /* Intel Hyperscan */
@@ -1337,6 +1334,11 @@ typedef struct {
ndpi_protocol_breed_t protocol_breed;
} ndpi_protocol_match;
typedef struct {
char *string_to_match, *hyperscan_string_to_match;
ndpi_protocol_category_t protocol_category;
} ndpi_category_match;
typedef struct {
u_int32_t network;
u_int8_t cidr;
+31 -39
View File
@@ -1,7 +1,7 @@
/*
* ndpi_content_match.c
*
* Copyright (C) 2011-18 - ntop.org
* Copyright (C) 2011-19 - ntop.org
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
@@ -8382,7 +8382,7 @@ https://www.regular-expressions.info/email.html
/* ****************************************************** */
ndpi_protocol_match host_match[] = {
static ndpi_protocol_match host_match[] = {
{ "s3.ll.dash.row.aiv-cdn.net", NULL, "s3\\.ll\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN },
{ "s3-dub.cf.dash.row.aiv-cdn.net", NULL, "s3-dub\\.cf\\.dash\\.row\\.aiv-cdn\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN },
{ "dmqdd6hw24ucf.cloudfront.net", NULL, "dmqdd6hw24ucf\\.cloudfront\\.net", "AmazonVideo", NDPI_PROTOCOL_AMAZON_VIDEO, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN },
@@ -8425,16 +8425,9 @@ ndpi_protocol_match host_match[] = {
{ ".xboxlive.com", NULL, "xboxlive" TLD, "Xbox", NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN },
{ ".xboxlive.com.akadns.net", NULL, "xboxlive" TLD, "Xbox", NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN },
{ ".xboxlive.com.c.footprint.net", NULL, "xboxlive" TLD, "Xbox", NDPI_PROTOCOL_XBOX, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN },
{ ".edgecastcdn.net", NULL, "egdecastcdn" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE },
{ ".hwcdn.net", NULL, "hwcdn" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE },
{ ".llnwd.net", NULL, "llnwd" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE },
{ ".llns.net", NULL, "llns" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE },
{ ".fastly.net", NULL, "fastly" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE },
{ ".akamaiedge.net", NULL, "akamaiedge" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_MEDIA, NDPI_PROTOCOL_SAFE },
{ "e13555.b.akamaiedge.net", NULL, "e13555\\.b\\.akamaiedge" TLD, "Playstation", NDPI_PROTOCOL_PLAYSTATION, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN },
{ "e1800.d.akamaiedge.net", NULL, "e1800\\.d\\.akamaiedge" TLD, "Playstation", NDPI_PROTOCOL_PLAYSTATION, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN },
{ "e1879.e7.akamaiedge.net", NULL, "e1879\\.e7\\.akamaiedge" TLD, "Playstation", NDPI_PROTOCOL_PLAYSTATION, NDPI_PROTOCOL_CATEGORY_GAME, NDPI_PROTOCOL_FUN },
{ ".vultr.com", NULL, "vultr" TLD, "Unknown", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE },
{ ".cnn.c", NULL, "\\.cnn" TLD, "CNN", NDPI_PROTOCOL_CNN, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE },
{ ".cnn.net", NULL, NULL, "CNN", NDPI_PROTOCOL_CNN, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_SAFE },
@@ -8779,39 +8772,10 @@ ndpi_protocol_match host_match[] = {
/*
VidTO streaming service
NOTE: this is a possible candidate for NDPI_PROTOCOL_GENERIC
*/
{ ".vidto.me", NULL, "\\.vidto" TLD, "VidTO", NDPI_PROTOCOL_PPSTREAM, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ ".vidto.se", NULL, NULL, "VidTO", NDPI_PROTOCOL_PPSTREAM, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
/*
Not so popular services without a protocl dissector (and thus not worth a protocolId) that are handled/detected by categpory rather
than by protocol id. They are bound to a generic protocol (NDPI_PROTOCOL_GENERIC) and placed onto the right category
*/
{ "quickplay.com", NULL, "quickplay" TLD, "QuickPlay", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ ".iqiyi.com", NULL, "\\.iqiyi" TLD, "iQIYI", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ ".qiyi.com", NULL, "\\.qiyi" TLD, "iQIYI", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ ".71.am", NULL, "\\.71" TLD, "iQIYI", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ ".qiyipic.com", NULL, "\\.qiyipic" TLD, "iQIYI", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ ".1kxun.", NULL, "\\.1kxun\\.", "1kxun", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ "tcad.wedolook.com", NULL, "tcad\\.wedolook" TLD, "1kxun", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ "baidu.com", NULL, "baidu" TLD, "baidu", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_WEB, NDPI_PROTOCOL_FUN },
{ "icq.", NULL, "icq" TLD, "ICQ", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE },
{ ".icq.", NULL, NULL, "ICQ", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE },
/* RapidVideo streaming */
{ ".rapidvideo.com", NULL, "\\.rapidvideo" TLD, "RapidVideo", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
{ ".playercdn.net", NULL, "\\.playercdn" TLD, "RapidVideo", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_STREAMING, NDPI_PROTOCOL_FUN },
/* showmax.com video streaming */
{ "showmax.com", NULL, "showmax" TLD, "Showmax", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN },
{ "showmax.akamaized.net", NULL, "showmax\\.akamaized" TLD, "Showmax", NDPI_PROTOCOL_GENERIC, NDPI_PROTOCOL_CATEGORY_VIDEO, NDPI_PROTOCOL_FUN },
{ "snapcraft.io", NULL, "snapcraft\\.io" TLD, "UbuntuONE", NDPI_PROTOCOL_UBUNTUONE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE },
{ "ubuntu.com", NULL, "ubuntu\\.com" TLD, "UbuntuONE", NDPI_PROTOCOL_UBUNTUONE, NDPI_PROTOCOL_CATEGORY_CLOUD, NDPI_PROTOCOL_ACCEPTABLE },
@@ -8822,6 +8786,33 @@ ndpi_protocol_match host_match[] = {
{ NULL, NULL, NULL, 0 }
};
/* ******************************************************************** */
static ndpi_category_match category_match[] = {
{ ".edgecastcdn.net", "egdecastcdn" TLD, NDPI_PROTOCOL_CATEGORY_MEDIA },
{ ".hwcdn.net", "hwcdn" TLD, NDPI_PROTOCOL_CATEGORY_MEDIA },
{ ".llnwd.net", "llnwd" TLD, NDPI_PROTOCOL_CATEGORY_MEDIA },
{ ".llns.net", "llns" TLD, NDPI_PROTOCOL_CATEGORY_MEDIA },
{ ".fastly.net", "fastly" TLD, NDPI_PROTOCOL_CATEGORY_MEDIA },
{ ".akamaiedge.net", "akamaiedge" TLD, NDPI_PROTOCOL_CATEGORY_MEDIA },
{ ".vultr.com", "vultr" TLD, NDPI_PROTOCOL_CATEGORY_CLOUD },
{ "baidu.com", "baidu" TLD, NDPI_PROTOCOL_CATEGORY_WEB },
{ "icq.com", "icq" TLD, NDPI_PROTOCOL_CATEGORY_CHAT },
{ "quickplay.com", "quickplay" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ ".iqiyi.com", "\\.iqiyi" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ ".qiyi.com", "\\.qiyi" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ ".71.am", "\\.71" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ ".qiyipic.com", "\\.qiyipic" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ ".1kxun.", "\\.1kxun\\.", NDPI_PROTOCOL_CATEGORY_STREAMING },
{ "tcad.wedolook.com", "tcad\\.wedolook" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ ".rapidvideo.com", "\\.rapidvideo" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ ".playercdn.net", "\\.playercdn" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ "showmax.com", "showmax" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ "showmax.akamaized.net", "showmax\\.akamaized" TLD, NDPI_PROTOCOL_CATEGORY_STREAMING },
{ NULL, NULL, 0 }
};
/* ******************************************************************** */
/*
@@ -8882,7 +8873,8 @@ static const char *ndpi_en_bigrams[] = {
"mw", "nw", "ow", "pw", "rw", "sw", "tw", "uw", "ww", "xw", "yw", "zw", "ax", "ex", "ix", "nx", "ox",
"rx", "ux", "xx", "yx", "ay", "by", "cy", "dy", "ey", "fy", "gy", "hy", "ky", "ly", "my", "ny", "oy",
"py", "ry", "sy", "ty", "uy", "vy", "wy", "xy", "yy", "zy", "az", "bz", "cz", "dz", "ez", "gz", "iz",
"lz", "nz", "oz", "pz", "rz", "tz", "uz", "zz", NULL };
"lz", "nz", "oz", "pz", "rz", "tz", "uz", "zz",
NULL };
/* ******************************************************************** */
+1096 -1104
View File
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -1101,7 +1101,7 @@ int tlsDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s
NDPI_LOG_DBG2(ndpi_struct, "***** [SSL] %s\n", certificate);
#endif
ndpi_protocol_match_result ret_match;
u_int32_t subproto = ndpi_match_host_subprotocol(ndpi_struct, flow, certificate,
u_int16_t subproto = ndpi_match_host_subprotocol(ndpi_struct, flow, certificate,
strlen(certificate),
&ret_match,
NDPI_PROTOCOL_TLS);
@@ -1117,7 +1117,7 @@ int tlsDetectProtocolFromCertificate(struct ndpi_detection_module_struct *ndpi_s
ndpi_set_detected_protocol(ndpi_struct, flow, subproto,
ndpi_tls_refine_master_protocol(ndpi_struct, flow, NDPI_PROTOCOL_TLS));
return(rc); /* Fix courtesy of Gianluca Costa <g.costa@xplico.org> */
return(rc);
}
if(ndpi_is_tls_tor(ndpi_struct, flow, certificate) != 0)