Refactored nDPI subprotocol handling and aimini protocol detection. (#1156)
* Refactored and merged callback buffer routines for non-udp-tcp / udp / tcp / tcp-wo-payload. Signed-off-by: Toni Uhlig <matzeton@googlemail.com> * Try to detect one subprotocol if a detected protocol can have one. * This adds a performance overhead due to much more protocol detection routine calls. See #1148 for more information. Signed-off-by: Toni Uhlig <matzeton@googlemail.com> * Refactor subprotocol handling (1/2). Signed-off-by: Toni Uhlig <matzeton@googlemail.com> * Refactor subprotocol handling (2/2). Signed-off-by: Toni Uhlig <matzeton@googlemail.com> * Prevent some code duplication by using macros for ndpi_int_one_line_struct string comparision. Signed-off-by: Toni Uhlig <matzeton@googlemail.com> * Refactored aimini HTTP detection parts (somehow related to #1148). Signed-off-by: Toni Uhlig <matzeton@googlemail.com> * Added aimini client/server test pcap. Signed-off-by: Toni Uhlig <matzeton@googlemail.com> * Removed master protocol as it was only used for STUN and via also removed API function ndpi_get_protocol_id_master_proto * Adjusted Python code to conform to the changes made during the refactoring process. Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
This commit is contained in:
@@ -246,7 +246,7 @@ void init_doh_bins() {
|
||||
|
||||
u_int check_bin_doh_similarity(struct ndpi_bin *bin, float *similarity) {
|
||||
u_int i;
|
||||
float lowest_similarity = 9999999999;
|
||||
float lowest_similarity = 9999999999.0f;
|
||||
|
||||
for(i=0; i<NUM_DOH_BINS; i++) {
|
||||
*similarity = ndpi_bin_similarity(&doh_ndpi_bins[i], bin, 0);
|
||||
|
||||
+2
-3
@@ -716,7 +716,6 @@ struct ndpi_packet_struct {
|
||||
uint64_t current_time_ms;
|
||||
|
||||
uint16_t detected_protocol_stack[2];
|
||||
uint8_t detected_subprotocol_stack[2];
|
||||
uint16_t protocol_stack_info;
|
||||
|
||||
struct ndpi_int_one_line_struct line[64];
|
||||
@@ -870,9 +869,9 @@ typedef enum {
|
||||
typedef struct ndpi_proto_defaults {
|
||||
char *protoName;
|
||||
ndpi_protocol_category_t protoCategory;
|
||||
uint8_t can_have_a_subprotocol;
|
||||
u_int16_t * subprotocols;
|
||||
size_t subprotocol_count;
|
||||
uint16_t protoId, protoIdx;
|
||||
uint16_t master_tcp_protoId[2], master_udp_protoId[2]; /* The main protocols on which this sub-protocol sits on */
|
||||
uint16_t tcp_default_ports[5], udp_default_ports[5];
|
||||
ndpi_protocol_breed_t protoBreed;
|
||||
void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow);
|
||||
|
||||
@@ -94,11 +94,10 @@ class NDPIProtoDefaultsT(Structure):
|
||||
_fields_ = [
|
||||
("protoName", c_char_p),
|
||||
("protoCategory", c_uint),
|
||||
("can_have_a_subprotocol", c_uint8),
|
||||
("subprotocols", c_uint16_p),
|
||||
("subprotocol_count", c_uint32),
|
||||
("protoId", c_uint16),
|
||||
("protoIdx", c_uint16),
|
||||
("master_tcp_protoId", c_uint16 * 2),
|
||||
("master_udp_protoId", c_uint16 * 2),
|
||||
("protoBreed", c_uint),
|
||||
("func", CFUNCTYPE(None, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
|
||||
]
|
||||
@@ -653,7 +652,6 @@ class NDPIUdpHdr(Structure):
|
||||
class NDPIPacketStructStack(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('detected_subprotocol_stack', c_uint8 * ndpi.ndpi_wrap_ndpi_procol_size()),
|
||||
('protocol_stack_info', c_uint16)
|
||||
]
|
||||
|
||||
|
||||
@@ -31,6 +31,10 @@
|
||||
#include "ndpi_api.h"
|
||||
#include "ndpi_protocols.h"
|
||||
|
||||
/* used by ndpi_set_proto_subprotocols */
|
||||
#define NDPI_PROTOCOL_NO_MORE_SUBPROTOCOLS (-1)
|
||||
#define NDPI_PROTOCOL_MATCHED_BY_CONTENT (-2)
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
@@ -95,11 +99,11 @@ extern "C" {
|
||||
struct ndpi_flow_struct *flow,
|
||||
ndpi_protocol_category_t protocol_category);
|
||||
|
||||
extern void ndpi_set_proto_subprotocols(struct ndpi_detection_module_struct *ndpi_mod,
|
||||
int protoId, ...);
|
||||
|
||||
extern void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod,
|
||||
ndpi_protocol_breed_t protoBreed, u_int16_t protoId,
|
||||
u_int8_t can_have_a_subprotocol,
|
||||
u_int16_t tcp_alias_protoId[2],
|
||||
u_int16_t udp_alias_protoId[2], char *protoName,
|
||||
ndpi_protocol_breed_t protoBreed, u_int16_t protoId, char *protoName,
|
||||
ndpi_protocol_category_t protoCategory,
|
||||
ndpi_port_range *tcpDefPorts,
|
||||
ndpi_port_range *udpDefPorts);
|
||||
@@ -127,11 +131,7 @@ extern "C" {
|
||||
extern u_int8_t ndpi_is_proto(ndpi_protocol proto, u_int16_t p);
|
||||
|
||||
extern u_int16_t ndpi_get_lower_proto(ndpi_protocol p);
|
||||
extern int ndpi_get_protocol_id_master_proto(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
u_int16_t protocol_id,
|
||||
u_int16_t** tcp_master_proto,
|
||||
u_int16_t** udp_master_proto);
|
||||
#/* NDPI_PROTOCOL_NETBIOS */
|
||||
/* NDPI_PROTOCOL_NETBIOS */
|
||||
int ndpi_netbios_name_interpret(char *in, size_t inlen, char *out, u_int out_len);
|
||||
|
||||
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
|
||||
|
||||
@@ -809,6 +809,24 @@ struct ndpi_flow_udp_struct {
|
||||
|
||||
/* ************************************************** */
|
||||
|
||||
#define LINE_EQUALS(ndpi_int_one_line_struct, string_to_compare) \
|
||||
((ndpi_int_one_line_struct).len == strlen(string_to_compare) && \
|
||||
LINE_CMP(ndpi_int_one_line_struct, string_to_compare, strlen(string_to_compare)) == 1)
|
||||
|
||||
#define LINE_STARTS(ndpi_int_one_line_struct, string_to_compare) \
|
||||
((ndpi_int_one_line_struct).len >= strlen(string_to_compare) && \
|
||||
LINE_CMP(ndpi_int_one_line_struct, string_to_compare, strlen(string_to_compare)) == 1)
|
||||
|
||||
#define LINE_ENDS(ndpi_int_one_line_struct, string_to_compare) \
|
||||
((ndpi_int_one_line_struct).len >= strlen(string_to_compare) && \
|
||||
memcmp((ndpi_int_one_line_struct).ptr + \
|
||||
((ndpi_int_one_line_struct).len - strlen(string_to_compare)), \
|
||||
string_to_compare, strlen(string_to_compare)) == 0)
|
||||
|
||||
#define LINE_CMP(ndpi_int_one_line_struct, string_to_compare, string_to_compare_length) \
|
||||
((ndpi_int_one_line_struct).ptr != NULL && \
|
||||
memcmp((ndpi_int_one_line_struct).ptr, string_to_compare, string_to_compare_length) == 0)
|
||||
|
||||
struct ndpi_int_one_line_struct {
|
||||
const u_int8_t *ptr;
|
||||
u_int16_t len;
|
||||
@@ -825,7 +843,6 @@ struct ndpi_packet_struct {
|
||||
u_int64_t current_time_ms;
|
||||
|
||||
u_int16_t detected_protocol_stack[NDPI_PROTOCOL_SIZE];
|
||||
u_int8_t detected_subprotocol_stack[NDPI_PROTOCOL_SIZE];
|
||||
u_int16_t protocol_stack_info;
|
||||
|
||||
struct ndpi_int_one_line_struct line[NDPI_MAX_PARSE_LINES_PER_PACKET];
|
||||
@@ -987,9 +1004,9 @@ typedef enum {
|
||||
typedef struct ndpi_proto_defaults {
|
||||
char *protoName;
|
||||
ndpi_protocol_category_t protoCategory;
|
||||
u_int8_t can_have_a_subprotocol;
|
||||
u_int16_t * subprotocols;
|
||||
u_int32_t subprotocol_count;
|
||||
u_int16_t protoId, protoIdx;
|
||||
u_int16_t master_tcp_protoId[2], master_udp_protoId[2]; /* The main protocols on which this sub-protocol sits on */
|
||||
u_int16_t tcp_default_ports[MAX_DEFAULT_PORTS], udp_default_ports[MAX_DEFAULT_PORTS];
|
||||
ndpi_protocol_breed_t protoBreed;
|
||||
void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow);
|
||||
|
||||
@@ -710,7 +710,7 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
|
||||
#ifdef COSINE_SIMILARITY
|
||||
best_similarity = -1;
|
||||
#else
|
||||
best_similarity = 99999999999;
|
||||
best_similarity = 99999999999.0f;
|
||||
#endif
|
||||
|
||||
for(j=0; j<num_clusters; j++) {
|
||||
|
||||
+602
-631
File diff suppressed because it is too large
Load Diff
+23
-48
@@ -33,22 +33,10 @@
|
||||
static void ndpi_int_aimini_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow/* , */
|
||||
/* ndpi_protocol_type_t protocol_type */)
|
||||
{
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_AIMINI, NDPI_PROTOCOL_UNKNOWN);
|
||||
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_AIMINI);
|
||||
}
|
||||
|
||||
|
||||
static u_int8_t is_special_aimini_host(struct ndpi_int_one_line_struct host_line)
|
||||
{
|
||||
if (host_line.ptr != NULL && host_line.len >= NDPI_STATICSTRING_LEN("X.X.X.X.aimini.net")) {
|
||||
if ((get_u_int32_t(host_line.ptr, 0) & htonl(0x00ff00ff)) == htonl(0x002e002e) &&
|
||||
(get_u_int32_t(host_line.ptr, 4) & htonl(0x00ff00ff)) == htonl(0x002e002e) &&
|
||||
memcmp(&host_line.ptr[8], "aimini.net", NDPI_STATICSTRING_LEN("aimini.net")) == 0) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void ndpi_search_aimini(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
|
||||
{
|
||||
struct ndpi_packet_struct *packet = &flow->packet;
|
||||
@@ -232,45 +220,32 @@ void ndpi_search_aimini(struct ndpi_detection_module_struct *ndpi_struct, struct
|
||||
ndpi_int_aimini_add_connection(ndpi_struct, flow);
|
||||
return;
|
||||
}
|
||||
} else if (packet->tcp != NULL) {
|
||||
if ((packet->payload_packet_len > NDPI_STATICSTRING_LEN("GET /player/") &&
|
||||
(memcmp(packet->payload, "GET /player/", NDPI_STATICSTRING_LEN("GET /player/")) == 0)) ||
|
||||
(packet->payload_packet_len > NDPI_STATICSTRING_LEN("GET /play/?fid=") &&
|
||||
(memcmp(packet->payload, "GET /play/?fid=", NDPI_STATICSTRING_LEN("GET /play/?fid=")) == 0))) {
|
||||
NDPI_LOG_DBG2(ndpi_struct, "HTTP packet detected\n");
|
||||
ndpi_parse_packet_line_info(ndpi_struct, flow);
|
||||
if (packet->host_line.ptr != NULL && packet->host_line.len > 11
|
||||
&& (memcmp(&packet->host_line.ptr[packet->host_line.len - 11], ".aimini.net", 11) == 0)) {
|
||||
}
|
||||
|
||||
if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP) {
|
||||
if (flow->http.method == NDPI_HTTP_METHOD_GET)
|
||||
{
|
||||
if ((LINE_STARTS(packet->http_url_name, "/download/") == 1 ||
|
||||
LINE_STARTS(packet->http_url_name, "/player/") == 1 ||
|
||||
LINE_STARTS(packet->http_url_name, "/play/") == 1 ||
|
||||
LINE_STARTS(packet->http_url_name, "/member/") == 1) &&
|
||||
(LINE_ENDS(packet->host_line, ".aimini.net") == 1 ||
|
||||
LINE_ENDS(packet->host_line, ".aimini.com") == 1))
|
||||
{
|
||||
NDPI_LOG_INFO(ndpi_struct, "found AIMINI HTTP traffic\n");
|
||||
ndpi_int_aimini_add_connection(ndpi_struct, flow);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (packet->payload_packet_len > 100) {
|
||||
if (memcmp(packet->payload, "GET /", NDPI_STATICSTRING_LEN("GET /")) == 0) {
|
||||
if (memcmp(&packet->payload[NDPI_STATICSTRING_LEN("GET /")], "play/",
|
||||
NDPI_STATICSTRING_LEN("play/")) == 0 ||
|
||||
memcmp(&packet->payload[NDPI_STATICSTRING_LEN("GET /")], "download/",
|
||||
NDPI_STATICSTRING_LEN("download/")) == 0) {
|
||||
ndpi_parse_packet_line_info(ndpi_struct, flow);
|
||||
if (is_special_aimini_host(packet->host_line) == 1) {
|
||||
NDPI_LOG_INFO(ndpi_struct,
|
||||
"found AIMINI HTTP traffic\n");
|
||||
ndpi_int_aimini_add_connection(ndpi_struct, flow);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else if (memcmp(packet->payload, "POST /", NDPI_STATICSTRING_LEN("POST /")) == 0) {
|
||||
if (memcmp(&packet->payload[NDPI_STATICSTRING_LEN("POST /")], "upload/",
|
||||
NDPI_STATICSTRING_LEN("upload/")) == 0) {
|
||||
ndpi_parse_packet_line_info(ndpi_struct, flow);
|
||||
if (is_special_aimini_host(packet->host_line) == 1) {
|
||||
NDPI_LOG_INFO(ndpi_struct,
|
||||
"found AIMINI HTTP traffic detected.\n");
|
||||
ndpi_int_aimini_add_connection(ndpi_struct, flow);
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else if (flow->http.method == NDPI_HTTP_METHOD_POST)
|
||||
{
|
||||
if ((LINE_STARTS(packet->http_url_name, "/upload/") == 1 ||
|
||||
LINE_STARTS(packet->http_url_name, "/member/") == 1) &&
|
||||
(LINE_ENDS(packet->host_line, ".aimini.net") == 1 ||
|
||||
LINE_ENDS(packet->host_line, ".aimini.com") == 1))
|
||||
{
|
||||
NDPI_LOG_INFO(ndpi_struct, "found AIMINI HTTP traffic\n");
|
||||
ndpi_int_aimini_add_connection(ndpi_struct, flow);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -573,7 +573,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
|
||||
**/
|
||||
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN) {
|
||||
/* Avoid putting as subprotocol a "core" protocol such as SSL or DNS */
|
||||
if(ndpi_struct->proto_defaults[flow->guessed_protocol_id].can_have_a_subprotocol == 0) {
|
||||
if(ndpi_struct->proto_defaults[flow->guessed_protocol_id].subprotocol_count == 0) {
|
||||
flow->detected_protocol_stack[1] = flow->guessed_protocol_id;
|
||||
if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
|
||||
flow->detected_protocol_stack[0] = flow->guessed_host_protocol_id;
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,6 @@
|
||||
HTTP 133 86722 4
|
||||
|
||||
1 TCP 10.101.0.2:28501 <-> 10.102.0.2:80 [proto: 99.7/Aimini.HTTP][cat: Web/5][38 pkts/36756 bytes <-> 34 pkts/28010 bytes][Goodput ratio: 94/93][0.00 sec][Host: www.aimini.net][bytes ratio: 0.135 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 0/0 1/0 0/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 967/824 1514/1514 664/699][URL: www.aimini.net/member/signup/][StatusCode: 0][User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17][PLAIN TEXT (GET /member/signup/ HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,4,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,83,0,0]
|
||||
2 TCP 10.101.0.2:28503 <-> 10.102.0.2:80 [proto: 7/HTTP][cat: Web/5][12 pkts/1920 bytes <-> 18 pkts/13384 bytes][Goodput ratio: 63/92][0.00 sec][Host: www.aimini.net][bytes ratio: -0.749 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 0/0 0/0 0/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 160/744 658/1514 223/696][URL: www.aimini.net/search/?q=pictures&sca=][StatusCode: 200][User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17][PLAIN TEXT (GET /search/)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0,0]
|
||||
3 TCP 10.101.0.2:28502 <-> 10.102.0.2:80 [proto: 7/HTTP][cat: Web/5][10 pkts/2690 bytes <-> 8 pkts/1544 bytes][Goodput ratio: 78/70][0.00 sec][Host: www.aimini.com][bytes ratio: 0.271 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 0/0 0/0 0/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 269/193 585/339 255/132][URL: www.aimini.com/webcounter/w.php?___hm=.net_SignUp_&_lh_=http://www.aimini.net/member/signup/&__Refer_=http://www.aimini.net/][StatusCode: 200][User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17][PLAIN TEXT (GET /webcounter/w.php)][Plen Bins: 0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
||||
4 TCP 10.101.0.2:28504 <-> 10.102.0.2:80 [proto: 7/HTTP][cat: Web/5][8 pkts/1556 bytes <-> 5 pkts/862 bytes][Goodput ratio: 70/66][0.00 sec][Host: www.aimini.com][bytes ratio: 0.287 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 0/0 0/0 0/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 194/172 596/339 232/136][URL: www.aimini.com/webcounter/w.php?___hm=.net_Search_&_lh_=http://www.aimini.net/search/?q=pictures&sca=&__Refer_=http://www.aimini.net/][StatusCode: 200][User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17][PLAIN TEXT (GET /webcounter/w.php)][Plen Bins: 0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
|
||||
Reference in New Issue
Block a user