Refactored nDPI subprotocol handling and aimini protocol detection. (#1156)

* Refactored and merged callback buffer routines for non-udp-tcp / udp / tcp / tcp-wo-payload.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

* Try to detect one subprotocol if a detected protocol can have one.

 * This adds a performance overhead due to much more protocol detection routine calls.
   See #1148 for more information.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

* Refactor subprotocol handling (1/2).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

* Refactor subprotocol handling (2/2).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

* Prevent some code duplication by using macros for ndpi_int_one_line_struct string comparision.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

* Refactored aimini HTTP detection parts (somehow related to #1148).

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

* Added aimini client/server test pcap.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>

* Removed master protocol as it was only used for STUN and via also removed API function ndpi_get_protocol_id_master_proto

 * Adjusted Python code to conform to the changes made during the refactoring process.

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
This commit is contained in:
Toni
2021-03-23 11:46:12 +01:00
committed by GitHub
parent 0cae9bf4a4
commit b040407683
11 changed files with 667 additions and 701 deletions
+1 -1
View File
@@ -246,7 +246,7 @@ void init_doh_bins() {
u_int check_bin_doh_similarity(struct ndpi_bin *bin, float *similarity) {
u_int i;
float lowest_similarity = 9999999999;
float lowest_similarity = 9999999999.0f;
for(i=0; i<NUM_DOH_BINS; i++) {
*similarity = ndpi_bin_similarity(&doh_ndpi_bins[i], bin, 0);
+2 -3
View File
@@ -716,7 +716,6 @@ struct ndpi_packet_struct {
uint64_t current_time_ms;
uint16_t detected_protocol_stack[2];
uint8_t detected_subprotocol_stack[2];
uint16_t protocol_stack_info;
struct ndpi_int_one_line_struct line[64];
@@ -870,9 +869,9 @@ typedef enum {
typedef struct ndpi_proto_defaults {
char *protoName;
ndpi_protocol_category_t protoCategory;
uint8_t can_have_a_subprotocol;
u_int16_t * subprotocols;
size_t subprotocol_count;
uint16_t protoId, protoIdx;
uint16_t master_tcp_protoId[2], master_udp_protoId[2]; /* The main protocols on which this sub-protocol sits on */
uint16_t tcp_default_ports[5], udp_default_ports[5];
ndpi_protocol_breed_t protoBreed;
void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow);
+2 -4
View File
@@ -94,11 +94,10 @@ class NDPIProtoDefaultsT(Structure):
_fields_ = [
("protoName", c_char_p),
("protoCategory", c_uint),
("can_have_a_subprotocol", c_uint8),
("subprotocols", c_uint16_p),
("subprotocol_count", c_uint32),
("protoId", c_uint16),
("protoIdx", c_uint16),
("master_tcp_protoId", c_uint16 * 2),
("master_udp_protoId", c_uint16 * 2),
("protoBreed", c_uint),
("func", CFUNCTYPE(None, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
]
@@ -653,7 +652,6 @@ class NDPIUdpHdr(Structure):
class NDPIPacketStructStack(Structure):
_pack_ = 1
_fields_ = [
('detected_subprotocol_stack', c_uint8 * ndpi.ndpi_wrap_ndpi_procol_size()),
('protocol_stack_info', c_uint16)
]
+9 -9
View File
@@ -31,6 +31,10 @@
#include "ndpi_api.h"
#include "ndpi_protocols.h"
/* used by ndpi_set_proto_subprotocols */
#define NDPI_PROTOCOL_NO_MORE_SUBPROTOCOLS (-1)
#define NDPI_PROTOCOL_MATCHED_BY_CONTENT (-2)
#ifdef __cplusplus
extern "C" {
#endif
@@ -95,11 +99,11 @@ extern "C" {
struct ndpi_flow_struct *flow,
ndpi_protocol_category_t protocol_category);
extern void ndpi_set_proto_subprotocols(struct ndpi_detection_module_struct *ndpi_mod,
int protoId, ...);
extern void ndpi_set_proto_defaults(struct ndpi_detection_module_struct *ndpi_mod,
ndpi_protocol_breed_t protoBreed, u_int16_t protoId,
u_int8_t can_have_a_subprotocol,
u_int16_t tcp_alias_protoId[2],
u_int16_t udp_alias_protoId[2], char *protoName,
ndpi_protocol_breed_t protoBreed, u_int16_t protoId, char *protoName,
ndpi_protocol_category_t protoCategory,
ndpi_port_range *tcpDefPorts,
ndpi_port_range *udpDefPorts);
@@ -127,11 +131,7 @@ extern "C" {
extern u_int8_t ndpi_is_proto(ndpi_protocol proto, u_int16_t p);
extern u_int16_t ndpi_get_lower_proto(ndpi_protocol p);
extern int ndpi_get_protocol_id_master_proto(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t protocol_id,
u_int16_t** tcp_master_proto,
u_int16_t** udp_master_proto);
#/* NDPI_PROTOCOL_NETBIOS */
/* NDPI_PROTOCOL_NETBIOS */
int ndpi_netbios_name_interpret(char *in, size_t inlen, char *out, u_int out_len);
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
+20 -3
View File
@@ -809,6 +809,24 @@ struct ndpi_flow_udp_struct {
/* ************************************************** */
#define LINE_EQUALS(ndpi_int_one_line_struct, string_to_compare) \
((ndpi_int_one_line_struct).len == strlen(string_to_compare) && \
LINE_CMP(ndpi_int_one_line_struct, string_to_compare, strlen(string_to_compare)) == 1)
#define LINE_STARTS(ndpi_int_one_line_struct, string_to_compare) \
((ndpi_int_one_line_struct).len >= strlen(string_to_compare) && \
LINE_CMP(ndpi_int_one_line_struct, string_to_compare, strlen(string_to_compare)) == 1)
#define LINE_ENDS(ndpi_int_one_line_struct, string_to_compare) \
((ndpi_int_one_line_struct).len >= strlen(string_to_compare) && \
memcmp((ndpi_int_one_line_struct).ptr + \
((ndpi_int_one_line_struct).len - strlen(string_to_compare)), \
string_to_compare, strlen(string_to_compare)) == 0)
#define LINE_CMP(ndpi_int_one_line_struct, string_to_compare, string_to_compare_length) \
((ndpi_int_one_line_struct).ptr != NULL && \
memcmp((ndpi_int_one_line_struct).ptr, string_to_compare, string_to_compare_length) == 0)
struct ndpi_int_one_line_struct {
const u_int8_t *ptr;
u_int16_t len;
@@ -825,7 +843,6 @@ struct ndpi_packet_struct {
u_int64_t current_time_ms;
u_int16_t detected_protocol_stack[NDPI_PROTOCOL_SIZE];
u_int8_t detected_subprotocol_stack[NDPI_PROTOCOL_SIZE];
u_int16_t protocol_stack_info;
struct ndpi_int_one_line_struct line[NDPI_MAX_PARSE_LINES_PER_PACKET];
@@ -987,9 +1004,9 @@ typedef enum {
typedef struct ndpi_proto_defaults {
char *protoName;
ndpi_protocol_category_t protoCategory;
u_int8_t can_have_a_subprotocol;
u_int16_t * subprotocols;
u_int32_t subprotocol_count;
u_int16_t protoId, protoIdx;
u_int16_t master_tcp_protoId[2], master_udp_protoId[2]; /* The main protocols on which this sub-protocol sits on */
u_int16_t tcp_default_ports[MAX_DEFAULT_PORTS], udp_default_ports[MAX_DEFAULT_PORTS];
ndpi_protocol_breed_t protoBreed;
void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow);
+1 -1
View File
@@ -710,7 +710,7 @@ int ndpi_cluster_bins(struct ndpi_bin *bins, u_int16_t num_bins,
#ifdef COSINE_SIMILARITY
best_similarity = -1;
#else
best_similarity = 99999999999;
best_similarity = 99999999999.0f;
#endif
for(j=0; j<num_clusters; j++) {
+602 -631
View File
File diff suppressed because it is too large Load Diff
+23 -48
View File
@@ -33,22 +33,10 @@
static void ndpi_int_aimini_add_connection(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow/* , */
/* ndpi_protocol_type_t protocol_type */)
{
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_AIMINI, NDPI_PROTOCOL_UNKNOWN);
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HTTP, NDPI_PROTOCOL_AIMINI);
}
static u_int8_t is_special_aimini_host(struct ndpi_int_one_line_struct host_line)
{
if (host_line.ptr != NULL && host_line.len >= NDPI_STATICSTRING_LEN("X.X.X.X.aimini.net")) {
if ((get_u_int32_t(host_line.ptr, 0) & htonl(0x00ff00ff)) == htonl(0x002e002e) &&
(get_u_int32_t(host_line.ptr, 4) & htonl(0x00ff00ff)) == htonl(0x002e002e) &&
memcmp(&host_line.ptr[8], "aimini.net", NDPI_STATICSTRING_LEN("aimini.net")) == 0) {
return 1;
}
}
return 0;
}
void ndpi_search_aimini(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct *packet = &flow->packet;
@@ -232,45 +220,32 @@ void ndpi_search_aimini(struct ndpi_detection_module_struct *ndpi_struct, struct
ndpi_int_aimini_add_connection(ndpi_struct, flow);
return;
}
} else if (packet->tcp != NULL) {
if ((packet->payload_packet_len > NDPI_STATICSTRING_LEN("GET /player/") &&
(memcmp(packet->payload, "GET /player/", NDPI_STATICSTRING_LEN("GET /player/")) == 0)) ||
(packet->payload_packet_len > NDPI_STATICSTRING_LEN("GET /play/?fid=") &&
(memcmp(packet->payload, "GET /play/?fid=", NDPI_STATICSTRING_LEN("GET /play/?fid=")) == 0))) {
NDPI_LOG_DBG2(ndpi_struct, "HTTP packet detected\n");
ndpi_parse_packet_line_info(ndpi_struct, flow);
if (packet->host_line.ptr != NULL && packet->host_line.len > 11
&& (memcmp(&packet->host_line.ptr[packet->host_line.len - 11], ".aimini.net", 11) == 0)) {
}
if (flow->detected_protocol_stack[0] == NDPI_PROTOCOL_HTTP) {
if (flow->http.method == NDPI_HTTP_METHOD_GET)
{
if ((LINE_STARTS(packet->http_url_name, "/download/") == 1 ||
LINE_STARTS(packet->http_url_name, "/player/") == 1 ||
LINE_STARTS(packet->http_url_name, "/play/") == 1 ||
LINE_STARTS(packet->http_url_name, "/member/") == 1) &&
(LINE_ENDS(packet->host_line, ".aimini.net") == 1 ||
LINE_ENDS(packet->host_line, ".aimini.com") == 1))
{
NDPI_LOG_INFO(ndpi_struct, "found AIMINI HTTP traffic\n");
ndpi_int_aimini_add_connection(ndpi_struct, flow);
return;
}
}
if (packet->payload_packet_len > 100) {
if (memcmp(packet->payload, "GET /", NDPI_STATICSTRING_LEN("GET /")) == 0) {
if (memcmp(&packet->payload[NDPI_STATICSTRING_LEN("GET /")], "play/",
NDPI_STATICSTRING_LEN("play/")) == 0 ||
memcmp(&packet->payload[NDPI_STATICSTRING_LEN("GET /")], "download/",
NDPI_STATICSTRING_LEN("download/")) == 0) {
ndpi_parse_packet_line_info(ndpi_struct, flow);
if (is_special_aimini_host(packet->host_line) == 1) {
NDPI_LOG_INFO(ndpi_struct,
"found AIMINI HTTP traffic\n");
ndpi_int_aimini_add_connection(ndpi_struct, flow);
return;
}
}
} else if (memcmp(packet->payload, "POST /", NDPI_STATICSTRING_LEN("POST /")) == 0) {
if (memcmp(&packet->payload[NDPI_STATICSTRING_LEN("POST /")], "upload/",
NDPI_STATICSTRING_LEN("upload/")) == 0) {
ndpi_parse_packet_line_info(ndpi_struct, flow);
if (is_special_aimini_host(packet->host_line) == 1) {
NDPI_LOG_INFO(ndpi_struct,
"found AIMINI HTTP traffic detected.\n");
ndpi_int_aimini_add_connection(ndpi_struct, flow);
return;
}
}
} else if (flow->http.method == NDPI_HTTP_METHOD_POST)
{
if ((LINE_STARTS(packet->http_url_name, "/upload/") == 1 ||
LINE_STARTS(packet->http_url_name, "/member/") == 1) &&
(LINE_ENDS(packet->host_line, ".aimini.net") == 1 ||
LINE_ENDS(packet->host_line, ".aimini.com") == 1))
{
NDPI_LOG_INFO(ndpi_struct, "found AIMINI HTTP traffic\n");
ndpi_int_aimini_add_connection(ndpi_struct, flow);
return;
}
}
}
+1 -1
View File
@@ -573,7 +573,7 @@ static void check_content_type_and_change_protocol(struct ndpi_detection_module_
**/
if(flow->detected_protocol_stack[1] == NDPI_PROTOCOL_UNKNOWN) {
/* Avoid putting as subprotocol a "core" protocol such as SSL or DNS */
if(ndpi_struct->proto_defaults[flow->guessed_protocol_id].can_have_a_subprotocol == 0) {
if(ndpi_struct->proto_defaults[flow->guessed_protocol_id].subprotocol_count == 0) {
flow->detected_protocol_stack[1] = flow->guessed_protocol_id;
if(flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN)
flow->detected_protocol_stack[0] = flow->guessed_host_protocol_id;
Binary file not shown.
+6
View File
@@ -0,0 +1,6 @@
HTTP 133 86722 4
1 TCP 10.101.0.2:28501 <-> 10.102.0.2:80 [proto: 99.7/Aimini.HTTP][cat: Web/5][38 pkts/36756 bytes <-> 34 pkts/28010 bytes][Goodput ratio: 94/93][0.00 sec][Host: www.aimini.net][bytes ratio: 0.135 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 0/0 1/0 0/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 967/824 1514/1514 664/699][URL: www.aimini.net/member/signup/][StatusCode: 0][User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17][PLAIN TEXT (GET /member/signup/ HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,4,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,83,0,0]
2 TCP 10.101.0.2:28503 <-> 10.102.0.2:80 [proto: 7/HTTP][cat: Web/5][12 pkts/1920 bytes <-> 18 pkts/13384 bytes][Goodput ratio: 63/92][0.00 sec][Host: www.aimini.net][bytes ratio: -0.749 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 0/0 0/0 0/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 160/744 658/1514 223/696][URL: www.aimini.net/search/?q=pictures&sca=][StatusCode: 200][User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17][PLAIN TEXT (GET /search/)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,16,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,67,0,0]
3 TCP 10.101.0.2:28502 <-> 10.102.0.2:80 [proto: 7/HTTP][cat: Web/5][10 pkts/2690 bytes <-> 8 pkts/1544 bytes][Goodput ratio: 78/70][0.00 sec][Host: www.aimini.com][bytes ratio: 0.271 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 0/0 0/0 0/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 269/193 585/339 255/132][URL: www.aimini.com/webcounter/w.php?___hm=.net_SignUp_&_lh_=http://www.aimini.net/member/signup/&__Refer_=http://www.aimini.net/][StatusCode: 200][User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17][PLAIN TEXT (GET /webcounter/w.php)][Plen Bins: 0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 TCP 10.101.0.2:28504 <-> 10.102.0.2:80 [proto: 7/HTTP][cat: Web/5][8 pkts/1556 bytes <-> 5 pkts/862 bytes][Goodput ratio: 70/66][0.00 sec][Host: www.aimini.com][bytes ratio: 0.287 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 0/0 0/0 0/0][Pkt Len c2s/s2c min/avg/max/stddev: 60/60 194/172 596/339 232/136][URL: www.aimini.com/webcounter/w.php?___hm=.net_Search_&_lh_=http://www.aimini.net/search/?q=pictures&sca=&__Refer_=http://www.aimini.net/][StatusCode: 200][User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.17) Gecko/20110420 Firefox/3.6.17][PLAIN TEXT (GET /webcounter/w.php)][Plen Bins: 0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]