fragments management added (#1122)
Management of tcp segments managements. Co-authored-by: ragostino <ragostino73@gmail.com> Co-authored-by: Luca Deri <lucaderi@users.noreply.github.com>
This commit is contained in:
co-authored by
ragostino
Luca Deri
parent
ee94534906
commit
b70ad0e2f1
@@ -27,6 +27,7 @@
|
||||
#include "ndpi_define.h"
|
||||
#include "ndpi_protocol_ids.h"
|
||||
|
||||
#include "ndpi_utils.h"
|
||||
/* NDPI_LOG_LEVEL */
|
||||
typedef enum {
|
||||
NDPI_LOG_ERROR,
|
||||
@@ -430,6 +431,12 @@ struct ndpi_vxlanhdr {
|
||||
/* ******************* ********************* ****************** */
|
||||
/* ************************************************************ */
|
||||
|
||||
typedef struct message {
|
||||
u_int8_t *buffer;
|
||||
u_int buffer_len, buffer_used, max_expected;
|
||||
u_int32_t next_seq[2]; /* Directions */
|
||||
} message_t;
|
||||
|
||||
/* NDPI_PROTOCOL_BITTORRENT */
|
||||
typedef struct spinlock {
|
||||
volatile int val;
|
||||
@@ -667,11 +674,9 @@ struct ndpi_flow_tcp_struct {
|
||||
u_int32_t telnet_stage:2; // 0 - 2
|
||||
|
||||
struct {
|
||||
struct {
|
||||
u_int8_t *buffer;
|
||||
u_int buffer_len, buffer_used;
|
||||
u_int32_t next_seq[2]; /* Directions */
|
||||
} message;
|
||||
message_t message;
|
||||
|
||||
void* srv_cert_fingerprint_ctx; /* SHA-1 */
|
||||
|
||||
/* NDPI_PROTOCOL_TLS */
|
||||
u_int8_t hello_processed:1, certificate_processed:1, subprotocol_detected:1,
|
||||
@@ -1182,9 +1187,19 @@ struct ndpi_flow_struct {
|
||||
*/
|
||||
u_int32_t next_tcp_seq_nr[2];
|
||||
|
||||
#ifdef FRAG_MAN
|
||||
/* tcp_segments lists */
|
||||
u_int8_t tcp_segments_management:1;
|
||||
u_int8_t not_sorted[2],must_free[2]; // 0: client->server and 1: server->client
|
||||
uint32_t trigger[2]; // the seq waited number to start to reassembly
|
||||
fragments_wrapper_t tcp_segments_list[2];
|
||||
#endif // FRAG_MAN
|
||||
|
||||
// -----------------------------------------
|
||||
|
||||
u_int8_t max_extra_packets_to_check;
|
||||
u_int8_t num_extra_packets_checked;
|
||||
u_int8_t num_processed_pkts; /* <= WARNING it can wrap but we do expect people to giveup earlier */
|
||||
u_int16_t num_processed_pkts; /* <= WARNING it can wrap but we do expect people to giveup earlier */
|
||||
|
||||
int (*extra_packets_func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow);
|
||||
|
||||
@@ -1272,7 +1287,8 @@ struct ndpi_flow_struct {
|
||||
} tls_quic;
|
||||
|
||||
struct {
|
||||
u_int8_t num_udp_pkts, num_processed_pkts, num_binding_requests;
|
||||
u_int8_t num_udp_pkts, num_binding_requests;
|
||||
u_int16_t num_processed_pkts;
|
||||
} stun;
|
||||
|
||||
/* We can have STUN over SSL/TLS thus they need to live together */
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
|
||||
|
||||
#ifndef __NDPI_UTILS_H__
|
||||
#define __NDPI_UTILS_H__
|
||||
|
||||
#include "ndpi_define.h"
|
||||
|
||||
#define MYDBG(m, ...) \
|
||||
printf(" DBG[%s:%s:%u]: \t" m "\n", __FILE__, __FUNCTION__, __LINE__, ##__VA_ARGS__);
|
||||
|
||||
|
||||
// #define NDPI_ENABLE_DEBUG_POINTER_MESSAGES
|
||||
// #define NDPI_ENABLE_DEBUG_INFO_MESSAGES
|
||||
// #define NDPI_ENABLE_DEBUG_TRACE_MESSAGES
|
||||
|
||||
#ifdef NDPI_ENABLE_DEBUG_POINTER_MESSAGES
|
||||
#define DBGPOINTER(m, args...) MYDBG(m, ##args)
|
||||
#else
|
||||
#define DBGPOINTER(m, args...)
|
||||
#endif
|
||||
|
||||
#ifdef NDPI_ENABLE_DEBUG_INFO_MESSAGES
|
||||
#define DBGINFO(m, args...) MYDBG(m, ##args)
|
||||
#else
|
||||
#define DBGINFO(m, args...)
|
||||
#endif
|
||||
|
||||
#ifdef NDPI_ENABLE_DEBUG_TRACE_MESSAGES
|
||||
#define DBGTRACER(m, args...) MYDBG(m, ##args)
|
||||
#else
|
||||
#define DBGTRACER(m, args...)
|
||||
#endif
|
||||
|
||||
// FRAGMENTATION
|
||||
typedef struct {
|
||||
uint32_t offset;
|
||||
size_t len;
|
||||
void *data;
|
||||
} fragment_t;
|
||||
|
||||
typedef struct fragment_wrapper {
|
||||
uint16_t id;
|
||||
uint8_t l4_protocol;
|
||||
uint32_t initial_offset;
|
||||
uint16_t ct_frag;
|
||||
#ifdef NDPI_DETECTION_SUPPORT_IPV6
|
||||
char *flow_label; // IP6
|
||||
#endif
|
||||
char gap[200];
|
||||
fragment_t **fragments_list;
|
||||
} fragments_wrapper_t;
|
||||
|
||||
typedef struct fragments_buffer {
|
||||
u_int8_t *buffer;
|
||||
u_int buffer_len, buffer_used;
|
||||
} fragments_buffer_t;
|
||||
|
||||
// SORTING
|
||||
typedef struct {
|
||||
int sort_value;
|
||||
int item_index;
|
||||
} sorter_index_item_t;
|
||||
|
||||
/* ***************************************************** */
|
||||
|
||||
extern void ins_sort_array(sorter_index_item_t arr[], int len);
|
||||
extern void shell_sort_array(sorter_index_item_t arr[], int len);
|
||||
extern void free_fragment(fragments_wrapper_t *frag);
|
||||
|
||||
extern void printRawData(const uint8_t *ptr, size_t len);
|
||||
//extern uint8_t add_segment_to_buffer( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph, uint32_t waited);
|
||||
//extern uint8_t check_for_sequence( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph);
|
||||
|
||||
#endif
|
||||
+170
-41
@@ -66,6 +66,11 @@ static void (*_ndpi_flow_free)(void *ptr);
|
||||
static void *(*_ndpi_malloc)(size_t size);
|
||||
static void (*_ndpi_free)(void *ptr);
|
||||
|
||||
#ifdef FRAG_MAN
|
||||
extern void add_segment_to_buffer( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph);
|
||||
extern uint8_t check_for_sequence( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph);
|
||||
#endif // FRAG_MAN
|
||||
|
||||
/* ****************************************** */
|
||||
|
||||
/* Forward */
|
||||
@@ -3490,6 +3495,40 @@ int ndpi_handle_ipv6_extension_headers(struct ndpi_detection_module_struct *ndpi
|
||||
#endif /* NDPI_DETECTION_SUPPORT_IPV6 */
|
||||
|
||||
static u_int8_t ndpi_iph_is_valid_and_not_fragmented(const struct ndpi_iphdr *iph, const u_int16_t ipsize) {
|
||||
|
||||
#ifdef FRAG_MAN
|
||||
/*
|
||||
the logic has been inverted!!! returned value:
|
||||
0: not fragmented (instead of fragmented)
|
||||
1: packet too small
|
||||
2: fragmented and last, reassemble
|
||||
3: fragmented but not the last, add to buffer
|
||||
*/
|
||||
u_int16_t tot_len = ntohs(iph->tot_len);
|
||||
if ( ipsize < iph->ihl * 4 || ipsize < tot_len || tot_len < iph->ihl * 4 )
|
||||
// packet too small
|
||||
return(1);
|
||||
else if ((iph->frag_off & htons(0x2000)) != 0) {
|
||||
// MF=1 : this is a fragment and not the last -> add to buffer
|
||||
//printf("DBG(ndpi_iph_is_valid_and_not_fragmented): ipv4 fragment and not the last! (off=%u) \n", (htons(iph->frag_off) & 0x1FFF)<<3);
|
||||
|
||||
// MUST add to buffer
|
||||
return(3);
|
||||
} else if ((iph->frag_off & htons(0x1FFF)) != 0) {
|
||||
// MF=0, this is (a fragment, but) the last fragment!
|
||||
//printf("DBG(ndpi_iph_is_valid_and_not_fragmented): ipv4 fragment and the last! (0ff=%u) \n", (htons(iph->frag_off) & 0x1FFF)<<3);
|
||||
|
||||
// MUST to reassemble the packet!
|
||||
return(2);
|
||||
}
|
||||
return (0);
|
||||
|
||||
#else // FRAG_MAN
|
||||
/*
|
||||
returned value:
|
||||
0: fragmented
|
||||
1: not fragmented
|
||||
*/
|
||||
//#ifdef REQUIRE_FULL_PACKETS
|
||||
if(ipsize < iph->ihl * 4 || ipsize < ntohs(iph->tot_len) || ntohs(iph->tot_len) < iph->ihl * 4 ||
|
||||
(iph->frag_off & htons(0x1FFF)) != 0) {
|
||||
@@ -3498,8 +3537,21 @@ static u_int8_t ndpi_iph_is_valid_and_not_fragmented(const struct ndpi_iphdr *ip
|
||||
//#endif
|
||||
|
||||
return(1);
|
||||
|
||||
#endif // FRAG_MAN
|
||||
}
|
||||
|
||||
/*
|
||||
extract the l4 payload, if available
|
||||
returned value:
|
||||
FRAG_MAN
|
||||
0: ok, extracted
|
||||
1: packet too small
|
||||
2,3: fragmented, ....
|
||||
else
|
||||
0: ok, extracted
|
||||
1: error or not available
|
||||
*/
|
||||
static u_int8_t ndpi_detection_get_l4_internal(struct ndpi_detection_module_struct *ndpi_str, const u_int8_t *l3,
|
||||
u_int16_t l3_len, const u_int8_t **l4_return, u_int16_t *l4_len_return,
|
||||
u_int8_t *l4_protocol_return, u_int32_t flags) {
|
||||
@@ -3542,6 +3594,27 @@ static u_int8_t ndpi_detection_get_l4_internal(struct ndpi_detection_module_stru
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef FRAG_MAN
|
||||
if(iph != NULL) {
|
||||
u_int8_t check4Frag = ndpi_iph_is_valid_and_not_fragmented(iph, l3_len);
|
||||
/* 0: not fragmented; 1: too small; 2,3: fragmented */
|
||||
if (!check4Frag) {
|
||||
u_int16_t len = ntohs(iph->tot_len);
|
||||
u_int16_t hlen = (iph->ihl * 4);
|
||||
|
||||
l4ptr = (((const u_int8_t *) iph) + hlen);
|
||||
|
||||
if(len == 0)
|
||||
len = l3_len;
|
||||
|
||||
l4len = (len > hlen) ? (len - hlen) : 0;
|
||||
l4protocol = iph->protocol;
|
||||
}
|
||||
else
|
||||
return check4Frag;
|
||||
}
|
||||
#else //FRAGMAN
|
||||
/* 0: fragmented; 1: not fragmented */
|
||||
if(iph != NULL && ndpi_iph_is_valid_and_not_fragmented(iph, l3_len)) {
|
||||
u_int16_t len = ntohs(iph->tot_len);
|
||||
u_int16_t hlen = (iph->ihl * 4);
|
||||
@@ -3554,6 +3627,7 @@ static u_int8_t ndpi_detection_get_l4_internal(struct ndpi_detection_module_stru
|
||||
l4len = (len > hlen) ? (len - hlen) : 0;
|
||||
l4protocol = iph->protocol;
|
||||
}
|
||||
#endif //FRAGMAN
|
||||
#ifdef NDPI_DETECTION_SUPPORT_IPV6
|
||||
else if(iph_v6 != NULL && (l3_len - sizeof(struct ndpi_ipv6hdr)) >= ntohs(iph_v6->ip6_hdr.ip6_un1_plen)) {
|
||||
l4ptr = (((const u_int8_t *) iph_v6) + sizeof(struct ndpi_ipv6hdr));
|
||||
@@ -3683,53 +3757,63 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str
|
||||
* idea: reset detection state if a connection is unknown
|
||||
*/
|
||||
if(flow->packet.tcp->syn != 0 && flow->packet.tcp->ack == 0 && flow->init_finished != 0 &&
|
||||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
|
||||
u_int8_t backup;
|
||||
u_int16_t backup1, backup2;
|
||||
flow->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN) {
|
||||
|
||||
u_int8_t backup;
|
||||
u_int16_t backup1, backup2;
|
||||
|
||||
if(flow->http.url) {
|
||||
ndpi_free(flow->http.url);
|
||||
flow->http.url = NULL;
|
||||
}
|
||||
#ifdef FRAG_MAN
|
||||
/* initialize the buffer to manage segments for a new http/dns connection */
|
||||
flow->tcp_segments_management=1;
|
||||
for (int i=0; i<2; i++ ) {
|
||||
// reset counter tcp segments management lists
|
||||
flow->tcp_segments_list[i].ct_frag=0;
|
||||
}
|
||||
#endif // FRAG_MAN
|
||||
|
||||
if(flow->http.content_type) {
|
||||
ndpi_free(flow->http.content_type);
|
||||
flow->http.content_type = NULL;
|
||||
}
|
||||
if(flow->http.url) {
|
||||
ndpi_free(flow->http.url);
|
||||
flow->http.url = NULL;
|
||||
}
|
||||
|
||||
if(flow->http.request_content_type) {
|
||||
ndpi_free(flow->http.request_content_type);
|
||||
flow->http.request_content_type = NULL;
|
||||
}
|
||||
if(flow->http.content_type) {
|
||||
ndpi_free(flow->http.content_type);
|
||||
flow->http.content_type = NULL;
|
||||
}
|
||||
|
||||
if(flow->http.user_agent) {
|
||||
ndpi_free(flow->http.user_agent);
|
||||
flow->http.user_agent = NULL;
|
||||
}
|
||||
if(flow->http.request_content_type) {
|
||||
ndpi_free(flow->http.request_content_type);
|
||||
flow->http.request_content_type = NULL;
|
||||
}
|
||||
|
||||
if(flow->kerberos_buf.pktbuf) {
|
||||
ndpi_free(flow->kerberos_buf.pktbuf);
|
||||
flow->kerberos_buf.pktbuf = NULL;
|
||||
}
|
||||
if(flow->http.user_agent) {
|
||||
ndpi_free(flow->http.user_agent);
|
||||
flow->http.user_agent = NULL;
|
||||
}
|
||||
|
||||
if(flow->l4.tcp.tls.message.buffer) {
|
||||
ndpi_free(flow->l4.tcp.tls.message.buffer);
|
||||
flow->l4.tcp.tls.message.buffer = NULL;
|
||||
flow->l4.tcp.tls.message.buffer_len = flow->l4.tcp.tls.message.buffer_used = 0;
|
||||
}
|
||||
if(flow->kerberos_buf.pktbuf) {
|
||||
ndpi_free(flow->kerberos_buf.pktbuf);
|
||||
flow->kerberos_buf.pktbuf = NULL;
|
||||
}
|
||||
|
||||
backup = flow->num_processed_pkts;
|
||||
backup1 = flow->guessed_protocol_id;
|
||||
backup2 = flow->guessed_host_protocol_id;
|
||||
memset(flow, 0, sizeof(*(flow)));
|
||||
if(flow->l4.tcp.tls.message.buffer) {
|
||||
ndpi_free(flow->l4.tcp.tls.message.buffer);
|
||||
flow->l4.tcp.tls.message.buffer = NULL;
|
||||
flow->l4.tcp.tls.message.buffer_len = flow->l4.tcp.tls.message.buffer_used = 0;
|
||||
}
|
||||
|
||||
/* Restore pointers */
|
||||
flow->num_processed_pkts = backup;
|
||||
flow->guessed_protocol_id = backup1;
|
||||
flow->guessed_host_protocol_id = backup2;
|
||||
flow->packet.tcp = (struct ndpi_tcphdr *) l4ptr;
|
||||
backup = flow->num_processed_pkts;
|
||||
backup1 = flow->guessed_protocol_id;
|
||||
backup2 = flow->guessed_host_protocol_id;
|
||||
memset(flow, 0, sizeof(*(flow)));
|
||||
|
||||
NDPI_LOG_DBG(ndpi_str, "tcp syn packet for unknown protocol, reset detection state\n");
|
||||
/* Restore pointers */
|
||||
flow->num_processed_pkts = backup;
|
||||
flow->guessed_protocol_id = backup1;
|
||||
flow->guessed_host_protocol_id = backup2;
|
||||
flow->packet.tcp = (struct ndpi_tcphdr *) l4ptr;
|
||||
|
||||
NDPI_LOG_DBG(ndpi_str, "tcp syn packet for unknown protocol, reset detection state\n");
|
||||
}
|
||||
} else {
|
||||
/* tcp header not complete */
|
||||
@@ -3752,10 +3836,19 @@ static int ndpi_init_packet_header(struct ndpi_detection_module_struct *ndpi_str
|
||||
|
||||
/* ************************************************ */
|
||||
|
||||
#ifdef FRAG_MAN
|
||||
uint8_t ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
|
||||
struct ndpi_flow_struct *flow) {
|
||||
#else // FRAG_MAN
|
||||
void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
|
||||
struct ndpi_flow_struct *flow) {
|
||||
#endif // FRAG_MAN
|
||||
if(!flow) {
|
||||
#ifdef FRAG_MAN
|
||||
return 0;
|
||||
#else // FRAG_MAN
|
||||
return;
|
||||
#endif // FRAG_MAN
|
||||
} else {
|
||||
/* const for gcc code optimization and cleaner code */
|
||||
struct ndpi_packet_struct *packet = &flow->packet;
|
||||
@@ -3764,7 +3857,7 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
|
||||
const struct ndpi_ipv6hdr *iphv6 = packet->iphv6;
|
||||
#endif
|
||||
const struct ndpi_tcphdr *tcph = packet->tcp;
|
||||
const struct ndpi_udphdr *udph = flow->packet.udp;
|
||||
const struct ndpi_udphdr *udph = packet->udp;
|
||||
|
||||
packet->tcp_retransmission = 0, packet->packet_direction = 0;
|
||||
|
||||
@@ -3797,15 +3890,25 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
|
||||
if(tcph->syn != 0 && tcph->ack == 0 && flow->l4.tcp.seen_syn == 0 && flow->l4.tcp.seen_syn_ack == 0 &&
|
||||
flow->l4.tcp.seen_ack == 0) {
|
||||
flow->l4.tcp.seen_syn = 1;
|
||||
}
|
||||
} else
|
||||
if(tcph->syn != 0 && tcph->ack != 0 && flow->l4.tcp.seen_syn == 1 && flow->l4.tcp.seen_syn_ack == 0 &&
|
||||
flow->l4.tcp.seen_ack == 0) {
|
||||
flow->l4.tcp.seen_syn_ack = 1;
|
||||
}
|
||||
} else
|
||||
if(tcph->syn == 0 && tcph->ack == 1 && flow->l4.tcp.seen_syn == 1 && flow->l4.tcp.seen_syn_ack == 1 &&
|
||||
flow->l4.tcp.seen_ack == 0) {
|
||||
flow->l4.tcp.seen_ack = 1;
|
||||
}
|
||||
|
||||
#ifdef FRAG_MAN
|
||||
// check sequence, if there is missing packet, add it to buffer
|
||||
if ( check_for_sequence(flow, tcph) ) {
|
||||
// if here added segment to list for next elaboration
|
||||
// and skip extra processing for after...
|
||||
return 0;
|
||||
}
|
||||
#endif //FRAG_MAN
|
||||
|
||||
if((flow->next_tcp_seq_nr[0] == 0 && flow->next_tcp_seq_nr[1] == 0) ||
|
||||
(flow->next_tcp_seq_nr[0] == 0 || flow->next_tcp_seq_nr[1] == 0)) {
|
||||
/* initialize tcp sequence counters */
|
||||
@@ -3873,6 +3976,9 @@ void ndpi_connection_tracking(struct ndpi_detection_module_struct *ndpi_str,
|
||||
flow->byte_counter[packet->packet_direction] += packet->payload_packet_len;
|
||||
}
|
||||
}
|
||||
#ifdef FRAG_MAN
|
||||
return 1;
|
||||
#endif // FRAG_MAN
|
||||
}
|
||||
|
||||
/* ************************************************ */
|
||||
@@ -4256,7 +4362,12 @@ void ndpi_process_extra_packet(struct ndpi_detection_module_struct *ndpi_str, st
|
||||
|
||||
/* detect traffic for tcp or udp only */
|
||||
flow->src = src, flow->dst = dst;
|
||||
|
||||
#ifdef FRAG_MAN
|
||||
if ( ndpi_connection_tracking(ndpi_str, flow) ) {
|
||||
#else // FRAG_MAN
|
||||
ndpi_connection_tracking(ndpi_str, flow);
|
||||
#endif // FRAG_MAN
|
||||
|
||||
/* call the extra packet function (which may add more data/info to flow) */
|
||||
if(flow->extra_packets_func) {
|
||||
@@ -4266,8 +4377,13 @@ void ndpi_process_extra_packet(struct ndpi_detection_module_struct *ndpi_str, st
|
||||
if(++flow->num_extra_packets_checked == flow->max_extra_packets_to_check)
|
||||
flow->extra_packets_func = NULL; /* Enough packets detected */
|
||||
}
|
||||
#ifdef FRAG_MAN
|
||||
}
|
||||
#endif // FRAG_MAN
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* ********************************************************************************* */
|
||||
|
||||
int ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_str, const char *ip_address_and_mask,
|
||||
@@ -4620,6 +4736,9 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
|
||||
}
|
||||
|
||||
flow->num_processed_pkts++;
|
||||
#ifdef FRAG_MAN
|
||||
flow->tcp_segments_management=1;
|
||||
#endif // FRAG_MAN
|
||||
|
||||
/* Init default */
|
||||
ret.master_protocol = flow->detected_protocol_stack[1],
|
||||
@@ -4896,6 +5015,16 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
|
||||
flow->fail_with_unknown = 1;
|
||||
|
||||
invalidate_ptr:
|
||||
#ifdef FRAG_MAN
|
||||
if (flow->must_free[flow->packet.packet_direction] &&
|
||||
flow->packet.payload_packet_len>0 && flow->packet.payload) {
|
||||
// if the payload is allocated for segments reassembling, it must be free
|
||||
ndpi_free((void*)flow->packet.payload);
|
||||
// flow->packet.payload=NULL; done after
|
||||
flow->packet.payload_packet_len=0;
|
||||
flow->must_free[flow->packet.packet_direction]=0;
|
||||
}
|
||||
#endif // FRAG_MAN
|
||||
/*
|
||||
Invalidate packet memory to avoid accessing the pointers below
|
||||
when the packet is no longer accessible
|
||||
|
||||
@@ -1791,8 +1791,384 @@ ndpi_http_method ndpi_http_str2method(const char* method, u_int16_t method_len)
|
||||
return(NDPI_HTTP_METHOD_UNKNOWN);
|
||||
}
|
||||
|
||||
#define ARRAYSZ_255 255
|
||||
void printRawData(const uint8_t *ptr, size_t len) {
|
||||
uint8_t *p=(uint8_t*)ptr;
|
||||
DBGINFO("ptr=%p, len=%llu", ptr, (unsigned long long)len)
|
||||
if (p && len>0) {
|
||||
size_t ctLines=0,i,j;
|
||||
char line1[ARRAYSZ_255]={0}, line2[ARRAYSZ_255]={0}, temp[ARRAYSZ_255];
|
||||
snprintf(line1,sizeof(line1),"\t%05X",(unsigned int)(16*ctLines));
|
||||
for (i=0; i<len; i++) {
|
||||
|
||||
if (i>0 && i%16==0) {
|
||||
printf("%s\t%s\n", line1,line2);
|
||||
ctLines++;
|
||||
snprintf(line1,ARRAYSZ_255,"\t%05X",(unsigned int)(16*ctLines));
|
||||
snprintf(line2,ARRAYSZ_255,"%s","");
|
||||
}
|
||||
snprintf(temp,ARRAYSZ_255," %02X", (uint8_t)*p);
|
||||
strncat(line1, temp, 3);
|
||||
snprintf(temp,ARRAYSZ_255,"%c", (isprint(*p)? *(p):'.'));
|
||||
strncat(line2, temp, 1);
|
||||
p++;
|
||||
}
|
||||
uint8_t exv= i%16;
|
||||
for(j=exv;exv>0 && j<16;j++) {
|
||||
strncat(line1, " ", 3);
|
||||
}
|
||||
printf("%s\t%s\n", line1,line2);
|
||||
}
|
||||
return;
|
||||
}
|
||||
/* ******************************************************************** */
|
||||
|
||||
// the struct is defined in ndpi_utils.h
|
||||
|
||||
void ins_sort_array(sorter_index_item_t arr[], int len) {
|
||||
DBGINFO("sorting no. %u items", (unsigned)len)
|
||||
for (int i=1; i<len; i++) {
|
||||
for (int j=i; j>0 && arr[j].sort_value<arr[j-1].sort_value; j--) {
|
||||
sorter_index_item_t temp = arr[j];
|
||||
arr[j] = arr[j-1];
|
||||
arr[j-1] = temp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void shell_sort_array(sorter_index_item_t arr[], int n) {
|
||||
// Rearrange elements at each n/2, n/4, n/8, ... intervals
|
||||
DBGINFO("sorting no. %u items", (unsigned)n)
|
||||
for (int interval = n / 2; interval > 0; interval /= 2) {
|
||||
for (int i = interval; i < n; i += 1) {
|
||||
sorter_index_item_t temp = arr[i];
|
||||
int j;
|
||||
for (j = i; j >= interval && arr[j - interval].sort_value > temp.sort_value; j -= interval) {
|
||||
arr[j] = arr[j - interval];
|
||||
DBGTRACER("exchanged item no. %d (%d) with: %d (%d)", j, arr[j].sort_value, j-interval, temp.sort_value)
|
||||
}
|
||||
DBGTRACER("item no. %d value: %d", j, temp.sort_value)
|
||||
arr[j] = temp;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ******************************************************************** */
|
||||
#ifdef FRAG_MAN
|
||||
|
||||
void free_fragment(fragments_wrapper_t *frag)
|
||||
{ /*
|
||||
*
|
||||
typedef struct fragment_wrapper {
|
||||
uint16_t id;
|
||||
uint8_t l4_protocol;
|
||||
uint8_t ct_frag;
|
||||
#ifdef NDPI_DETECTION_SUPPORT_IPV6
|
||||
char *flow_label; // IP6
|
||||
#endif
|
||||
fragment_t **fragments_list;
|
||||
} fragments_wrapper_t;
|
||||
*
|
||||
* */
|
||||
if (frag) {
|
||||
DBGTRACER("(frag:%p) freeing fragments list -> %p",frag, frag->fragments_list)
|
||||
if (frag->fragments_list) {
|
||||
DBGTRACER("fragments are %u.",frag->ct_frag)
|
||||
for ( int y=0;y<frag->ct_frag;y++) {
|
||||
if (frag->fragments_list[y]) {
|
||||
if (frag->fragments_list[y]->data) {
|
||||
DBGPOINTER("freeing fragment item %d -> %p",y, frag->fragments_list[y])
|
||||
ndpi_free(frag->fragments_list[y]->data);
|
||||
}
|
||||
ndpi_free(frag->fragments_list[y]);
|
||||
}
|
||||
}
|
||||
DBGPOINTER("freeing fragments list -> %p",frag->fragments_list)
|
||||
ndpi_free(frag->fragments_list);
|
||||
frag->fragments_list= NULL;
|
||||
}
|
||||
//reset counter and initial offset
|
||||
frag->ct_frag=0;
|
||||
frag->initial_offset=0;
|
||||
}
|
||||
}
|
||||
|
||||
uint8_t add_segment_to_buffer( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph, uint32_t new_waited_seq) {
|
||||
|
||||
DBGINFO("[flow:%p], dir: %d, seq:%u, ack:%u, len: %ubytes",flow,flow->packet.packet_direction, ntohl(tcph->seq), ntohl(tcph->ack_seq), flow->packet.payload_packet_len)
|
||||
|
||||
if (flow->tcp_segments_management) {
|
||||
fragments_wrapper_t *fragW= &flow->tcp_segments_list[flow->packet.packet_direction];
|
||||
DBGTRACER("tcp segments management enabled (list container: %p)",fragW)
|
||||
|
||||
if ( fragW->ct_frag == 0 ) {
|
||||
if (fragW->fragments_list) { free_fragment(fragW); }
|
||||
// initialize the offset with the first fragment seq number
|
||||
fragW->initial_offset= new_waited_seq;
|
||||
DBGTRACER("initialized initial_offset: %u)",fragW->initial_offset)
|
||||
}
|
||||
|
||||
if ( flow->packet.payload_packet_len>0 ) {
|
||||
// allocate memory for pointer
|
||||
size_t new_len= (1+fragW->ct_frag) * sizeof(fragment_t*);
|
||||
DBGTRACER("actual fragment list ct=%d, new size: %llu", fragW->ct_frag, (unsigned long long)new_len)
|
||||
//fragW->fragments_list= realloc(fragW->fragments_list,new_len);
|
||||
fragW->fragments_list= ndpi_realloc(fragW->fragments_list,(fragW->ct_frag * sizeof(fragment_t*)),new_len);
|
||||
DBGPOINTER("fragments_list initialized for item no. %u, list->%p i-esimo->%p", fragW->ct_frag, fragW->fragments_list, fragW->fragments_list[fragW->ct_frag])
|
||||
if ( !fragW->fragments_list ) {
|
||||
flow->tcp_segments_management= 0;
|
||||
fprintf(stderr, "[%8u] Not enough memory for new fragment \n", flow->packet_counter);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// allocate memory for item
|
||||
fragment_t *new_frag= (fragment_t*)ndpi_calloc(1, sizeof(fragment_t));
|
||||
DBGPOINTER("new_frag=> %p",new_frag)
|
||||
if ( !new_frag ) {
|
||||
flow->tcp_segments_management= 0;
|
||||
free_fragment(fragW);
|
||||
fprintf(stderr, "[%8u] Not enough memory for new fragment \n", flow->packet_counter);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// fill item with offsetm len and data fragment/segment
|
||||
new_frag->offset= ( 0xffffffff & ntohl(tcph->seq)) - fragW->initial_offset;
|
||||
DBGTRACER("offset calculation: seq %u, init: %u, offset result: %u", ntohl(tcph->seq), fragW->initial_offset, new_frag->offset)
|
||||
new_frag->len= flow->packet.payload_packet_len;
|
||||
|
||||
new_frag->data= (void*)ndpi_calloc(new_frag->len, sizeof(char));
|
||||
DBGPOINTER("new_frag->data=> %p",new_frag->data)
|
||||
if ( new_frag->data ) {
|
||||
memcpy(new_frag->data,flow->packet.payload,new_frag->len);
|
||||
fragW->fragments_list[fragW->ct_frag++]= new_frag;
|
||||
} else {
|
||||
flow->tcp_segments_management= 0;
|
||||
ndpi_free(new_frag);
|
||||
free_fragment(fragW);
|
||||
fprintf(stderr, "[%8u] Not enough memory for new fragment data \n", flow->packet_counter);
|
||||
return 0;
|
||||
}
|
||||
|
||||
DBGINFO("item no. %u: %p->%p [off:%u, len:%u, data:%p]", fragW->ct_frag, fragW->fragments_list, *fragW->fragments_list, (unsigned int) new_frag->offset, (unsigned int)new_frag->len, new_frag->data)
|
||||
return fragW->ct_frag;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
//TODO: manage partial retrasmission
|
||||
|
||||
uint32_t reassembly_fragment(struct ndpi_flow_struct *const flow, struct ndpi_tcphdr const * tcph, uint8_t **ret_buffer, size_t *len_buffer)
|
||||
{
|
||||
DBGTRACER("tcph:%p, ret_buffer:%p, len_buffer:%u",tcph,ret_buffer,len_buffer)
|
||||
|
||||
// reassemble the fragments...
|
||||
uint32_t ret_value=0;
|
||||
uint16_t last_item=0;
|
||||
size_t length=0,tot_length=0; //, offset=0;
|
||||
fragments_wrapper_t *fragW= &flow->tcp_segments_list[flow->packet.packet_direction];
|
||||
|
||||
// phase 1: calculate the size and fill the indexes array
|
||||
DBGINFO("phase 1: init sorter, calculate the size of buffer to reassemble: %u items", fragW->ct_frag)
|
||||
sorter_index_item_t *sorted_indexes= ndpi_calloc( fragW->ct_frag, sizeof(sorter_index_item_t) );
|
||||
DBGPOINTER("sorted_indexes=> %p",sorted_indexes)
|
||||
if ( !sorted_indexes ) {
|
||||
fprintf(stderr, "[%8u] Not enough memory to sort the %u segments \n",
|
||||
flow->packet_counter, fragW->ct_frag);
|
||||
free_fragment(fragW);
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (int i=0; i<fragW->ct_frag; i++) {
|
||||
fragment_t *item = (fragment_t*)fragW->fragments_list[i];
|
||||
|
||||
sorted_indexes[i].sort_value= item->offset;
|
||||
sorted_indexes[i].item_index= i;
|
||||
tot_length += item->len;
|
||||
DBGTRACER("segment (%d): len:%lu, offset: %u => partial buffer len: %lu", i, (long unsigned int)item->len, (unsigned int)item->offset, (long unsigned int)tot_length)
|
||||
}
|
||||
|
||||
// phase 2: sorts fragments and check fragments and sequences
|
||||
DBGINFO(" phase 2 sorting %d segments and checking",fragW->ct_frag)
|
||||
if (fragW->ct_frag>1) shell_sort_array(sorted_indexes, fragW->ct_frag);
|
||||
|
||||
// checks
|
||||
for (uint i=0; i<fragW->ct_frag; i++) {
|
||||
fragment_t *item = (fragment_t*)fragW->fragments_list[ sorted_indexes[i].item_index ];
|
||||
|
||||
// 1: no segment offset can be > tot_length
|
||||
DBGTRACER("checking %d/%d element: offset=%lu vs t_length=%lu",i,sorted_indexes[i].item_index,(unsigned long)item->offset,(unsigned long)tot_length)
|
||||
if ( (item->offset) > (uint32_t)tot_length ) {
|
||||
// update the last index of elements to elaborate
|
||||
DBGINFO("stop processing at %d/%d element: len= %u; offset= %u",i,sorted_indexes[i].item_index,(unsigned)length,(unsigned)item->offset)
|
||||
|
||||
tot_length= length;
|
||||
ret_value= (fragW->initial_offset)+(item->offset); // set the first offset to wait for the next segment
|
||||
break;
|
||||
}
|
||||
// 2: for every len(buffer) must exists a offset fragment
|
||||
else if ( (item->offset) != (uint32_t)length ) {
|
||||
// update the last index of elements to elaborate
|
||||
DBGINFO("checking %d/%d element: stop processing! len: %u; n_offset: %u", i,sorted_indexes[i].item_index,(unsigned)length,(unsigned)item->offset)
|
||||
|
||||
tot_length= length;
|
||||
ret_value= (fragW->initial_offset)+(item->offset); // set the first offset to wait for the next segment
|
||||
break;
|
||||
} else {
|
||||
// continue to sum length data bytes
|
||||
length+= item->len;
|
||||
last_item= i;
|
||||
}
|
||||
}
|
||||
last_item++; // index to number aligment
|
||||
|
||||
// phase 3: allocate memory and fill the buffer
|
||||
DBGINFO("phase 3: allocate memory for %u items and fill the buffer tot: %lu", last_item, (unsigned long int)tot_length)
|
||||
uint8_t *buffer= ndpi_calloc(tot_length, sizeof(uint8_t));
|
||||
DBGPOINTER("buffer (len:%lu)=> %p",(unsigned long)tot_length,buffer)
|
||||
if ( !buffer ) {
|
||||
fprintf(stderr, "[%8u] Not enough memory for buffer for %u segments \n", flow->packet_counter,last_item);
|
||||
free_fragment(fragW);
|
||||
ndpi_free(sorted_indexes);
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (uint i=0; i<last_item; i++) {
|
||||
fragment_t *item = (fragment_t*)fragW->fragments_list[sorted_indexes[i].item_index];
|
||||
|
||||
//uint_t blk_sz= (tot_length - item->offset) > item->len ) ? item->len : tot_length-item->offset
|
||||
DBGINFO("copying data item no:%u of len: %lu to buffer: %p (offset:%lu)", sorted_indexes[i].item_index, (unsigned long int)item->len, buffer, (unsigned long int)item->offset)
|
||||
memcpy((void*)(buffer+item->offset),item->data, item->len);
|
||||
|
||||
// free memory item
|
||||
ndpi_free(item->data);
|
||||
item->data=NULL;
|
||||
ndpi_free(item);
|
||||
fragW->fragments_list[sorted_indexes[i].item_index]=NULL;
|
||||
}
|
||||
|
||||
if ( last_item==fragW->ct_frag ) {
|
||||
DBGTRACER("all processed: free all memory!")
|
||||
free_fragment(fragW);
|
||||
} else {
|
||||
// phase 4: re-organize the other segments, updating the list
|
||||
fragment_t **fragW_old_list = fragW->fragments_list;
|
||||
fragW->fragments_list= ndpi_calloc((fragW->ct_frag-last_item), sizeof(struct fragment_t*));
|
||||
DBGPOINTER("old segments list: %p, new segments list: %p.",fragW_old_list,fragW->fragments_list)
|
||||
if (!fragW->fragments_list) {
|
||||
fprintf(stderr, "[%8u] Not enough memory for new segments list \n", flow->packet_counter);
|
||||
free_fragment(fragW);
|
||||
ndpi_free(buffer);
|
||||
ndpi_free(sorted_indexes);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// re-fill the new segments list, updating the offsets
|
||||
for (uint i=last_item; i<fragW->ct_frag; i++) {
|
||||
fragW->fragments_list[i-last_item] = (fragment_t*)fragW_old_list[sorted_indexes[i].item_index];
|
||||
fragW->fragments_list[i-last_item]->offset-= tot_length;
|
||||
DBGTRACER("moving the item (%p), index %u - to position %u of new segments list; new offset: %u.", fragW->fragments_list[i-last_item], sorted_indexes[i].item_index, i-last_item, fragW->fragments_list[i-last_item]->offset )
|
||||
}
|
||||
|
||||
// update the fragments countes
|
||||
fragW->ct_frag-= last_item;
|
||||
fragW->initial_offset+= tot_length;
|
||||
DBGINFO("updated counter: %d and i_offset: %u.",(unsigned)fragW->ct_frag, (unsigned)fragW->initial_offset)
|
||||
|
||||
DBGPOINTER("freeing old segments list: %p ",fragW_old_list)
|
||||
ndpi_free(fragW_old_list);
|
||||
}
|
||||
|
||||
if ( sorted_indexes) {
|
||||
DBGPOINTER("freeing sorter indexes: %p ",sorted_indexes)
|
||||
ndpi_free(sorted_indexes);
|
||||
}
|
||||
|
||||
if(len_buffer != NULL) {
|
||||
*len_buffer = tot_length;
|
||||
}
|
||||
if(ret_buffer != NULL) {
|
||||
*ret_buffer = (u_int8_t *) buffer;
|
||||
flow->must_free[flow->packet.packet_direction]=1;
|
||||
DBGINFO("retrieved the buffer of segments (len:%lu) %p",*len_buffer,*ret_buffer)
|
||||
} else {
|
||||
DBGPOINTER("freeing buffer=> %p",buffer)
|
||||
ndpi_free(buffer);
|
||||
}
|
||||
DBGINFO("returning: %d",ret_value)
|
||||
return ret_value;
|
||||
}
|
||||
|
||||
uint8_t check_for_sequence( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph)
|
||||
{
|
||||
uint8_t *ret_buffer=NULL;
|
||||
size_t len_buffer=0;
|
||||
|
||||
DBGINFO("## sorted flags: %d/%d ",flow->not_sorted[0],flow->not_sorted[1])
|
||||
if ( flow->next_tcp_seq_nr[flow->packet.packet_direction] ) {
|
||||
uint32_t *trigger;
|
||||
uint8_t *not_sorted;
|
||||
|
||||
// use pointers to allow the modification
|
||||
not_sorted = &flow->not_sorted[flow->packet.packet_direction];
|
||||
trigger = &flow->trigger[flow->packet.packet_direction];
|
||||
|
||||
DBGTRACER("dir:%d, trg:%u, next:%u", flow->packet.packet_direction,*trigger, flow->next_tcp_seq_nr[flow->packet.packet_direction])
|
||||
|
||||
uint32_t waited= (*not_sorted && *trigger) ? ndpi_min(*trigger,flow->next_tcp_seq_nr[flow->packet.packet_direction]) : flow->next_tcp_seq_nr[flow->packet.packet_direction];
|
||||
if ( waited<(0xffffffff & ntohl(tcph->seq))) {
|
||||
// segment not in order... almost 1 has been skipped! add this fragment to buffer
|
||||
DBGINFO("received a segment (seq:%u) over the waited (next:%u)", (0xffffffff & ntohl(tcph->seq)), waited)
|
||||
|
||||
if ( add_segment_to_buffer(flow, tcph, waited) ) {
|
||||
DBGTRACER("segment (seq:%u) bufferized, waiting for (next:%u)", (0xffffffff & ntohl(tcph->seq)), waited)
|
||||
|
||||
// set flag a save the waited sequence number
|
||||
*not_sorted=1;
|
||||
*trigger= *trigger ? ndpi_min(flow->next_tcp_seq_nr[flow->packet.packet_direction],*trigger):flow->next_tcp_seq_nr[flow->packet.packet_direction];
|
||||
DBGINFO("set flag and trigger[%d]: %u",flow->packet.packet_direction,*trigger)
|
||||
}
|
||||
return 1;
|
||||
|
||||
} else if (waited>(0xffffffff & ntohl(tcph->seq))) {
|
||||
DBGINFO("received a segment (seq:%u) minus than the waited (next:%u): retransmission!!", (0xffffffff & ntohl(tcph->seq)), flow->next_tcp_seq_nr[flow->packet.packet_direction])
|
||||
|
||||
flow->packet.tcp_retransmission = 1;
|
||||
|
||||
/* CHECK IF PARTIAL RETRY IS HAPPENING */
|
||||
if((flow->next_tcp_seq_nr[flow->packet.packet_direction] - ntohl(tcph->seq) <
|
||||
flow->packet.payload_packet_len)) {
|
||||
/* num_retried_bytes actual_payload_len hold info about the partial retry
|
||||
analyzer which require this info can make use of this info
|
||||
Other analyzer can use packet->payload_packet_len */
|
||||
flow->packet.num_retried_bytes = (u_int16_t)(flow->next_tcp_seq_nr[flow->packet.packet_direction] - ntohl(tcph->seq));
|
||||
flow->packet.actual_payload_len = flow->packet.payload_packet_len - flow->packet.num_retried_bytes;
|
||||
flow->next_tcp_seq_nr[flow->packet.packet_direction] = ntohl(tcph->seq) + flow->packet.payload_packet_len;
|
||||
DBGINFO("partial_bytes:%u",flow->packet.num_retried_bytes)
|
||||
|
||||
//TODO: manage this!!
|
||||
}
|
||||
|
||||
} else {
|
||||
DBGTRACER("seq (%u) and waited (%u) matched! sorted flag: %d", (0xffffffff & ntohl(tcph->seq)), flow->next_tcp_seq_nr[flow->packet.packet_direction], *not_sorted)
|
||||
if ( *not_sorted ) {
|
||||
if ( add_segment_to_buffer(flow, tcph, 0) ) {
|
||||
*trigger= reassembly_fragment(flow,tcph,&ret_buffer,&len_buffer);
|
||||
*not_sorted=(*trigger>0);
|
||||
|
||||
if (len_buffer>0) {
|
||||
// the previous pointers must not be free, because managed in other part
|
||||
flow->packet.payload_packet_len= len_buffer;
|
||||
flow->packet.payload= ret_buffer;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
#endif // FRAG_MAN
|
||||
|
||||
|
||||
#define ROR64(x,r) (((x)>>(r))|((x)<<(64-(r))))
|
||||
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user