LINE_CALL: add detection of LINE voip calls (#1761)

These flows are classifed as `LINE_CALL`; another option was
`RTP/LINE_CALL`. No sure about the best solution...

Extend LINE domains list.

Remove RTP dead code.
This commit is contained in:
Ivan Nardi
2022-10-06 17:09:26 +02:00
committed by GitHub
parent e4dcec560e
commit 1796a1d814
59 changed files with 196 additions and 343 deletions
+1
View File
@@ -344,6 +344,7 @@ typedef enum {
NDPI_PROTOCOL_SYNCTHING = 313,
NDPI_PROTOCOL_CRYNET = 314,
NDPI_PROTOCOL_LINE = 315,
NDPI_PROTOCOL_LINE_CALL = 316,
#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"
+1
View File
@@ -243,6 +243,7 @@ void init_fastcgi_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_
void init_natpmp_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_syncthing_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_crynet_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
void init_line_dissector(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask);
/* ndpi_main.c */
extern u_int32_t ndpi_ip_port_hash_funct(u_int32_t ip, u_int16_t port);
+4
View File
@@ -824,6 +824,10 @@ struct ndpi_flow_udp_struct {
/* NDPI_PROTOCOL_IMO */
u_int8_t imo_last_one_byte_pkt, imo_last_byte;
/* NDPI_PROTOCOL_LINE_CALL */
u_int8_t line_pkts[2];
u_int8_t line_base_cnt[2];
};
/* ************************************************** */
+5
View File
@@ -1726,6 +1726,11 @@ static ndpi_protocol_match host_match[] =
{ "discovery.syncthing.net", "Syncthing", NDPI_PROTOCOL_SYNCTHING, NDPI_PROTOCOL_CATEGORY_DOWNLOAD_FT, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ ".line-apps.com", "Line", NDPI_PROTOCOL_LINE, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ ".line-scdn.net", "Line", NDPI_PROTOCOL_LINE, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "line.me", "Line", NDPI_PROTOCOL_LINE, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ "line-website.com", "Line", NDPI_PROTOCOL_LINE, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ ".linecorp.com", "Line", NDPI_PROTOCOL_LINE, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
{ ".line.naver.jp", "Line", NDPI_PROTOCOL_LINE, NDPI_PROTOCOL_CATEGORY_CHAT, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_DEFAULT_LEVEL },
#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_content_match_host_match.c.inc"
+7
View File
@@ -2001,6 +2001,10 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
"Line", NDPI_PROTOCOL_CATEGORY_CHAT,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_LINE_CALL,
"LineCall", NDPI_PROTOCOL_CATEGORY_VOIP,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
#ifdef CUSTOM_NDPI_PROTOCOLS
@@ -4687,6 +4691,9 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str) {
/* CryNetwork */
init_crynet_dissector(ndpi_str, &a, detection_bitmask);
/* Line voip */
init_line_dissector(ndpi_str, &a, detection_bitmask);
#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_main_init.c"
#endif
+91
View File
@@ -0,0 +1,91 @@
/*
* line.c
*
* Copyright (C) 2022 - ntop.org
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "ndpi_protocol_ids.h"
#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_LINE_CALL
#include "ndpi_api.h"
extern int is_valid_rtp_payload_type(uint8_t type);
static void ndpi_int_line_add_connection(struct ndpi_detection_module_struct * const ndpi_struct,
struct ndpi_flow_struct * const flow)
{
NDPI_LOG_INFO(ndpi_struct, "found LineCall\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_UNKNOWN,
NDPI_PROTOCOL_LINE_CALL, NDPI_CONFIDENCE_DPI);
}
void ndpi_search_line(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct const * const packet = &ndpi_struct->packet;
NDPI_LOG_DBG(ndpi_struct, "searching LineCall\n");
/* Some "random" UDP packets before the standard RTP stream:
it seems that the 4th bytes of these packets is some kind of packet
number. Look for 4 packets per direction with consecutive numbers. */
if(packet->payload_packet_len > 10) {
if(flow->l4.udp.line_pkts[packet->packet_direction] == 0) {
flow->l4.udp.line_base_cnt[packet->packet_direction] = packet->payload[3];
flow->l4.udp.line_pkts[packet->packet_direction] += 1;
return;
} else {
/* It might be a RTP/RTCP packet. Ignore it and keep looking for the
LINE packet numbers */
/* Basic RTP detection */
if((packet->payload[0] >> 6) == 2 && /* Version 2 */
(packet->payload[1] == 201 || /* RTCP, Receiver Report */
packet->payload[1] == 200 || /* RTCP, Sender Report */
is_valid_rtp_payload_type(packet->payload[1] & 0x7F)) /* RTP */) {
NDPI_LOG_DBG(ndpi_struct, "Probably RTP; keep looking for LINE");
return;
} else {
if((u_int8_t)(flow->l4.udp.line_base_cnt[packet->packet_direction] +
flow->l4.udp.line_pkts[packet->packet_direction]) == packet->payload[3]) {
flow->l4.udp.line_pkts[packet->packet_direction] += 1;
if(flow->l4.udp.line_pkts[0] >= 4 && flow->l4.udp.line_pkts[1] >= 4)
ndpi_int_line_add_connection(ndpi_struct, flow);
return;
}
}
}
}
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
return;
}
void init_line_dissector(struct ndpi_detection_module_struct *ndpi_struct,
u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask)
{
ndpi_set_bitmask_protocol_detection("LineCall", ndpi_struct, detection_bitmask, *id,
NDPI_PROTOCOL_LINE_CALL,
ndpi_search_line,
NDPI_SELECTION_BITMASK_PROTOCOL_V4_V6_UDP_WITH_PAYLOAD,
SAVE_DETECTION_BITMASK_AS_UNKNOWN,
ADD_TO_DETECTION_BITMASK);
*id += 1;
}
+1 -6
View File
@@ -47,6 +47,7 @@ extern int processClientServerHello(struct ndpi_detection_module_struct *ndpi_st
extern int http_process_user_agent(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
const u_int8_t *ua_ptr, u_int16_t ua_ptr_len);
extern int is_valid_rtp_payload_type(uint8_t type);
/* Versions */
#define V_1 0x00000001
@@ -1609,12 +1610,6 @@ static int eval_extra_processing(struct ndpi_detection_module_struct *ndpi_struc
return 0;
}
static int is_valid_rtp_payload_type(uint8_t type)
{
/* https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml */
return type <= 34 || (type >= 96 && type <= 127);
}
static void ndpi_search_quic(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow);
static int ndpi_search_quic_extra(struct ndpi_detection_module_struct *ndpi_struct,
+15 -264
View File
@@ -71,6 +71,12 @@ static u_int8_t isValidMSRTPType(u_int8_t payloadType) {
}
}
int is_valid_rtp_payload_type(uint8_t type)
{
/* https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml */
return type <= 34 || (type >= 96 && type <= 127);
}
/* *************************************************************** */
static void ndpi_rtp_search(struct ndpi_detection_module_struct *ndpi_struct,
@@ -96,14 +102,16 @@ static void ndpi_rtp_search(struct ndpi_detection_module_struct *ndpi_struct,
if((payload_len >= 12)
&& (((payload[0] & 0xFF) == 0x80) || ((payload[0] & 0xFF) == 0xA0)) /* RTP magic byte[1] */
&& ((payload_type < 72) || (payload_type > 76))
&& ((payload_type <= 34)
|| ((payload_type >= 96) && (payload_type <= 127))
/* http://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml */
)
&& (is_valid_rtp_payload_type(payload_type))
) {
NDPI_LOG_INFO(ndpi_struct, "Found RTP\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RTP, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
return;
if(flow->l4.udp.line_pkts[0] >= 2 && flow->l4.udp.line_pkts[1] >= 2) {
/* It seems that it is a LINE stuff; let its dissector to evaluate */
return;
} else {
NDPI_LOG_INFO(ndpi_struct, "Found RTP\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RTP, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
return;
}
} else if((payload_len >= 12)
&& (((payload[0] & 0xFF) == 0x80) || ((payload[0] & 0xFF) == 0xA0)) /* RTP magic byte[1] */
&& (payloadType = isValidMSRTPType(payload[1] & 0xFF))) {
@@ -141,263 +149,6 @@ void ndpi_search_rtp(struct ndpi_detection_module_struct *ndpi_struct, struct nd
/* *************************************************************** */
#if 0
/* Original (messy) OpenDPI code */
#define RTP_MAX_OUT_OF_ORDER 11
static void ndpi_int_rtp_add_connection(struct ndpi_detection_module_struct
*ndpi_struct, struct ndpi_flow_struct *flow)
{
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_RTP, NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI);
}
/*
* maintenance of current highest sequence number, cycle count, packet counter
* adapted from RFC3550 Appendix A.1
*
* In their formulation, it is not possible to represent "no packets sent yet". This is fixed here by defining
* baseseq to be the sequence number of the first packet minus 1 (in other words, the sequence number of the
* zeroth packet).
*
* Note: As described in the RFC, the number of packets received includes retransmitted packets.
* This means the "packets lost" count (seq_num-isn+1)-received can become negative.
*
* include_current_packet should be
* 1, if the current packet should count towards the total, or
* 0, if it it regarded as belonging to the previous reporting interval
*/
#if !defined(WIN32)
static inline
#elif defined(MINGW_GCC)
__mingw_forceinline static
#else
__forceinline static
#endif
void init_seq(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow,
u_int8_t direction, u_int16_t seq, u_int8_t include_current_packet)
{
flow->rtp_seqnum[direction] = seq;
NDPI_LOG_DBG(ndpi_struct, "rtp_seqnum[%u] = %u\n", direction, seq);
}
/* returns difference between old and new highest sequence number */
#if !defined(WIN32)
static inline
#elif defined(MINGW_GCC)
__mingw_forceinline static
#else
__forceinline static
#endif
u_int16_t update_seq(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow,
u_int8_t direction, u_int16_t seq)
{
u_int16_t delta = seq - flow->rtp_seqnum[direction];
if(delta < RTP_MAX_OUT_OF_ORDER) { /* in order, with permissible gap */
flow->rtp_seqnum[direction] = seq;
NDPI_LOG_DBG(ndpi_struct, "rtp_seqnum[%u] = %u (increased by %u)\n",
direction, seq, delta);
return delta;
} else {
NDPI_LOG_DBG(ndpi_struct, "retransmission (dir %u, seqnum %u)\n",
direction, seq);
return 0;
}
}
static void ndpi_rtp_search(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
const u_int8_t * payload, const u_int16_t payload_len)
{
struct ndpi_packet_struct *packet = &flow->packet;
u_int8_t stage;
u_int16_t seqnum = ntohs(get_u_int16_t(payload, 2));
NDPI_LOG_DBG(ndpi_struct, "search rtp\n");
if(payload_len == 4 && get_u_int32_t(packet->payload, 0) == 0 && flow->packet_counter < 8) {
NDPI_LOG_DBG(ndpi_struct, "need next packet, maybe ClearSea out calls\n");
return;
}
if(payload_len == 5 && memcmp(payload, "hello", 5) == 0) {
NDPI_LOG_DBG(ndpi_struct,
"need next packet, initial hello packet of SIP out calls.\n");
return;
}
if(payload_len == 1 && payload[0] == 0) {
NDPI_LOG_DBG(ndpi_struct,
"need next packet, payload_packet_len == 1 && payload[0] == 0.\n");
return;
}
if(payload_len == 3 && memcmp(payload, "png", 3) == 0) {
/* weird packet found in Ninja GlobalIP trace */
NDPI_LOG_DBG(ndpi_struct, "skipping packet with len = 3 and png payload\n");
return;
}
if(payload_len < 12) {
NDPI_LOG_DBG(ndpi_struct, "minimal packet size for rtp packets: 12\n");
goto exclude_rtp;
}
if(payload_len == 12 && get_u_int32_t(payload, 0) == 0 && get_u_int32_t(payload, 4) == 0 && get_u_int32_t(payload, 8) == 0) {
NDPI_LOG_DBG(ndpi_struct, "skipping packet with len = 12 and only 0-bytes\n");
return;
}
if((payload[0] & 0xc0) == 0xc0 || (payload[0] & 0xc0) == 0x40 || (payload[0] & 0xc0) == 0x00) {
NDPI_LOG_DBG(ndpi_struct, "version = 3 || 1 || 0, maybe first rtp packet\n");
return;
}
if((payload[0] & 0xc0) != 0x80) {
NDPI_LOG_DBG(ndpi_struct, "rtp version must be 2, first two bits of a packets must be 10\n");
goto exclude_rtp;
}
/* rtp_payload_type are the last seven bits of the second byte */
if(flow->rtp_payload_type[packet->packet_direction] != (payload[1] & 0x7F)) {
NDPI_LOG_DBG(ndpi_struct, "payload_type has changed, reset stages\n");
packet->packet_direction == 0 ? (flow->rtp_stage1 = 0) : (flow->rtp_stage2 = 0);
}
/* first bit of first byte is not part of payload_type */
flow->rtp_payload_type[packet->packet_direction] = payload[1] & 0x7F;
stage = (packet->packet_direction == 0 ? flow->rtp_stage1 : flow->rtp_stage2);
if(stage > 0) {
NDPI_LOG_DBG(ndpi_struct, "stage = %u\n", packet->packet_direction == 0 ? flow->rtp_stage1 : flow->rtp_stage2);
if(flow->rtp_ssid[packet->packet_direction] != get_u_int32_t(payload, 8)) {
NDPI_LOG_DBG(ndpi_struct, "ssid has changed, goto exclude rtp\n");
goto exclude_rtp;
}
if(seqnum == flow->rtp_seqnum[packet->packet_direction]) {
NDPI_LOG_DBG(ndpi_struct, "maybe \"retransmission\", need next packet\n");
return;
} else if((u_int16_t) (seqnum - flow->rtp_seqnum[packet->packet_direction]) < RTP_MAX_OUT_OF_ORDER) {
NDPI_LOG_DBG(ndpi_struct,
"new packet has larger sequence number (within valid range)\n");
update_seq(ndpi_struct, flow, packet->packet_direction, seqnum);
} else if((u_int16_t) (flow->rtp_seqnum[packet->packet_direction] - seqnum) < RTP_MAX_OUT_OF_ORDER) {
NDPI_LOG_DBG(ndpi_struct,
"new packet has smaller sequence number (within valid range)\n");
init_seq(ndpi_struct, flow, packet->packet_direction, seqnum, 1);
} else {
NDPI_LOG_DBG(ndpi_struct,
"sequence number diff is too big, goto exclude rtp.\n");
goto exclude_rtp;
}
} else {
NDPI_LOG_DBG(ndpi_struct, "rtp_ssid[%u] = %u\n", packet->packet_direction,
flow->rtp_ssid[packet->packet_direction]);
flow->rtp_ssid[packet->packet_direction] = get_u_int32_t(payload, 8);
if(flow->packet_counter < 3) {
NDPI_LOG_DBG(ndpi_struct, "packet_counter < 3, need next packet\n");
}
init_seq(ndpi_struct, flow, packet->packet_direction, seqnum, 1);
}
if(seqnum <= 3) {
NDPI_LOG_DBG(ndpi_struct, "sequence_number = %u, too small, need next packet, return\n", seqnum);
return;
}
if(stage == 3) {
NDPI_LOG_DBG(ndpi_struct, "add connection I\n");
ndpi_int_rtp_add_connection(ndpi_struct, flow);
} else {
packet->packet_direction == 0 ? flow->rtp_stage1++ : flow->rtp_stage2++;
NDPI_LOG_DBG(ndpi_struct, "stage[%u]++; need next packet\n",
packet->packet_direction);
}
return;
exclude_rtp:
if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_STUN
|| /* packet->real_protocol_read_only == NDPI_PROTOCOL_STUN */) {
NDPI_LOG_DBG(ndpi_struct, "STUN: is detected, need next packet\n");
return;
}
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
}
/* *************************************************************** */
void ndpi_search_rtp(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow)
{
struct ndpi_packet_struct *packet = &flow->packet;
if(packet->udp) {
ndpi_rtp_search(ndpi_struct, flow, packet->payload, packet->payload_packet_len);
} else if(packet->tcp) {
/* skip special packets seen at yahoo traces */
if(packet->payload_packet_len >= 20 && ntohs(get_u_int16_t(packet->payload, 2)) + 20 == packet->payload_packet_len &&
packet->payload[0] == 0x90 && packet->payload[1] >= 0x01 && packet->payload[1] <= 0x07) {
if(flow->packet_counter == 2)
flow->l4.tcp.rtp_special_packets_seen = 1;
NDPI_LOG_DBG(ndpi_struct,
"skipping STUN-like, special yahoo packets with payload[0] == 0x90.\n");
return;
}
/* TODO the rtp detection sometimes doesn't exclude rtp
* so for TCP flows only run the detection if STUN has been
* detected (or RTP is already detected)
* If flows will be seen which start directly with RTP
* we can remove this restriction
*/
if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_STUN
|| packet->detected_protocol_stack[0] == NDPI_PROTOCOL_RTP) {
/* RTP may be encapsulated in TCP packets */
if(packet->payload_packet_len >= 2 && ntohs(get_u_int16_t(packet->payload, 0)) + 2 == packet->payload_packet_len) {
/* TODO there could be several RTP packets in a single TCP packet so maybe the detection could be
* improved by checking only the RTP packet of given length */
ndpi_rtp_search(ndpi_struct, flow, packet->payload + 2, packet->payload_packet_len - 2);
return;
}
}
if(packet->detected_protocol_stack[0] == NDPI_PROTOCOL_UNKNOWN && flow->l4.tcp.rtp_special_packets_seen == 1) {
if(packet->payload_packet_len >= 4 && ntohl(get_u_int32_t(packet->payload, 0)) + 4 == packet->payload_packet_len) {
/* TODO there could be several RTP packets in a single TCP packet so maybe the detection could be
* improved by checking only the RTP packet of given length */
ndpi_rtp_search(ndpi_struct, flow, packet->payload + 4, packet->payload_packet_len - 4);
return;
}
}
if(NDPI_FLOW_PROTOCOL_EXCLUDED(ndpi_struct, flow, NDPI_PROTOCOL_STUN)) {
NDPI_EXCLUDE_PROTO(ndpi_struct, flow);
} else {
NDPI_LOG_DBG(ndpi_struct, "STUN not yet excluded, need next packet\n");
}
}
}
#endif
/* *************************************************************** */
void init_rtp_dissector(struct ndpi_detection_module_struct *ndpi_struct,
u_int32_t *id, NDPI_PROTOCOL_BITMASK *detection_bitmask) {
ndpi_set_bitmask_protocol_detection("RTP", ndpi_struct, detection_bitmask, *id,