From abce6d4023c2b76f23e75deff446121687abe5d9 Mon Sep 17 00:00:00 2001 From: Toni Date: Mon, 27 May 2024 09:46:32 +0200 Subject: [PATCH] Improved Kafka dissector. (#2456) * detect more Kafka request packet's * requires less flow memory * same detection behavior as before e.g. no asym detection implemented (can be done by dissecting responses, requires more effort) Signed-off-by: Toni Uhlig Co-authored-by: Nardi Ivan --- src/include/ndpi_typedefs.h | 6 --- src/lib/protocols/kafka.c | 51 +++++++++++++-------- tests/cfgs/default/pcap/kafka.pcapng | Bin 3240 -> 8676 bytes tests/cfgs/default/result/kafka.pcapng.out | 31 +++++++++---- 4 files changed, 53 insertions(+), 35 deletions(-) diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index ffc98ecd1..a2e10878f 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -835,9 +835,6 @@ struct ndpi_flow_tcp_struct { /* NDPI_PROTOCOL_SSH */ u_int32_t ssh_stage:3; - /* NDPI_PROTOCOL_KAFKA */ - u_int32_t kafka_stage:1; - /* NDPI_PROTOCOL_VNC */ u_int32_t vnc_stage:2; // 0 - 3 @@ -891,9 +888,6 @@ struct ndpi_flow_tcp_struct { /* NDPI_PROTOCOL_RADMIN */ u_int32_t radmin_stage:1; - - /* NDPI_PROTOCOL_KAFKA */ - u_int32_t kafka_correlation_id; }; /* ************************************************** */ diff --git a/src/lib/protocols/kafka.c b/src/lib/protocols/kafka.c index cffd1f32f..abf0ae3ca 100644 --- a/src/lib/protocols/kafka.c +++ b/src/lib/protocols/kafka.c @@ -29,6 +29,14 @@ #include "ndpi_api.h" #include "ndpi_private.h" +static void ndpi_int_kafka_add_connection(struct ndpi_detection_module_struct *ndpi_struct, + struct ndpi_flow_struct *flow) +{ + NDPI_LOG_INFO(ndpi_struct, "found Apache Kafka\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_APACHE_KAFKA, + NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI); +} + static void ndpi_search_kafka(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow) { @@ -41,32 +49,37 @@ static void ndpi_search_kafka(struct ndpi_detection_module_struct *ndpi_struct, * API keys: https://kafka.apache.org/protocol.html#protocol_api_keys * API versions: https://cwiki.apache.org/confluence/display/KAFKA/Kafka+APIs */ - if (packet->payload_packet_len > 40 && - ntohl(get_u_int32_t(packet->payload, 0)) == (u_int32_t)(packet->payload_packet_len-4)) + if (packet->payload_packet_len < 8 /* min. required packet length */ || + ntohl(get_u_int32_t(packet->payload, 0)) != (uint32_t)(packet->payload_packet_len - 4)) { - /* Request */ - if (!flow->l4.tcp.kafka_stage && - current_pkt_from_client_to_server(ndpi_struct, flow) && - ntohs(get_u_int16_t(packet->payload, 4)) < 75 && /* API key */ - ntohs(get_u_int16_t(packet->payload, 6)) < 16 /* API version */) + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + return; + } + + /* Request */ + if (ntohs(get_u_int16_t(packet->payload, 4)) < 75 && /* API key */ + ntohs(get_u_int16_t(packet->payload, 6)) < 16 /* API version */) + { + if (packet->payload_packet_len < 14) { - flow->l4.tcp.kafka_correlation_id = ntohl(get_u_int16_t(packet->payload, 8)); - flow->l4.tcp.kafka_stage = 1; - return; + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + return; } - /* Response */ - if (flow->l4.tcp.kafka_stage == 1 && - current_pkt_from_server_to_client(ndpi_struct, flow)) + const uint16_t client_id_len = ntohs(get_u_int16_t(packet->payload, 12)); + if (client_id_len + 12 + 2 > packet->payload_packet_len) { - if (ntohl(get_u_int16_t(packet->payload, 4)) == flow->l4.tcp.kafka_correlation_id) - { - NDPI_LOG_INFO(ndpi_struct, "found Apache Kafka\n"); - ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_APACHE_KAFKA, - NDPI_PROTOCOL_UNKNOWN, NDPI_CONFIDENCE_DPI); + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); return; - } } + if (ndpi_is_printable_buffer(&packet->payload[14], client_id_len) == 0) + { + NDPI_EXCLUDE_PROTO(ndpi_struct, flow); + return; + } + + ndpi_int_kafka_add_connection(ndpi_struct, flow); + return; } NDPI_EXCLUDE_PROTO(ndpi_struct, flow); diff --git a/tests/cfgs/default/pcap/kafka.pcapng b/tests/cfgs/default/pcap/kafka.pcapng index bcdabe98abeddda0e685720138a416084ebd7d97..88ddd53a217bb4c39698b4b102b17d0100e1a8d2 100644 GIT binary patch literal 8676 zcmeHM3vg7`89uvt5CVCM5FnVFhY&Ek?B+p28YB;+f+P@2z(Qej5_0Z zcK-9a_nz;6eCOPA&xS`tgl}YwmE_$tNrSRqPC+b=EwDN2S4&xXW0u~aGa9mW)mDeq zZ8Hb6QdVMhS6j^{mz28H=C*oPn%uS1rA&RgK1)iy&E}OFMBQaF*P5!W7RlrhS9Qkw z>+h*|cWe7WOocGmo)JZOnzxl4iscihsuF^SS ztQZcNa5SIghrVdv8y#cadjq@F=}T@oX1^43=#h?GU!3M*O((R~goP*AGhR7&>$?dU zG9X5`ii-AZ*R!hd#(>b3!L;2$%4a zTRMG&3;Y?rI3H`cD6oN>NBC$h!#Hbcbe6+&at^shi#(JCAs+*GG8Td#Lz@&Dg&);CUqS5EjMrXx#PejHr||qn-8>*AqGi>O%$tUHFq>DW=Zt3r58s=70M)#3u@ty#A0 zjQNYI`ThL4fXa(27ps4v;->wztvQ>> zwhpj?+xF`RCnoHF|5db>VVw0e`j6`X=F_*b4k+`E;um>w#iZf6x$S<6pRr_U{B)6s z-zyZqkM5DAwJagTldHIDR&d#xyK1Yi5OukY~l^+V9<0{Qw_x$NsLyZ0qP zC)wAJeSf}I^XP|Wo5G`Z#H}N$Pa2*_A#;gGC*D|h0|NeNi z2GEqqwskkpT~E{|xn^Hd^1nyP*JoN4U!VOrxn=u>I#%9LUaQ{IO67Ytb{k4-Fo?3J z-KF-m*hkZQT6vcd_QTK5T|fN0EPp=vb<=ML{d)Fe!Ug_B^6Lpq2YQzQcOBuQwG05e zAV0p#XlMJayvtIw+`nuW;|02n2tFwB%raD(rZjy0Ujxi@YC&Q4P zl$PYSnyt15tHr+`w|nr(itYlJTQbg;46}06vvLjDlH1edu{*0RxstTp>9$qd9FoUb zPgyJc8crtgA@6djL*q#K-gP<%^(dTGC3V#|zbh4 zanVy{GTZE;N2rZHm)lwGHq|*yb!48RFfMpa)vAu%r28Vm;XmRA&G9w}CF01Y)c8{m zpibhyUuCPBa%hhK?Bw7a-zpJ53flXkd~wZjPyZfK0Y|`(EaC~qpsy|B;K+2q4U9dP zmpB+{wA&!0Uzf}o($A8ea_UvINv`4W3savW4)4+{>!09oUlerg;vB{orr>a2H0g*| z699s>Bi@BZSq-gOnU|9ibJuEntFUSRR{ZnG)T-WLr3TJQM84yXb&8%)pTB@M0$ z=x&3Me#(AtH~eMz$Fbny_j~W){2)K{%Reao>UlZ2<;;O`In8S~^dd_Nlz&mIE7){^!H=|6U$~edM*tJzFi3 z3Ogz=wjH;Sf3lBmpYZhl;_msVlc*CvyUmMzp0H<(#al{4(vgF++smKR4vMvD3)ocy zUOK`3XM(k1;bl8-;d@L?5QT*>$j54|?qefiF-BN+db8EdVomrD zV5L>ZkHd8}rUp}J22Sbebx1mqZfg6TH!c~${`i@so3vTcP4@Ky_}b!&KGCbPpLE+T z#wH92dkn??qS}gXhBg-zO5$hlpT-*qQDI6zIXD)QUiXXCp-+&fys?D7Y4b z0>;sw4;W0G#mk5B@@c$$9M8mgJpS9^&rK8$OcKRp6c{C5w5EuHj^#0Cf+!>uz)M5{ zqZoKV5%sliOy6}Oc$LqL`!_nur#y)t3!dL7lk;dXPBmRjmUyR9Mv;% zR?m+&0YvL5uH}m7@ok} zLO%F0BSzteJO?esM;+n5=(&X|4iZ21403P_hEXv3u2lA}LYqXLIEFd)0AWl&7Z4*3 z7)!YknCHb&0s~)%<0&+dYv&+)KzWbwlJH0uvU?3g(MfNs>8(JUM4h-!rSV46NorJd z5_8WHj=8tlhk0=o?hN$pVYEZYzS)TD=;lwen^7lGCvKlNwuAfpWIzn^`AWsUO zIjtGA$kUo5uXf>fr`cquJv{x3!`K8zv2Nk~hID6HQ+<(n^@4j#R+pP^ZItg<>wLVu z^Pj6K3l6uoZ+(5*EvN0w&qChhnTGl3d2SckN3l@sOFVovyAW*>b>jAA#vbHxj!^8A zyFRhoGDZ6l{vHh*yS%OIcKT{?44G&vvM3p zpv5$lU2D?d;L3WU^pGk%wITP9`7`bf4I{+-Fpv2dML!YqFDeo9uAwonB>nJ4QDROd z0sjPZd&j(!kt1Nd$T7kLV#KJ|?55t~F(&-{ZzEvDXL3J3reMf^?t-6J_+r$1lco!0 jJDYj~+p+4IkuYLk;&zNyFl1k?fUnLC^%cjsATa(9`VJF3 delta 390 zcmaFjyh1XNi8U9SiNz3ET|+9Ba8-q^wk9tN183yP4(!7$? zqGE-DqWrSV6ri({Diu;79%W+S0ISt3k1l71vN?eISb*$uP{;xSE0B#0Kz0BCO;TnK diff --git a/tests/cfgs/default/result/kafka.pcapng.out b/tests/cfgs/default/result/kafka.pcapng.out index ee0deb38d..47bdc80f3 100644 --- a/tests/cfgs/default/result/kafka.pcapng.out +++ b/tests/cfgs/default/result/kafka.pcapng.out @@ -1,11 +1,14 @@ -DPI Packets (TCP): 6 (6.00 pkts/flow) -Confidence DPI : 1 (flows) -Num dissector calls: 150 (150.00 diss/flow) +Guessed flow protos: 1 + +DPI Packets (TCP): 16 (1.78 pkts/flow) +Confidence Match by port : 1 (flows) +Confidence DPI : 8 (flows) +Num dissector calls: 222 (24.67 diss/flow) LRU cache ookla: 0/0/0 (insert/search/found) -LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/3/0 (insert/search/found) LRU cache stun: 0/0/0 (insert/search/found) LRU cache tls_cert: 0/0/0 (insert/search/found) -LRU cache mining: 0/0/0 (insert/search/found) +LRU cache mining: 0/1/0 (insert/search/found) LRU cache msteams: 0/0/0 (insert/search/found) LRU cache stun_zoom: 0/0/0 (insert/search/found) Automa host: 0/0 (search/found) @@ -13,15 +16,23 @@ Automa domain: 0/0 (search/found) Automa tls cert: 0/0 (search/found) Automa risk mask: 0/0 (search/found) Automa common alpns: 0/0 (search/found) -Patricia risk mask: 0/0 (search/found) +Patricia risk mask: 14/0 (search/found) Patricia risk mask IPv6: 0/0 (search/found) Patricia risk: 0/0 (search/found) Patricia risk IPv6: 0/0 (search/found) -Patricia protocols: 2/0 (search/found) +Patricia protocols: 18/0 (search/found) Patricia protocols IPv6: 0/0 (search/found) -Kafka 19 2237 1 +Kafka 41 7067 9 -Acceptable 19 2237 1 +Acceptable 41 7067 9 - 1 TCP 127.0.0.1:46136 <-> 127.0.0.1:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: RPC/16][12 pkts/1107 bytes <-> 7 pkts/1130 bytes][Goodput ratio: 28/58][13.63 sec][bytes ratio: -0.010 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 800/288 6849/1049 2039/441][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 92/161 206/512 42/149][PLAIN TEXT (console)][Plen Bins: 12,38,12,12,12,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 1 TCP 127.0.0.1:46136 <-> 127.0.0.1:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 4][cat: RPC/16][12 pkts/1107 bytes <-> 7 pkts/1130 bytes][Goodput ratio: 28/58][13.63 sec][bytes ratio: -0.010 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 800/288 6849/1049 2039/441][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 92/161 206/512 42/149][Risk: ** Probing attempt **][Risk Score: 50][Risk Info: TCP connection with unidirectional traffic][PLAIN TEXT (console)][Plen Bins: 12,38,12,12,12,0,0,0,0,0,0,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 2 TCP 172.16.17.101:38176 <-> 172.30.0.237:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 1][cat: RPC/16][3 pkts/1408 bytes <-> 2 pkts/254 bytes][Goodput ratio: 86/48][0.58 sec][Risk: ** Probing attempt **][Risk Score: 50][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][PLAIN TEXT (timestamp)][Plen Bins: 0,40,0,0,0,0,0,0,0,0,0,40,0,0,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 3 TCP 172.30.0.237:9092 <-> 172.16.17.101:58052 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 5][cat: RPC/16][4 pkts/974 bytes <-> 1 pkts/110 bytes][Goodput ratio: 73/40][599.70 sec][PLAIN TEXT (172.30.0.237)][Plen Bins: 0,20,0,60,0,0,0,0,0,0,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 4 TCP 172.16.17.101:49280 <-> 172.30.0.237:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 1][cat: RPC/16][2 pkts/201 bytes <-> 3 pkts/788 bytes][Goodput ratio: 34/75][899.84 sec][Risk: ** Probing attempt **][Risk Score: 50][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][PLAIN TEXT (172.30.0.237)][Plen Bins: 20,20,0,40,0,0,0,0,0,0,20,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 5 TCP 172.16.17.101:56556 <-> 172.30.0.237:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 1][cat: RPC/16][1 pkts/91 bytes <-> 1 pkts/416 bytes][Goodput ratio: 27/84][0.03 sec][Risk: ** Probing attempt **][Risk Score: 50][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 50,0,0,0,0,0,0,0,0,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 6 TCP 172.16.17.101:40042 <-> 172.30.0.237:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 1][cat: RPC/16][1 pkts/110 bytes <-> 1 pkts/186 bytes][Goodput ratio: 40/64][0.03 sec][Risk: ** Probing attempt **][Risk Score: 50][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][PLAIN TEXT (172.30.0.237)][Plen Bins: 0,50,0,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 7 TCP 172.16.17.101:53768 -> 172.30.0.237:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 1][cat: RPC/16][1 pkts/110 bytes -> 0 pkts/0 bytes][Goodput ratio: 40/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 8 TCP 172.16.17.101:53052 -> 172.30.0.237:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 1][cat: RPC/16][1 pkts/91 bytes -> 0 pkts/0 bytes][Goodput ratio: 27/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 9 TCP 172.16.17.101:58300 -> 172.30.0.237:9092 [proto: 377/Kafka][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 1][cat: RPC/16][1 pkts/91 bytes -> 0 pkts/0 bytes][Goodput ratio: 27/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]