wireshark: lua: export some metadata
Export some metadata (for the moment, SNI and TLS fingerprints) to Wireshark/tshark via extcap. Note that: * metadata are exported only once per flow * metadata are exported (all together) when nDPI stopped processing the flow Still room for a lot of improvements! In particular: * we need to add some boundary checks (if we are going to export other attributes) * we should try to have a variable length trailer
This commit is contained in:
+68
-1
@@ -216,6 +216,18 @@ struct receiver {
|
|||||||
struct receiver *receivers = NULL, *topReceivers = NULL;
|
struct receiver *receivers = NULL, *topReceivers = NULL;
|
||||||
|
|
||||||
#define WIRESHARK_NTOP_MAGIC 0x19680924
|
#define WIRESHARK_NTOP_MAGIC 0x19680924
|
||||||
|
#define WIRESHARK_METADATA_SIZE 256
|
||||||
|
|
||||||
|
#define WIRESHARK_METADATA_SERVERNAME 0x01
|
||||||
|
#define WIRESHARK_METADATA_JA3C 0x02
|
||||||
|
#define WIRESHARK_METADATA_JA3S 0x03
|
||||||
|
#define WIRESHARK_METADATA_JA4C 0x04
|
||||||
|
|
||||||
|
struct ndpi_packet_tlv {
|
||||||
|
u_int16_t type;
|
||||||
|
u_int16_t length;
|
||||||
|
unsigned char data[];
|
||||||
|
};
|
||||||
|
|
||||||
PACK_ON
|
PACK_ON
|
||||||
struct ndpi_packet_trailer {
|
struct ndpi_packet_trailer {
|
||||||
@@ -224,6 +236,9 @@ struct ndpi_packet_trailer {
|
|||||||
ndpi_risk flow_risk;
|
ndpi_risk flow_risk;
|
||||||
u_int16_t flow_score;
|
u_int16_t flow_score;
|
||||||
char name[16];
|
char name[16];
|
||||||
|
/* TLV of attributes. Having a max and fixed size for all the metadata
|
||||||
|
is not efficient but greatly improves detection of the trailer by Wireshark */
|
||||||
|
unsigned char metadata[WIRESHARK_METADATA_SIZE];
|
||||||
} PACK_OFF;
|
} PACK_OFF;
|
||||||
|
|
||||||
static pcap_dumper_t *extcap_dumper = NULL;
|
static pcap_dumper_t *extcap_dumper = NULL;
|
||||||
@@ -4424,6 +4439,7 @@ static void ndpi_process_packet(u_char *args,
|
|||||||
const u_char *packet) {
|
const u_char *packet) {
|
||||||
struct ndpi_proto p;
|
struct ndpi_proto p;
|
||||||
ndpi_risk flow_risk;
|
ndpi_risk flow_risk;
|
||||||
|
struct ndpi_flow_info *flow;
|
||||||
u_int16_t thread_id = *((u_int16_t*)args);
|
u_int16_t thread_id = *((u_int16_t*)args);
|
||||||
|
|
||||||
/* allocate an exact size buffer to check overflows */
|
/* allocate an exact size buffer to check overflows */
|
||||||
@@ -4434,7 +4450,7 @@ static void ndpi_process_packet(u_char *args,
|
|||||||
}
|
}
|
||||||
|
|
||||||
memcpy(packet_checked, packet, header->caplen);
|
memcpy(packet_checked, packet, header->caplen);
|
||||||
p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked, &flow_risk);
|
p = ndpi_workflow_process_packet(ndpi_thread_info[thread_id].workflow, header, packet_checked, &flow_risk, &flow);
|
||||||
|
|
||||||
if(!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
|
if(!pcap_start.tv_sec) pcap_start.tv_sec = header->ts.tv_sec, pcap_start.tv_usec = header->ts.tv_usec;
|
||||||
pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
|
pcap_end.tv_sec = header->ts.tv_sec, pcap_end.tv_usec = header->ts.tv_usec;
|
||||||
@@ -4495,6 +4511,57 @@ static void ndpi_process_packet(u_char *args,
|
|||||||
trailer->flow_score = htons(ndpi_risk2score(flow_risk, &cli_score, &srv_score));
|
trailer->flow_score = htons(ndpi_risk2score(flow_risk, &cli_score, &srv_score));
|
||||||
trailer->master_protocol = htons(p.master_protocol), trailer->app_protocol = htons(p.app_protocol);
|
trailer->master_protocol = htons(p.master_protocol), trailer->app_protocol = htons(p.app_protocol);
|
||||||
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
|
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
|
||||||
|
|
||||||
|
/* Metadata */
|
||||||
|
/* Metadata are (all) available in `flow` only after nDPI completed its work!
|
||||||
|
We export them only once */
|
||||||
|
/* TODO: boundary check. Right now there is always enough room, but we should check it if we are
|
||||||
|
going to extend the list of the metadata exported */
|
||||||
|
struct ndpi_packet_tlv *tlv = (struct ndpi_packet_tlv *)trailer->metadata;
|
||||||
|
int tot_len = 0;
|
||||||
|
if(flow && flow->detection_completed == 1) {
|
||||||
|
if(flow->host_server_name[0] != '\0') {
|
||||||
|
tlv->type = ntohs(WIRESHARK_METADATA_SERVERNAME);
|
||||||
|
tlv->length = ntohs(sizeof(flow->host_server_name));
|
||||||
|
memcpy(tlv->data, flow->host_server_name, sizeof(flow->host_server_name));
|
||||||
|
/* TODO: boundary check */
|
||||||
|
tot_len += 4 + htons(tlv->length);
|
||||||
|
tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
|
||||||
|
}
|
||||||
|
if(flow->ssh_tls.ja3_client[0] != '\0') {
|
||||||
|
tlv->type = ntohs(WIRESHARK_METADATA_JA3C);
|
||||||
|
tlv->length = ntohs(sizeof(flow->ssh_tls.ja3_client));
|
||||||
|
memcpy(tlv->data, flow->ssh_tls.ja3_client, sizeof(flow->ssh_tls.ja3_client));
|
||||||
|
/* TODO: boundary check */
|
||||||
|
tot_len += 4 + htons(tlv->length);
|
||||||
|
tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
|
||||||
|
}
|
||||||
|
if(flow->ssh_tls.ja3_server[0] != '\0') {
|
||||||
|
tlv->type = ntohs(WIRESHARK_METADATA_JA3S);
|
||||||
|
tlv->length = ntohs(sizeof(flow->ssh_tls.ja3_server));
|
||||||
|
memcpy(tlv->data, flow->ssh_tls.ja3_server, sizeof(flow->ssh_tls.ja3_server));
|
||||||
|
/* TODO: boundary check */
|
||||||
|
tot_len += 4 + htons(tlv->length);
|
||||||
|
tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
|
||||||
|
}
|
||||||
|
if(flow->ssh_tls.ja4_client[0] != '\0') {
|
||||||
|
tlv->type = ntohs(WIRESHARK_METADATA_JA4C);
|
||||||
|
tlv->length = ntohs(sizeof(flow->ssh_tls.ja4_client));
|
||||||
|
memcpy(tlv->data, flow->ssh_tls.ja4_client, sizeof(flow->ssh_tls.ja4_client));
|
||||||
|
/* TODO: boundary check */
|
||||||
|
tot_len += 4 + htons(tlv->length);
|
||||||
|
tlv = (struct ndpi_packet_tlv *)&trailer->metadata[tot_len];
|
||||||
|
}
|
||||||
|
|
||||||
|
flow->detection_completed = 2; /* Avoid exporting metadata again.
|
||||||
|
If we really want to have the metadata on Wireshark for *all*
|
||||||
|
the future packets of this flow, simply remove that assignment */
|
||||||
|
}
|
||||||
|
/* Last: padding */
|
||||||
|
tlv->type = 0;
|
||||||
|
tlv->length = ntohs(WIRESHARK_METADATA_SIZE - tot_len - 4);
|
||||||
|
/* The remaining bytes are already set to 0 */
|
||||||
|
|
||||||
crc = (uint32_t*)&extcap_buf[h.caplen+sizeof(struct ndpi_packet_trailer)];
|
crc = (uint32_t*)&extcap_buf[h.caplen+sizeof(struct ndpi_packet_trailer)];
|
||||||
*crc = ndpi_crc32((const void*)extcap_buf, h.caplen+sizeof(struct ndpi_packet_trailer));
|
*crc = ndpi_crc32((const void*)extcap_buf, h.caplen+sizeof(struct ndpi_packet_trailer));
|
||||||
h.caplen += delta, h.len += delta;
|
h.caplen += delta, h.len += delta;
|
||||||
|
|||||||
@@ -1440,7 +1440,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
|||||||
const struct pcap_pkthdr *header,
|
const struct pcap_pkthdr *header,
|
||||||
const u_char *packet,
|
const u_char *packet,
|
||||||
pkt_timeval when,
|
pkt_timeval when,
|
||||||
ndpi_risk *flow_risk) {
|
ndpi_risk *flow_risk,
|
||||||
|
struct ndpi_flow_info **flow_ext) {
|
||||||
struct ndpi_flow_info *flow = NULL;
|
struct ndpi_flow_info *flow = NULL;
|
||||||
struct ndpi_flow_struct *ndpi_flow = NULL;
|
struct ndpi_flow_struct *ndpi_flow = NULL;
|
||||||
u_int8_t proto;
|
u_int8_t proto;
|
||||||
@@ -1702,6 +1703,7 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
*flow_risk = flow->risk;
|
*flow_risk = flow->risk;
|
||||||
|
*flow_ext = flow;
|
||||||
|
|
||||||
return(flow->detected_protocol);
|
return(flow->detected_protocol);
|
||||||
}
|
}
|
||||||
@@ -1851,7 +1853,8 @@ static uint32_t ndpi_is_valid_gre_tunnel(const struct pcap_pkthdr *header,
|
|||||||
struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
|
struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
|
||||||
const struct pcap_pkthdr *header,
|
const struct pcap_pkthdr *header,
|
||||||
const u_char *packet,
|
const u_char *packet,
|
||||||
ndpi_risk *flow_risk) {
|
ndpi_risk *flow_risk,
|
||||||
|
struct ndpi_flow_info **flow) {
|
||||||
/*
|
/*
|
||||||
* Declare pointers to packet headers
|
* Declare pointers to packet headers
|
||||||
*/
|
*/
|
||||||
@@ -1900,6 +1903,7 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
|
|||||||
u_int8_t vlan_packet = 0;
|
u_int8_t vlan_packet = 0;
|
||||||
|
|
||||||
*flow_risk = 0 /* NDPI_NO_RISK */;
|
*flow_risk = 0 /* NDPI_NO_RISK */;
|
||||||
|
*flow = NULL;
|
||||||
|
|
||||||
/* Increment raw packet counter */
|
/* Increment raw packet counter */
|
||||||
workflow->stats.raw_packet_count++;
|
workflow->stats.raw_packet_count++;
|
||||||
@@ -2376,7 +2380,7 @@ struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
|
|||||||
return(packet_processing(workflow, time_ms, vlan_id, tunnel_type, iph, iph6,
|
return(packet_processing(workflow, time_ms, vlan_id, tunnel_type, iph, iph6,
|
||||||
header->caplen - ip_offset,
|
header->caplen - ip_offset,
|
||||||
header->caplen, header, packet, header->ts,
|
header->caplen, header, packet, header->ts,
|
||||||
flow_risk));
|
flow_risk, flow));
|
||||||
}
|
}
|
||||||
|
|
||||||
/* *********************************************** */
|
/* *********************************************** */
|
||||||
|
|||||||
@@ -416,7 +416,8 @@ void ndpi_free_flow_info_half(struct ndpi_flow_info *flow);
|
|||||||
struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
|
struct ndpi_proto ndpi_workflow_process_packet(struct ndpi_workflow * workflow,
|
||||||
const struct pcap_pkthdr *header,
|
const struct pcap_pkthdr *header,
|
||||||
const u_char *packet,
|
const u_char *packet,
|
||||||
ndpi_risk *flow_risk);
|
ndpi_risk *flow_risk,
|
||||||
|
struct ndpi_flow_info **flow);
|
||||||
|
|
||||||
|
|
||||||
/* Flow callback for completed flows, before the flow memory will be freed. */
|
/* Flow callback for completed flows, before the flow memory will be freed. */
|
||||||
|
|||||||
@@ -126,9 +126,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
|
|||||||
|
|
||||||
if(packet_checked) {
|
if(packet_checked) {
|
||||||
ndpi_risk flow_risk;
|
ndpi_risk flow_risk;
|
||||||
|
struct ndpi_flow_info *flow = NULL; /* unused */
|
||||||
|
|
||||||
memcpy(packet_checked, pkt, header->caplen);
|
memcpy(packet_checked, pkt, header->caplen);
|
||||||
ndpi_workflow_process_packet(workflow, header, packet_checked, &flow_risk);
|
ndpi_workflow_process_packet(workflow, header, packet_checked, &flow_risk, &flow);
|
||||||
free(packet_checked);
|
free(packet_checked);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|||||||
ndpi_serialization_format serialization_format;
|
ndpi_serialization_format serialization_format;
|
||||||
NDPI_PROTOCOL_BITMASK enabled_bitmask;
|
NDPI_PROTOCOL_BITMASK enabled_bitmask;
|
||||||
ndpi_risk flow_risk;
|
ndpi_risk flow_risk;
|
||||||
|
struct ndpi_flow_info *flow = NULL; /* unused */
|
||||||
const u_char *pkt;
|
const u_char *pkt;
|
||||||
struct pcap_pkthdr *header;
|
struct pcap_pkthdr *header;
|
||||||
int r, rc;
|
int r, rc;
|
||||||
@@ -95,7 +96,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
|
|||||||
header = NULL;
|
header = NULL;
|
||||||
r = pcap_next_ex(pcap_handle, &header, &pkt);
|
r = pcap_next_ex(pcap_handle, &header, &pkt);
|
||||||
while (r > 0) {
|
while (r > 0) {
|
||||||
ndpi_workflow_process_packet(w, header, pkt, &flow_risk);
|
ndpi_workflow_process_packet(w, header, pkt, &flow_risk, &flow);
|
||||||
r = pcap_next_ex(pcap_handle, &header, &pkt);
|
r = pcap_next_ex(pcap_handle, &header, &pkt);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+54
-2
@@ -38,6 +38,25 @@ ndpi_fds.name = ProtoField.new("nDPI Protocol Name", "ndpi.proto
|
|||||||
ndpi_fds.flow_risk = ProtoField.new("nDPI Flow Risk", "ndpi.flow_risk", ftypes.UINT64, nil, base.HEX)
|
ndpi_fds.flow_risk = ProtoField.new("nDPI Flow Risk", "ndpi.flow_risk", ftypes.UINT64, nil, base.HEX)
|
||||||
ndpi_fds.flow_score = ProtoField.new("nDPI Flow Score", "ndpi.flow_score", ftypes.UINT32)
|
ndpi_fds.flow_score = ProtoField.new("nDPI Flow Score", "ndpi.flow_score", ftypes.UINT32)
|
||||||
|
|
||||||
|
ndpi_fds.metadata_list = ProtoField.new("nDPI Metadata List", "ndpi.metadata_list", ftypes.NONE)
|
||||||
|
ndpi_fds.metadata = ProtoField.new("nDPI Metadata", "ndpi.metadata", ftypes.NONE)
|
||||||
|
local mtd_types = {
|
||||||
|
[0] = "Padding",
|
||||||
|
[1] = "Server Name",
|
||||||
|
[2] = "JA3C",
|
||||||
|
[3] = "JA3S",
|
||||||
|
[4] = "JA4C"
|
||||||
|
}
|
||||||
|
ndpi_fds.metadata_type = ProtoField.new("nDPI Metadata Type", "ndpi.metadata.type", ftypes.UINT16, mtd_types)
|
||||||
|
ndpi_fds.metadata_length = ProtoField.new("nDPI Metadata Length", "ndpi.metadata.length", ftypes.UINT16)
|
||||||
|
-- Generic field
|
||||||
|
ndpi_fds.metadata_value = ProtoField.new("nDPI Metadata Value", "ndpi.metadata.value", ftypes.BYTES)
|
||||||
|
-- Specific fields
|
||||||
|
ndpi_fds.metadata_server_name = ProtoField.new("nDPI Server Name", "ndpi.metadata.server_name", ftypes.STRING)
|
||||||
|
ndpi_fds.metadata_ja3c = ProtoField.new("nDPI JA3C", "ndpi.metadata.ja3c", ftypes.STRING)
|
||||||
|
ndpi_fds.metadata_ja3s = ProtoField.new("nDPI JA3S", "ndpi.metadata.ja3s", ftypes.STRING)
|
||||||
|
ndpi_fds.metadata_ja4c = ProtoField.new("nDPI JA4C", "ndpi.metadata.ja4c", ftypes.STRING)
|
||||||
|
|
||||||
|
|
||||||
local flow_risks = {}
|
local flow_risks = {}
|
||||||
--- You can't use a 64 bit integer "as-is" as mask: we choose to use UInt64 object instead
|
--- You can't use a 64 bit integer "as-is" as mask: we choose to use UInt64 object instead
|
||||||
@@ -1055,9 +1074,9 @@ function ndpi_proto.dissector(tvb, pinfo, tree)
|
|||||||
|
|
||||||
if(magic == "19:68:09:24") then
|
if(magic == "19:68:09:24") then
|
||||||
local ndpikey, srckey, dstkey, flowkey, flow_risk
|
local ndpikey, srckey, dstkey, flowkey, flow_risk
|
||||||
local flow_risk_tree
|
local flow_risk_tree, metadata_list_tree, metadata_tree
|
||||||
local name
|
local name
|
||||||
local trailer_tvb = tvb(tvb:len() - 38, 34) -- The last 4 bytes are the CRC. Even if nDPI needs to update it, it is not part of the nDPI trailer, strictly speaking
|
local trailer_tvb = tvb(tvb:len() - 294 , 290) -- The last 4 bytes are the CRC. Even if nDPI needs to update it, it is not part of the nDPI trailer, strictly speaking
|
||||||
local ndpi_subtree = tree:add(ndpi_proto, trailer_tvb, "nDPI Protocol")
|
local ndpi_subtree = tree:add(ndpi_proto, trailer_tvb, "nDPI Protocol")
|
||||||
|
|
||||||
ndpi_subtree:add(ndpi_fds.magic, trailer_tvb(0, 4))
|
ndpi_subtree:add(ndpi_fds.magic, trailer_tvb(0, 4))
|
||||||
@@ -1113,6 +1132,39 @@ function ndpi_proto.dissector(tvb, pinfo, tree)
|
|||||||
--print(network_protocol .. "/" .. application_protocol .. "/".. name)
|
--print(network_protocol .. "/" .. application_protocol .. "/".. name)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
-- Metadata
|
||||||
|
local offset = 34
|
||||||
|
metadata_list_tree = ndpi_subtree:add(ndpi_fds.metadata_list, trailer_tvb(offset, 256))
|
||||||
|
while offset + 4 < 294 do
|
||||||
|
|
||||||
|
local mtd_type = trailer_tvb(offset, 2):int();
|
||||||
|
local mtd_length = trailer_tvb(offset + 2, 2):int();
|
||||||
|
|
||||||
|
metadata_tree = metadata_list_tree:add(ndpi_fds.metadata, trailer_tvb(offset, 4 + mtd_length))
|
||||||
|
metadata_tree:add(ndpi_fds.metadata_type, trailer_tvb(offset, 2))
|
||||||
|
metadata_tree:add(ndpi_fds.metadata_length, trailer_tvb(offset + 2, 2))
|
||||||
|
|
||||||
|
-- Specific fields: there is definitely a better way...
|
||||||
|
if mtd_type == 1 then
|
||||||
|
metadata_tree:append_text(" ServerName: " .. trailer_tvb(offset + 4, mtd_length):string())
|
||||||
|
metadata_tree:add(ndpi_fds.metadata_server_name, trailer_tvb(offset + 4, mtd_length))
|
||||||
|
elseif mtd_type == 2 then
|
||||||
|
metadata_tree:append_text(" JA3C: " .. trailer_tvb(offset + 4, mtd_length):string())
|
||||||
|
metadata_tree:add(ndpi_fds.metadata_ja3c, trailer_tvb(offset + 4, mtd_length))
|
||||||
|
elseif mtd_type == 3 then
|
||||||
|
metadata_tree:append_text(" JA3S: " .. trailer_tvb(offset + 4, mtd_length):string())
|
||||||
|
metadata_tree:add(ndpi_fds.metadata_ja3s, trailer_tvb(offset + 4, mtd_length))
|
||||||
|
elseif mtd_type == 4 then
|
||||||
|
metadata_tree:append_text(" JA4C: " .. trailer_tvb(offset + 4, mtd_length):string())
|
||||||
|
metadata_tree:add(ndpi_fds.metadata_ja4c, trailer_tvb(offset + 4, mtd_length))
|
||||||
|
else
|
||||||
|
-- Generic field
|
||||||
|
metadata_tree:add(ndpi_fds.metadata_value, trailer_tvb(offset + 4, mtd_length))
|
||||||
|
end
|
||||||
|
|
||||||
|
offset = offset + 4 + mtd_length
|
||||||
|
end
|
||||||
|
|
||||||
if(compute_flows_stats and pinfo.visited == false) then
|
if(compute_flows_stats and pinfo.visited == false) then
|
||||||
ndpikey = tostring(slen(name))
|
ndpikey = tostring(slen(name))
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user