Implemented custom category loading.

Modified ndpiReader with -c <file> (use mining_hosts.txt as example) for loading name-based categories
This commit is contained in:
Luca
2018-04-26 21:10:59 +02:00
parent 79f3199339
commit 3adccc98f7
7 changed files with 762 additions and 249 deletions
+262
View File
@@ -0,0 +1,262 @@
# NoCoin Filter List
# Blocking Web Browser Bitcoin Mining
#
# Homepage: https://github.com/hoshsadiq/adblock-nocoin-list/
# Contribute: https://github.com/hoshsadiq/adblock-nocoin-list/issues
# Changelog: https://github.com/hoshsadiq/adblock-nocoin-list/commits/master/hosts.txt
# License: https://mit-license.org/
#
# Last modified: 24 April 2018
#
cnhv.co 99
coin-hive.com 99
coinhive.com 99
authedmine.com 99
gus.host 99
load.jsecoin.com 99
miner.pr0gramm.com 99
minemytraffic.com 99
ppoi.org 99
projectpoi.com 99
crypto-loot.com 99
coinerra.com 99
coin-have.com 99
minero.pw 99
minero-proxy-01.now.sh 99
minero-proxy-02.now.sh 99
minero-proxy-03.now.sh 99
api.inwemo.com 99
rocks.io 99
adminer.com 99
ad-miner.com 99
jsccnn.com 99
jscdndel.com 99
coinhiveproxy.com 99
coinblind.com 99
coinnebula.com 99
monerominer.rocks 99
cdn.cloudcoins.co 99
coinlab.biz 99
go.megabanners.cf 99
baiduccdn1.com 99
wsp.marketgid.com 99
papoto.com 99
flare-analytics.com 99
www.sparechange.io 99
static.sparechange.io 99
miner.nablabee.com 99
m.anyfiles.ovh 99
www.freecontent.bid 99
www.freecontent.loan 99
www.freecontent.win 99
www.cryptonoter.com 99
www.mutuza.win 99
cryweb.github.io 99
crywebber.github.io 99
crypto-webminer.com 99
cdn.adless.io 99
hegrinhar.com 99
verresof.com 99
hemnes.win 99
tidafors.xyz 99
moneone.ga 99
plexcoin.info 99
www.monkeyminer.net 99
go2.mercy.ga 99
coinpirate.cf 99
d.cpufan.club 99
krb.devphp.org.ua 99
nfwebminer.com 99
cfcdist.gdn 99
node.cfcdist.gdn 99
webxmr.com 99
xmr.mining.best 99
webminepool.com 99
webminepool.tk 99
hive.tubetitties.com 99
playerassets.info 99
tokyodrift.ga 99
webassembly.stream 99
okeyletsgo.ml 99
candid.zone 99
webmine.pro 99
andlache.com 99
bablace.com 99
bewaslac.com 99
biberukalap.com 99
bowithow.com 99
butcalve.com 99
evengparme.com 99
gridiogrid.com 99
hatcalter.com 99
kedtise.com 99
ledinund.com 99
nathetsof.com 99
renhertfo.com 99
rintindown.com 99
sparnove.com 99
witthethim.com 99
1q2w3.fun 99
1q2w3.me 99
cryptoloot.pro 99
bjorksta.men 99
crypto.csgocpu.com 99
noblock.pro 99
miner.cryptobara.com 99
digger.cryptobara.com 99
dev.cryptobara.com 99
reservedoffers.club 99
mine.torrent.pw 99
host.d-ns.ga 99
abc.pema.cl 99
mine.nahnoji.cz 99
webmine.cz 99
intactoffers.club 99
analytics.blue 99
smectapop12.pl 99
berserkpl.net.pl 99
hodlers.party 99
hodling.faith 99
chainblock.science 99
minescripts.info 99
cdn.minescripts.info 99
miner.nablabee.com 99
wss.nablabee.com 99
clickwith.bid 99
dronml.ml 99
niematego.tk 99
tulip18.com 99
p.estream.to 99
didnkinrab.com 99
ledhenone.com 99
losital.ru 99
mebablo.com 99
moonsade.com 99
nebabrop.com 99
pearno.com 99
rintinwa.com 99
willacrit.com 99
www2.adfreetv.ch 99
minr.pw 99
new.minr.pw 99
test.minr.pw 99
staticsfs.host 99
cdn-code.host 99
g-content.bid 99
ad.g-content.bid 99
cdn.static-cnt.bid 99
cnt.statistic.date 99
cdn.jquery-uim.download 99
cdn-jquery.host 99
p1.interestingz.pw 99
kippbeak.cf 99
pasoherb.gq 99
axoncoho.tk 99
depttake.ga 99
flophous.cf 99
pr0gram.org 99
authedmine.eu 99
www.monero-miner.com 99
www.datasecu.download 99
www.jquery-cdn.download 99
www.etzbnfuigipwvs.ru 99
www.terethat.ru 99
freshrefresher.com 99
api.pzoifaum.info 99
ws.pzoifaum.info 99
api.bhzejltg.info 99
ws.bhzejltg.info 99
d.cfcnet.top 99
vip.cfcnet.top 99
eu.cfcnet.top 99
as.cfcnet.top 99
us.cfcnet.top 99
eu.cfcdist.loan 99
as.cfcdist.loan 99
us.cfcdist.loan 99
gustaver.ddns.net 99
worker.salon.com 99
s2.appelamule.com 99
mepirtedic.com 99
cdn.streambeam.io 99
adzjzewsma.cf 99
ffinwwfpqi.gq 99
ininmacerad.pro 99
mhiobjnirs.gq 99
open-hive-server-1.pp.ua 99
pool.hws.ru 99
pool.etn.spacepools.org 99
api.aalbbh84.info 99
www.aymcsx.ru 99
aeros01.tk 99
aeros02.tk 99
aeros03.tk 99
aeros04.tk 99
aeros05.tk 99
aeros06.tk 99
aeros07.tk 99
aeros08.tk 99
aeros09.tk 99
aeros10.tk 99
aeros11.tk 99
aeros12.tk 99
npcdn1.now.sh 99
mxcdn2.now.sh 99
sxcdn6.now.sh 99
mxcdn1.now.sh 99
sxcdn02.now.sh 99
sxcdn4.now.sh 99
jqcdn2.herokuapp.com 99
sxcdn1.herokuapp.com 99
sxcdn5.herokuapp.com 99
wpcdn1.herokuapp.com 99
jqcdn01.herokuapp.com 99
jqcdn03.herokuapp.com 99
1q2w3.website 99
video.videos.vidto.me 99
video.streaming.estream.to 99
eth-pocket.de 99
xvideosharing.site 99
bestcoinsignals.com 99
eucsoft.com 99
traviilo.com 99
wasm24.ru 99
xmr.cool 99
api.netflare.info 99
cdnjs.cloudflane.com 99
www.cloudflane.com 99
hide.ovh 99
graftpool.ovh 99
encoding.ovh 99
altavista.ovh 99
scaleway.ovh 99
nexttime.ovh 99
never.ovh 99
2giga.download 99
support.2giga.link 99
webminerpool.com 99
minercry.pt 99
adplusplus.fr 99
# Does not seem to work anymore, but keeping it here just in case it gets revived
azvjudwr.info 99
jroqvbvw.info 99
jyhfuqoh.info 99
kdowqlpt.info 99
xbasfbno.info 99
# These were Obfuscated
1beb2a44.space 99
300ca0d0.space 99
310ca263.space 99
320ca3f6.space 99
330ca589.space 99
340ca71c.space 99
360caa42.space 99
370cabd5.space 99
3c0cb3b4.space 99
3d0cb547.space 99
imageshack.us 99
img265.imageshack.us 99
+144 -105
View File
@@ -1,7 +1,7 @@
/*
* ndpiReader.c
*
* Copyright (C) 2011-17 - ntop.org
* Copyright (C) 2011-18 - ntop.org
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
@@ -67,6 +67,7 @@ static FILE *results_file = NULL;
static char *results_path = NULL;
static char * bpfFilter = NULL; /**< bpf filter */
static char *_protoFilePath = NULL; /**< Protocol file path */
static char *_customCategoryFilePath= NULL; /**< Custom categories file path */
#ifdef HAVE_JSON_C
static char *_statsFilePath = NULL; /**< Top stats file path */
static char *_diagnoseFilePath = NULL; /**< Top stats file path */
@@ -104,8 +105,8 @@ static u_int32_t num_flows;
static struct ndpi_detection_module_struct *ndpi_info_mod = NULL;
struct flow_info {
struct ndpi_flow_info *flow;
u_int16_t thread_id;
struct ndpi_flow_info *flow;
u_int16_t thread_id;
};
static struct flow_info *all_flows;
@@ -230,7 +231,7 @@ static void help(u_int long_help) {
printf("ndpiReader -i <file|device> [-f <filter>][-s <duration>][-m <duration>]\n"
" [-p <protos>][-l <loops> [-q][-d][-h][-t][-v <level>]\n"
" [-n <threads>] [-w <file>] [-j <file>] [-x <file>] \n\n"
" [-n <threads>][-w <file>][-c <file>][-j <file>][-x <file>]\n\n"
"Usage:\n"
" -i <file.pcap|device> | Specify a pcap file/playlist to read packets from or a\n"
" | device for live capture (comma-separated list)\n"
@@ -249,6 +250,7 @@ static void help(u_int long_help) {
" -q | Quiet mode\n"
" -t | Dissect GTP/TZSP tunnels\n"
" -r | Print nDPI version and git revision\n"
" -c <path> | Load custom categories from the specified file\n"
" -w <path> | Write test output on the specified file. This is useful for\n"
" | testing purposes in order to compare results across runs\n"
" -h | This help\n"
@@ -276,7 +278,7 @@ static void help(u_int long_help) {
" --fifo <path to file or pipe>\n"
" --debug\n"
" --dbg-proto proto|num[,...]\n"
);
);
#endif
if(long_help) {
@@ -360,7 +362,7 @@ int cmpFlows(const void *_a, const void *_b) {
uint64_t a_size = fa->src2dst_bytes + fa->dst2src_bytes;
uint64_t b_size = fb->src2dst_bytes + fb->dst2src_bytes;
if(a_size != b_size)
return a_size < b_size ? 1 : -1;
return a_size < b_size ? 1 : -1;
// copy from ndpi_workflow_node_cmp();
@@ -449,7 +451,7 @@ static void parseOptions(int argc, char **argv) {
if(trace) fprintf(trace, " #### %s #### \n", __FUNCTION__);
#endif
while ((opt = getopt_long(argc, argv, "df:g:i:hp:l:s:tv:V:n:j:rp:w:q0123:456:7:89:m:b:x:", longopts, &option_idx)) != EOF) {
while ((opt = getopt_long(argc, argv, "c:df:g:i:hp:l:s:tv:V:n:j:rp:w:q0123:456:7:89:m:b:x:", longopts, &option_idx)) != EOF) {
#ifdef DEBUG_TRACE
if(trace) fprintf(trace, " #### -%c [%s] #### \n", opt, optarg ? optarg : "");
#endif
@@ -507,6 +509,10 @@ static void parseOptions(int argc, char **argv) {
_protoFilePath = optarg;
break;
case 'c':
_customCategoryFilePath = optarg;
break;
case 's':
capture_for = atoi(optarg);
capture_until = capture_for + time(NULL);
@@ -528,8 +534,8 @@ static void parseOptions(int argc, char **argv) {
nDPI_LogLevel = atoi(optarg);
if(nDPI_LogLevel < 0) nDPI_LogLevel = 0;
if(nDPI_LogLevel > 3) {
nDPI_LogLevel = 3;
_debug_protocols = strdup("all");
nDPI_LogLevel = 3;
_debug_protocols = strdup("all");
}
break;
@@ -738,7 +744,7 @@ static void printFlow(u_int16_t id, struct ndpi_flow_info *flow, u_int16_t threa
flow->bidirectional ? "<->" : "->",
(flow->ip_version == 6) ? "[" : "",
flow->dst_name, (flow->ip_version == 6) ? "]" : "", ntohs(flow->dst_port)
);
);
if(flow->vlan_id > 0) fprintf(out, "[VLAN: %u]", flow->vlan_id);
@@ -834,9 +840,9 @@ static void node_print_unknown_proto_walker(const void *node, ndpi_VISIT which,
if(flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN) return;
if((which == ndpi_preorder) || (which == ndpi_leaf)) {
/* Avoid walking the same node multiple times */
all_flows[num_flows].thread_id = thread_id, all_flows[num_flows].flow = flow;
num_flows++;
/* Avoid walking the same node multiple times */
all_flows[num_flows].thread_id = thread_id, all_flows[num_flows].flow = flow;
num_flows++;
}
}
@@ -851,9 +857,9 @@ static void node_print_known_proto_walker(const void *node, ndpi_VISIT which, in
if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_UNKNOWN) return;
if((which == ndpi_preorder) || (which == ndpi_leaf)) {
/* Avoid walking the same node multiple times */
all_flows[num_flows].thread_id = thread_id, all_flows[num_flows].flow = flow;
num_flows++;
/* Avoid walking the same node multiple times */
all_flows[num_flows].thread_id = thread_id, all_flows[num_flows].flow = flow;
num_flows++;
}
}
@@ -948,7 +954,7 @@ void updateScanners(struct single_flow_info **scanners, u_int32_t saddr,
/* *********************************************** */
int updateIpTree(u_int32_t key, u_int8_t version,
addr_node **vrootp, const char *proto) {
addr_node **vrootp, const char *proto) {
addr_node *q;
addr_node **rootp = vrootp;
@@ -1116,7 +1122,7 @@ static int receivers_sort_asc(void *_a, void *_b) {
/* ***************************************************** */
/*@brief removes first (size - max) elements from hash table.
* hash table is ordered in ascending order.
*/
*/
static struct receiver *cutBackTo(struct receiver **receivers, u_int32_t size, u_int32_t max) {
struct receiver *r, *tmp;
int i=0;
@@ -1143,7 +1149,7 @@ static struct receiver *cutBackTo(struct receiver **receivers, u_int32_t size, u
* if element already in the second table
* then updates its value
* else adds it to the second table
*/
*/
static void mergeTables(struct receiver **primary, struct receiver **secondary) {
struct receiver *r, *s, *tmp;
@@ -1191,7 +1197,7 @@ static void deleteReceivers(struct receiver *receivers) {
* }
* else
* update table1
*/
*/
static void updateReceivers(struct receiver **receivers, u_int32_t dst_addr,
u_int8_t version, u_int32_t num_pkts,
struct receiver **topReceivers) {
@@ -1202,7 +1208,7 @@ static void updateReceivers(struct receiver **receivers, u_int32_t dst_addr,
HASH_FIND_INT(*receivers, (int *)&dst_addr, r);
if(r == NULL) {
if(((size = HASH_COUNT(*receivers)) < MAX_TABLE_SIZE_1)
|| ((a = acceptable(num_pkts)) != 0)){
|| ((a = acceptable(num_pkts)) != 0)){
r = (struct receiver *)malloc(sizeof(struct receiver));
if(!r) return;
@@ -1320,7 +1326,7 @@ static void port_stats_walker(const void *node, ndpi_VISIT which, int depth, voi
if(((r = strcmp(ipProto2Name(flow->protocol), "TCP")) == 0)
&& (flow->src2dst_packets == 1) && (flow->dst2src_packets == 0)) {
updateScanners(&scannerHosts, flow->src_ip, flow->ip_version, dport);
updateScanners(&scannerHosts, flow->src_ip, flow->ip_version, dport);
}
updateReceivers(&receivers, flow->dst_ip, flow->ip_version,
@@ -1463,6 +1469,39 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
if(_protoFilePath != NULL)
ndpi_load_protocols_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _protoFilePath);
if(_customCategoryFilePath) {
FILE *fd = fopen(_customCategoryFilePath, "r");
if(fd) {
while(fd) {
char buffer[512], *line, *name, *category;
int i;
if(!(line = fgets(buffer, sizeof(buffer), fd)))
break;
if(((i = strlen(line)) <= 1) || (line[0] == '#'))
continue;
else
line[i-1] = '\0';
name = strtok(line, "\t");
if(name) {
category = strtok(NULL, "\t");
if(category) {
// printf("Loading %s\t%s\n", name, category);
ndpi_load_hostname_category(ndpi_thread_info[thread_id].workflow->ndpi_struct,
name, (ndpi_protocol_category_t)atoi(category));
}
}
}
ndpi_enable_loaded_categories(ndpi_thread_info[thread_id].workflow->ndpi_struct);
} else
printf("ERROR: Unable to read file %s\n", _customCategoryFilePath);
}
}
@@ -2045,8 +2084,8 @@ static void printResults(u_int64_t tot_usec) {
total_flows += ndpi_thread_info[thread_id].workflow->num_allocated_flows;
if((all_flows = (struct flow_info*)malloc(sizeof(struct flow_info)*total_flows)) == NULL) {
printf("Fatal error: not enough memory\n");
exit(-1);
printf("Fatal error: not enough memory\n");
exit(-1);
}
if(!json_flag) fprintf(out, "\n");
@@ -2130,12 +2169,12 @@ static void printResults(u_int64_t tot_usec) {
saveScannerStats(&jObj_stats, &scannerHosts);
if((count = HASH_COUNT(topReceivers)) == 0){
HASH_SORT(receivers, receivers_sort);
saveReceiverStats(&jObj_stats, &receivers, cumulative_stats.ip_packet_count);
HASH_SORT(receivers, receivers_sort);
saveReceiverStats(&jObj_stats, &receivers, cumulative_stats.ip_packet_count);
}
else{
HASH_SORT(topReceivers, receivers_sort);
saveReceiverStats(&jObj_stats, &topReceivers, cumulative_stats.ip_packet_count);
HASH_SORT(topReceivers, receivers_sort);
saveReceiverStats(&jObj_stats, &topReceivers, cumulative_stats.ip_packet_count);
}
u_int64_t total_src_addr = getTopStats(srcStats);
@@ -2151,7 +2190,7 @@ static void printResults(u_int64_t tot_usec) {
#endif
}
free_stats:
free_stats:
if(scannerHosts) {
deleteScanners(scannerHosts);
scannerHosts = NULL;
@@ -2213,7 +2252,7 @@ static int getNextPcapFileFromPlaylist(u_int16_t thread_id, char filename[], u_i
return -1;
}
next_line:
next_line:
if(fgets(filename, filename_len, playlist_fp[thread_id])) {
int l = strlen(filename);
if(filename[0] == '\0' || filename[0] == '#') goto next_line;
@@ -2272,7 +2311,7 @@ static pcap_t * openPcapFileOrDevice(u_int16_t thread_id, const u_char * pcap_fi
if(strstr((char*)pcap_file, (char*)".pcap"))
printf("ERROR: could not open pcap file %s: %s\n", pcap_file, pcap_error_buffer);
else if((getNextPcapFileFromPlaylist(thread_id, filename, sizeof(filename)) != 0)
|| ((pcap_handle = pcap_open_offline(filename, pcap_error_buffer)) == NULL)) {
|| ((pcap_handle = pcap_open_offline(filename, pcap_error_buffer)) == NULL)) {
printf("ERROR: could not open playlist %s: %s\n", filename, pcap_error_buffer);
exit(-1);
} else {
@@ -2361,8 +2400,8 @@ static void pcap_process_packet(u_char *args,
&& ((extcap_packet_filter == (u_int16_t)-1)
|| (p.app_protocol == extcap_packet_filter)
|| (p.master_protocol == extcap_packet_filter)
)
) {
)
) {
struct pcap_pkthdr h;
uint32_t *crc, delta = sizeof(struct ndpi_packet_trailer) + 4 /* ethernet trailer */;
struct ndpi_packet_trailer *trailer;
@@ -2454,7 +2493,7 @@ void * processing_thread(void *_thread_id) {
#endif
if((!json_flag) && (!quiet_mode)) printf("Running thread %ld...\n", thread_id);
pcap_loop:
pcap_loop:
runPcapLoop(thread_id);
if(playlist_fp[thread_id] != NULL) { /* playlist: read next file */
@@ -2881,8 +2920,8 @@ void getSourcePorts(struct json_object *jObj_stat, int srcPortArray[], int size,
if((flows_packets > FLOWS_PACKETS_THRESHOLD)
&& (flows_percent >= FLOWS_PERCENT_THRESHOLD)
&& packets_number >= threshold) {
&& (flows_percent >= FLOWS_PERCENT_THRESHOLD)
&& packets_number >= threshold) {
if((res = json_object_object_get_ex(src_pkts_stat, "port", &jObj_port)) == 0) {
fprintf(stderr, "ERROR: can't get \"port\", use -x flag only with .json files generated by ndpiReader -b flag.\n");
exit(-1);
@@ -2967,7 +3006,7 @@ void getScannerHosts(struct json_object *jObj_stat, int duration,
#ifdef HAVE_JSON_C
void getDestinationHosts(struct json_object *jObj_stat, int duration,
const char *dstHostArray[16], int size) {
const char *dstHostArray[16], int size) {
int j;
for(j=0; j<json_object_array_length(jObj_stat); j++) {
@@ -3119,7 +3158,7 @@ static void produceBpfFilter(char *filePath) {
jObj_bpfFilter = json_object_new_object();
bpf_filter_pkt_peak_filter(&jObj_bpfFilter, filterSrcPorts, PORT_ARRAY_SIZE,
filterSrcHosts, HOST_ARRAY_SIZE, filterPktDstHosts, HOST_ARRAY_SIZE/2);
filterSrcHosts, HOST_ARRAY_SIZE, filterPktDstHosts, HOST_ARRAY_SIZE/2);
bpf_filter_host_peak_filter(&jObj_bpfFilter, filterDstHosts, HOST_ARRAY_SIZE);
@@ -3142,55 +3181,55 @@ static void produceBpfFilter(char *filePath) {
#ifdef APP_HAS_OWN_MAIN
int orginal_main(int argc, char **argv) {
#else
int main(int argc, char **argv) {
int main(int argc, char **argv) {
#endif
int i;
int i;
if(ndpi_get_api_version() != NDPI_API_VERSION) {
printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
return(-1);
}
if(ndpi_get_api_version() != NDPI_API_VERSION) {
printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
return(-1);
}
automataUnitTest();
automataUnitTest();
ndpi_info_mod = ndpi_init_detection_module();
if (ndpi_info_mod == NULL) return -1;
ndpi_info_mod = ndpi_init_detection_module();
if (ndpi_info_mod == NULL) return -1;
memset(ndpi_thread_info, 0, sizeof(ndpi_thread_info));
memset(ndpi_thread_info, 0, sizeof(ndpi_thread_info));
parseOptions(argc, argv);
parseOptions(argc, argv);
if(bpf_filter_flag) {
if(bpf_filter_flag) {
#ifdef HAVE_JSON_C
produceBpfFilter(_diagnoseFilePath);
return 0;
produceBpfFilter(_diagnoseFilePath);
return 0;
#endif
}
if((!json_flag) && (!quiet_mode)) {
printf("\n-----------------------------------------------------------\n"
"* NOTE: This is demo app to show *some* nDPI features.\n"
"* In this demo we have implemented only some basic features\n"
"* just to show you what you can do with the library. Feel \n"
"* free to extend it and send us the patches for inclusion\n"
"------------------------------------------------------------\n\n");
printf("Using nDPI (%s) [%d thread(s)]\n", ndpi_revision(), num_threads);
}
signal(SIGINT, sigproc);
for(i=0; i<num_loops; i++)
test_lib();
if(results_path) free(results_path);
if(results_file) fclose(results_file);
if(extcap_dumper) pcap_dump_close(extcap_dumper);
if(ndpi_info_mod) ndpi_exit_detection_module(ndpi_info_mod);
return 0;
}
if((!json_flag) && (!quiet_mode)) {
printf("\n-----------------------------------------------------------\n"
"* NOTE: This is demo app to show *some* nDPI features.\n"
"* In this demo we have implemented only some basic features\n"
"* just to show you what you can do with the library. Feel \n"
"* free to extend it and send us the patches for inclusion\n"
"------------------------------------------------------------\n\n");
printf("Using nDPI (%s) [%d thread(s)]\n", ndpi_revision(), num_threads);
}
signal(SIGINT, sigproc);
for(i=0; i<num_loops; i++)
test_lib();
if(results_path) free(results_path);
if(results_file) fclose(results_file);
if(extcap_dumper) pcap_dump_close(extcap_dumper);
if(ndpi_info_mod) ndpi_exit_detection_module(ndpi_info_mod);
return 0;
}
#ifdef WIN32
#ifndef __GNUC__
#define EPOCHFILETIME (116444736000000000i64)
@@ -3201,42 +3240,42 @@ int main(int argc, char **argv) {
/**
@brief Timezone
**/
struct timezone {
int tz_minuteswest; /* minutes W of Greenwich */
int tz_dsttime; /* type of dst correction */
};
struct timezone {
int tz_minuteswest; /* minutes W of Greenwich */
int tz_dsttime; /* type of dst correction */
};
/**
@brief Set time
**/
int gettimeofday(struct timeval *tv, struct timezone *tz) {
FILETIME ft;
LARGE_INTEGER li;
__int64 t;
static int tzflag;
int gettimeofday(struct timeval *tv, struct timezone *tz) {
FILETIME ft;
LARGE_INTEGER li;
__int64 t;
static int tzflag;
if(tv) {
GetSystemTimeAsFileTime(&ft);
li.LowPart = ft.dwLowDateTime;
li.HighPart = ft.dwHighDateTime;
t = li.QuadPart; /* In 100-nanosecond intervals */
t -= EPOCHFILETIME; /* Offset to the Epoch time */
t /= 10; /* In microseconds */
tv->tv_sec = (long)(t / 1000000);
tv->tv_usec = (long)(t % 1000000);
}
if(tz) {
if(!tzflag) {
_tzset();
tzflag++;
if(tv) {
GetSystemTimeAsFileTime(&ft);
li.LowPart = ft.dwLowDateTime;
li.HighPart = ft.dwHighDateTime;
t = li.QuadPart; /* In 100-nanosecond intervals */
t -= EPOCHFILETIME; /* Offset to the Epoch time */
t /= 10; /* In microseconds */
tv->tv_sec = (long)(t / 1000000);
tv->tv_usec = (long)(t % 1000000);
}
tz->tz_minuteswest = _timezone / 60;
tz->tz_dsttime = _daylight;
}
if(tz) {
if(!tzflag) {
_tzset();
tzflag++;
}
return 0;
}
tz->tz_minuteswest = _timezone / 60;
tz->tz_dsttime = _daylight;
}
return 0;
}
#endif /* WIN32 */
+99 -92
View File
@@ -1,7 +1,7 @@
/*
* ndpi_util.c
*
* Copyright (C) 2011-17 - ntop.org
* Copyright (C) 2011-18 - ntop.org
*
* This file is part of nDPI, an open source deep packet inspection
* library based on the OpenDPI and PACE technology by ipoque GmbH
@@ -114,49 +114,49 @@ static uint16_t ndpi_get_proto_id(struct ndpi_detection_module_struct *ndpi_mod,
char *e;
unsigned long p = strtol(name,&e,0);
if(e && !*e) {
if(p < NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS &&
ndpi_mod->proto_defaults[p].protoName) return (uint16_t)p;
return NDPI_PROTOCOL_UNKNOWN;
if(p < NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS &&
ndpi_mod->proto_defaults[p].protoName) return (uint16_t)p;
return NDPI_PROTOCOL_UNKNOWN;
}
for(proto_id=NDPI_PROTOCOL_UNKNOWN; proto_id < NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS; proto_id++) {
if(ndpi_mod->proto_defaults[proto_id].protoName &&
!strcasecmp(ndpi_mod->proto_defaults[proto_id].protoName,name))
return proto_id;
if(ndpi_mod->proto_defaults[proto_id].protoName &&
!strcasecmp(ndpi_mod->proto_defaults[proto_id].protoName,name))
return proto_id;
}
return NDPI_PROTOCOL_UNKNOWN;
}
static NDPI_PROTOCOL_BITMASK debug_bitmask;
static char _proto_delim[] = " \t,:;";
static int parse_debug_proto(struct ndpi_detection_module_struct *ndpi_mod, char *str) {
char *n;
uint16_t proto;
char op=1;
for(n = strtok(str,_proto_delim); n && *n; n = strtok(NULL,_proto_delim)) {
if(*n == '-') {
op = 0;
n++;
} else if(*n == '+') {
op = 1;
n++;
}
if(!strcmp(n,"all")) {
if(op)
NDPI_BITMASK_SET_ALL(debug_bitmask);
else
NDPI_BITMASK_RESET(debug_bitmask);
continue;
}
proto = ndpi_get_proto_id(ndpi_mod, n);
if(proto == NDPI_PROTOCOL_UNKNOWN && strcmp(n,"unknown") && strcmp(n,"0")) {
fprintf(stderr,"Invalid protocol %s\n",n);
return 1;
}
if(op)
NDPI_BITMASK_ADD(debug_bitmask,proto);
else
NDPI_BITMASK_DEL(debug_bitmask,proto);
}
return 0;
char *n;
uint16_t proto;
char op=1;
for(n = strtok(str,_proto_delim); n && *n; n = strtok(NULL,_proto_delim)) {
if(*n == '-') {
op = 0;
n++;
} else if(*n == '+') {
op = 1;
n++;
}
if(!strcmp(n,"all")) {
if(op)
NDPI_BITMASK_SET_ALL(debug_bitmask);
else
NDPI_BITMASK_RESET(debug_bitmask);
continue;
}
proto = ndpi_get_proto_id(ndpi_mod, n);
if(proto == NDPI_PROTOCOL_UNKNOWN && strcmp(n,"unknown") && strcmp(n,"0")) {
fprintf(stderr,"Invalid protocol %s\n",n);
return 1;
}
if(op)
NDPI_BITMASK_ADD(debug_bitmask,proto);
else
NDPI_BITMASK_DEL(debug_bitmask,proto);
}
return 0;
}
/* ***************************************************** */
@@ -164,16 +164,17 @@ return 0;
extern char *_debug_protocols;
static int _debug_protocols_ok = 0;
struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs, pcap_t * pcap_handle) {
struct ndpi_workflow* ndpi_workflow_init(const struct ndpi_workflow_prefs * prefs,
pcap_t * pcap_handle) {
set_ndpi_malloc(malloc_wrapper), set_ndpi_free(free_wrapper);
set_ndpi_flow_malloc(NULL), set_ndpi_flow_free(NULL);
/* TODO: just needed here to init ndpi malloc wrapper */
struct ndpi_detection_module_struct * module = ndpi_init_detection_module();
struct ndpi_workflow * workflow = ndpi_calloc(1, sizeof(struct ndpi_workflow));
workflow->pcap_handle = pcap_handle;
workflow->prefs = *prefs;
workflow->prefs = *prefs;
workflow->ndpi_struct = module;
if(workflow->ndpi_struct == NULL) {
@@ -183,16 +184,19 @@ struct ndpi_workflow * ndpi_workflow_init(const struct ndpi_workflow_prefs * pre
module->ndpi_log_level = nDPI_LogLevel;
if(_debug_protocols != NULL && ! _debug_protocols_ok) {
if(parse_debug_proto(module,_debug_protocols))
exit(-1);
_debug_protocols_ok = 1;
if(parse_debug_proto(module,_debug_protocols))
exit(-1);
_debug_protocols_ok = 1;
}
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
NDPI_BITMASK_RESET(module->debug_bitmask);
if(_debug_protocols_ok)
module->debug_bitmask = debug_bitmask;
module->debug_bitmask = debug_bitmask;
#endif
workflow->ndpi_flows_root = ndpi_calloc(workflow->prefs.num_roots, sizeof(void *));
return workflow;
}
@@ -232,20 +236,20 @@ int ndpi_workflow_node_cmp(const void *a, const void *b) {
if(fa->protocol < fb->protocol ) return(-1); else { if(fa->protocol > fb->protocol ) return(1); }
if(
(
(
(fa->src_ip == fb->src_ip )
&& (fa->src_port == fb->src_port)
&& (fa->dst_ip == fb->dst_ip )
&& (fa->dst_port == fb->dst_port)
)
||
(
||
(
(fa->src_ip == fb->dst_ip )
&& (fa->src_port == fb->dst_port)
&& (fa->dst_ip == fb->src_ip )
&& (fa->dst_port == fb->src_port)
)
)
)
return(0);
if(fa->src_ip < fb->src_ip ) return(-1); else { if(fa->src_ip > fb->src_ip ) return(1); }
@@ -367,24 +371,24 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp);
/* to avoid two nodes in one binary tree for a flow */
int is_changed = 0;
if(ret == NULL)
{
u_int32_t orig_src_ip = flow.src_ip;
u_int16_t orig_src_port = flow.src_port;
u_int32_t orig_dst_ip = flow.dst_ip;
u_int16_t orig_dst_port = flow.dst_port;
/* to avoid two nodes in one binary tree for a flow */
int is_changed = 0;
if(ret == NULL)
{
u_int32_t orig_src_ip = flow.src_ip;
u_int16_t orig_src_port = flow.src_port;
u_int32_t orig_dst_ip = flow.dst_ip;
u_int16_t orig_dst_port = flow.dst_port;
flow.src_ip = orig_dst_ip;
flow.src_port = orig_dst_port;
flow.dst_ip = orig_src_ip;
flow.dst_port = orig_src_port;
flow.src_ip = orig_dst_ip;
flow.src_port = orig_dst_port;
flow.dst_ip = orig_src_ip;
flow.dst_port = orig_src_port;
is_changed = 1;
is_changed = 1;
ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp);
}
ret = ndpi_tfind(&flow, &workflow->ndpi_flows_root[idx], ndpi_workflow_node_cmp);
}
if(ret == NULL) {
if(workflow->stats.ndpi_flow_count == workflow->prefs.max_ndpi_flows) {
@@ -449,26 +453,26 @@ static struct ndpi_flow_info *get_ndpi_flow_info(struct ndpi_workflow * workflow
} else {
struct ndpi_flow_info *flow = *(struct ndpi_flow_info**)ret;
if (is_changed) {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
else
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
}
else {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
else
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
}
if (is_changed) {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
else
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
}
else {
if(flow->src_ip == iph->saddr
&& flow->dst_ip == iph->daddr
&& flow->src_port == htons(*sport)
&& flow->dst_port == htons(*dport)
)
*src = flow->src_id, *dst = flow->dst_id, *src_to_dst_direction = 1;
else
*src = flow->dst_id, *dst = flow->src_id, *src_to_dst_direction = 0, flow->bidirectional = 1;
}
return flow;
}
}
@@ -514,6 +518,8 @@ static struct ndpi_flow_info *get_ndpi_flow_info6(struct ndpi_workflow * workflo
void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_flow_info *flow) {
if(!flow->ndpi_flow) return;
/* printf("CATEGORY %u\n", flow->detected_protocol.category); */
snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s",
flow->ndpi_flow->host_server_name);
@@ -633,8 +639,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
}
if(ndpi_flow->num_extra_packets_checked < ndpi_flow->max_extra_packets_to_check) {
ndpi_process_extra_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
if (ndpi_flow->check_extra_packets == 0) {
flow->check_extra_packets = 0;
process_ndpi_collected_info(workflow, flow);
@@ -649,9 +655,10 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
return(flow->detected_protocol);
}
flow->detected_protocol = ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
flow->detected_protocol =
ndpi_detection_process_packet(workflow->ndpi_struct, ndpi_flow,
iph ? (uint8_t *)iph : (uint8_t *)iph6,
ipsize, time, src, dst);
if((flow->detected_protocol.app_protocol != NDPI_PROTOCOL_UNKNOWN)
|| ((proto == IPPROTO_UDP) && ((flow->src2dst_packets + flow->dst2src_packets) > 8))
@@ -663,8 +670,8 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
flow->check_extra_packets = 1;
if(flow->detected_protocol.app_protocol == NDPI_PROTOCOL_UNKNOWN)
flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct,
flow->ndpi_flow);
flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct,
flow->ndpi_flow);
process_ndpi_collected_info(workflow, flow);
}
@@ -739,7 +746,7 @@ struct ndpi_proto ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
/*** check Data Link type ***/
const int datalink_type = pcap_datalink(workflow->pcap_handle);
datalink_check:
datalink_check:
switch(datalink_type) {
case DLT_NULL:
if(ntohl(*((u_int32_t*)&packet[eth_offset])) == 2)
@@ -875,7 +882,7 @@ struct ndpi_proto ndpi_workflow_process_packet (struct ndpi_workflow * workflow,
workflow->stats.vlan_count += vlan_packet;
iph_check:
iph_check:
/* Check and set IP header size and total packet length */
iph = (struct ndpi_iphdr *) &packet[ip_offset];
+35 -18
View File
@@ -36,8 +36,8 @@ extern "C" {
*/
#define NDPI_API_VERSION 1
#define SIZEOF_ID_STRUCT (sizeof(struct ndpi_id_struct))
#define SIZEOF_FLOW_STRUCT (sizeof(struct ndpi_flow_struct))
#define SIZEOF_ID_STRUCT ( sizeof(struct ndpi_id_struct) )
#define SIZEOF_FLOW_STRUCT ( sizeof(struct ndpi_flow_struct) )
#define NDPI_DETECTION_ONLY_IPV4 ( 1 << 0 )
#define NDPI_DETECTION_ONLY_IPV6 ( 1 << 1 )
@@ -125,7 +125,8 @@ extern "C" {
* @return the nDPI protocol ID
*
*/
u_int16_t ndpi_network_ptree_match(struct ndpi_detection_module_struct *ndpi_struct, struct in_addr *pin);
u_int16_t ndpi_network_ptree_match(struct ndpi_detection_module_struct *ndpi_struct,
struct in_addr *pin);
/**
@@ -135,7 +136,8 @@ extern "C" {
* @par match = the struct passed to match the protocol
*
*/
void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol_match *match);
void ndpi_init_protocol_match(struct ndpi_detection_module_struct *ndpi_mod,
ndpi_protocol_match *match);
/**
* Returns a new initialized detection module
@@ -163,7 +165,8 @@ extern "C" {
* @par port = unsigned int for the port number
*
*/
void ndpi_enable_cache(struct ndpi_detection_module_struct *ndpi_mod, char* host, u_int port);
void ndpi_enable_cache(struct ndpi_detection_module_struct *ndpi_mod,
char* host, u_int port);
/**
@@ -193,7 +196,8 @@ extern "C" {
const NDPI_PROTOCOL_BITMASK *detection_bitmask,
const u_int32_t idx,
u_int16_t ndpi_protocol_id,
void (*func) (struct ndpi_detection_module_struct *, struct ndpi_flow_struct *flow),
void (*func) (struct ndpi_detection_module_struct *,
struct ndpi_flow_struct *flow),
const NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_bitmask,
u_int8_t b_save_bitmask_unknow,
u_int8_t b_add_detection_bitmask);
@@ -514,7 +518,8 @@ extern "C" {
* @par proto = the struct ndpi_protocol contain the protocols name
* @return the protocol category
*/
ndpi_protocol_category_t ndpi_get_proto_category(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol proto);
ndpi_protocol_category_t ndpi_get_proto_category(struct ndpi_detection_module_struct *ndpi_mod,
ndpi_protocol proto);
/**
* Get the protocol name associated to the ID
@@ -535,7 +540,8 @@ extern "C" {
* @return the breed ID associated to the protocol
*
*/
ndpi_protocol_breed_t ndpi_get_proto_breed(struct ndpi_detection_module_struct *ndpi_struct, u_int16_t proto);
ndpi_protocol_breed_t ndpi_get_proto_breed(struct ndpi_detection_module_struct *ndpi_struct,
u_int16_t proto);
/**
@@ -546,7 +552,8 @@ extern "C" {
* @return the string name of the breed ID
*
*/
char* ndpi_get_proto_breed_name(struct ndpi_detection_module_struct *ndpi_struct, ndpi_protocol_breed_t breed_id);
char* ndpi_get_proto_breed_name(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_protocol_breed_t breed_id);
/**
@@ -594,7 +601,8 @@ extern "C" {
* -1 else
*
*/
int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod, char* path);
int ndpi_load_protocols_file(struct ndpi_detection_module_struct *ndpi_mod,
char* path);
/**
@@ -623,7 +631,8 @@ extern "C" {
* @par automa = the automa to match
*
*/
void ndpi_set_automa(struct ndpi_detection_module_struct *ndpi_struct, void* automa);
void ndpi_set_automa(struct ndpi_detection_module_struct *ndpi_struct,
void* automa);
#ifdef NDPI_PROTOCOL_HTTP
@@ -635,7 +644,8 @@ extern "C" {
* @return the HTTP method information about the flow
*
*/
ndpi_http_method ndpi_get_http_method(struct ndpi_detection_module_struct *ndpi_mod, struct ndpi_flow_struct *flow);
ndpi_http_method ndpi_get_http_method(struct ndpi_detection_module_struct *ndpi_mod,
struct ndpi_flow_struct *flow);
/**
@@ -646,7 +656,8 @@ extern "C" {
* @return the HTTP method information about the flow
*
*/
char* ndpi_get_http_url(struct ndpi_detection_module_struct *ndpi_mod, struct ndpi_flow_struct *flow);
char* ndpi_get_http_url(struct ndpi_detection_module_struct *ndpi_mod,
struct ndpi_flow_struct *flow);
/**
@@ -657,7 +668,8 @@ extern "C" {
* @return the HTTP method information about the flow
*
*/
char* ndpi_get_http_content_type(struct ndpi_detection_module_struct *ndpi_mod, struct ndpi_flow_struct *flow);
char* ndpi_get_http_content_type(struct ndpi_detection_module_struct *ndpi_mod,
struct ndpi_flow_struct *flow);
#endif
@@ -736,8 +748,13 @@ extern "C" {
*
*/
int ndpi_match_string(void *_automa, char *string_to_match);
void ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_struct,
char *ip_address_and_mask, ndpi_protocol_category_t category);
void ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, ndpi_protocol_category_t category);
void ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct);
/**
* Add a string to match to an automata
*
@@ -749,13 +766,13 @@ extern "C" {
*/
int ndpi_match_string_id(void *_automa, char *string_to_match, unsigned long *id);
/* Utility functions to set ndpi malloc/free/print wrappers */
void set_ndpi_malloc(void* (*__ndpi_malloc)(size_t size));
void set_ndpi_free(void (*__ndpi_free)(void *ptr));
void set_ndpi_flow_malloc(void* (*__ndpi_flow_malloc)(size_t size));
void set_ndpi_flow_free(void (*__ndpi_flow_free)(void *ptr));
void set_ndpi_debug_function(struct ndpi_detection_module_struct *ndpi_str, ndpi_debug_function_ptr ndpi_debug_printf);
void set_ndpi_debug_function(struct ndpi_detection_module_struct *ndpi_str,
ndpi_debug_function_ptr ndpi_debug_printf);
void * ndpi_malloc(size_t size);
void * ndpi_calloc(unsigned long count, size_t size);
void ndpi_free(void *ptr);
+6
View File
@@ -820,6 +820,7 @@ typedef struct _ndpi_automa {
typedef struct ndpi_proto {
u_int16_t master_protocol /* e.g. HTTP */, app_protocol /* e.g. FaceBook */;
ndpi_protocol_category_t category;
} ndpi_protocol;
#define NDPI_PROTOCOL_NULL { NDPI_PROTOCOL_UNKNOWN , NDPI_PROTOCOL_UNKNOWN }
@@ -884,6 +885,11 @@ struct ndpi_detection_module_struct {
subprotocol_automa, /* Used for HTTP subprotocol_detection */
bigrams_automa, impossible_bigrams_automa; /* TOR */
struct {
ndpi_automa hostnames, hostnames_shadow;
void *ipAddresses, *ipAddresses_shadow; /* Patricia */
} custom_categories;
/* IP-based protocol detection */
void *protocols_ptree;
+1 -1
View File
@@ -1,7 +1,7 @@
/*
* ndpi_content_match.c
*
* Copyright (C) 2011-2017 - ntop.org
* Copyright (C) 2011-18 - ntop.org
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
+215 -33
View File
@@ -655,9 +655,7 @@ static int removeDefaultPort(ndpi_port_range *range,
static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_struct,
ndpi_automa *automa,
char *value, int protocol_id,
ndpi_protocol_breed_t breed)
{
char *value, int protocol_id) {
AC_PATTERN_t ac_pattern;
if(protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS+NDPI_MAX_NUM_CUSTOM_PROTOCOLS)) {
@@ -682,24 +680,22 @@ static int ndpi_string_to_automa(struct ndpi_detection_module_struct *ndpi_struc
static int ndpi_add_host_url_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
char *value, int protocol_id,
ndpi_protocol_breed_t breed)
ndpi_protocol_breed_t breed /* UNUSED */)
{
#ifdef DEBUG
NDPI_LOG_DEBUG2(ndpi_struct, "[NDPI] Adding [%s][%d]\n", value, protocol_id);
#endif
return(ndpi_string_to_automa(ndpi_struct, &ndpi_struct->host_automa,
value, protocol_id, breed));
return(ndpi_string_to_automa(ndpi_struct, &ndpi_struct->host_automa, value, protocol_id));
}
/* ****************************************************** */
int ndpi_add_content_subprotocol(struct ndpi_detection_module_struct *ndpi_struct,
char *value, int protocol_id,
ndpi_protocol_breed_t breed)
{
ndpi_protocol_breed_t breed /* UNUSED */) {
return(ndpi_string_to_automa(ndpi_struct, &ndpi_struct->content_automa,
value, protocol_id, breed));
value, protocol_id));
}
/* ****************************************************** */
@@ -840,12 +836,12 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
for(i=0; ndpi_en_bigrams[i] != NULL; i++)
ndpi_string_to_automa(ndpi_mod, &ndpi_mod->bigrams_automa,
(char*)ndpi_en_bigrams[i],
1, NDPI_PROTOCOL_UNRATED);
1);
for(i=0; ndpi_en_impossible_bigrams[i] != NULL; i++)
ndpi_string_to_automa(ndpi_mod, &ndpi_mod->impossible_bigrams_automa,
(char*)ndpi_en_impossible_bigrams[i],
1, NDPI_PROTOCOL_UNRATED);
1);
}
/* ******************************************************************** */
@@ -1859,7 +1855,8 @@ static int fill_prefix_v4(prefix_t *p, struct in_addr *a, int b, int mb) {
/* ******************************************* */
u_int16_t ndpi_network_ptree_match(struct ndpi_detection_module_struct *ndpi_struct, struct in_addr *pin /* network byte order */) {
u_int16_t ndpi_network_ptree_match(struct ndpi_detection_module_struct *ndpi_struct,
struct in_addr *pin /* network byte order */) {
prefix_t prefix;
patricia_node_t *node;
@@ -1943,19 +1940,18 @@ static int ndpi_add_host_ip_subprotocol(struct ndpi_detection_module_struct *ndp
int bits = 32;
char *ptr = strrchr(value, '/');
if (ptr)
{
ptr[0] = '\0';
ptr++;
if (atoi(ptr)>=0 && atoi(ptr)<=32)
bits = atoi(ptr);
}
if(ptr) {
ptr[0] = '\0';
ptr++;
if(atoi(ptr)>=0 && atoi(ptr)<=32)
bits = atoi(ptr);
}
inet_pton(AF_INET, value, &pin);
if((node = add_to_ptree(ndpi_struct->protocols_ptree, AF_INET, &pin, bits)) != NULL)
node->value.user_value = protocol_id;
return 0;
}
@@ -2047,11 +2043,20 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(void) {
ndpi_str->ndpi_num_supported_protocols = NDPI_MAX_SUPPORTED_PROTOCOLS;
ndpi_str->ndpi_num_custom_protocols = 0;
ndpi_str->host_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->content_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->bigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->host_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->content_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->bigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->impossible_bigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->custom_categories.hostnames.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->custom_categories.ipAddresses = ndpi_New_Patricia(32 /* IPv4 */);
ndpi_str->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */);
if((ndpi_str->custom_categories.ipAddresses == NULL)
|| (ndpi_str->custom_categories.ipAddresses_shadow == NULL))
return(NULL);
ndpi_init_protocol_defaults(ndpi_str);
for(i=0; i<NUM_CUSTOM_CATEGORIES; i++)
@@ -2169,6 +2174,18 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct
destroy_hyperscan(ndpi_struct);
#endif
if(ndpi_struct->custom_categories.hostnames.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t*)ndpi_struct->custom_categories.hostnames.ac_automa);
if(ndpi_struct->custom_categories.hostnames_shadow.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t*)ndpi_struct->custom_categories.hostnames_shadow.ac_automa);
if(ndpi_struct->custom_categories.ipAddresses != NULL)
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->custom_categories.ipAddresses, free_ptree_data);
if(ndpi_struct->custom_categories.ipAddresses_shadow != NULL)
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->custom_categories.ipAddresses_shadow, free_ptree_data);
ndpi_free(ndpi_struct);
}
}
@@ -3627,7 +3644,7 @@ static u_int16_t ndpi_guess_host_protocol_id(struct ndpi_detection_module_struct
ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow) {
ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN };
ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
if(flow == NULL) return(ret);
@@ -3724,17 +3741,125 @@ void ndpi_process_extra_packet(struct ndpi_detection_module_struct *ndpi_struct,
/* ********************************************************************************* */
void ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_struct,
char *ip_address_and_mask, ndpi_protocol_category_t category) {
patricia_node_t *node;
struct in_addr pin;
int bits = 32;
char *ptr = strrchr(ip_address_and_mask, '/');
if(ptr) {
ptr[0] = '\0';
ptr++;
if (atoi(ptr)>=0 && atoi(ptr)<=32)
bits = atoi(ptr);
}
inet_pton(AF_INET, ip_address_and_mask, &pin);
if((node = add_to_ptree(ndpi_struct->custom_categories.ipAddresses_shadow,
AF_INET, &pin, bits)) != NULL)
node->value.user_value = (int)category;
}
/* ********************************************************************************* */
void ndpi_load_hostname_category(struct ndpi_detection_module_struct *ndpi_struct,
char *name, ndpi_protocol_category_t category) {
AC_PATTERN_t ac_pattern;
if(name == NULL) return;
/* printf("===> Loading %s as %u\n", name, category); */
if(ndpi_struct->custom_categories.hostnames_shadow.ac_automa == NULL) return;
ac_pattern.astring = name, ac_pattern.length = strlen(ac_pattern.astring);
ac_pattern.rep.number = (int)category;
ac_automata_add(ndpi_struct->custom_categories.hostnames_shadow.ac_automa, &ac_pattern);
ac_automata_finalize(ndpi_struct->custom_categories.hostnames_shadow.ac_automa);
}
/* ********************************************************************************* */
void ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct) {
/* Free */
ac_automata_release((AC_AUTOMATA_t*)ndpi_struct->custom_categories.hostnames.ac_automa);
ndpi_Destroy_Patricia((patricia_tree_t*)ndpi_struct->custom_categories.ipAddresses, free_ptree_data);
/* Finalize */
ac_automata_finalize((AC_AUTOMATA_t*)ndpi_struct->custom_categories.hostnames_shadow.ac_automa);
/* Swap */
ndpi_struct->custom_categories.hostnames.ac_automa = ndpi_struct->custom_categories.hostnames_shadow.ac_automa;
ndpi_struct->custom_categories.ipAddresses = ndpi_struct->custom_categories.ipAddresses_shadow;
/* Realloc */
ndpi_struct->custom_categories.hostnames_shadow.ac_automa = ac_automata_init(ac_match_handler);
ndpi_struct->custom_categories.ipAddresses_shadow = ndpi_New_Patricia(32 /* IPv4 */);
}
/* ********************************************************************************* */
static void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
ndpi_protocol *ret) {
if(flow->packet.iph) {
prefix_t prefix;
patricia_node_t *node;
/* Make sure all in network byte order otherwise compares wont work */
fill_prefix_v4(&prefix, (struct in_addr *)&flow->packet.iph->saddr,
32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
if(!node) {
fill_prefix_v4(&prefix, (struct in_addr *)&flow->packet.iph->daddr,
32, ((patricia_tree_t*)ndpi_struct->protocols_ptree)->maxbits);
node = ndpi_patricia_search_best(ndpi_struct->custom_categories.ipAddresses, &prefix);
}
if(node) {
ret->category = (ndpi_protocol_category_t)node->value.user_value;
return;
}
}
if(flow->host_server_name[0] != '\0') {
unsigned long id;
int rc = ndpi_match_string_id(ndpi_struct->custom_categories.hostnames.ac_automa, (char *)flow->host_server_name, &id);
if(rc == 0) {
ret->category = (ndpi_protocol_category_t)id;
return;
}
}
if(flow->protos.ssl.server_certificate[0] != '\0') {
unsigned long id;
int rc = ndpi_match_string_id(ndpi_struct->custom_categories.hostnames.ac_automa, (char *)flow->protos.ssl.server_certificate, &id);
if(rc == 0) {
ret->category = (ndpi_protocol_category_t)id;
return;
}
}
ret->category = ndpi_get_proto_category(ndpi_struct, *ret);
}
/* ********************************************************************************* */
ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
const unsigned char *packet,
const unsigned short packetlen,
const u_int64_t current_tick_l,
struct ndpi_id_struct *src,
struct ndpi_id_struct *dst)
{
struct ndpi_id_struct *dst) {
NDPI_SELECTION_BITMASK_PROTOCOL_SIZE ndpi_selection_packet;
u_int32_t a;
ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN };
ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
if(ndpi_struct->ndpi_log_level >= NDPI_LOG_TRACE)
NDPI_LOG(flow ? flow->detected_protocol_stack[0]:NDPI_PROTOCOL_UNKNOWN,
@@ -3825,6 +3950,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
if(flow->guessed_protocol_id >= (NDPI_MAX_SUPPORTED_PROTOCOLS-1)) {
/* This is a custom protocol and it has priority over everything else */
ret.master_protocol = NDPI_PROTOCOL_UNKNOWN, ret.app_protocol = flow->guessed_host_protocol_id;
ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
return(ret);
}
@@ -3835,6 +3961,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
ret = ndpi_detection_giveup(ndpi_struct, flow);
}
ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
return(ret);
}
} else {
@@ -3851,6 +3978,7 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
/* This is a custom protocol and it has priority over everything else */
ret.master_protocol = NDPI_PROTOCOL_UNKNOWN, ret.app_protocol = flow->guessed_host_protocol_id;
ndpi_check_flow_func(ndpi_struct, flow, &ndpi_selection_packet);
ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
return(ret);
}
@@ -3878,9 +4006,12 @@ ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct
} else
ret.app_protocol = flow->detected_protocol_stack[0];
ndpi_fill_protocol_category(ndpi_struct, flow, &ret);
return(ret);
}
/* ********************************************************************************* */
u_int32_t ndpi_bytestream_to_number(const u_int8_t * str, u_int16_t max_chars_to_read, u_int16_t * bytes_read)
{
u_int32_t val;
@@ -3896,6 +4027,8 @@ u_int32_t ndpi_bytestream_to_number(const u_int8_t * str, u_int16_t max_chars_to
return (val);
}
/* ********************************************************************************* */
u_int32_t ndpi_bytestream_dec_or_hex_to_number(const u_int8_t * str, u_int16_t max_chars_to_read, u_int16_t * bytes_read)
{
u_int32_t val;
@@ -3929,6 +4062,7 @@ u_int32_t ndpi_bytestream_dec_or_hex_to_number(const u_int8_t * str, u_int16_t m
return (val);
}
/* ********************************************************************************* */
u_int64_t ndpi_bytestream_to_number64(const u_int8_t * str, u_int16_t max_chars_to_read, u_int16_t * bytes_read)
{
@@ -3945,6 +4079,8 @@ u_int64_t ndpi_bytestream_to_number64(const u_int8_t * str, u_int16_t max_chars_
return (val);
}
/* ********************************************************************************* */
u_int64_t ndpi_bytestream_dec_or_hex_to_number64(const u_int8_t * str, u_int16_t max_chars_to_read, u_int16_t * bytes_read)
{
u_int64_t val;
@@ -3978,6 +4114,7 @@ u_int64_t ndpi_bytestream_dec_or_hex_to_number64(const u_int8_t * str, u_int16_t
return (val);
}
/* ********************************************************************************* */
u_int32_t ndpi_bytestream_to_ipv4(const u_int8_t * str, u_int16_t max_chars_to_read, u_int16_t * bytes_read)
{
@@ -4015,6 +4152,8 @@ u_int32_t ndpi_bytestream_to_ipv4(const u_int8_t * str, u_int16_t max_chars_to_r
return htonl(val);
}
/* ********************************************************************************* */
/* internal function for every detection to parse one packet and to increase the info buffer */
void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow)
@@ -4266,6 +4405,8 @@ void ndpi_parse_packet_line_info(struct ndpi_detection_module_struct *ndpi_struc
}
}
/* ********************************************************************************* */
void ndpi_parse_packet_line_info_any(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow)
{
@@ -4309,6 +4450,7 @@ void ndpi_parse_packet_line_info_any(struct ndpi_detection_module_struct *ndpi_s
}
}
/* ********************************************************************************* */
u_int16_t ndpi_check_for_email_address(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow, u_int16_t counter)
@@ -4391,6 +4533,8 @@ u_int16_t ndpi_check_for_email_address(struct ndpi_detection_module_struct *ndpi
}
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
/* ********************************************************************************* */
void ndpi_debug_get_last_log_function_line(struct ndpi_detection_module_struct
*ndpi_struct, const char **file, const char **func, u_int32_t * line)
{
@@ -4406,12 +4550,17 @@ void ndpi_debug_get_last_log_function_line(struct ndpi_detection_module_struct
*line = ndpi_struct->ndpi_debug_print_line;
}
#endif
/* ********************************************************************************* */
u_int8_t ndpi_detection_get_l4(const u_int8_t * l3, u_int16_t l3_len, const u_int8_t ** l4_return, u_int16_t * l4_len_return,
u_int8_t * l4_protocol_return, u_int32_t flags)
{
return ndpi_detection_get_l4_internal(NULL, l3, l3_len, l4_return, l4_len_return, l4_protocol_return, flags);
}
/* ********************************************************************************* */
void ndpi_set_detected_protocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int16_t upper_detected_protocol,
@@ -4437,11 +4586,15 @@ void ndpi_set_detected_protocol(struct ndpi_detection_module_struct *ndpi_struct
}
}
/* ********************************************************************************* */
u_int16_t ndpi_get_flow_masterprotocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow) {
return(flow->detected_protocol_stack[1]);
}
/* ********************************************************************************* */
void ndpi_int_change_flow_protocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int16_t upper_detected_protocol,
@@ -4451,6 +4604,8 @@ void ndpi_int_change_flow_protocol(struct ndpi_detection_module_struct *ndpi_str
flow->detected_protocol_stack[0] = upper_detected_protocol, flow->detected_protocol_stack[1] = lower_detected_protocol;
}
/* ********************************************************************************* */
void ndpi_int_change_packet_protocol(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int16_t upper_detected_protocol,
@@ -4464,9 +4619,12 @@ void ndpi_int_change_packet_protocol(struct ndpi_detection_module_struct *ndpi_s
if(!packet)
return;
packet->detected_protocol_stack[0] = upper_detected_protocol, packet->detected_protocol_stack[1] = lower_detected_protocol;
packet->detected_protocol_stack[0] = upper_detected_protocol,
packet->detected_protocol_stack[1] = lower_detected_protocol;
}
/* ********************************************************************************* */
/* generic function for changing the protocol
*
* what it does is:
@@ -4490,6 +4648,8 @@ void ndpi_int_change_protocol(struct ndpi_detection_module_struct *ndpi_struct,
upper_detected_protocol, lower_detected_protocol);
}
/* ********************************************************************************* */
/* change protocol only if guessing is active */
/* void ndpi_guess_change_protocol(struct ndpi_detection_module_struct *ndpi_struct, */
/* struct ndpi_flow_struct *flow) */
@@ -4509,6 +4669,8 @@ void ndpi_int_change_protocol(struct ndpi_detection_module_struct *ndpi_struct,
/* } */
/* } */
/* ********************************************************************************* */
/* turns a packet back to unknown */
void ndpi_int_reset_packet_protocol(struct ndpi_packet_struct *packet) {
int a;
@@ -4517,6 +4679,8 @@ void ndpi_int_reset_packet_protocol(struct ndpi_packet_struct *packet) {
packet->detected_protocol_stack[a] = NDPI_PROTOCOL_UNKNOWN;
}
/* ********************************************************************************* */
void ndpi_int_reset_protocol(struct ndpi_flow_struct *flow) {
if(flow) {
int a;
@@ -4527,16 +4691,22 @@ void ndpi_int_reset_protocol(struct ndpi_flow_struct *flow) {
}
}
/* ********************************************************************************* */
void NDPI_PROTOCOL_IP_clear(ndpi_ip_addr_t * ip) {
memset(ip, 0, sizeof(ndpi_ip_addr_t));
}
/* ********************************************************************************* */
/* NTOP */
int NDPI_PROTOCOL_IP_is_set(const ndpi_ip_addr_t * ip)
{
return memcmp(ip, "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", sizeof(ndpi_ip_addr_t)) != 0;
}
/* ********************************************************************************* */
/* check if the source ip address in packet and ip are equal */
/* NTOP */
int ndpi_packet_src_ip_eql(const struct ndpi_packet_struct *packet, const ndpi_ip_addr_t * ip)
@@ -4563,6 +4733,8 @@ int ndpi_packet_src_ip_eql(const struct ndpi_packet_struct *packet, const ndpi_i
return 0;
}
/* ********************************************************************************* */
/* check if the destination ip address in packet and ip are equal */
int ndpi_packet_dst_ip_eql(const struct ndpi_packet_struct *packet, const ndpi_ip_addr_t * ip)
{
@@ -4588,6 +4760,8 @@ int ndpi_packet_dst_ip_eql(const struct ndpi_packet_struct *packet, const ndpi_i
return 0;
}
/* ********************************************************************************* */
/* get the source ip address from packet and put it into ip */
/* NTOP */
void ndpi_packet_src_ip_get(const struct ndpi_packet_struct *packet, ndpi_ip_addr_t * ip)
@@ -4611,6 +4785,8 @@ void ndpi_packet_src_ip_get(const struct ndpi_packet_struct *packet, ndpi_ip_add
ip->ipv4 = packet->iph->saddr;
}
/* ********************************************************************************* */
/* get the destination ip address from packet and put it into ip */
/* NTOP */
void ndpi_packet_dst_ip_get(const struct ndpi_packet_struct *packet, ndpi_ip_addr_t * ip)
@@ -4633,6 +4809,8 @@ void ndpi_packet_dst_ip_get(const struct ndpi_packet_struct *packet, ndpi_ip_add
ip->ipv4 = packet->iph->daddr;
}
/* ********************************************************************************* */
#ifdef NDPI_ENABLE_DEBUG_MESSAGES
/* get the string representation of ip
* returns a pointer to a static string
@@ -4663,6 +4841,7 @@ char *ndpi_get_ip_string(struct ndpi_detection_module_struct *ndpi_struct,
}
/* ********************************************************************************* */
/* get the string representation of the source ip address from packet */
char *ndpi_get_packet_src_ip_string(struct ndpi_detection_module_struct *ndpi_struct,
@@ -4673,6 +4852,8 @@ char *ndpi_get_packet_src_ip_string(struct ndpi_detection_module_struct *ndpi_st
return ndpi_get_ip_string(ndpi_struct, &ip);
}
/* ********************************************************************************* */
/* get the string representation of the destination ip address from packet */
char *ndpi_get_packet_dst_ip_string(struct ndpi_detection_module_struct *ndpi_struct,
const struct ndpi_packet_struct *packet)
@@ -4681,11 +4862,12 @@ char *ndpi_get_packet_dst_ip_string(struct ndpi_detection_module_struct *ndpi_st
ndpi_packet_dst_ip_get(packet, &ip);
return ndpi_get_ip_string(ndpi_struct, &ip);
}
#endif /* NDPI_ENABLE_DEBUG_MESSAGES */
#endif /* NDPI_ENABLE_DEBUG_MESSAGES */
/* ****************************************************** */
u_int16_t ntohs_ndpi_bytestream_to_number(const u_int8_t * str, u_int16_t max_chars_to_read, u_int16_t * bytes_read)
u_int16_t ntohs_ndpi_bytestream_to_number(const u_int8_t * str,
u_int16_t max_chars_to_read, u_int16_t * bytes_read)
{
u_int16_t val = ndpi_bytestream_to_number(str, max_chars_to_read, bytes_read);
return ntohs(val);
@@ -4711,7 +4893,7 @@ ndpi_protocol ndpi_guess_undetected_protocol(struct ndpi_detection_module_struct
u_int32_t dhost /* host byte order */, u_int16_t dport) {
u_int32_t rc;
struct in_addr addr;
ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN };
ndpi_protocol ret = { NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED };
u_int8_t user_defined_proto;
if((proto == IPPROTO_TCP) || (proto == IPPROTO_UDP)) {