Added NDPI_MALICIOUS_SHA1 flow risk. (#1142)

* An external file which contains known malicious SSL certificate SHA-1 hashes
   can be loaded via ndpi_load_malicious_sha1_file(...)

Signed-off-by: Toni Uhlig <matzeton@googlemail.com>
This commit is contained in:
Toni
2021-02-26 17:00:05 +01:00
committed by GitHub
parent fba61adf5e
commit 16890a6632
8 changed files with 3894 additions and 14 deletions
+11 -2
View File
@@ -68,6 +68,7 @@ static char * bpfFilter = NULL; /**< bpf filter */
static char *_protoFilePath = NULL; /**< Protocol file path */
static char *_customCategoryFilePath= NULL; /**< Custom categories file path */
static char *_maliciousJA3Path = NULL; /**< Malicious JA3 signatures */
static char *_maliciousSHA1Path = NULL; /**< Malicious SSL certificate SHA1 fingerprints */
static char *_riskyDomainFilePath = NULL; /**< Risky domain files */
static u_int8_t live_capture = 0;
static u_int8_t undetected_flows_deleted = 0;
@@ -439,7 +440,7 @@ static void help(u_int long_help) {
"[-f <filter>][-s <duration>][-m <duration>][-b <num bin clusters>]\n"
" [-p <protos>][-l <loops> [-q][-d][-J][-h][-D][-e <len>][-t][-v <level>]\n"
" [-n <threads>][-w <file>][-c <file>][-C <file>][-j <file>][-x <file>]\n"
" [-r <file>][-j <file>][-T <num>][-U <num>] [-x <domain>]\n\n"
" [-r <file>][-j <file>][-S <file>][-T <num>][-U <num>] [-x <domain>]\n\n"
"Usage:\n"
" -i <file.pcap|device> | Specify a pcap file/playlist to read packets from or a\n"
" | device for live capture (comma-separated list)\n"
@@ -471,6 +472,7 @@ static void help(u_int long_help) {
" -C <path> | Write output in CSV format on the specified file\n"
" -r <path> | Load risky domain file\n"
" -j <path> | Load malicious JA3 fingeprints\n"
" -S <path> | Load malicious SSL certificate SHA1 fingerprints\n"
" -w <path> | Write test output on the specified file. This is useful for\n"
" | testing purposes in order to compare results across runs\n"
" -h | This help\n"
@@ -765,7 +767,7 @@ static void parseOptions(int argc, char **argv) {
}
#endif
while((opt = getopt_long(argc, argv, "b:e:c:C:dDf:g:i:Ij:hp:P:l:r:s:tu:v:V:n:Jrp:x:w:q0123:456:7:89:m:T:U:",
while((opt = getopt_long(argc, argv, "b:e:c:C:dDf:g:i:Ij:S:hp:P:l:r:s:tu:v:V:n:Jrp:x:w:q0123:456:7:89:m:T:U:",
longopts, &option_idx)) != EOF) {
#ifdef DEBUG_TRACE
if(trace) fprintf(trace, " #### -%c [%s] #### \n", opt, optarg ? optarg : "");
@@ -802,6 +804,10 @@ static void parseOptions(int argc, char **argv) {
_maliciousJA3Path = optarg;
break;
case 'S':
_maliciousSHA1Path = optarg;
break;
case 'm':
pcap_analysis_duration = atol(optarg);
break;
@@ -2069,6 +2075,9 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
if(_maliciousJA3Path)
ndpi_load_malicious_ja3_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _maliciousJA3Path);
if(_maliciousSHA1Path)
ndpi_load_malicious_sha1_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _maliciousSHA1Path);
ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct);
if(enable_doh_dot_detection)
File diff suppressed because it is too large Load Diff
+9
View File
@@ -715,6 +715,15 @@ extern "C" {
*/
int ndpi_load_malicious_ja3_file(struct ndpi_detection_module_struct *ndpi_str, const char *path);
/**
* Read a file and load the list of malicious SSL certificate SHA1 fingerprints.
* @par ndpi_mod = the detection module
* @par path = the path of the file
* @return 0 if the file is loaded correctly;
* -1 else
*/
int ndpi_load_malicious_sha1_file(struct ndpi_detection_module_struct *ndpi_str, const char *path);
/**
* Get the total number of the supported protocols
*
+2 -1
View File
@@ -92,6 +92,7 @@ typedef enum {
NDPI_RISKY_ASN,
NDPI_RISKY_DOMAIN,
NDPI_MALICIOUS_JA3,
NDPI_MALICIOUS_SHA1,
/* Leave this as last member */
@@ -1101,7 +1102,7 @@ struct ndpi_detection_module_struct {
subprotocol_automa, /* Used for HTTP subprotocol_detection */
bigrams_automa, impossible_bigrams_automa, /* TOR */
risky_domain_automa, tls_cert_subject_automa,
malicious_ja3_automa;
malicious_ja3_automa, malicious_sha1_automa;
/* IMPORTANT: please update ndpi_finalize_initialization() whenever you add a new automa */
struct {
+69 -2
View File
@@ -2193,6 +2193,7 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs
ndpi_str->impossible_bigrams_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->tls_cert_subject_automa.ac_automa = ac_automata_init(ac_match_handler);
ndpi_str->malicious_ja3_automa.ac_automa = NULL; /* Initialized on demand */
ndpi_str->malicious_sha1_automa.ac_automa = NULL; /* Initialized on demand */
ndpi_str->risky_domain_automa.ac_automa = NULL; /* Initialized on demand */
if((sizeof(categories) / sizeof(char *)) != NDPI_PROTOCOL_NUM_CATEGORIES) {
@@ -2253,7 +2254,11 @@ void ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str)
case 5:
automa = &ndpi_str->malicious_ja3_automa;
break;
case 6:
automa = &ndpi_str->malicious_sha1_automa;
break;
default:
return;
}
@@ -2519,6 +2524,9 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) {
if(ndpi_str->malicious_ja3_automa.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->malicious_ja3_automa.ac_automa, 0);
if(ndpi_str->malicious_sha1_automa.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->malicious_sha1_automa.ac_automa, 0);
if(ndpi_str->custom_categories.hostnames.ac_automa != NULL)
ac_automata_release((AC_AUTOMATA_t *) ndpi_str->custom_categories.hostnames.ac_automa,
1 /* free patterns strings memory */);
@@ -2968,7 +2976,7 @@ int ndpi_load_risk_domain_file(struct ndpi_detection_module_struct *ndpi_str, co
/*
* Format:
*
* <domain name>[,<other info>]
* <ja3 hash>[,<other info>]
*
*/
int ndpi_load_malicious_ja3_file(struct ndpi_detection_module_struct *ndpi_str, const char *path) {
@@ -3015,6 +3023,65 @@ int ndpi_load_malicious_ja3_file(struct ndpi_detection_module_struct *ndpi_str,
/* ******************************************************************** */
/*
* Format:
*
* <sha1 hash>
* <other info>,<sha1 hash>
* <other info>,<sha1 hash>[,<other info>[...]]
*
*/
int ndpi_load_malicious_sha1_file(struct ndpi_detection_module_struct *ndpi_str, const char *path)
{
char buffer[128];
char *first_comma, *second_comma;
FILE *fd;
size_t len;
int num = 0;
if (ndpi_str->malicious_sha1_automa.ac_automa == NULL)
ndpi_str->malicious_sha1_automa.ac_automa = ac_automata_init(ac_match_handler);
fd = fopen(path, "r");
if (fd == NULL) {
NDPI_LOG_ERR(ndpi_str, "Unable to open file %s [%s]\n", path, strerror(errno));
return -1;
}
while (fgets(buffer, sizeof(buffer), fd) != NULL) {
len = strlen(buffer);
if (len <= 1 || buffer[0] == '#')
continue;
first_comma = strchr(buffer, ',');
if (first_comma != NULL) {
first_comma++;
second_comma = strchr(first_comma, ',');
if (second_comma == NULL)
second_comma = &buffer[len - 1];
} else {
first_comma = &buffer[0];
second_comma = &buffer[len - 1];
}
if ((second_comma - first_comma) != 40)
continue;
second_comma[0] = '\0';
for (size_t i = 0; i < 40; ++i)
first_comma[i] = toupper(first_comma[i]);
if (ndpi_add_string_to_automa(ndpi_str->malicious_sha1_automa.ac_automa, first_comma) >= 0)
num++;
}
return num;
}
/* ******************************************************************** */
/*
Format:
<tcp|udp>:<port>,<tcp|udp>:<port>,.....@<proto>
+3
View File
@@ -1766,6 +1766,9 @@ const char* ndpi_risk2str(ndpi_risk_enum risk) {
case NDPI_MALICIOUS_JA3:
return("Possibly Malicious JA3 Fingerprint");
case NDPI_MALICIOUS_SHA1:
return("Possibly Malicious SSL Certificate SHA1 Fingerprint");
default:
snprintf(buf, sizeof(buf), "%d", (int)risk);
return(buf);
+20 -8
View File
@@ -621,17 +621,29 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct,
flow->l4.tcp.tls.fingerprint_set = 1;
#ifdef DEBUG_TLS
{
int i;
printf("[TLS] SHA-1: ");
for(i=0;i<20;i++)
printf("%s%02X", (i > 0) ? ":" : "", flow->protos.tls_quic_stun.tls_quic.sha1_certificate_fingerprint[i]);
printf("\n");
uint8_t * sha1 = flow->protos.tls_quic_stun.tls_quic.sha1_certificate_fingerprint;
const size_t sha1_siz = sizeof(flow->protos.tls_quic_stun.tls_quic.sha1_certificate_fingerprint);
char sha1_str[sha1_siz * 2 + 1];
static const char hexalnum[] = "0123456789ABCDEF";
for (size_t i = 0; i < sha1_siz; ++i) {
u_int8_t lower = (sha1[i] & 0x0F);
u_int8_t upper = (sha1[i] & 0xF0) >> 4;
sha1_str[i*2] = hexalnum[upper];
sha1_str[i*2 + 1] = hexalnum[lower];
}
sha1_str[sha1_siz * 2] = '\0';
#ifdef DEBUG_TLS
printf("[TLS] SHA-1: %s\n", sha1_str);
#endif
if (ndpi_struct->malicious_sha1_automa.ac_automa != NULL) {
u_int16_t rc1 = ndpi_match_string(ndpi_struct->malicious_sha1_automa.ac_automa, sha1_str);
if(rc1 > 0)
NDPI_SET_BIT(flow->risk, NDPI_MALICIOUS_SHA1);
}
processCertificateElements(ndpi_struct, flow, certificates_offset, certificate_len);
}
+1 -1
View File
@@ -6,7 +6,7 @@ FUZZY_TESTING_ENABLED=@BUILD_FUZZTARGETS@
GCRYPT_ENABLED=@GCRYPT_ENABLED@
GCRYPT_PCAPS="gquic.pcap quic-23.pcap quic-24.pcap quic-27.pcap quic-28.pcap quic-29.pcap quic-mvfst-22.pcap quic-mvfst-27.pcap quic-mvfst-exp.pcap quic_q50.pcap quic_t50.pcap quic_t51.pcap quic_0RTT.pcap quic_interop_V.pcapng quic-33.pcapng doq.pcapng doq_adguard.pcapng dlt_ppp.pcap os_detected.pcapng"
READER="../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt -r ../example/risky_domains.txt -j ../example/ja3_fingerprints.csv"
READER="../example/ndpiReader -p ../example/protos.txt -c ../example/categories.txt -r ../example/risky_domains.txt -j ../example/ja3_fingerprints.csv -S ../example/sha1_fingerprints.csv"
RC=0
PCAPS=`cd pcap; /bin/ls *.pcap *.pcapng`