Implement DGA detection performances tracking workflow. (#1064)

* Implement dga evaluation helper.

* Add test set for DGA classification.

* Add DGA classification performances tracking as part of Travis.

* Add DGA evaluation doc.

* Fix CI on OSX.

* Add missing backquote.
This commit is contained in:
Zied Aouini
2020-11-16 21:17:16 +01:00
committed by GitHub
parent 76bb83085b
commit 3d8fd42307
8 changed files with 67745 additions and 2 deletions
+1
View File
@@ -205,6 +205,7 @@ script:
- cd tests
- ./do.sh
- ./do-unit.sh
- ./do-dga.sh
#after_success:
#- cd ${TRAVIS_BUILD_DIR}
+2 -2
View File
@@ -7,7 +7,7 @@ AM_INIT_AUTOMAKE([foreign subdir-objects])
AC_PREFIX_DEFAULT(/usr)
EXTRA_TARGETS="example tests tests/unit"
EXTRA_TARGETS="example tests tests/unit tests/dga"
AC_ARG_WITH(only-libndpi, AS_HELP_STRING([--with-only-libndpi], [Build only libndpi (no examples, tests etc)]))
AS_IF([test "${with_only_libndpi+set}" = set],[
EXTRA_TARGETS=""
@@ -201,7 +201,7 @@ if test "${with_pcre+set}" = set; then :
fi
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/dga/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
AC_CONFIG_HEADERS(src/include/ndpi_config.h)
AC_SUBST(GIT_RELEASE)
AC_SUBST(NDPI_MAJOR)
+36
View File
@@ -0,0 +1,36 @@
CC=@CC@
CXX=@CXX@
SRCHOME=../../src
CFLAGS=-g -fPIC -DPIC -I$(SRCHOME)/include @JSONC_CFLAGS@ @PCAP_INC@ @CFLAGS@
LIBNDPI=$(SRCHOME)/lib/libndpi.a
LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @ADDITIONAL_LIBS@ @JSONC_LIBS@ -lpthread -lm @LDFLAGS@
HEADERS=$(SRCHOME)/include/ndpi_api.h $(SRCHOME)/include/ndpi_typedefs.h $(SRCHOME)/include/ndpi_protocol_ids.h
OBJS=dga_evaluate
PREFIX?=@prefix@
all: dga_evaluate
EXECUTABLE_SOURCES := dga_evaluate.c
COMMON_SOURCES := $(filter-out $(EXECUTABLE_SOURCES),$(wildcard *.c ))
dga_evaluate: $(LIBNDPI) dga_evaluate.o
$(CC) $(CFLAGS) dga_evaluate.o -o $@ $(LDFLAGS)
%.o: %.c $(HEADERS) Makefile
$(CC) $(CFLAGS) -c $< -o $@
clean:
/bin/rm -f *.o dga_evaluate
/bin/rm -f .*.o.cmd .*.o.d
/bin/rm -rf build
install:
echo ""
distdir:
distclean: clean
/bin/rm -f Makefile
+70
View File
@@ -0,0 +1,70 @@
# DGA detection testing workflow
## Overview
nDPI provides a set of threat detection features available through NDPI_RISK detection.
As part of these features, we provide DGA detection.
Domain generation algorithms (DGA) are algorithms seen in various families of malware that are used
to periodically generate a large number of domain names that can be used as rendezvous points with
their command and control servers.
DGA detection heuristic is implemented [**here**](https://github.com/ntop/nDPI/blob/328ff2465709372c595cb25d99135aa515da3c5a/src/lib/ndpi_main.c#L6729).
DGA performances test and tracking allows us to detect automatically if a modification is harmful.
The modification can be a simple threshold change or a future lightweight ML approach.
## Used data
Original used dataset is a collection of legit and DGA domains (balanced) that can be obtained as follow:
```shell
wget https://raw.githubusercontent.com/chrmor/DGA_domains_dataset/master/dga_domains_full.csv
```
We split the dataset into DGA and NON-DGA and we keep 10% of each as test set and 90% as training set.
```shell
python3 -m pip install pandas
python3 -m pip install sklearn
```
Instruction using python3
```python3
from sklearn.model_selection import train_test_split
import pandas as pd
df = pd.read_csv("dga_domains_full.csv", header=None, names=["type", "family", "domain"])
df_dga = df[df.type=="dga"]
df_non_dga = df[df.type=="legit"]
train_non_dga, test_non_dga = train_test_split(df_non_dga, test_size=0.1, shuffle=True, random_state=27)
train_dga, test_dga = train_test_split(df_dga, test_size=0.1, shuffle=True, random_state=27)
test_dga["domain"].to_csv("test_dga.csv", header=False, index=False)
test_non_dga["domain"].to_csv("test_non_dga.csv", header=False, index=False)
train_dga["domain"].to_csv("train_dga.csv", header=False, index=False)
test_non_dga["domain"].to_csv("test_non_dga.csv", header=False, index=False)
```
**Detection approach must be built on top of training set only, test set must be kept as unseen cases for testing**
## dga_evaluate
After nDPI compilation, you can use dga_evaluate helper to check number of detections out of an input file.
```shell
dga_evaluate <file name>
```
You can evaluate your modifications performances before submitting it as follows:
```shell
./do-dga.sh
```
If your modifications decreases baseline performances, test will fails.
If not (well done), test passed and you must update the baseline metrics with your obtained ones.
+81
View File
@@ -0,0 +1,81 @@
/*
* dga.c
*
* Copyright (C) 2019-20 - ntop.org
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include <stdint.h>
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <unistd.h>
#include <ndpi_api.h>
#include <ndpi_main.h>
#include <ndpi_typedefs.h>
#include "ndpi_config.h"
void help() {
printf("dga_evaluate <file name>\n");
exit(0);
}
/* *********************************************** */
int main(int argc, char **argv) {
FILE *fd;
char buffer[512];
if(argc != 2) help();
fd = fopen(argv[1], "r");
if(fd == NULL) {
printf("Unable to open file %s\n", argv[1]);
exit(0);
}
if (ndpi_get_api_version() != NDPI_API_VERSION) {
printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
return -1;
}
/* Initialize nDPI detection module*/
NDPI_PROTOCOL_BITMASK all;
struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(ndpi_no_prefs);
assert(ndpi_str != NULL);
NDPI_BITMASK_SET_ALL(all);
ndpi_set_protocol_detection_bitmask2(ndpi_str, &all);
ndpi_finalize_initalization(ndpi_str);
assert(ndpi_str != NULL);
int num_detections = 0;
while(fgets(buffer, sizeof(buffer), fd) != NULL) {
char *hostname;
hostname = strtok(buffer, "\n");
if (ndpi_check_dga_name(ndpi_str, NULL, hostname, 1)) num_detections++;
}
fclose(fd);
ndpi_exit_detection_module(ndpi_str);
printf("%i", num_detections);
return 0;
}
+33750
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+65
View File
@@ -0,0 +1,65 @@
#!/bin/sh
cd "$(dirname "${0}")"
# Baseline performances ------------------------------------------------------------------------------------------------
# Important notes: BASE values must be integers examples and represents percentage (e.g. 79%, 98%).
BASE_ACCURACY=66
BASE_PRECISION=86
BASE_RECALL=38
# ----------------------------------------------------------------------------------------------------------------------
DGA_EVALUATE="./dga/dga_evaluate"
DGA_DATA="dga/test_dga.csv"
NON_DGA_DATA="dga/test_non_dga.csv"
DGA_DATA_SIZE=0
NON_DGA_DATA_SIZE=0
DATA_SIZE=0
RC=0
get_evaluation_data_size() {
DGA_DATA_SIZE=`wc -l dga/test_dga.csv | awk '{split($0,a," "); print a[1]}'`
NON_DGA_DATA_SIZE=`wc -l dga/test_non_dga.csv | awk '{split($0,a," "); print a[1]}'`
DATA_SIZE=$(( $NON_DGA_DATA_SIZE + $DGA_DATA_SIZE ))
}
evaluate_ndpi_dga_detection() {
# DGA detection is a binary classification problem, We evaluate the following metrics:
# Accuracy: (TP + TN) / (TP + TN + FN + FP)
# Precision: TP / (TP + FP)
# Recall: TP / (TP + FN)
TP=`$DGA_EVALUATE dga/test_dga.csv`
FN=$(( $DGA_DATA_SIZE - $TP ))
FP=`$DGA_EVALUATE dga/test_non_dga.csv`
TN=$(( $NON_DGA_DATA_SIZE - $FP ))
ACCURACY=`echo "print(int(((${TP} + ${TN})/(${TP} + ${TN} + ${FP} + ${FN}))*100))" | python3`
PRECISION=`echo "print(int(((${TP})/(${TP} + ${FP}))*100))" | python3`
RECALL=`echo "print(int(((${TP})/(${TP} + ${FN}))*100))" | python3`
# In case modified version of classification algorithm decreases performances, test do not pass.
if [ $ACCURACY -lt $BASE_ACCURACY ]; then
printf "ERROR: Your modifications decreased DGA classifier accuracy: 0.${BASE_ACCURACY} decreased to 0.${ACCURACY}!\n"
RC=1
fi
if [ $PRECISION -lt $BASE_PRECISION ]; then
printf "ERROR: Your modifications decreased DGA classifier precision: 0.${BASE_PRECISION} decreased to 0.${PRECISION}!\n"
RC=1
fi
if [ $RECALL -lt $BASE_RECALL ]; then
printf "ERROR: Your modifications decreased DGA classifier recall: 0.${BASE_RECALL} decreased to 0.${RECALL}!\n"
RC=1
fi
# Finally we print the current performances, upgrade BASE_ metrics in case you improved it.
echo "DGA detection performances report:"
echo "Accuracy=0.$ACCURACY"
echo "Precision=0.$PRECISION"
echo "Recall=0.$RECALL"
}
get_evaluation_data_size
evaluate_ndpi_dga_detection
exit $RC