Implement DGA detection performances tracking workflow. (#1064)
* Implement dga evaluation helper. * Add test set for DGA classification. * Add DGA classification performances tracking as part of Travis. * Add DGA evaluation doc. * Fix CI on OSX. * Add missing backquote.
This commit is contained in:
@@ -205,6 +205,7 @@ script:
|
||||
- cd tests
|
||||
- ./do.sh
|
||||
- ./do-unit.sh
|
||||
- ./do-dga.sh
|
||||
|
||||
#after_success:
|
||||
#- cd ${TRAVIS_BUILD_DIR}
|
||||
|
||||
+2
-2
@@ -7,7 +7,7 @@ AM_INIT_AUTOMAKE([foreign subdir-objects])
|
||||
|
||||
AC_PREFIX_DEFAULT(/usr)
|
||||
|
||||
EXTRA_TARGETS="example tests tests/unit"
|
||||
EXTRA_TARGETS="example tests tests/unit tests/dga"
|
||||
AC_ARG_WITH(only-libndpi, AS_HELP_STRING([--with-only-libndpi], [Build only libndpi (no examples, tests etc)]))
|
||||
AS_IF([test "${with_only_libndpi+set}" = set],[
|
||||
EXTRA_TARGETS=""
|
||||
@@ -201,7 +201,7 @@ if test "${with_pcre+set}" = set; then :
|
||||
fi
|
||||
|
||||
|
||||
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
|
||||
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/dga/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
|
||||
AC_CONFIG_HEADERS(src/include/ndpi_config.h)
|
||||
AC_SUBST(GIT_RELEASE)
|
||||
AC_SUBST(NDPI_MAJOR)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
CC=@CC@
|
||||
CXX=@CXX@
|
||||
|
||||
SRCHOME=../../src
|
||||
|
||||
CFLAGS=-g -fPIC -DPIC -I$(SRCHOME)/include @JSONC_CFLAGS@ @PCAP_INC@ @CFLAGS@
|
||||
LIBNDPI=$(SRCHOME)/lib/libndpi.a
|
||||
LDFLAGS=$(LIBNDPI) @PCAP_LIB@ @LIBS@ @ADDITIONAL_LIBS@ @JSONC_LIBS@ -lpthread -lm @LDFLAGS@
|
||||
HEADERS=$(SRCHOME)/include/ndpi_api.h $(SRCHOME)/include/ndpi_typedefs.h $(SRCHOME)/include/ndpi_protocol_ids.h
|
||||
OBJS=dga_evaluate
|
||||
PREFIX?=@prefix@
|
||||
|
||||
all: dga_evaluate
|
||||
|
||||
EXECUTABLE_SOURCES := dga_evaluate.c
|
||||
COMMON_SOURCES := $(filter-out $(EXECUTABLE_SOURCES),$(wildcard *.c ))
|
||||
|
||||
dga_evaluate: $(LIBNDPI) dga_evaluate.o
|
||||
$(CC) $(CFLAGS) dga_evaluate.o -o $@ $(LDFLAGS)
|
||||
|
||||
%.o: %.c $(HEADERS) Makefile
|
||||
$(CC) $(CFLAGS) -c $< -o $@
|
||||
|
||||
clean:
|
||||
/bin/rm -f *.o dga_evaluate
|
||||
/bin/rm -f .*.o.cmd .*.o.d
|
||||
/bin/rm -rf build
|
||||
|
||||
install:
|
||||
echo ""
|
||||
|
||||
distdir:
|
||||
|
||||
|
||||
distclean: clean
|
||||
/bin/rm -f Makefile
|
||||
@@ -0,0 +1,70 @@
|
||||
# DGA detection testing workflow
|
||||
|
||||
|
||||
## Overview
|
||||
|
||||
nDPI provides a set of threat detection features available through NDPI_RISK detection.
|
||||
|
||||
As part of these features, we provide DGA detection.
|
||||
|
||||
Domain generation algorithms (DGA) are algorithms seen in various families of malware that are used
|
||||
to periodically generate a large number of domain names that can be used as rendezvous points with
|
||||
their command and control servers.
|
||||
|
||||
DGA detection heuristic is implemented [**here**](https://github.com/ntop/nDPI/blob/328ff2465709372c595cb25d99135aa515da3c5a/src/lib/ndpi_main.c#L6729).
|
||||
|
||||
DGA performances test and tracking allows us to detect automatically if a modification is harmful.
|
||||
|
||||
The modification can be a simple threshold change or a future lightweight ML approach.
|
||||
|
||||
## Used data
|
||||
|
||||
Original used dataset is a collection of legit and DGA domains (balanced) that can be obtained as follow:
|
||||
|
||||
```shell
|
||||
wget https://raw.githubusercontent.com/chrmor/DGA_domains_dataset/master/dga_domains_full.csv
|
||||
```
|
||||
|
||||
We split the dataset into DGA and NON-DGA and we keep 10% of each as test set and 90% as training set.
|
||||
|
||||
```shell
|
||||
python3 -m pip install pandas
|
||||
python3 -m pip install sklearn
|
||||
```
|
||||
|
||||
Instruction using python3
|
||||
|
||||
```python3
|
||||
from sklearn.model_selection import train_test_split
|
||||
import pandas as pd
|
||||
|
||||
df = pd.read_csv("dga_domains_full.csv", header=None, names=["type", "family", "domain"])
|
||||
df_dga = df[df.type=="dga"]
|
||||
df_non_dga = df[df.type=="legit"]
|
||||
train_non_dga, test_non_dga = train_test_split(df_non_dga, test_size=0.1, shuffle=True, random_state=27)
|
||||
train_dga, test_dga = train_test_split(df_dga, test_size=0.1, shuffle=True, random_state=27)
|
||||
|
||||
test_dga["domain"].to_csv("test_dga.csv", header=False, index=False)
|
||||
test_non_dga["domain"].to_csv("test_non_dga.csv", header=False, index=False)
|
||||
train_dga["domain"].to_csv("train_dga.csv", header=False, index=False)
|
||||
test_non_dga["domain"].to_csv("test_non_dga.csv", header=False, index=False)
|
||||
```
|
||||
|
||||
**Detection approach must be built on top of training set only, test set must be kept as unseen cases for testing**
|
||||
|
||||
## dga_evaluate
|
||||
|
||||
After nDPI compilation, you can use dga_evaluate helper to check number of detections out of an input file.
|
||||
|
||||
```shell
|
||||
dga_evaluate <file name>
|
||||
```
|
||||
|
||||
You can evaluate your modifications performances before submitting it as follows:
|
||||
|
||||
```shell
|
||||
./do-dga.sh
|
||||
```
|
||||
|
||||
If your modifications decreases baseline performances, test will fails.
|
||||
If not (well done), test passed and you must update the baseline metrics with your obtained ones.
|
||||
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* dga.c
|
||||
*
|
||||
* Copyright (C) 2019-20 - ntop.org
|
||||
*
|
||||
* nDPI is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* nDPI is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdarg.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <ndpi_api.h>
|
||||
#include <ndpi_main.h>
|
||||
#include <ndpi_typedefs.h>
|
||||
#include "ndpi_config.h"
|
||||
|
||||
|
||||
void help() {
|
||||
printf("dga_evaluate <file name>\n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
/* *********************************************** */
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
FILE *fd;
|
||||
char buffer[512];
|
||||
|
||||
if(argc != 2) help();
|
||||
fd = fopen(argv[1], "r");
|
||||
|
||||
if(fd == NULL) {
|
||||
printf("Unable to open file %s\n", argv[1]);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
if (ndpi_get_api_version() != NDPI_API_VERSION) {
|
||||
printf("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Initialize nDPI detection module*/
|
||||
NDPI_PROTOCOL_BITMASK all;
|
||||
struct ndpi_detection_module_struct *ndpi_str = ndpi_init_detection_module(ndpi_no_prefs);
|
||||
assert(ndpi_str != NULL);
|
||||
NDPI_BITMASK_SET_ALL(all);
|
||||
ndpi_set_protocol_detection_bitmask2(ndpi_str, &all);
|
||||
ndpi_finalize_initalization(ndpi_str);
|
||||
assert(ndpi_str != NULL);
|
||||
int num_detections = 0;
|
||||
|
||||
|
||||
while(fgets(buffer, sizeof(buffer), fd) != NULL) {
|
||||
char *hostname;
|
||||
hostname = strtok(buffer, "\n");
|
||||
if (ndpi_check_dga_name(ndpi_str, NULL, hostname, 1)) num_detections++;
|
||||
}
|
||||
|
||||
fclose(fd);
|
||||
ndpi_exit_detection_module(ndpi_str);
|
||||
printf("%i", num_detections);
|
||||
return 0;
|
||||
}
|
||||
|
||||
+33750
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/bin/sh
|
||||
|
||||
cd "$(dirname "${0}")"
|
||||
|
||||
# Baseline performances ------------------------------------------------------------------------------------------------
|
||||
# Important notes: BASE values must be integers examples and represents percentage (e.g. 79%, 98%).
|
||||
BASE_ACCURACY=66
|
||||
BASE_PRECISION=86
|
||||
BASE_RECALL=38
|
||||
# ----------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
DGA_EVALUATE="./dga/dga_evaluate"
|
||||
DGA_DATA="dga/test_dga.csv"
|
||||
NON_DGA_DATA="dga/test_non_dga.csv"
|
||||
DGA_DATA_SIZE=0
|
||||
NON_DGA_DATA_SIZE=0
|
||||
DATA_SIZE=0
|
||||
RC=0
|
||||
|
||||
get_evaluation_data_size() {
|
||||
DGA_DATA_SIZE=`wc -l dga/test_dga.csv | awk '{split($0,a," "); print a[1]}'`
|
||||
NON_DGA_DATA_SIZE=`wc -l dga/test_non_dga.csv | awk '{split($0,a," "); print a[1]}'`
|
||||
DATA_SIZE=$(( $NON_DGA_DATA_SIZE + $DGA_DATA_SIZE ))
|
||||
}
|
||||
|
||||
evaluate_ndpi_dga_detection() {
|
||||
# DGA detection is a binary classification problem, We evaluate the following metrics:
|
||||
# Accuracy: (TP + TN) / (TP + TN + FN + FP)
|
||||
# Precision: TP / (TP + FP)
|
||||
# Recall: TP / (TP + FN)
|
||||
|
||||
TP=`$DGA_EVALUATE dga/test_dga.csv`
|
||||
FN=$(( $DGA_DATA_SIZE - $TP ))
|
||||
FP=`$DGA_EVALUATE dga/test_non_dga.csv`
|
||||
TN=$(( $NON_DGA_DATA_SIZE - $FP ))
|
||||
|
||||
ACCURACY=`echo "print(int(((${TP} + ${TN})/(${TP} + ${TN} + ${FP} + ${FN}))*100))" | python3`
|
||||
PRECISION=`echo "print(int(((${TP})/(${TP} + ${FP}))*100))" | python3`
|
||||
RECALL=`echo "print(int(((${TP})/(${TP} + ${FN}))*100))" | python3`
|
||||
|
||||
# In case modified version of classification algorithm decreases performances, test do not pass.
|
||||
if [ $ACCURACY -lt $BASE_ACCURACY ]; then
|
||||
printf "ERROR: Your modifications decreased DGA classifier accuracy: 0.${BASE_ACCURACY} decreased to 0.${ACCURACY}!\n"
|
||||
RC=1
|
||||
fi
|
||||
if [ $PRECISION -lt $BASE_PRECISION ]; then
|
||||
printf "ERROR: Your modifications decreased DGA classifier precision: 0.${BASE_PRECISION} decreased to 0.${PRECISION}!\n"
|
||||
RC=1
|
||||
fi
|
||||
if [ $RECALL -lt $BASE_RECALL ]; then
|
||||
printf "ERROR: Your modifications decreased DGA classifier recall: 0.${BASE_RECALL} decreased to 0.${RECALL}!\n"
|
||||
RC=1
|
||||
fi
|
||||
|
||||
# Finally we print the current performances, upgrade BASE_ metrics in case you improved it.
|
||||
echo "DGA detection performances report:"
|
||||
echo "Accuracy=0.$ACCURACY"
|
||||
echo "Precision=0.$PRECISION"
|
||||
echo "Recall=0.$RECALL"
|
||||
}
|
||||
|
||||
get_evaluation_data_size
|
||||
evaluate_ndpi_dga_detection
|
||||
|
||||
exit $RC
|
||||
Reference in New Issue
Block a user