Complete rework of nDPI Python bindings (cffi API, automatic generation, packaging and CI integration)
This commit is contained in:
@@ -43,6 +43,33 @@ jobs:
|
||||
path: coverage_report
|
||||
retention-days: 7
|
||||
|
||||
python-bindings:
|
||||
name: Python Bindings (ubuntu-latest)
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Install Ubuntu Prerequisites
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
|
||||
- name: Build nDPI library
|
||||
run: |
|
||||
env CC=gcc CFLAGS='-Werror' ./autogen.sh
|
||||
make
|
||||
- name: Generate Python bindings
|
||||
run: |
|
||||
cd python
|
||||
python3 -m pip install --upgrade pip
|
||||
python3 -m pip install -r requirements.txt
|
||||
python3 setup.py install
|
||||
cd ..
|
||||
- name: Test Python Bindings
|
||||
run: |
|
||||
cd python
|
||||
python3 tests.py
|
||||
|
||||
test-scripts:
|
||||
name: Test Utils (ubuntu-latest)
|
||||
runs-on: ubuntu-latest
|
||||
@@ -267,7 +294,6 @@ jobs:
|
||||
make all
|
||||
make -C example ndpiSimpleIntegration
|
||||
make -C rrdtool
|
||||
make -C python
|
||||
- name: Install nDPI
|
||||
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
|
||||
run: |
|
||||
@@ -320,7 +346,6 @@ jobs:
|
||||
make all &&
|
||||
make -C example ndpiSimpleIntegration &&
|
||||
make -C rrdtool &&
|
||||
make -C python &&
|
||||
./tests/do.sh &&
|
||||
./tests/do-unit.sh
|
||||
"
|
||||
@@ -345,7 +370,6 @@ jobs:
|
||||
make all &&
|
||||
make -C example ndpiSimpleIntegration &&
|
||||
make -C rrdtool &&
|
||||
make -C python &&
|
||||
./tests/do.sh &&
|
||||
./tests/do-unit.sh
|
||||
"
|
||||
@@ -370,7 +394,6 @@ jobs:
|
||||
make all &&
|
||||
make -C example ndpiSimpleIntegration &&
|
||||
make -C rrdtool &&
|
||||
make -C python &&
|
||||
./tests/do.sh &&
|
||||
./tests/do-unit.sh
|
||||
"
|
||||
|
||||
@@ -61,7 +61,6 @@ stamp-h1
|
||||
/fuzz/fuzz_ndpi_reader_with_main
|
||||
/fuzz/fuzz_process_packet
|
||||
/fuzz/fuzz_process_packet_with_main
|
||||
/python/Makefile
|
||||
/tests/dga/Makefile
|
||||
/tests/do.sh
|
||||
/tests/unit/Makefile
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ endif
|
||||
pkgconfigdir = $(libdir)/pkgconfig
|
||||
pkgconfig_DATA = libndpi.pc
|
||||
|
||||
EXTRA_DIST = README.md CHANGELOG.md CONTRIBUTING.md autogen.sh wireshark python windows utils packages example/MacOS doc/api.rst doc/conf.py doc/flow_risks.rst doc/guide/nDPI_QuickStartGuide.pages doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst doc/Makefile doc/what_is_ndpi.rst
|
||||
EXTRA_DIST = README.md CHANGELOG.md CONTRIBUTING.md autogen.sh wireshark windows utils packages example/MacOS doc/api.rst doc/conf.py doc/flow_risks.rst doc/guide/nDPI_QuickStartGuide.pages doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst doc/Makefile doc/what_is_ndpi.rst
|
||||
|
||||
changelog:
|
||||
git log --since={`curl -s https://github.com/ntop/ndpi/releases | grep datetime | head -n1 | egrep -o "[0-9]+\-[0-9]+\-[0-9]+"`} --name-only --pretty=format:" - %s" | grep "^ " > CHANGELOG.latest
|
||||
|
||||
+1
-1
@@ -285,7 +285,7 @@ dnl> ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lcurl"
|
||||
dnl> AC_DEFINE_UNQUOTED(HAVE_CURL, 1, [curl is present])
|
||||
dnl> fi
|
||||
|
||||
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/performance/Makefile tests/dga/Makefile rrdtool/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
|
||||
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/performance/Makefile tests/dga/Makefile rrdtool/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile fuzz/Makefile src/include/ndpi_api.h])
|
||||
AC_CONFIG_FILES([tests/do.sh], [chmod +x tests/do.sh])
|
||||
AC_CONFIG_HEADERS(src/include/ndpi_config.h)
|
||||
AC_SUBST(GIT_RELEASE)
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
CC=@CC@
|
||||
CFLAGS=-I. -I../src/include -I./src/lib/third_party/include @CFLAGS@ @CUSTOM_NDPI@ -shared
|
||||
#LIBNDPI=../src/lib/libndpi.so.@NDPI_VERSION_SHORT@
|
||||
LIBNDPI=../src/lib/libndpi.a
|
||||
LDFLAGS=@LDFLAGS@
|
||||
LIBS=$(CFILE) $(LIBNDPI) -lpcap @LIBS@ @ADDITIONAL_LIBS@
|
||||
SHARE = -soname,ndpi_wrap
|
||||
SO=ndpi_wrap.so
|
||||
OBJS = ndpi_wrap.o
|
||||
CFILE = ndpi_wrap.c
|
||||
PIC = -fPIC
|
||||
PREFIX?=/usr/local
|
||||
.PHONY: all
|
||||
|
||||
UNAME_S := $(shell uname -s)
|
||||
ifeq ($(UNAME_S),Darwin)#do something for linux
|
||||
SHARE=-install_name,ndpiReader.so
|
||||
endif
|
||||
|
||||
all: $(SO)
|
||||
|
||||
$(SO): $(CFILE) $(LIBNDPI) Makefile
|
||||
$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(PIC) -Wl,$(SHARE) -o $@ $(LIBS)
|
||||
# ln -s $(LIBNDPI) .
|
||||
|
||||
clean:
|
||||
/bin/rm -f $(SO)
|
||||
@@ -0,0 +1,79 @@
|
||||
# ndpi
|
||||
|
||||
This package contains Python bindings for nDPI. nDPI is an Open and Extensible LGPLv3 Deep Packet Inspection Library.
|
||||
|
||||
**ndpi** is implemented using [**CFFI**][cffi] (out-of-line API mode). Consequently, it is fast and [**PyPy**][pypy]
|
||||
compliant.
|
||||
|
||||
## Installation
|
||||
|
||||
### Build nDPI
|
||||
|
||||
``` bash
|
||||
git clone --branch dev https://github.com/ntop/nDPI.git
|
||||
cd nDPI
|
||||
./autogen.sh
|
||||
./configure
|
||||
make
|
||||
```
|
||||
|
||||
### Install ndpi package
|
||||
|
||||
``` bash
|
||||
cd python
|
||||
python3 -m install -r dev_requirements.txt
|
||||
python3 setup.py install
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### API
|
||||
``` python
|
||||
from ndpi import NDPI, NDPIFlow
|
||||
|
||||
nDPI = NDPI()
|
||||
|
||||
# You per flow processing here
|
||||
# ...
|
||||
|
||||
ndpi_flow = NDPIFlow()
|
||||
nDPI.process_packet(ndpi_flow, ip_bytes, time_ms)
|
||||
nDPI.giveup(ndpi_flow) # If you want to guess it instead (DPI fallback)
|
||||
```
|
||||
|
||||
### Example Application
|
||||
|
||||
[ndpi_example.py][ndpi_example] is provided to demonstrate how **ndpi** can be integrated within your Python application.
|
||||
|
||||
``` bash
|
||||
Using nDPI 4.3.0-3532-8dd70b70
|
||||
usage: ndpi_example.py [-h] [-u] input
|
||||
|
||||
positional arguments:
|
||||
input input pcap file path
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
-u, --include-unknowns
|
||||
```
|
||||
|
||||
Example with a Skype capture file
|
||||
|
||||
``` bash
|
||||
python ndpi_example.py -u ../tests/pcap/skype.pcap
|
||||
```
|
||||
|
||||
## Related projects
|
||||
|
||||
The provided example is for demo purposes only, For additional features (live capture, multiplatform support,
|
||||
multiprocessing, ML based classification, system visibility, etc.), please check nDPI based
|
||||
framework, [**NFStream**][nfstream].
|
||||
## License
|
||||
|
||||
This project is licensed under the LGPLv3 License - see the [**License**][license] file for details.
|
||||
|
||||
[license]: https://github.com/ntop/nDPI/blob/dev/COPYING
|
||||
[cffi]: https://cffi.readthedocs.io/en/latest/
|
||||
[pypy]: https://www.pypy.org/
|
||||
[nfstream]: https://github.com/nfstream/nfstream
|
||||
[ndpi_example]: https://github.com/ntop/nDPI/blob/dev/python/ndpi_example.py
|
||||
@@ -1,43 +0,0 @@
|
||||
nDPI Python bindings
|
||||
--------------------
|
||||
|
||||
This directory contains the Python3 bindings for nDPI. We provide both cffi and ctypes based bindings.
|
||||
|
||||
**cffi bindings**
|
||||
|
||||
Files:
|
||||
|
||||
* ndpi.py
|
||||
|
||||
Example (using `nfstream <https://github.com/aouinizied/nfstream>`_ package):
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip3 install nfstream
|
||||
python3 flow_printer.py <interface>
|
||||
python3 flow_printer.py <pcap_file>
|
||||
|
||||
Code courtesy:
|
||||
|
||||
* Zied Aouini
|
||||
|
||||
**ctypes bindings**
|
||||
|
||||
Files:
|
||||
|
||||
* ndpi_typestruct.py
|
||||
* ndpi_wrap.c
|
||||
* Makefile.in
|
||||
|
||||
Example:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip3 install scapy
|
||||
python3 ndpi_example.py <interface>
|
||||
python3 ndpi_example.py <pcap_file>
|
||||
|
||||
Code courtesy:
|
||||
|
||||
* Massimo Puddu
|
||||
* Zied Aouini
|
||||
@@ -0,0 +1,2 @@
|
||||
cffi>=1.15.0
|
||||
dpkt>=1.9.7.2
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
flow_printer.py
|
||||
Copyright (C) 2019-20 - NFStream Developers
|
||||
This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/).
|
||||
NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
|
||||
version.
|
||||
NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
|
||||
You should have received a copy of the GNU Lesser General Public License along with NFStream.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
from nfstream import NFStreamer
|
||||
import sys
|
||||
|
||||
# Example must run with nfstream >= 6.1.1
|
||||
path = sys.argv[1]
|
||||
flow_streamer = NFStreamer(source=path, statistical_analysis=False, performance_report=1)
|
||||
result = {}
|
||||
try:
|
||||
for flow in flow_streamer:
|
||||
print(flow)
|
||||
try:
|
||||
result[flow.application_name] += flow.bidirectional_packets
|
||||
except KeyError:
|
||||
result[flow.application_name] = flow.bidirectional_packets
|
||||
print("\nSummary (Application Name: Packets):")
|
||||
print(result)
|
||||
except KeyboardInterrupt:
|
||||
print("\nSummary (Application Name: Packets):")
|
||||
print(result)
|
||||
print("Terminated.")
|
||||
-1346
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,21 @@
|
||||
"""
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
__init__.py
|
||||
Copyright (C) 2011-22 - ntop.org
|
||||
This file is part of nDPI, an open source deep packet inspection library.
|
||||
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
|
||||
version.
|
||||
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
|
||||
You should have received a copy of the GNU Lesser General Public License along with NFStream.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
from .ndpi import NDPI, NDPIFlow
|
||||
|
||||
|
||||
__author__ = """Zied Aouini"""
|
||||
__email__ = 'aouinizied@gmail.com'
|
||||
__version__ = '4.3.0'
|
||||
@@ -0,0 +1,101 @@
|
||||
"""
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
ndpi.py
|
||||
Copyright (C) 2011-22 - ntop.org
|
||||
This file is part of nDPI, an open source deep packet inspection library.
|
||||
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
|
||||
version.
|
||||
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
|
||||
You should have received a copy of the GNU Lesser General Public License along with NFStream.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
from collections import namedtuple
|
||||
from _ndpi import ffi, lib
|
||||
|
||||
|
||||
ndpi_protocol = namedtuple('NDPIProtocol', ['C',
|
||||
'master_protocol',
|
||||
'app_protocol',
|
||||
'category'])
|
||||
|
||||
ndpi_confidence = namedtuple('NDPIConfidence', ['id',
|
||||
'name'])
|
||||
|
||||
|
||||
class NDPI(object):
|
||||
__slots__ = ("_api_version",
|
||||
"_revision",
|
||||
"_detection_module")
|
||||
|
||||
def __init__(self):
|
||||
self._detection_module = lib.ndpi_init_detection_module(0)
|
||||
if self._detection_module == ffi.NULL:
|
||||
raise MemoryError("Unable to instantiate NDPI object")
|
||||
lib.ndpi_py_setup_detection_module(self._detection_module)
|
||||
|
||||
@property
|
||||
def api_version(self):
|
||||
return lib.ndpi_get_api_version()
|
||||
|
||||
@property
|
||||
def revision(self):
|
||||
return ffi.string(lib.ndpi_revision()).decode('utf-8', errors='ignore')
|
||||
|
||||
def process_packet(self, flow, packet, packet_time_ms):
|
||||
p = lib.ndpi_detection_process_packet(self._detection_module,
|
||||
flow._C,
|
||||
packet,
|
||||
len(packet),
|
||||
int(packet_time_ms))
|
||||
return ndpi_protocol(C=p,
|
||||
master_protocol=p.master_protocol,
|
||||
app_protocol=p.app_protocol,
|
||||
category=p.category)
|
||||
|
||||
def giveup(self, flow, enable_guess=True):
|
||||
p = lib.ndpi_detection_giveup(self._detection_module,
|
||||
flow._C,
|
||||
enable_guess,
|
||||
ffi.new("uint8_t*", 0))
|
||||
return ndpi_protocol(C=p,
|
||||
master_protocol=p.master_protocol,
|
||||
app_protocol=p.app_protocol,
|
||||
category=p.category)
|
||||
|
||||
def protocol_name(self, protocol):
|
||||
buf = ffi.new("char[40]")
|
||||
lib.ndpi_protocol2name(self._detection_module, protocol.C, buf, ffi.sizeof(buf))
|
||||
return ffi.string(buf).decode('utf-8', errors='ignore')
|
||||
|
||||
def protocol_category_name(self, protocol):
|
||||
return ffi.string(lib.ndpi_category_get_name(self._detection_module,
|
||||
protocol.C.category)).decode('utf-8',
|
||||
errors='ignore')
|
||||
|
||||
def __del__(self):
|
||||
if self._detection_module != ffi.NULL:
|
||||
lib.ndpi_exit_detection_module(self._detection_module)
|
||||
|
||||
|
||||
class NDPIFlow(object):
|
||||
__slots__ = ("_C")
|
||||
|
||||
@property
|
||||
def confidence(self):
|
||||
confidence = self._C.confidence
|
||||
return ndpi_confidence(id=confidence,
|
||||
name=ffi.string(lib.ndpi_confidence_get_name(confidence)).decode('utf-8',
|
||||
errors='ignore'))
|
||||
|
||||
def __init__(self):
|
||||
self._C = lib.ndpi_py_initialize_flow()
|
||||
|
||||
def __del__(self):
|
||||
if self._C != ffi.NULL:
|
||||
lib.ndpi_flow_free(self._C)
|
||||
self._C = ffi.NULL
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
"""
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
ndpi_build.py
|
||||
Copyright (C) 2011-22 - ntop.org
|
||||
This file is part of nDPI, an open source deep packet inspection library.
|
||||
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
|
||||
version.
|
||||
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
|
||||
You should have received a copy of the GNU Lesser General Public License along with NFStream.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
from cffi import FFI
|
||||
import subprocess
|
||||
import pathlib
|
||||
|
||||
NDPI_INCLUDES = """
|
||||
#include "ndpi_main.h"
|
||||
#include "ndpi_typedefs.h"
|
||||
#include "ndpi_api.h"
|
||||
"""
|
||||
|
||||
NDPI_HELPERS = """
|
||||
// nDPI cffi helper functions (function naming convention ndpi_py_*)
|
||||
|
||||
void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod) {
|
||||
if (mod == NULL) {
|
||||
return;
|
||||
} else {
|
||||
NDPI_PROTOCOL_BITMASK protos;
|
||||
NDPI_BITMASK_SET_ALL(protos); // Set bitmask for ALL protocols
|
||||
ndpi_set_protocol_detection_bitmask2(mod, &protos);
|
||||
ndpi_finalize_initialization(mod);
|
||||
}
|
||||
};
|
||||
|
||||
struct ndpi_flow_struct * ndpi_py_initialize_flow(void) {
|
||||
struct ndpi_flow_struct * ndpi_flow = NULL;
|
||||
ndpi_flow = (struct ndpi_flow_struct *)ndpi_flow_malloc(SIZEOF_FLOW_STRUCT);
|
||||
memset(ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
|
||||
return ndpi_flow;
|
||||
};
|
||||
"""
|
||||
|
||||
|
||||
NDPI_APIS = """
|
||||
u_int16_t ndpi_get_api_version(void);
|
||||
char* ndpi_revision(void);
|
||||
struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs prefs);
|
||||
void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct);
|
||||
void ndpi_flow_free(void *ptr);
|
||||
ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow,
|
||||
const unsigned char *packet,
|
||||
const unsigned short packetlen,
|
||||
const u_int64_t packet_time_ms);
|
||||
ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
|
||||
struct ndpi_flow_struct *flow,
|
||||
u_int8_t enable_guess,
|
||||
u_int8_t *protocol_was_guessed);
|
||||
void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod);
|
||||
struct ndpi_flow_struct * ndpi_py_initialize_flow(void);
|
||||
char* ndpi_protocol2name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol proto, char *buf, u_int buf_len);
|
||||
const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol_category_t category);
|
||||
const char* ndpi_confidence_get_name(ndpi_confidence_t confidence);
|
||||
"""
|
||||
|
||||
ffi_builder = FFI()
|
||||
|
||||
|
||||
INCLUDE_DIR = pathlib.Path(__file__)\
|
||||
.parent.resolve().parent.resolve().parent.resolve().\
|
||||
joinpath("src").joinpath("include")
|
||||
|
||||
LIBRARY_DIR = pathlib.Path(__file__)\
|
||||
.parent.resolve().parent.resolve().parent.resolve().\
|
||||
joinpath("src").joinpath("lib")
|
||||
|
||||
|
||||
NDPI_CDEF = subprocess.run(["gcc",
|
||||
"-DNDPI_LIB_COMPILATION",
|
||||
"-DNDPI_CFFI_PREPROCESSING",
|
||||
"-DNDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED",
|
||||
"-E", "-x", "c", "-P", "-C",
|
||||
str(INCLUDE_DIR.joinpath("ndpi_typedefs.h"))],
|
||||
capture_output=True
|
||||
).stdout.decode('utf-8',
|
||||
errors='ignore')
|
||||
|
||||
NDPI_MODULE_STRUCT_CDEF = NDPI_CDEF.split("//CFFI.NDPI_MODULE_STRUCT")[1]
|
||||
|
||||
|
||||
NDPI_PACKED = subprocess.run(["gcc",
|
||||
"-DNDPI_LIB_COMPILATION", "-DNDPI_CFFI_PREPROCESSING",
|
||||
"-E", "-x", "c", "-P", "-C",
|
||||
str(INCLUDE_DIR.joinpath("ndpi_typedefs.h"))],
|
||||
capture_output=True
|
||||
).stdout.decode('utf-8',
|
||||
errors='ignore')
|
||||
|
||||
NDPI_PACKED_STRUCTURES = NDPI_PACKED.split("//CFFI.NDPI_PACKED_STRUCTURES")[1]
|
||||
|
||||
NDPI_SOURCE = NDPI_INCLUDES + NDPI_MODULE_STRUCT_CDEF + NDPI_HELPERS
|
||||
|
||||
|
||||
ffi_builder.set_source("_ndpi",
|
||||
NDPI_SOURCE,
|
||||
libraries=["ndpi"],
|
||||
library_dirs=[str(LIBRARY_DIR)],
|
||||
include_dirs=[str(INCLUDE_DIR)])
|
||||
|
||||
|
||||
ffi_builder.cdef("""
|
||||
typedef uint64_t u_int64_t;
|
||||
typedef uint32_t u_int32_t;
|
||||
typedef uint16_t u_int16_t;
|
||||
typedef uint8_t u_int8_t;
|
||||
typedef uint8_t u_char;
|
||||
typedef unsigned u_int;
|
||||
struct in_addr {
|
||||
unsigned long s_addr;
|
||||
};
|
||||
struct in6_addr {
|
||||
unsigned char s6_addr[16];
|
||||
};
|
||||
""")
|
||||
|
||||
ffi_builder.cdef(NDPI_PACKED_STRUCTURES, packed=True)
|
||||
ffi_builder.cdef(NDPI_CDEF)
|
||||
ffi_builder.cdef(NDPI_APIS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ffi_builder.compile(verbose=True)
|
||||
Executable → Regular
+144
-250
@@ -1,273 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
file: ndpi_example.py
|
||||
This file is part of nDPI.
|
||||
|
||||
Copyright (C) 2011-19 - ntop.org
|
||||
Copyright (C) 2019 - Zied Aouini <aouinizied@gmail.com> (Incremental improvements)
|
||||
|
||||
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License
|
||||
as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
||||
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
ndpi_example.py
|
||||
Copyright (C) 2011-22 - ntop.org
|
||||
This file is part of nDPI, an open source deep packet inspection library.
|
||||
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
|
||||
version.
|
||||
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along with nDPI.
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
|
||||
You should have received a copy of the GNU Lesser General Public License along with NFStream.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
from ndpi_typestruct import *
|
||||
from ctypes import *
|
||||
from scapy.all import *
|
||||
import sys
|
||||
|
||||
# ------- return type & pcapstruct to declare -------
|
||||
from collections import namedtuple
|
||||
from ndpi import NDPI, NDPIFlow
|
||||
import argparse
|
||||
import socket
|
||||
import dpkt
|
||||
|
||||
|
||||
class WorkFlow(Structure):
|
||||
_fields_ = [("src_ip", c_uint32),
|
||||
("dst_ip", c_uint32),
|
||||
("src_port", c_uint16),
|
||||
("dst_port", c_uint16),
|
||||
("protocol", c_uint8),
|
||||
("packets", c_uint32),
|
||||
("detected_protocol", NDPIProtocol),
|
||||
("detection_completed", c_uint8),
|
||||
("id", c_uint32),
|
||||
("src_id", POINTER(NDPIIdStruct)),
|
||||
("dst_id", POINTER(NDPIIdStruct)),
|
||||
("flow", POINTER(NDPIFlowStruct))]
|
||||
FLOW_KEY = "{} {}:{} <-> {}:{}"
|
||||
FLOW_STR = " {} {} [protocol:{}] [category:{}] [confidence:{}] [{} packets/{} bytes]"
|
||||
|
||||
|
||||
CMCFUN = CFUNCTYPE(c_int, c_void_p, c_void_p)
|
||||
GUESS = CFUNCTYPE(None, c_void_p, c_int32, c_int, c_void_p)
|
||||
FREE = CFUNCTYPE(None, c_void_p)
|
||||
PROTOCOL_UNKNWON = 0
|
||||
|
||||
|
||||
def node_proto_guess_walker(nodo, which, depth, user_data):
|
||||
global ndpi_info_mod
|
||||
class Flow(object):
|
||||
__slots__ = ("index",
|
||||
"pkts",
|
||||
"bytes",
|
||||
"detected_protocol",
|
||||
"ndpi_flow")
|
||||
|
||||
flow = cast(nodo, POINTER(POINTER(WorkFlow))).contents.contents
|
||||
if which == 0 or which == 3: # execute only preorder operation of the tree
|
||||
if flow.detection_completed == 0: # order for tree operation
|
||||
flow.detected_protocol = ndpi.ndpi_detection_giveup(ndpi_info_mod,
|
||||
flow.flow,
|
||||
1,
|
||||
cast(addressof(c_uint8(0)), POINTER(c_uint8)))
|
||||
count_protocol[flow.detected_protocol.app_protocol] += flow.packets
|
||||
def __init__(self):
|
||||
self.pkts = 0
|
||||
self.detected_protocol = None
|
||||
self.bytes = 0
|
||||
self.ndpi_flow = None
|
||||
|
||||
|
||||
def py_cmp_fun(a, b):
|
||||
fa = cast(a, POINTER(WorkFlow))
|
||||
fb = cast(b, POINTER(WorkFlow))
|
||||
|
||||
if fa.contents.src_ip < fb.contents.src_ip: return -1
|
||||
elif fa.contents.src_ip > fb.contents.src_ip: return 1
|
||||
if fa.contents.src_port < fb.contents.src_port: return -1
|
||||
elif fa.contents.src_port > fb.contents.src_port: return 1
|
||||
if fa.contents.dst_ip < fb.contents.dst_ip: return -1
|
||||
elif fa.contents.dst_ip > fb.contents.dst_ip: return 1
|
||||
if fa.contents.dst_port < fb.contents.dst_port: return -1
|
||||
elif fa.contents.dst_port > fb.contents.dst_port: return 1
|
||||
if fa.contents.protocol < fb.contents.protocol: return -1
|
||||
elif fa.contents.protocol > fb.contents.protocol: return 1
|
||||
return 0
|
||||
ppacket = namedtuple('ParsedPacket', ['src_ip',
|
||||
'src_port',
|
||||
'dst_ip',
|
||||
'dst_port',
|
||||
'protocol',
|
||||
'ip_version',
|
||||
'ip_bytes'])
|
||||
|
||||
|
||||
def freer(a):
|
||||
pass
|
||||
|
||||
|
||||
cmp_func = CMCFUN(py_cmp_fun)
|
||||
guess_walker = GUESS(node_proto_guess_walker)
|
||||
free_walk = FREE(freer)
|
||||
|
||||
# --------------------------------------
|
||||
|
||||
# number of analyzed packet
|
||||
packet_number = 0
|
||||
flow_count = 0
|
||||
max_num_udp_dissected_pkts = 16
|
||||
max_num_tcp_dissected_pkts = 10
|
||||
flows_root = c_void_p(None)
|
||||
flows_root_ref = pointer(flows_root)
|
||||
count_protocol = (c_int32 * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + ndpi.ndpi_wrap_ndpi_max_num_custom_protocols() + 1))()
|
||||
lista = [] # used to avoid impropriate memory deallocation from python
|
||||
|
||||
|
||||
# check ndpi version
|
||||
if ndpi.ndpi_get_api_version() != ndpi.ndpi_wrap_get_api_version():
|
||||
print("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n")
|
||||
sys.exit(-1)
|
||||
|
||||
# create data structure of ndpi
|
||||
ndpi_info_mod = ndpi.ndpi_init_detection_module()
|
||||
if ndpi_info_mod is None:
|
||||
sys.exit(-1)
|
||||
else:
|
||||
ndpi_ndpi_finalize_initalization(ndpi_info_mod)
|
||||
|
||||
|
||||
def ip2int(ip):
|
||||
"""
|
||||
Convert an IP string to long and then c_uint32
|
||||
"""
|
||||
packedIP = socket.inet_aton(ip)
|
||||
return int(struct.unpack("!I", packedIP)[0])
|
||||
|
||||
|
||||
def get_flow(pkt):
|
||||
global flows_root
|
||||
global flows_root_ref
|
||||
global flow_count
|
||||
|
||||
ip_packet = pkt[1]
|
||||
ip_protocol = ip_packet.proto
|
||||
transport_packet = None
|
||||
|
||||
if ip_protocol == 6 or ip_protocol == 17:
|
||||
transport_packet = pkt[2]
|
||||
if transport_packet is not None:
|
||||
# to avoid two nodes in one binary tree for a flow
|
||||
ip_src = ip2int(ip_packet.src)
|
||||
ip_dst = ip2int(ip_packet.dst)
|
||||
src_port = transport_packet.sport
|
||||
dst_port = transport_packet.dport
|
||||
else:
|
||||
return None
|
||||
# set flows to correct type and data
|
||||
ndpi_flow = pointer(NDPIFlowStruct())
|
||||
memset(ndpi_flow, 0, sizeof(NDPIFlowStruct))
|
||||
if ip_src > ip_dst:
|
||||
flow = WorkFlow(ip_src, ip_dst, src_port, dst_port, int(ip_packet.proto), 0, NDPIProtocol(), 0, 0,
|
||||
pointer(NDPIIdStruct()), pointer(NDPIIdStruct()), ndpi_flow)
|
||||
else:
|
||||
flow = WorkFlow(ip_dst, ip_src, dst_port, src_port, int(ip_packet.proto), 0, NDPIProtocol(), 0, 0,
|
||||
pointer(NDPIIdStruct()), pointer(NDPIIdStruct()), ndpi_flow)
|
||||
flow_ref = pointer(flow)
|
||||
res = ndpi.ndpi_tfind(flow_ref, flows_root_ref, cmp_func)
|
||||
if res is None:
|
||||
ndpi.ndpi_tsearch(flow_ref, pointer(flows_root), cmp_func) # add
|
||||
lista.append(flow)
|
||||
flow_count += 1
|
||||
return pointer(flow)
|
||||
flow = cast(res, POINTER(POINTER(WorkFlow))).contents
|
||||
return flow
|
||||
|
||||
|
||||
def packetcaptured(pkt):
|
||||
global packet_number
|
||||
global ndpi_info_mod
|
||||
|
||||
flow = None
|
||||
h = PcapPktHdr()
|
||||
|
||||
# getting flow
|
||||
def inet_to_str(inet):
|
||||
""" get string representation of IP address """
|
||||
try:
|
||||
flow = get_flow(pkt)
|
||||
except AttributeError:
|
||||
pass # ignore packet
|
||||
if flow is None: return
|
||||
|
||||
# filling pcap_pkthdr
|
||||
h.len = h.caplen = len(pkt)
|
||||
h.ts.tv_sec = int(pkt["IP"].time/1000000)
|
||||
h.ts.tv_usec = int(pkt["IP"].time)
|
||||
|
||||
# real work
|
||||
if int(pkt[1].frag) == 0: # not fragmented packet
|
||||
flow.contents.packets += 1
|
||||
packet_number += 1
|
||||
# get ndpi_iphdr address
|
||||
iphdr_addr = cast(c_char_p(pkt[1].build()), c_void_p)
|
||||
ndpi_flow = flow.contents.flow
|
||||
|
||||
if flow.contents.detection_completed is 0:
|
||||
flow.contents.detected_protocol = ndpi.ndpi_detection_process_packet(ndpi_info_mod,
|
||||
ndpi_flow,
|
||||
cast(iphdr_addr, POINTER(c_uint8)),
|
||||
int(pkt[1].len),
|
||||
h.ts.tv_usec,
|
||||
flow.contents.src_id,
|
||||
flow.contents.dst_id)
|
||||
|
||||
flow1 = flow.contents.detected_protocol
|
||||
|
||||
valid = False
|
||||
|
||||
if flow.contents.protocol == 6: valid = flow.contents.packets > max_num_tcp_dissected_pkts
|
||||
elif flow.contents.protocol == 17: valid = flow.contents.packets > max_num_udp_dissected_pkts
|
||||
|
||||
# should we continue anylizing packet or not?
|
||||
if valid or flow1.app_protocol is not 0:
|
||||
if valid or flow1.master_protocol is not 91: # or # 91 is NDPI_PROTOCOL_TLS
|
||||
flow.contents.detection_completed = 1 # protocol found
|
||||
if flow1.app_protocol is 0:
|
||||
flow.contents.detected_protocol = ndpi.ndpi_detection_giveup(ndpi_info_mod,
|
||||
ndpi_flow,
|
||||
1,
|
||||
cast(addressof(c_uint8(0)),
|
||||
POINTER(c_uint8)))
|
||||
return socket.inet_ntop(socket.AF_INET, inet)
|
||||
except ValueError:
|
||||
return socket.inet_ntop(socket.AF_INET6, inet)
|
||||
|
||||
|
||||
def result():
|
||||
global flows_root_ref
|
||||
global ndpi_info_mod
|
||||
print('\nnumber of analyzed packet: ' + str(packet_number))
|
||||
print('number of flows: ' + str(flow_count))
|
||||
|
||||
ndpi.ndpi_twalk(flows_root_ref.contents, guess_walker, None)
|
||||
|
||||
print('\nDetected protocols:')
|
||||
for i in range(0, ndpi.ndpi_get_num_supported_protocols(ndpi_info_mod)):
|
||||
if count_protocol[i] > 0:
|
||||
print("{}: {} packets".format(
|
||||
cast(ndpi.ndpi_get_proto_name(ndpi_info_mod, i), c_char_p).value.decode('utf-8'),
|
||||
str(count_protocol[i])))
|
||||
|
||||
|
||||
def free(ndpi_struct):
|
||||
ndpi.ndpi_tdestroy(flows_root, free_walk)
|
||||
ndpi.ndpi_exit_detection_module(ndpi_struct)
|
||||
|
||||
|
||||
def initialize(ndpi_struct):
|
||||
all = NDPIProtocolBitMask()
|
||||
ndpi.ndpi_wrap_NDPI_BITMASK_SET_ALL(pointer(all))
|
||||
ndpi.ndpi_set_protocol_detection_bitmask2(ndpi_struct, pointer(all))
|
||||
|
||||
|
||||
print('Using nDPI ' + cast(ndpi.ndpi_revision(), c_char_p).value.decode("utf-8"))
|
||||
|
||||
initialize(ndpi_info_mod)
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print("\nUsage: ndpi_example.py <device>")
|
||||
sys.exit(0)
|
||||
|
||||
if "." in sys.argv[1]:
|
||||
print('Reading pcap from file ' + sys.argv[1] + '...')
|
||||
scapy_cap = None
|
||||
def parse_packet(pkt):
|
||||
""" parse packet and extract 5 tuple and IP bytes """
|
||||
try:
|
||||
scapy_cap = rdpcap(sys.argv[1])
|
||||
except FileNotFoundError:
|
||||
print("\nFile not found")
|
||||
except Scapy_Exception:
|
||||
print("\nBad pcap")
|
||||
l2 = dpkt.ethernet.Ethernet(pkt)
|
||||
if isinstance(l2.data, dpkt.ip.IP):
|
||||
ip_version = 4
|
||||
elif isinstance(l2.data, dpkt.ip6.IP6):
|
||||
ip_version = 6
|
||||
else:
|
||||
return
|
||||
except dpkt.dpkt.NeedData:
|
||||
return
|
||||
|
||||
l3 = l2.data
|
||||
stop_decoding = False
|
||||
while not stop_decoding:
|
||||
if isinstance(l3.data, dpkt.tcp.TCP):
|
||||
l4 = l3.data
|
||||
proto = "TCP"
|
||||
stop_decoding = True
|
||||
elif isinstance(l3.data, dpkt.udp.UDP):
|
||||
l4 = l3.data
|
||||
proto = "UDP"
|
||||
stop_decoding = True
|
||||
elif isinstance(l3.data, dpkt.ip6.IP6):
|
||||
l3 = l3.data
|
||||
else:
|
||||
return
|
||||
|
||||
return ppacket(src_ip=inet_to_str(l3.src), src_port=l4.sport,
|
||||
dst_ip=inet_to_str(l3.dst), dst_port=l4.dport,
|
||||
protocol=proto, ip_version=ip_version,
|
||||
ip_bytes=bytes(l3))
|
||||
|
||||
|
||||
def ppkt_to_flow_key(ppkt):
|
||||
""" create a consistent direction agnostic flow keyfrom a parsed packet """
|
||||
if ppkt.src_ip < ppkt.dst_ip:
|
||||
k = FLOW_KEY.format(ppkt.protocol, ppkt.src_ip, ppkt.src_port, ppkt.dst_ip, ppkt.dst_port)
|
||||
else:
|
||||
for packet in scapy_cap:
|
||||
packetcaptured(packet)
|
||||
else:
|
||||
print('Capturing live traffic from device ' + sys.argv[1] + '...')
|
||||
try:
|
||||
sniff(iface=sys.argv[1], prn=packetcaptured)
|
||||
except KeyboardInterrupt:
|
||||
print('\nInterrupted\n')
|
||||
except PermissionError:
|
||||
sys.exit('\nRoot privilege required for live capture on interface: {}\n'.format(sys.argv[1]))
|
||||
if ppkt.src_ip == ppkt.dst_ip:
|
||||
if ppkt.src_port <= ppkt.dst_port:
|
||||
k = FLOW_KEY.format(ppkt.protocol, ppkt.src_ip, ppkt.src_port, ppkt.dst_ip, ppkt.dst_port)
|
||||
else:
|
||||
k = FLOW_KEY.format(ppkt.protocol, ppkt.dst_ip, ppkt.dst_port, ppkt.src_ip, ppkt.src_port)
|
||||
else:
|
||||
k = FLOW_KEY.format(ppkt.protocol, ppkt.dst_ip, ppkt.dst_port, ppkt.src_ip, ppkt.src_port)
|
||||
return k
|
||||
|
||||
|
||||
result()
|
||||
free(ndpi_info_mod)
|
||||
def parse_arguments():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("input", help="input pcap file path")
|
||||
parser.add_argument('-u', '--include-unknowns', action='store_true')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
nDPI = NDPI() # As simple as that. :)
|
||||
flow_cache = {} # We store the flows in a dictionary.
|
||||
flow_count = 0 # Flow counter
|
||||
print("Using nDPI {}".format(nDPI.revision))
|
||||
args = parse_arguments()
|
||||
|
||||
with open(args.input, 'rb') as pcap_file:
|
||||
capture = dpkt.pcap.Reader(pcap_file) # We use dpkt pcap capture handler
|
||||
for time, packet in capture:
|
||||
time_ms = int(time * 1000) # Convert packet timestamp to milliseconds
|
||||
ppkt = parse_packet(packet)
|
||||
if ppkt is not None: # If we succeed to parse the packet
|
||||
key = ppkt_to_flow_key(ppkt)
|
||||
try: # Try a Flow update
|
||||
flow = flow_cache[key]
|
||||
flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms)
|
||||
flow.pkts += 1
|
||||
flow.bytes += len(packet)
|
||||
except KeyError: # New Flow
|
||||
flow = Flow()
|
||||
flow.index = flow_count
|
||||
flow_count += 1
|
||||
flow.ndpi_flow = NDPIFlow() # We create an nDPIFlow object per Flow
|
||||
flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms)
|
||||
flow.pkts += 1
|
||||
flow.bytes += len(packet)
|
||||
flow_cache[key] = flow
|
||||
|
||||
print(" Detected flows:")
|
||||
unknown_flows = []
|
||||
for key, flow in flow_cache.items(): # Iterate over all flows in flow cache
|
||||
if flow.detected_protocol.app_protocol == PROTOCOL_UNKNWON: # Didn't succeed to identigy it using DPI
|
||||
flow.detected_protocol = nDPI.giveup(flow.ndpi_flow) # We try to guess it (port matching, LRU, etc.)
|
||||
FLOW_EXPORT = FLOW_STR.format(flow.index,
|
||||
key,
|
||||
nDPI.protocol_name(flow.detected_protocol),
|
||||
nDPI.protocol_category_name(flow.detected_protocol),
|
||||
flow.ndpi_flow.confidence.name,
|
||||
flow.pkts,
|
||||
flow.bytes)
|
||||
if flow.detected_protocol.app_protocol != PROTOCOL_UNKNWON:
|
||||
print(FLOW_EXPORT) # We start by printing detected flows
|
||||
else:
|
||||
# Format it for later
|
||||
unknown_flows.append(FLOW_EXPORT)
|
||||
if args.include_unknowns:
|
||||
print(" Unknown flows:")
|
||||
for unknown_flow in unknown_flows: # Dump unknown flows
|
||||
print(unknown_flow)
|
||||
|
||||
@@ -1,780 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
"""
|
||||
file: ndpi_typestruct.py
|
||||
This file is part of nDPI.
|
||||
|
||||
Copyright (C) 2011-19 - ntop.org
|
||||
Copyright (C) 2019 - Zied Aouini <aouinizied@gmail.com> (Incremental improvements)
|
||||
|
||||
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License
|
||||
as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
|
||||
|
||||
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License along with nDPI.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
|
||||
from ctypes import CDLL, Structure, c_uint16, c_int, c_ulong, c_uint32, CFUNCTYPE, c_void_p, POINTER, c_char_p, c_uint8
|
||||
from ctypes import c_char, c_uint, c_int16, c_longlong, c_size_t, Union, c_ubyte, c_uint64, c_int32, c_ushort, cast
|
||||
from os.path import abspath, dirname
|
||||
ndpi = CDLL(dirname(abspath(__file__)) + '/ndpi_wrap.so')
|
||||
|
||||
# ----------------------------------------------- Structures -----------------------------------------------------------
|
||||
|
||||
|
||||
class NDPIDetectionModuleStruct(Structure):
|
||||
pass
|
||||
|
||||
|
||||
class NDPIFlowStruct(Structure):
|
||||
pass
|
||||
|
||||
|
||||
class NDPIProtocol(Structure):
|
||||
_fields_ = [
|
||||
("master_protocol", c_uint16),
|
||||
("app_protocol", c_uint16),
|
||||
("category", c_int)
|
||||
]
|
||||
|
||||
|
||||
class TimeVal(Structure):
|
||||
_fields_ = [("tv_sec", c_ulong), ("tv_usec", c_ulong)]
|
||||
|
||||
|
||||
class PcapPktHdr(Structure):
|
||||
_fields_ = [("ts", TimeVal), ("caplen", c_uint32), ("len", c_uint32)]
|
||||
|
||||
|
||||
class NDPIMask(Structure):
|
||||
_fields_ = [("fds_bits", c_uint32)]
|
||||
|
||||
|
||||
class NDPIProtocolBitMask(Structure):
|
||||
_fields_ = [("fds_bits", NDPIMask * ndpi.ndpi_wrap_ndpi_num_fds_bits())]
|
||||
|
||||
|
||||
class NDPISubprotocolConfStruct(Structure):
|
||||
_fields_ = [("func", CFUNCTYPE(c_void_p, POINTER(NDPIDetectionModuleStruct), c_char_p, c_char_p, c_int))]
|
||||
|
||||
|
||||
class NDPIAutoma(Structure):
|
||||
_fields_ = [
|
||||
("ac_automa", c_void_p),
|
||||
("ac_automa_finalized", c_uint8)
|
||||
]
|
||||
|
||||
|
||||
class NDPINode(Structure):
|
||||
pass
|
||||
|
||||
|
||||
NDPINode._fields_ = [
|
||||
('key', POINTER(c_char)),
|
||||
('left', POINTER(NDPINode)),
|
||||
('right', POINTER(NDPINode)),
|
||||
]
|
||||
|
||||
|
||||
class NDPICallFunctionStruct(Structure):
|
||||
_fields_ = [
|
||||
("detection_bitmask", NDPIProtocolBitMask),
|
||||
("excluded_protocol_bitmask", NDPIProtocolBitMask),
|
||||
("ndpi_selection_bitmask", c_uint32),
|
||||
("func", CFUNCTYPE(None, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
|
||||
("detection_feature", c_uint8)
|
||||
]
|
||||
|
||||
|
||||
class NDPIProtoDefaultsT(Structure):
|
||||
_fields_ = [
|
||||
("protoName", c_char_p),
|
||||
("protoCategory", c_uint),
|
||||
("subprotocols", c_uint16_p),
|
||||
("subprotocol_count", c_uint32),
|
||||
("protoId", c_uint16),
|
||||
("protoIdx", c_uint16),
|
||||
("protoBreed", c_uint),
|
||||
("func", CFUNCTYPE(None, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
|
||||
]
|
||||
|
||||
|
||||
class NDPIDefaultsPortsTreeNodeT(Structure):
|
||||
_fields_ = [
|
||||
("proto", NDPIProtoDefaultsT),
|
||||
("customUserProto", c_uint8),
|
||||
("default_port", c_int16)
|
||||
]
|
||||
|
||||
|
||||
class SpinlockT(Structure):
|
||||
_fields_ = [("val", c_int)]
|
||||
|
||||
|
||||
class AtomicT(Structure):
|
||||
_fields_ = [("counter", c_int)]
|
||||
|
||||
|
||||
class TimeT(Structure):
|
||||
_fields_ = [("counter", c_longlong)]
|
||||
|
||||
|
||||
class HashIp4pNode(Structure):
|
||||
pass
|
||||
|
||||
|
||||
HashIp4pNode._fields_ = [
|
||||
("next", POINTER(HashIp4pNode)),
|
||||
("prev", POINTER(HashIp4pNode)),
|
||||
("lchg", TimeT),
|
||||
("port", c_uint16),
|
||||
("count", c_uint16, 12),
|
||||
("flag", c_uint16, 4),
|
||||
("ip", c_uint32)
|
||||
]
|
||||
|
||||
|
||||
class HashIp4p(Structure):
|
||||
_fields_ = [
|
||||
("top", POINTER(HashIp4pNode)),
|
||||
("lock",SpinlockT),
|
||||
("len", c_size_t)
|
||||
]
|
||||
|
||||
|
||||
class HashIp4pTable(Structure):
|
||||
_fields_ = [
|
||||
("size", c_size_t),
|
||||
("ipv6", c_int),
|
||||
("lock", SpinlockT),
|
||||
("count", AtomicT),
|
||||
("tbl", HashIp4p)
|
||||
]
|
||||
|
||||
|
||||
class BtAnnounce(Structure):
|
||||
_fields_ = [
|
||||
("hash", c_uint32 * 5),
|
||||
("ip", c_uint32 * 4),
|
||||
("time", c_uint32),
|
||||
("port", c_uint16),
|
||||
("name_len", c_uint8),
|
||||
("name", c_uint8 * 149)
|
||||
]
|
||||
|
||||
|
||||
class NDPILruCacheEntry(Structure):
|
||||
_fields_ = [
|
||||
("key", c_uint32),
|
||||
("is_full", c_uint32, 1),
|
||||
("value", c_uint32, 16),
|
||||
("pad", c_uint32, 15)
|
||||
]
|
||||
|
||||
|
||||
class NDPILruCache(Structure):
|
||||
_fields_ = [
|
||||
("num_entries", c_uint32),
|
||||
("entries", POINTER(NDPILruCacheEntry)),
|
||||
]
|
||||
|
||||
|
||||
class CacheEntry(Structure):
|
||||
pass
|
||||
|
||||
|
||||
CacheEntry._fields_ = [
|
||||
("item", c_void_p),
|
||||
("item_size", c_uint32),
|
||||
("prev", POINTER(CacheEntry)),
|
||||
("next", POINTER(CacheEntry))
|
||||
]
|
||||
|
||||
|
||||
class CacheEntryMap(Structure):
|
||||
pass
|
||||
|
||||
|
||||
CacheEntryMap._fields_ = [
|
||||
("entry", POINTER(CacheEntry)),
|
||||
("next", POINTER(CacheEntryMap)),
|
||||
]
|
||||
|
||||
|
||||
class Cache(Structure):
|
||||
_fields_ = [
|
||||
("size", c_uint32),
|
||||
("max_size", c_uint32),
|
||||
("head", POINTER(CacheEntry)),
|
||||
("tail", POINTER(CacheEntry)),
|
||||
("map", POINTER(POINTER(CacheEntryMap)))
|
||||
]
|
||||
|
||||
|
||||
class CustomCategories(Structure):
|
||||
_fields_ = [
|
||||
("hostnames", NDPIAutoma),
|
||||
("hostnames_shadow", NDPIAutoma),
|
||||
("ipAddresses", c_void_p),
|
||||
("ipAddresses_shadow", c_void_p),
|
||||
("categories_loaded", c_uint8),
|
||||
]
|
||||
|
||||
|
||||
NDPIDetectionModuleStruct._fields_ = [
|
||||
("detection_bitmask", NDPIProtocolBitMask),
|
||||
("current_ts", c_uint32),
|
||||
("custom_category_labels",
|
||||
(c_char * ndpi.ndpi_wrap_num_custom_categories()) * ndpi.ndpi_wrap_custom_category_label_len()),
|
||||
("callback_buffer", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
|
||||
("callback_buffer_size", c_uint32),
|
||||
("callback_buffer_tcp_no_payload", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
|
||||
("callback_buffer_size_tcp_no_payload", c_uint32),
|
||||
("callback_buffer_tcp_payload", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
|
||||
("callback_buffer_size_tcp_payload", c_uint32),
|
||||
("callback_buffer_udp", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
|
||||
("callback_buffer_size_udp", c_uint32),
|
||||
("callback_buffer_non_tcp_udp", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
|
||||
("callback_buffer_size_non_tcp_udp", c_uint32),
|
||||
("tcpRoot", POINTER(NDPIDefaultsPortsTreeNodeT)),
|
||||
("udpRoot", POINTER(NDPIDefaultsPortsTreeNodeT)),
|
||||
("ndpi_log_level", c_uint),
|
||||
("tcp_max_retransmission_window_size", c_uint32),
|
||||
("subprotocol_conf", NDPISubprotocolConfStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
|
||||
("ndpi_num_supported_protocols", c_uint),
|
||||
("ndpi_num_custom_protocols", c_uint),
|
||||
("host_automa", NDPIAutoma),
|
||||
("bigrams_automa", NDPIAutoma),
|
||||
("impossible_bigrams_automa", NDPIAutoma),
|
||||
("custom_categories", CustomCategories),
|
||||
("protocols_ptree", c_void_p),
|
||||
("ip_version_limit", c_uint8),
|
||||
("ookla_cache", POINTER(NDPILruCache)),
|
||||
("tinc_cache", POINTER(Cache)),
|
||||
("proto_defaults", NDPIProtoDefaultsT * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() +
|
||||
ndpi.ndpi_wrap_ndpi_max_num_custom_protocols())),
|
||||
("direction_detect_disable", c_uint8, 1),
|
||||
('_pad', c_uint8, 7),
|
||||
('packet', NDPIPacketStruct),
|
||||
]
|
||||
|
||||
|
||||
class U6Addr(Union):
|
||||
_fields_ = [
|
||||
("u6_addr8", c_uint8 * 16),
|
||||
("u6_addr16", c_uint16 * 8),
|
||||
("u6_addr32", c_uint32 * 4),
|
||||
("u6_addr64", c_uint64 * 2)
|
||||
]
|
||||
|
||||
|
||||
class NDPIIn6Addr(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [("u6_addr", U6Addr)]
|
||||
|
||||
|
||||
class NDPIIpAddrT(Union):
|
||||
_fields_ = [
|
||||
('ipv4', c_uint32),
|
||||
('ipv4_u_int8_t', c_uint8 * 4),
|
||||
('ipv6', NDPIIn6Addr),
|
||||
]
|
||||
|
||||
|
||||
class NDPIIdStruct(Structure):
|
||||
_fields_ = [
|
||||
('detected_protocol_bitmask', NDPIProtocolBitMask),
|
||||
('irc_port', c_uint16 * 8),
|
||||
('last_time_port_used', c_uint32 * 8),
|
||||
('irc_ts', c_uint32),
|
||||
('gnutella_ts', c_uint32),
|
||||
('battlefield_ts', c_uint32),
|
||||
('rtsp_timer', c_uint32),
|
||||
('jabber_stun_or_ft_ts', c_uint32),
|
||||
('directconnect_last_safe_access_time', c_uint32),
|
||||
('detected_directconnect_port', c_uint16),
|
||||
('detected_directconnect_udp_port', c_uint16),
|
||||
('detected_directconnect_ssl_port', c_uint16),
|
||||
('jabber_voice_stun_port', c_uint16 * 6),
|
||||
('jabber_file_transfer_port', c_uint16 * 2),
|
||||
('detected_gnutella_udp_port1', c_uint16),
|
||||
('detected_gnutella_udp_port2', c_uint16),
|
||||
('irc_number_of_port', c_uint8),
|
||||
('oscar_ssl_session_id', c_uint8 * 33),
|
||||
('jabber_voice_stun_used_ports', c_uint8),
|
||||
]
|
||||
|
||||
|
||||
class NDPIFlowTcpStruct(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('smtp_command_bitmask', c_uint16),
|
||||
('pop_command_bitmask', c_uint16),
|
||||
('wa_matched_so_far', c_uint8),
|
||||
('irc_stage', c_uint8),
|
||||
('h323_valid_packets', c_uint8),
|
||||
('gnutella_msg_id', c_uint8 * 3),
|
||||
('irc_3a_counter', c_uint32, 3),
|
||||
('irc_stage2', c_uint32, 5),
|
||||
('irc_direction', c_uint32, 2),
|
||||
('irc_0x1000_full', c_uint32, 1),
|
||||
('usenet_stage', c_uint32, 2),
|
||||
('http_stage', c_uint32, 2),
|
||||
('http_empty_line_seen', c_uint32, 1),
|
||||
('gnutella_stage', c_uint32, 2),
|
||||
('ssh_stage', c_uint32, 3),
|
||||
('vnc_stage', c_uint32, 2),
|
||||
('telnet_stage', c_uint32, 2),
|
||||
('tls_seen_client_cert', c_uint8, 1),
|
||||
('tls_seen_server_cert', c_uint8, 1),
|
||||
('tls_seen_certificate', c_uint8, 1),
|
||||
('tls_srv_cert_fingerprint_found', c_uint8, 1),
|
||||
('tls_srv_cert_fingerprint_processed', c_uint8, 1),
|
||||
('tls_stage', c_uint8, 2),
|
||||
('tls_record_offset', c_int16),
|
||||
('tls_fingerprint_len', c_int16),
|
||||
('tls_sha1_certificate_fingerprint', c_uint8 * 20),
|
||||
('postgres_stage', c_uint32, 3),
|
||||
('seen_syn', c_uint32, 1),
|
||||
('seen_syn_ack', c_uint32, 1),
|
||||
('seen_ack', c_uint32, 1),
|
||||
('icecast_stage', c_uint32, 1),
|
||||
('dofus_stage', c_uint32, 1),
|
||||
('fiesta_stage', c_uint32, 2),
|
||||
('wow_stage', c_uint32, 2),
|
||||
('shoutcast_stage', c_uint32, 2),
|
||||
('rtp_special_packets_seen', c_uint32, 1),
|
||||
('mail_pop_stage', c_uint32, 2),
|
||||
('mail_imap_stage', c_uint32, 3),
|
||||
('mail_imap_starttls', c_uint32, 2),
|
||||
('skype_packet_id', c_uint8),
|
||||
('lotus_notes_packet_id', c_uint8),
|
||||
('teamviewer_stage', c_uint8),
|
||||
('prev_zmq_pkt_len', c_uint8),
|
||||
('prev_zmq_pkt', c_char * 10),
|
||||
('ppstream_stage', c_uint32, 3),
|
||||
('memcached_matches', c_uint8),
|
||||
('nest_log_sink_matches', c_uint8),
|
||||
]
|
||||
|
||||
|
||||
class NDPIFlowUdpStruct(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('ppstream_stage', c_uint32, 3),
|
||||
('halflife2_stage', c_uint32, 2),
|
||||
('tftp_stage', c_uint32, 2),
|
||||
('aimini_stage', c_uint32, 5),
|
||||
('xbox_stage', c_uint32, 1),
|
||||
('skype_packet_id', c_uint8),
|
||||
('teamviewer_stage', c_uint8),
|
||||
('eaq_pkt_id', c_uint8),
|
||||
('eaq_sequence', c_uint32),
|
||||
('rx_conn_epoch', c_uint32),
|
||||
('rx_conn_id', c_uint32),
|
||||
('memcached_matches', c_uint8),
|
||||
('wireguard_stage', c_uint8),
|
||||
('wireguard_peer_index', c_uint32 * 2),
|
||||
('quic_reasm_buf', POINTER(c_uint8)),
|
||||
('quic_reasm_buf_len', c_uint32),
|
||||
('csgo_strid', c_uint8 * 18),
|
||||
('csgo_state', c_uint8),
|
||||
('csgo_s2', c_uint8),
|
||||
('csgo_id2', c_uint32),
|
||||
('rdp_to_srv', c_uint8 * 3),
|
||||
('rdp_from_srv', c_uint8 * 3),
|
||||
('rdp_to_srv_pkts,', c_uint8),
|
||||
('rdp_from_srv_pkts', c_uint8),
|
||||
('imo_last_one_byte_pkt,', c_uint8),
|
||||
('imo_last_byte', c_uint8),
|
||||
]
|
||||
|
||||
|
||||
class L4(Union):
|
||||
_fields_ = [("tcp", NDPIFlowTcpStruct), ("udp", NDPIFlowUdpStruct)]
|
||||
|
||||
|
||||
class Http(Structure):
|
||||
_fields_ = [
|
||||
("method", c_int),
|
||||
("request_version", c_uint8),
|
||||
("response_status_code", c_uint16),
|
||||
("url", c_char_p),
|
||||
("content_type", c_char_p),
|
||||
("user_agent", c_char_p),
|
||||
("detected_os", c_char_p),
|
||||
("nat_ip", c_char_p),
|
||||
]
|
||||
|
||||
|
||||
class Dns(Structure):
|
||||
_fields_ = [
|
||||
("num_queries", c_uint8),
|
||||
("num_answers", c_uint8),
|
||||
("reply_code", c_uint8),
|
||||
("is_query", c_uint8),
|
||||
("query_type", c_uint16),
|
||||
("query_class", c_uint16),
|
||||
("rsp_type", c_uint16),
|
||||
("rsp_addr", NDPIIpAddrT)
|
||||
]
|
||||
|
||||
|
||||
class Ntp(Structure):
|
||||
_fields_ = [("request_code", c_uint8),
|
||||
("version", c_uint8)]
|
||||
|
||||
|
||||
class Kerberos(Structure):
|
||||
_fields_ = [("cname", c_char * 24),
|
||||
("realm", c_char * 24)]
|
||||
|
||||
|
||||
class QuicSsl(Structure):
|
||||
_fields_ = [
|
||||
("ssl_version", c_uint16),
|
||||
("client_certificate", c_char * 64),
|
||||
("server_certificate", c_char * 64),
|
||||
("server_organization", c_char * 64),
|
||||
('notBefore', c_uint32),
|
||||
('notAfter', c_uint32),
|
||||
("ja3_client", c_char * 33),
|
||||
("ja3_server", c_char * 33),
|
||||
("server_cipher", c_uint16),
|
||||
("server_unsafe_cipher", c_int)
|
||||
]
|
||||
|
||||
|
||||
class Stun(Structure):
|
||||
_fields_ = [
|
||||
("num_udp_pkts", c_uint8),
|
||||
("num_processed_pkts", c_uint8),
|
||||
("num_binding_requests", c_uint8),
|
||||
]
|
||||
|
||||
|
||||
class Ssh(Structure):
|
||||
_fields_ = [
|
||||
("client_signature", c_char * 48),
|
||||
("server_signature", c_char * 48),
|
||||
("hassh_client", c_char * 33),
|
||||
("hassh_server", c_char * 33)
|
||||
]
|
||||
|
||||
|
||||
class Mdns(Structure):
|
||||
_fields_ = [("answer", c_char * 96)]
|
||||
|
||||
|
||||
class Ubntac2(Structure):
|
||||
_fields_ = [("version", c_char * 32)]
|
||||
|
||||
|
||||
class FtpImapPopSmtp(Structure):
|
||||
_fields_ = [
|
||||
("auth_found", c_uint8, 1),
|
||||
("auth_failed", c_uint8, 1),
|
||||
("auth_tls", c_uint8, 1),
|
||||
("auth_done", c_uint8, 1),
|
||||
("_pad", c_uint8, 4),
|
||||
("username", c_char * 32),
|
||||
("password", c_char * 16)
|
||||
]
|
||||
|
||||
class Bittorrent(Structure):
|
||||
_fields_ = [("hash", c_char * 20)]
|
||||
|
||||
|
||||
class Dhcp(Structure):
|
||||
_fields_ = [
|
||||
("fingerprint", c_char * 48),
|
||||
("class_ident", c_char * 48)
|
||||
]
|
||||
|
||||
|
||||
class Protos(Union):
|
||||
_fields_ = [
|
||||
("dns", Dns),
|
||||
("kerberos", Kerberos),
|
||||
("quic_ssl", QuicSsl),
|
||||
("ssh", Ssh),
|
||||
("mdns", Mdns),
|
||||
("ubntac2", Ubntac2),
|
||||
("bittorrent", Bittorrent),
|
||||
("dhcp", Dhcp)
|
||||
]
|
||||
|
||||
|
||||
class TincCacheEntry(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('src_address', c_uint32),
|
||||
('dst_address', c_uint32),
|
||||
('dst_port', c_uint16),
|
||||
]
|
||||
|
||||
|
||||
class NDPIIntOneLineStruct(Structure):
|
||||
_fields_ = [
|
||||
('ptr', POINTER(c_uint8)),
|
||||
('len', c_uint16),
|
||||
]
|
||||
|
||||
|
||||
class NDPIIphdr(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('ihl', c_uint8, 4),
|
||||
('version', c_uint8, 4),
|
||||
('tos', c_uint8),
|
||||
('tot_len', c_uint16),
|
||||
('id', c_uint16),
|
||||
('frag_off', c_uint16),
|
||||
('ttl', c_uint8),
|
||||
('protocol', c_uint8),
|
||||
('check', c_uint16),
|
||||
('saddr', c_uint32),
|
||||
('daddr', c_uint32)]
|
||||
|
||||
|
||||
class NDPIIp6Hdrctl(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('ip6_un1_flow', c_uint32),
|
||||
('ip6_un1_plen', c_uint16),
|
||||
('ip6_un1_nxt', c_uint8),
|
||||
('ip6_un1_hlim', c_uint8),
|
||||
]
|
||||
|
||||
|
||||
class NDPIIpv6hdr(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('ip6_hdr', NDPIIp6Hdrctl),
|
||||
('ip6_src', NDPIIn6Addr),
|
||||
('ip6_dst', NDPIIn6Addr),
|
||||
]
|
||||
|
||||
|
||||
class NDPITcpHdr(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('source', c_uint16),
|
||||
('dest', c_uint16),
|
||||
('seq', c_uint32),
|
||||
('ack_seq', c_uint32),
|
||||
('res1', c_uint16, 4),
|
||||
('doff', c_uint16, 4),
|
||||
('fin', c_uint16, 1),
|
||||
('syn', c_uint16, 1),
|
||||
('rst', c_uint16, 1),
|
||||
('psh', c_uint16, 1),
|
||||
('ack', c_uint16, 1),
|
||||
('urg', c_uint16, 1),
|
||||
('ece', c_uint16, 1),
|
||||
('cwr', c_uint16, 1),
|
||||
('window', c_uint16),
|
||||
('check', c_uint16),
|
||||
('urg_ptr', c_uint16),
|
||||
]
|
||||
|
||||
|
||||
class NDPIUdpHdr(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
('source', c_uint16),
|
||||
('dest', c_uint16),
|
||||
('len', c_uint16),
|
||||
('check', c_uint16),
|
||||
]
|
||||
|
||||
|
||||
class NDPIPacketStruct(Structure):
|
||||
_fields_ = [
|
||||
('iph', POINTER(NDPIIphdr)),
|
||||
('iphv6', POINTER(NDPIIpv6hdr)),
|
||||
('tcp', POINTER(NDPITcpHdr)),
|
||||
('udp', POINTER(NDPIUdpHdr)),
|
||||
('generic_l4_ptr', POINTER(c_uint8)),
|
||||
('payload', POINTER(c_uint8)),
|
||||
('current_timestamp_ms', c_uint64),
|
||||
('detected_protocol_stack', c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()),
|
||||
('line', NDPIIntOneLineStruct * 64),
|
||||
('host_line', NDPIIntOneLineStruct),
|
||||
('forwarded_line', NDPIIntOneLineStruct),
|
||||
('referer_line', NDPIIntOneLineStruct),
|
||||
('content_line', NDPIIntOneLineStruct),
|
||||
('accept_line', NDPIIntOneLineStruct),
|
||||
('user_agent_line', NDPIIntOneLineStruct),
|
||||
('http_url_name', NDPIIntOneLineStruct),
|
||||
('http_encoding', NDPIIntOneLineStruct),
|
||||
('http_transfer_encoding', NDPIIntOneLineStruct),
|
||||
('http_contentlen', NDPIIntOneLineStruct),
|
||||
('http_cookie', NDPIIntOneLineStruct),
|
||||
('http_origin', NDPIIntOneLineStruct),
|
||||
('http_x_session_type', NDPIIntOneLineStruct),
|
||||
('server_line', NDPIIntOneLineStruct),
|
||||
('http_method', NDPIIntOneLineStruct),
|
||||
('http_response', NDPIIntOneLineStruct),
|
||||
('http_num_headers', c_uint8),
|
||||
('l3_packet_len', c_uint16),
|
||||
('payload_packet_len', c_uint16),
|
||||
('parsed_lines', c_uint16),
|
||||
('empty_line_position', c_uint16),
|
||||
('tcp_retransmission', c_uint8),
|
||||
('packet_lines_parsed_complete', c_uint8, 1),
|
||||
('packet_direction', c_uint8, 1),
|
||||
('empty_line_position_set', c_uint8, 1),
|
||||
('http_check_content', c_uint8, 1),
|
||||
('pad', c_uint8, 4),
|
||||
]
|
||||
|
||||
|
||||
class NDPIFlowStructStack(Structure):
|
||||
_pack_ = 1
|
||||
_fields_ = [
|
||||
("detected_protocol_stack", c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()),
|
||||
]
|
||||
|
||||
|
||||
NDPIFlowStruct._fields_ = [
|
||||
("ndpi_flow_stack", NDPIFlowStructStack),
|
||||
("guessed_protocol_id", c_uint16),
|
||||
("guessed_host_protocol_id", c_uint16),
|
||||
("guessed_category", c_uint16),
|
||||
("guessed_header_category", c_uint16),
|
||||
("l4_proto", c_uint8),
|
||||
("protocol_id_already_guessed", c_uint8, 1),
|
||||
("host_already_guessed", c_uint8, 1),
|
||||
("init_finished", c_uint8, 1),
|
||||
("setup_packet_direction", c_uint8, 1),
|
||||
("packet_direction", c_uint8, 1),
|
||||
("check_extra_packets", c_uint8, 1),
|
||||
("next_tcp_seq_nr", c_uint32 * 2),
|
||||
("max_extra_packets_to_check", c_uint8),
|
||||
("num_extra_packets_checked", c_uint8),
|
||||
("num_processed_pkts", c_uint8),
|
||||
("extra_packets_func", CFUNCTYPE(c_int, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
|
||||
("l4", L4),
|
||||
("host_server_name", c_char * 80),
|
||||
("http", Http),
|
||||
("stun", Stun),
|
||||
("ftp_imap_pop_smtp", FtpImapPopSmtp),
|
||||
("protos", Protos),
|
||||
("excluded_protocol_bitmask", NDPIProtocolBitMask),
|
||||
("category", c_int),
|
||||
('redis_s2d_first_char', c_uint8),
|
||||
('redis_d2s_first_char', c_uint8),
|
||||
('packet_counter', c_uint16),
|
||||
('packet_direction_counter', c_uint16 * 2),
|
||||
('byte_counter', c_uint16 * 2),
|
||||
('bittorrent_stage', c_uint8),
|
||||
('directconnect_stage', c_uint8, 2),
|
||||
('http_detected', c_uint8, 1),
|
||||
('http_upper_protocol', c_uint16),
|
||||
('http_lower_protocol', c_uint16),
|
||||
('rtsprdt_stage', c_uint8, 2),
|
||||
('zattoo_stage', c_uint8, 3),
|
||||
('thunder_stage', c_uint8, 2),
|
||||
('oscar_ssl_voice_stage', c_uint8, 3),
|
||||
('oscar_video_voice', c_uint8, 1),
|
||||
('florensia_stage', c_uint8, 1),
|
||||
('socks5_stage', c_uint8, 2),
|
||||
('socks4_stage', c_uint8, 2),
|
||||
('edonkey_stage', c_uint8, 2),
|
||||
('ftp_control_stage', c_uint8, 2),
|
||||
('rtmp_stage', c_uint8, 2),
|
||||
('pando_stage', c_uint8, 3),
|
||||
('steam_stage1', c_uint16, 3),
|
||||
('steam_stage2', c_uint16, 2),
|
||||
('steam_stage3', c_uint16, 2),
|
||||
('pplive_stage1', c_uint8, 3),
|
||||
('pplive_stage2', c_uint8, 2),
|
||||
('pplive_stage3', c_uint8, 2),
|
||||
('starcraft_udp_stage', c_uint8, 3),
|
||||
('ovpn_session_id', c_uint8 * 8),
|
||||
('ovpn_counter', c_uint8),
|
||||
('key_mining_cache', c_uint32),
|
||||
('tinc_state', c_uint8),
|
||||
('TincCacheEntry', TincCacheEntry),
|
||||
('src', POINTER(NDPIIdStruct)),
|
||||
('dst', POINTER(NDPIIdStruct))
|
||||
]
|
||||
|
||||
# ----------------------------------------------- nDPI APIs ------------------------------------------------------------
|
||||
|
||||
""" ndpi_detection_giveup: Function to be called before we give up with detection for a given flow.
|
||||
This function reduces the NDPI_UNKNOWN_PROTOCOL detection. """
|
||||
ndpi.ndpi_detection_giveup.restype = NDPIProtocol
|
||||
ndpi.ndpi_detection_giveup.argtypes = [POINTER(NDPIDetectionModuleStruct),
|
||||
POINTER(NDPIFlowStruct), c_uint8,
|
||||
POINTER(c_uint8)]
|
||||
|
||||
""" ndpi_detection_process_packet: Processes one packet and returns the ID of the detected protocol.
|
||||
This is the MAIN PACKET PROCESSING FUNCTION. """
|
||||
ndpi.ndpi_detection_process_packet.restype = NDPIProtocol
|
||||
ndpi.ndpi_detection_process_packet.argtypes = [POINTER(NDPIDetectionModuleStruct),
|
||||
POINTER(NDPIFlowStruct),
|
||||
POINTER(c_ubyte),
|
||||
c_ushort,
|
||||
c_uint64,
|
||||
POINTER(NDPIIdStruct),
|
||||
POINTER(NDPIIdStruct)]
|
||||
|
||||
""" ndpi_ssl_version2str : Converts ssl version to readable string """
|
||||
ndpi.ndpi_ssl_version2str.restype = c_char_p
|
||||
ndpi.ndpi_ssl_version2str.argtypes = [c_int16, POINTER(c_uint8)]
|
||||
|
||||
""" ndpi_init_detection_module: Returns a new initialized detection module.
|
||||
Note that before you can use it you can still load hosts and do other things. As soon as you are ready to use
|
||||
it do not forget to call first ndpi_finalize_initalization() """
|
||||
ndpi.ndpi_init_detection_module.restype = POINTER(NDPIDetectionModuleStruct)
|
||||
|
||||
|
||||
def ndpi_ndpi_finalize_initalization(detection_module):
|
||||
""" ndpi_finalize_initalization: Completes the initialization (ndpi_revision >= 3.1)"""
|
||||
if cast(ndpi.ndpi_revision(), c_char_p).value.decode("utf-8")[:3] >= '3.1':
|
||||
ndpi.ndpi_finalize_initalization.restype = c_void_p
|
||||
ndpi.ndpi_finalize_initalization.argtypes = [POINTER(NDPIDetectionModuleStruct)]
|
||||
return ndpi.ndpi_finalize_initalization(detection_module)
|
||||
else:
|
||||
# ignore it
|
||||
return None
|
||||
|
||||
|
||||
""" ndpi_tfind: find a node, or return 0. """
|
||||
ndpi.ndpi_tfind.restype = c_void_p
|
||||
|
||||
""" ndpi_tsearch: ftp://ftp.cc.uoc.gr/mirrors/OpenBSD/src/lib/libc/stdlib/tsearch.c
|
||||
find or insert datum into search tree. """
|
||||
ndpi.ndpi_tsearch.restype = c_void_p
|
||||
ndpi.ndpi_tsearch.argtypes = [c_void_p, POINTER(c_void_p), CFUNCTYPE(c_int, c_void_p, c_void_p)]
|
||||
|
||||
""" ndpi_revision: Get the nDPI version release. """
|
||||
ndpi.ndpi_revision.restype = c_void_p
|
||||
|
||||
""" ndpi_get_proto_name: Get the protocol name associated to the ID."""
|
||||
ndpi.ndpi_get_proto_name.restype = c_void_p
|
||||
|
||||
""" ndpi_category_get_name: Get protocol category as string."""
|
||||
ndpi.ndpi_category_get_name.restype = c_void_p
|
||||
|
||||
""" ndpi_get_num_supported_protocols: Get the total number of the supported protocols."""
|
||||
ndpi.ndpi_get_num_supported_protocols.restype = c_uint
|
||||
|
||||
""" ndpi_wrap_NDPI_BITMASK_SET_ALL: memset((char *)(p), 0xFF, sizeof(*(p)))"""
|
||||
ndpi.ndpi_wrap_NDPI_BITMASK_SET_ALL.argtypes = [POINTER(NDPIProtocolBitMask)]
|
||||
|
||||
""" ndpi_set_protocol_detection_bitmask2: Sets the protocol bitmask2."""
|
||||
ndpi.ndpi_set_protocol_detection_bitmask2.argtypes = [POINTER(NDPIDetectionModuleStruct),
|
||||
POINTER(NDPIProtocolBitMask)]
|
||||
|
||||
""" ndpi_twalk: Walk the nodes of a tree. """
|
||||
ndpi.ndpi_twalk.argtypes = [c_void_p, CFUNCTYPE(None, c_void_p, c_int32, c_int, c_void_p), c_void_p]
|
||||
|
||||
""" ndpi_tdestroy: node destroy. """
|
||||
ndpi.ndpi_tdestroy.argtypes = [c_void_p, CFUNCTYPE(None, c_void_p)]
|
||||
@@ -1,60 +0,0 @@
|
||||
/*
|
||||
* ndpi_wrap.c
|
||||
*
|
||||
* Copyright (C) 2011-19 - ntop.org
|
||||
*
|
||||
* nDPI is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Lesser General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* nDPI is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Lesser General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Lesser General Public License
|
||||
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
#include "sys/types.h"
|
||||
#include "ndpi_config.h"
|
||||
#include "ndpi_main.h"
|
||||
|
||||
int ndpi_wrap_get_api_version(){
|
||||
return NDPI_API_VERSION;
|
||||
}
|
||||
|
||||
int ndpi_wrap_ndpi_num_fds_bits(){
|
||||
return NDPI_NUM_FDS_BITS;
|
||||
}
|
||||
|
||||
int ndpi_wrap_num_custom_categories(){
|
||||
return NUM_CUSTOM_CATEGORIES;
|
||||
}
|
||||
|
||||
int ndpi_wrap_custom_category_label_len(){
|
||||
return CUSTOM_CATEGORY_LABEL_LEN;
|
||||
}
|
||||
|
||||
int ndpi_wrap_ndpi_max_supported_protocols(){
|
||||
return NDPI_MAX_SUPPORTED_PROTOCOLS;
|
||||
}
|
||||
|
||||
int ndpi_wrap_ndpi_max_num_custom_protocols(){
|
||||
return NDPI_MAX_NUM_CUSTOM_PROTOCOLS;
|
||||
}
|
||||
|
||||
int ndpi_wrap_ndpi_procol_size(){
|
||||
return NDPI_PROTOCOL_SIZE;
|
||||
}
|
||||
|
||||
void ndpi_wrap_NDPI_BITMASK_SET_ALL(NDPI_PROTOCOL_BITMASK* bitmask){
|
||||
NDPI_ONE(bitmask);
|
||||
}
|
||||
|
||||
void dummy() {
|
||||
/* Dummy call just to cause linker to include the ndpi library */
|
||||
ndpi_tfind(NULL, NULL, NULL);
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1 @@
|
||||
cffi>=1.15.0
|
||||
@@ -0,0 +1,57 @@
|
||||
"""
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
setup.py
|
||||
Copyright (C) 2011-22 - ntop.org
|
||||
This file is part of nDPI, an open source deep packet inspection library.
|
||||
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
|
||||
version.
|
||||
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
|
||||
You should have received a copy of the GNU Lesser General Public License along with NFStream.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
from setuptools import setup
|
||||
import os
|
||||
|
||||
|
||||
this_directory = os.path.abspath(os.path.dirname(__file__))
|
||||
with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
|
||||
long_description = f.read()
|
||||
|
||||
|
||||
setup(
|
||||
name="ndpi",
|
||||
version='4.3.0',
|
||||
url='https://www.ntop.org/products/deep-packet-inspection/ndpi/',
|
||||
license='LGPLv3',
|
||||
description="Open and Extensible LGPLv3 Deep Packet Inspection Library",
|
||||
long_description=long_description,
|
||||
long_description_content_type='text/markdown',
|
||||
author='Zied Aouini',
|
||||
author_email='aouinizied@gmail.com',
|
||||
packages=['ndpi'],
|
||||
setup_requires=["cffi>=1.15.0"],
|
||||
cffi_modules=["ndpi/ndpi_build.py:ffi_builder"],
|
||||
install_requires=["cffi>=1.15.0"],
|
||||
platforms=["Linux",
|
||||
"Mac OS-X",
|
||||
"Windows",
|
||||
"Unix"],
|
||||
classifiers=[
|
||||
'Development Status :: 5 - Production/Stable',
|
||||
'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)',
|
||||
'Programming Language :: Python :: 3 :: Only',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: 3.9',
|
||||
'Programming Language :: Python :: 3.10',
|
||||
'Topic :: System :: Networking :: Monitoring',
|
||||
],
|
||||
project_urls={
|
||||
'GitHub': 'https://github.com/ntop/nDPI',
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
tests.py
|
||||
Copyright (C) 2011-22 - ntop.org
|
||||
This file is part of nDPI, an open source deep packet inspection library.
|
||||
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
|
||||
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
|
||||
version.
|
||||
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
|
||||
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
|
||||
You should have received a copy of the GNU Lesser General Public License along with NFStream.
|
||||
If not, see <http://www.gnu.org/licenses/>.
|
||||
------------------------------------------------------------------------------------------------------------------------
|
||||
"""
|
||||
|
||||
from ndpi import NDPI, NDPIFlow
|
||||
import time
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
nDPI = NDPI()
|
||||
ndpi_flow = NDPIFlow()
|
||||
nDPI.process_packet(ndpi_flow, b'', time.time())
|
||||
nDPI.giveup(ndpi_flow)
|
||||
print("nDPI Python bindings: OK")
|
||||
except Exception:
|
||||
raise AssertionError("nDPI Python bindings: KO")
|
||||
+38
-18
@@ -61,7 +61,6 @@ typedef enum {
|
||||
- nDPI/wireshark/ndpi.lua
|
||||
- ndpi_risk2str (in ndpi_utils.c)
|
||||
- doc/flow_risks.rst
|
||||
- ndpi_risk_enum (in python/ndpi.py)
|
||||
- ndpi_known_risks (ndpi_main.c)
|
||||
|
||||
To make sure the risk is also seen by ntopng:
|
||||
@@ -170,7 +169,12 @@ typedef struct node_t {
|
||||
/* NDPI_MASK_SIZE */
|
||||
typedef u_int32_t ndpi_ndpi_mask;
|
||||
|
||||
|
||||
/* NDPI_PROTO_BITMASK_STRUCT */
|
||||
#ifdef NDPI_CFFI_PREPROCESSING
|
||||
#define NDPI_NUM_FDS_BITS 16
|
||||
#endif
|
||||
|
||||
typedef struct ndpi_protocol_bitmask_struct {
|
||||
ndpi_ndpi_mask fds_bits[NDPI_NUM_FDS_BITS];
|
||||
} ndpi_protocol_bitmask_struct_t;
|
||||
@@ -181,6 +185,7 @@ typedef void (*ndpi_debug_function_ptr) (u_int32_t protocol, void *module_struct
|
||||
const char *func, unsigned line,
|
||||
const char *format, ...);
|
||||
|
||||
#ifndef NDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED
|
||||
/* ************************************************************ */
|
||||
/* ******************* NDPI NETWORKS HEADERS ****************** */
|
||||
/* ************************************************************ */
|
||||
@@ -198,6 +203,13 @@ typedef void (*ndpi_debug_function_ptr) (u_int32_t protocol, void *module_struct
|
||||
#define PACK_OFF __attribute__((packed))
|
||||
#endif
|
||||
|
||||
/* PLEASE DO NOT REMOVE OR CHANGE THE ORDER OF WHAT IS DELIMITED BY CFFI.NDPI_PACKED_STRUCTURES FLAG AS IT IS USED FOR
|
||||
PYTHON BINDINGS AUTO GENERATION */
|
||||
#ifdef NDPI_CFFI_PREPROCESSING
|
||||
#define PACK_ON
|
||||
#define PACK_OFF
|
||||
#endif
|
||||
//CFFI.NDPI_PACKED_STRUCTURES
|
||||
PACK_ON
|
||||
struct ndpi_chdlc
|
||||
{
|
||||
@@ -435,12 +447,6 @@ struct ndpi_dns_packet_header {
|
||||
u_int16_t additional_rrs;
|
||||
} PACK_OFF;
|
||||
|
||||
typedef union
|
||||
{
|
||||
u_int32_t ipv4;
|
||||
struct ndpi_in6_addr ipv6;
|
||||
} ndpi_ip_addr_t;
|
||||
|
||||
|
||||
/* +++++++++++++++++++++++ ICMP header +++++++++++++++++++++++ */
|
||||
|
||||
@@ -490,6 +496,22 @@ struct ndpi_vxlanhdr {
|
||||
/* ******************* ********************* ****************** */
|
||||
/* ************************************************************ */
|
||||
|
||||
PACK_ON struct tinc_cache_entry {
|
||||
u_int32_t src_address;
|
||||
u_int32_t dst_address;
|
||||
u_int16_t dst_port;
|
||||
} PACK_OFF;
|
||||
//CFFI.NDPI_PACKED_STRUCTURES
|
||||
#endif // NDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED
|
||||
|
||||
|
||||
typedef union
|
||||
{
|
||||
u_int32_t ipv4;
|
||||
struct ndpi_in6_addr ipv6;
|
||||
} ndpi_ip_addr_t;
|
||||
|
||||
|
||||
typedef struct message {
|
||||
u_int8_t *buffer;
|
||||
u_int buffer_len, buffer_used;
|
||||
@@ -499,12 +521,6 @@ typedef struct message {
|
||||
/* NDPI_PROTOCOL_TINC */
|
||||
#define TINC_CACHE_MAX_SIZE 10
|
||||
|
||||
PACK_ON struct tinc_cache_entry {
|
||||
u_int32_t src_address;
|
||||
u_int32_t dst_address;
|
||||
u_int16_t dst_port;
|
||||
} PACK_OFF;
|
||||
|
||||
/*
|
||||
In case the typedef below is modified, please update
|
||||
ndpi_http_method2str (ndpi_utils.c)
|
||||
@@ -968,6 +984,9 @@ typedef struct ndpi_proto {
|
||||
#define _NDPI_CONFIG_H_
|
||||
#endif
|
||||
|
||||
/* PLEASE DO NOT REMOVE OR CHANGE THE ORDER OF WHAT IS DELIMITED BY CFFI.NDPI_MODULE_STRUCT FLAG AS IT IS USED FOR
|
||||
PYTHON BINDINGS AUTO GENERATION */
|
||||
//CFFI.NDPI_MODULE_STRUCT
|
||||
typedef enum {
|
||||
ndpi_stun_cache,
|
||||
ndpi_hangout_cache
|
||||
@@ -1096,6 +1115,7 @@ struct ndpi_detection_module_struct {
|
||||
};
|
||||
|
||||
#endif /* NDPI_LIB_COMPILATION */
|
||||
//CFFI.NDPI_MODULE_STRUCT
|
||||
|
||||
typedef enum {
|
||||
ndpi_cipher_safe = NDPI_CIPHER_SAFE,
|
||||
@@ -1113,11 +1133,7 @@ struct tls_heuristics {
|
||||
u_int8_t is_safari_tls:1, is_firefox_tls:1, is_chrome_tls:1, notused:5;
|
||||
};
|
||||
|
||||
/*
|
||||
NOTE
|
||||
When the struct below is modified don't forget to update
|
||||
- ndpi_flow_struct (in python/ndpi.py)
|
||||
*/
|
||||
|
||||
struct ndpi_flow_struct {
|
||||
u_int16_t detected_protocol_stack[NDPI_PROTOCOL_SIZE];
|
||||
|
||||
@@ -1473,7 +1489,11 @@ typedef struct {
|
||||
|
||||
#define ndpi_private_deserializer ndpi_private_serializer
|
||||
|
||||
#ifdef NDPI_CFFI_PREPROCESSING
|
||||
typedef struct { char c[72]; } ndpi_serializer;
|
||||
#else
|
||||
typedef struct { char c[sizeof(ndpi_private_serializer)]; } ndpi_serializer;
|
||||
#endif
|
||||
|
||||
#define ndpi_deserializer ndpi_serializer
|
||||
|
||||
|
||||
@@ -12,12 +12,13 @@
|
||||
// #define NDPI_ENABLE_DEBUG_INFO_MESSAGES
|
||||
// #define NDPI_ENABLE_DEBUG_TRACE_MESSAGES
|
||||
|
||||
#ifndef NDPI_CFFI_PREPROCESSING
|
||||
extern void printRawData(const uint8_t *ptr, size_t len);
|
||||
//extern uint8_t add_segment_to_buffer( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph, uint32_t waited);
|
||||
//extern uint8_t check_for_sequence( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph);
|
||||
|
||||
extern u_int8_t ndpi_ends_with(char *str, char *ends);
|
||||
|
||||
#endif // NDPI_CFFI_PREPROCESSING
|
||||
/* **************************************** */
|
||||
|
||||
/* Can't call libc functions from kernel space, define some stub instead */
|
||||
|
||||
Reference in New Issue
Block a user