Complete rework of nDPI Python bindings (cffi API, automatic generation, packaging and CI integration)

This commit is contained in:
aouinizied
2022-03-22 13:19:27 +01:00
parent 93f723d50f
commit beef4f997b
22 changed files with 639 additions and 2570 deletions
+27 -4
View File
@@ -43,6 +43,33 @@ jobs:
path: coverage_report
retention-days: 7
python-bindings:
name: Python Bindings (ubuntu-latest)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
- name: Build nDPI library
run: |
env CC=gcc CFLAGS='-Werror' ./autogen.sh
make
- name: Generate Python bindings
run: |
cd python
python3 -m pip install --upgrade pip
python3 -m pip install -r requirements.txt
python3 setup.py install
cd ..
- name: Test Python Bindings
run: |
cd python
python3 tests.py
test-scripts:
name: Test Utils (ubuntu-latest)
runs-on: ubuntu-latest
@@ -267,7 +294,6 @@ jobs:
make all
make -C example ndpiSimpleIntegration
make -C rrdtool
make -C python
- name: Install nDPI
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
run: |
@@ -320,7 +346,6 @@ jobs:
make all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make -C python &&
./tests/do.sh &&
./tests/do-unit.sh
"
@@ -345,7 +370,6 @@ jobs:
make all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make -C python &&
./tests/do.sh &&
./tests/do-unit.sh
"
@@ -370,7 +394,6 @@ jobs:
make all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make -C python &&
./tests/do.sh &&
./tests/do-unit.sh
"
-1
View File
@@ -61,7 +61,6 @@ stamp-h1
/fuzz/fuzz_ndpi_reader_with_main
/fuzz/fuzz_process_packet
/fuzz/fuzz_process_packet_with_main
/python/Makefile
/tests/dga/Makefile
/tests/do.sh
/tests/unit/Makefile
+1 -1
View File
@@ -8,7 +8,7 @@ endif
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = libndpi.pc
EXTRA_DIST = README.md CHANGELOG.md CONTRIBUTING.md autogen.sh wireshark python windows utils packages example/MacOS doc/api.rst doc/conf.py doc/flow_risks.rst doc/guide/nDPI_QuickStartGuide.pages doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst doc/Makefile doc/what_is_ndpi.rst
EXTRA_DIST = README.md CHANGELOG.md CONTRIBUTING.md autogen.sh wireshark windows utils packages example/MacOS doc/api.rst doc/conf.py doc/flow_risks.rst doc/guide/nDPI_QuickStartGuide.pages doc/guide/nDPI_QuickStartGuide.pdf doc/img/logo.png doc/index.rst doc/Makefile doc/what_is_ndpi.rst
changelog:
git log --since={`curl -s https://github.com/ntop/ndpi/releases | grep datetime | head -n1 | egrep -o "[0-9]+\-[0-9]+\-[0-9]+"`} --name-only --pretty=format:" - %s" | grep "^ " > CHANGELOG.latest
+1 -1
View File
@@ -285,7 +285,7 @@ dnl> ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lcurl"
dnl> AC_DEFINE_UNQUOTED(HAVE_CURL, 1, [curl is present])
dnl> fi
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/performance/Makefile tests/dga/Makefile rrdtool/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile python/Makefile fuzz/Makefile src/include/ndpi_api.h])
AC_CONFIG_FILES([Makefile example/Makefile example/Makefile.dpdk tests/Makefile tests/unit/Makefile tests/performance/Makefile tests/dga/Makefile rrdtool/Makefile libndpi.pc src/include/ndpi_define.h src/lib/Makefile fuzz/Makefile src/include/ndpi_api.h])
AC_CONFIG_FILES([tests/do.sh], [chmod +x tests/do.sh])
AC_CONFIG_HEADERS(src/include/ndpi_config.h)
AC_SUBST(GIT_RELEASE)
-27
View File
@@ -1,27 +0,0 @@
CC=@CC@
CFLAGS=-I. -I../src/include -I./src/lib/third_party/include @CFLAGS@ @CUSTOM_NDPI@ -shared
#LIBNDPI=../src/lib/libndpi.so.@NDPI_VERSION_SHORT@
LIBNDPI=../src/lib/libndpi.a
LDFLAGS=@LDFLAGS@
LIBS=$(CFILE) $(LIBNDPI) -lpcap @LIBS@ @ADDITIONAL_LIBS@
SHARE = -soname,ndpi_wrap
SO=ndpi_wrap.so
OBJS = ndpi_wrap.o
CFILE = ndpi_wrap.c
PIC = -fPIC
PREFIX?=/usr/local
.PHONY: all
UNAME_S := $(shell uname -s)
ifeq ($(UNAME_S),Darwin)#do something for linux
SHARE=-install_name,ndpiReader.so
endif
all: $(SO)
$(SO): $(CFILE) $(LIBNDPI) Makefile
$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(PIC) -Wl,$(SHARE) -o $@ $(LIBS)
# ln -s $(LIBNDPI) .
clean:
/bin/rm -f $(SO)
+79
View File
@@ -0,0 +1,79 @@
# ndpi
This package contains Python bindings for nDPI. nDPI is an Open and Extensible LGPLv3 Deep Packet Inspection Library.
**ndpi** is implemented using [**CFFI**][cffi] (out-of-line API mode). Consequently, it is fast and [**PyPy**][pypy]
compliant.
## Installation
### Build nDPI
``` bash
git clone --branch dev https://github.com/ntop/nDPI.git
cd nDPI
./autogen.sh
./configure
make
```
### Install ndpi package
``` bash
cd python
python3 -m install -r dev_requirements.txt
python3 setup.py install
```
## Usage
### API
``` python
from ndpi import NDPI, NDPIFlow
nDPI = NDPI()
# You per flow processing here
# ...
ndpi_flow = NDPIFlow()
nDPI.process_packet(ndpi_flow, ip_bytes, time_ms)
nDPI.giveup(ndpi_flow) # If you want to guess it instead (DPI fallback)
```
### Example Application
[ndpi_example.py][ndpi_example] is provided to demonstrate how **ndpi** can be integrated within your Python application.
``` bash
Using nDPI 4.3.0-3532-8dd70b70
usage: ndpi_example.py [-h] [-u] input
positional arguments:
input input pcap file path
optional arguments:
-h, --help show this help message and exit
-u, --include-unknowns
```
Example with a Skype capture file
``` bash
python ndpi_example.py -u ../tests/pcap/skype.pcap
```
## Related projects
The provided example is for demo purposes only, For additional features (live capture, multiplatform support,
multiprocessing, ML based classification, system visibility, etc.), please check nDPI based
framework, [**NFStream**][nfstream].
## License
This project is licensed under the LGPLv3 License - see the [**License**][license] file for details.
[license]: https://github.com/ntop/nDPI/blob/dev/COPYING
[cffi]: https://cffi.readthedocs.io/en/latest/
[pypy]: https://www.pypy.org/
[nfstream]: https://github.com/nfstream/nfstream
[ndpi_example]: https://github.com/ntop/nDPI/blob/dev/python/ndpi_example.py
-43
View File
@@ -1,43 +0,0 @@
nDPI Python bindings
--------------------
This directory contains the Python3 bindings for nDPI. We provide both cffi and ctypes based bindings.
**cffi bindings**
Files:
* ndpi.py
Example (using `nfstream <https://github.com/aouinizied/nfstream>`_ package):
.. code-block:: bash
pip3 install nfstream
python3 flow_printer.py <interface>
python3 flow_printer.py <pcap_file>
Code courtesy:
* Zied Aouini
**ctypes bindings**
Files:
* ndpi_typestruct.py
* ndpi_wrap.c
* Makefile.in
Example:
.. code-block:: bash
pip3 install scapy
python3 ndpi_example.py <interface>
python3 ndpi_example.py <pcap_file>
Code courtesy:
* Massimo Puddu
* Zied Aouini
+2
View File
@@ -0,0 +1,2 @@
cffi>=1.15.0
dpkt>=1.9.7.2
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
------------------------------------------------------------------------------------------------------------------------
flow_printer.py
Copyright (C) 2019-20 - NFStream Developers
This file is part of NFStream, a Flexible Network Data Analysis Framework (https://www.nfstream.org/).
NFStream is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
version.
NFStream is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with NFStream.
If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
from nfstream import NFStreamer
import sys
# Example must run with nfstream >= 6.1.1
path = sys.argv[1]
flow_streamer = NFStreamer(source=path, statistical_analysis=False, performance_report=1)
result = {}
try:
for flow in flow_streamer:
print(flow)
try:
result[flow.application_name] += flow.bidirectional_packets
except KeyError:
result[flow.application_name] = flow.bidirectional_packets
print("\nSummary (Application Name: Packets):")
print(result)
except KeyboardInterrupt:
print("\nSummary (Application Name: Packets):")
print(result)
print("Terminated.")
-1346
View File
File diff suppressed because it is too large Load Diff
+21
View File
@@ -0,0 +1,21 @@
"""
------------------------------------------------------------------------------------------------------------------------
__init__.py
Copyright (C) 2011-22 - ntop.org
This file is part of nDPI, an open source deep packet inspection library.
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
version.
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with NFStream.
If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
from .ndpi import NDPI, NDPIFlow
__author__ = """Zied Aouini"""
__email__ = 'aouinizied@gmail.com'
__version__ = '4.3.0'
+101
View File
@@ -0,0 +1,101 @@
"""
------------------------------------------------------------------------------------------------------------------------
ndpi.py
Copyright (C) 2011-22 - ntop.org
This file is part of nDPI, an open source deep packet inspection library.
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
version.
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with NFStream.
If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
from collections import namedtuple
from _ndpi import ffi, lib
ndpi_protocol = namedtuple('NDPIProtocol', ['C',
'master_protocol',
'app_protocol',
'category'])
ndpi_confidence = namedtuple('NDPIConfidence', ['id',
'name'])
class NDPI(object):
__slots__ = ("_api_version",
"_revision",
"_detection_module")
def __init__(self):
self._detection_module = lib.ndpi_init_detection_module(0)
if self._detection_module == ffi.NULL:
raise MemoryError("Unable to instantiate NDPI object")
lib.ndpi_py_setup_detection_module(self._detection_module)
@property
def api_version(self):
return lib.ndpi_get_api_version()
@property
def revision(self):
return ffi.string(lib.ndpi_revision()).decode('utf-8', errors='ignore')
def process_packet(self, flow, packet, packet_time_ms):
p = lib.ndpi_detection_process_packet(self._detection_module,
flow._C,
packet,
len(packet),
int(packet_time_ms))
return ndpi_protocol(C=p,
master_protocol=p.master_protocol,
app_protocol=p.app_protocol,
category=p.category)
def giveup(self, flow, enable_guess=True):
p = lib.ndpi_detection_giveup(self._detection_module,
flow._C,
enable_guess,
ffi.new("uint8_t*", 0))
return ndpi_protocol(C=p,
master_protocol=p.master_protocol,
app_protocol=p.app_protocol,
category=p.category)
def protocol_name(self, protocol):
buf = ffi.new("char[40]")
lib.ndpi_protocol2name(self._detection_module, protocol.C, buf, ffi.sizeof(buf))
return ffi.string(buf).decode('utf-8', errors='ignore')
def protocol_category_name(self, protocol):
return ffi.string(lib.ndpi_category_get_name(self._detection_module,
protocol.C.category)).decode('utf-8',
errors='ignore')
def __del__(self):
if self._detection_module != ffi.NULL:
lib.ndpi_exit_detection_module(self._detection_module)
class NDPIFlow(object):
__slots__ = ("_C")
@property
def confidence(self):
confidence = self._C.confidence
return ndpi_confidence(id=confidence,
name=ffi.string(lib.ndpi_confidence_get_name(confidence)).decode('utf-8',
errors='ignore'))
def __init__(self):
self._C = lib.ndpi_py_initialize_flow()
def __del__(self):
if self._C != ffi.NULL:
lib.ndpi_flow_free(self._C)
self._C = ffi.NULL
+137
View File
@@ -0,0 +1,137 @@
"""
------------------------------------------------------------------------------------------------------------------------
ndpi_build.py
Copyright (C) 2011-22 - ntop.org
This file is part of nDPI, an open source deep packet inspection library.
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
version.
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with NFStream.
If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
from cffi import FFI
import subprocess
import pathlib
NDPI_INCLUDES = """
#include "ndpi_main.h"
#include "ndpi_typedefs.h"
#include "ndpi_api.h"
"""
NDPI_HELPERS = """
// nDPI cffi helper functions (function naming convention ndpi_py_*)
void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod) {
if (mod == NULL) {
return;
} else {
NDPI_PROTOCOL_BITMASK protos;
NDPI_BITMASK_SET_ALL(protos); // Set bitmask for ALL protocols
ndpi_set_protocol_detection_bitmask2(mod, &protos);
ndpi_finalize_initialization(mod);
}
};
struct ndpi_flow_struct * ndpi_py_initialize_flow(void) {
struct ndpi_flow_struct * ndpi_flow = NULL;
ndpi_flow = (struct ndpi_flow_struct *)ndpi_flow_malloc(SIZEOF_FLOW_STRUCT);
memset(ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
return ndpi_flow;
};
"""
NDPI_APIS = """
u_int16_t ndpi_get_api_version(void);
char* ndpi_revision(void);
struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs prefs);
void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_struct);
void ndpi_flow_free(void *ptr);
ndpi_protocol ndpi_detection_process_packet(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
const unsigned char *packet,
const unsigned short packetlen,
const u_int64_t packet_time_ms);
ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t enable_guess,
u_int8_t *protocol_was_guessed);
void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod);
struct ndpi_flow_struct * ndpi_py_initialize_flow(void);
char* ndpi_protocol2name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol proto, char *buf, u_int buf_len);
const char* ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol_category_t category);
const char* ndpi_confidence_get_name(ndpi_confidence_t confidence);
"""
ffi_builder = FFI()
INCLUDE_DIR = pathlib.Path(__file__)\
.parent.resolve().parent.resolve().parent.resolve().\
joinpath("src").joinpath("include")
LIBRARY_DIR = pathlib.Path(__file__)\
.parent.resolve().parent.resolve().parent.resolve().\
joinpath("src").joinpath("lib")
NDPI_CDEF = subprocess.run(["gcc",
"-DNDPI_LIB_COMPILATION",
"-DNDPI_CFFI_PREPROCESSING",
"-DNDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED",
"-E", "-x", "c", "-P", "-C",
str(INCLUDE_DIR.joinpath("ndpi_typedefs.h"))],
capture_output=True
).stdout.decode('utf-8',
errors='ignore')
NDPI_MODULE_STRUCT_CDEF = NDPI_CDEF.split("//CFFI.NDPI_MODULE_STRUCT")[1]
NDPI_PACKED = subprocess.run(["gcc",
"-DNDPI_LIB_COMPILATION", "-DNDPI_CFFI_PREPROCESSING",
"-E", "-x", "c", "-P", "-C",
str(INCLUDE_DIR.joinpath("ndpi_typedefs.h"))],
capture_output=True
).stdout.decode('utf-8',
errors='ignore')
NDPI_PACKED_STRUCTURES = NDPI_PACKED.split("//CFFI.NDPI_PACKED_STRUCTURES")[1]
NDPI_SOURCE = NDPI_INCLUDES + NDPI_MODULE_STRUCT_CDEF + NDPI_HELPERS
ffi_builder.set_source("_ndpi",
NDPI_SOURCE,
libraries=["ndpi"],
library_dirs=[str(LIBRARY_DIR)],
include_dirs=[str(INCLUDE_DIR)])
ffi_builder.cdef("""
typedef uint64_t u_int64_t;
typedef uint32_t u_int32_t;
typedef uint16_t u_int16_t;
typedef uint8_t u_int8_t;
typedef uint8_t u_char;
typedef unsigned u_int;
struct in_addr {
unsigned long s_addr;
};
struct in6_addr {
unsigned char s6_addr[16];
};
""")
ffi_builder.cdef(NDPI_PACKED_STRUCTURES, packed=True)
ffi_builder.cdef(NDPI_CDEF)
ffi_builder.cdef(NDPI_APIS)
if __name__ == "__main__":
ffi_builder.compile(verbose=True)
Executable → Regular
+144 -250
View File
@@ -1,273 +1,167 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
file: ndpi_example.py
This file is part of nDPI.
Copyright (C) 2011-19 - ntop.org
Copyright (C) 2019 - Zied Aouini <aouinizied@gmail.com> (Incremental improvements)
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License
as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
------------------------------------------------------------------------------------------------------------------------
ndpi_example.py
Copyright (C) 2011-22 - ntop.org
This file is part of nDPI, an open source deep packet inspection library.
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
version.
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with nDPI.
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with NFStream.
If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
from ndpi_typestruct import *
from ctypes import *
from scapy.all import *
import sys
# ------- return type & pcapstruct to declare -------
from collections import namedtuple
from ndpi import NDPI, NDPIFlow
import argparse
import socket
import dpkt
class WorkFlow(Structure):
_fields_ = [("src_ip", c_uint32),
("dst_ip", c_uint32),
("src_port", c_uint16),
("dst_port", c_uint16),
("protocol", c_uint8),
("packets", c_uint32),
("detected_protocol", NDPIProtocol),
("detection_completed", c_uint8),
("id", c_uint32),
("src_id", POINTER(NDPIIdStruct)),
("dst_id", POINTER(NDPIIdStruct)),
("flow", POINTER(NDPIFlowStruct))]
FLOW_KEY = "{} {}:{} <-> {}:{}"
FLOW_STR = " {} {} [protocol:{}] [category:{}] [confidence:{}] [{} packets/{} bytes]"
CMCFUN = CFUNCTYPE(c_int, c_void_p, c_void_p)
GUESS = CFUNCTYPE(None, c_void_p, c_int32, c_int, c_void_p)
FREE = CFUNCTYPE(None, c_void_p)
PROTOCOL_UNKNWON = 0
def node_proto_guess_walker(nodo, which, depth, user_data):
global ndpi_info_mod
class Flow(object):
__slots__ = ("index",
"pkts",
"bytes",
"detected_protocol",
"ndpi_flow")
flow = cast(nodo, POINTER(POINTER(WorkFlow))).contents.contents
if which == 0 or which == 3: # execute only preorder operation of the tree
if flow.detection_completed == 0: # order for tree operation
flow.detected_protocol = ndpi.ndpi_detection_giveup(ndpi_info_mod,
flow.flow,
1,
cast(addressof(c_uint8(0)), POINTER(c_uint8)))
count_protocol[flow.detected_protocol.app_protocol] += flow.packets
def __init__(self):
self.pkts = 0
self.detected_protocol = None
self.bytes = 0
self.ndpi_flow = None
def py_cmp_fun(a, b):
fa = cast(a, POINTER(WorkFlow))
fb = cast(b, POINTER(WorkFlow))
if fa.contents.src_ip < fb.contents.src_ip: return -1
elif fa.contents.src_ip > fb.contents.src_ip: return 1
if fa.contents.src_port < fb.contents.src_port: return -1
elif fa.contents.src_port > fb.contents.src_port: return 1
if fa.contents.dst_ip < fb.contents.dst_ip: return -1
elif fa.contents.dst_ip > fb.contents.dst_ip: return 1
if fa.contents.dst_port < fb.contents.dst_port: return -1
elif fa.contents.dst_port > fb.contents.dst_port: return 1
if fa.contents.protocol < fb.contents.protocol: return -1
elif fa.contents.protocol > fb.contents.protocol: return 1
return 0
ppacket = namedtuple('ParsedPacket', ['src_ip',
'src_port',
'dst_ip',
'dst_port',
'protocol',
'ip_version',
'ip_bytes'])
def freer(a):
pass
cmp_func = CMCFUN(py_cmp_fun)
guess_walker = GUESS(node_proto_guess_walker)
free_walk = FREE(freer)
# --------------------------------------
# number of analyzed packet
packet_number = 0
flow_count = 0
max_num_udp_dissected_pkts = 16
max_num_tcp_dissected_pkts = 10
flows_root = c_void_p(None)
flows_root_ref = pointer(flows_root)
count_protocol = (c_int32 * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + ndpi.ndpi_wrap_ndpi_max_num_custom_protocols() + 1))()
lista = [] # used to avoid impropriate memory deallocation from python
# check ndpi version
if ndpi.ndpi_get_api_version() != ndpi.ndpi_wrap_get_api_version():
print("nDPI Library version mismatch: please make sure this code and the nDPI library are in sync\n")
sys.exit(-1)
# create data structure of ndpi
ndpi_info_mod = ndpi.ndpi_init_detection_module()
if ndpi_info_mod is None:
sys.exit(-1)
else:
ndpi_ndpi_finalize_initalization(ndpi_info_mod)
def ip2int(ip):
"""
Convert an IP string to long and then c_uint32
"""
packedIP = socket.inet_aton(ip)
return int(struct.unpack("!I", packedIP)[0])
def get_flow(pkt):
global flows_root
global flows_root_ref
global flow_count
ip_packet = pkt[1]
ip_protocol = ip_packet.proto
transport_packet = None
if ip_protocol == 6 or ip_protocol == 17:
transport_packet = pkt[2]
if transport_packet is not None:
# to avoid two nodes in one binary tree for a flow
ip_src = ip2int(ip_packet.src)
ip_dst = ip2int(ip_packet.dst)
src_port = transport_packet.sport
dst_port = transport_packet.dport
else:
return None
# set flows to correct type and data
ndpi_flow = pointer(NDPIFlowStruct())
memset(ndpi_flow, 0, sizeof(NDPIFlowStruct))
if ip_src > ip_dst:
flow = WorkFlow(ip_src, ip_dst, src_port, dst_port, int(ip_packet.proto), 0, NDPIProtocol(), 0, 0,
pointer(NDPIIdStruct()), pointer(NDPIIdStruct()), ndpi_flow)
else:
flow = WorkFlow(ip_dst, ip_src, dst_port, src_port, int(ip_packet.proto), 0, NDPIProtocol(), 0, 0,
pointer(NDPIIdStruct()), pointer(NDPIIdStruct()), ndpi_flow)
flow_ref = pointer(flow)
res = ndpi.ndpi_tfind(flow_ref, flows_root_ref, cmp_func)
if res is None:
ndpi.ndpi_tsearch(flow_ref, pointer(flows_root), cmp_func) # add
lista.append(flow)
flow_count += 1
return pointer(flow)
flow = cast(res, POINTER(POINTER(WorkFlow))).contents
return flow
def packetcaptured(pkt):
global packet_number
global ndpi_info_mod
flow = None
h = PcapPktHdr()
# getting flow
def inet_to_str(inet):
""" get string representation of IP address """
try:
flow = get_flow(pkt)
except AttributeError:
pass # ignore packet
if flow is None: return
# filling pcap_pkthdr
h.len = h.caplen = len(pkt)
h.ts.tv_sec = int(pkt["IP"].time/1000000)
h.ts.tv_usec = int(pkt["IP"].time)
# real work
if int(pkt[1].frag) == 0: # not fragmented packet
flow.contents.packets += 1
packet_number += 1
# get ndpi_iphdr address
iphdr_addr = cast(c_char_p(pkt[1].build()), c_void_p)
ndpi_flow = flow.contents.flow
if flow.contents.detection_completed is 0:
flow.contents.detected_protocol = ndpi.ndpi_detection_process_packet(ndpi_info_mod,
ndpi_flow,
cast(iphdr_addr, POINTER(c_uint8)),
int(pkt[1].len),
h.ts.tv_usec,
flow.contents.src_id,
flow.contents.dst_id)
flow1 = flow.contents.detected_protocol
valid = False
if flow.contents.protocol == 6: valid = flow.contents.packets > max_num_tcp_dissected_pkts
elif flow.contents.protocol == 17: valid = flow.contents.packets > max_num_udp_dissected_pkts
# should we continue anylizing packet or not?
if valid or flow1.app_protocol is not 0:
if valid or flow1.master_protocol is not 91: # or # 91 is NDPI_PROTOCOL_TLS
flow.contents.detection_completed = 1 # protocol found
if flow1.app_protocol is 0:
flow.contents.detected_protocol = ndpi.ndpi_detection_giveup(ndpi_info_mod,
ndpi_flow,
1,
cast(addressof(c_uint8(0)),
POINTER(c_uint8)))
return socket.inet_ntop(socket.AF_INET, inet)
except ValueError:
return socket.inet_ntop(socket.AF_INET6, inet)
def result():
global flows_root_ref
global ndpi_info_mod
print('\nnumber of analyzed packet: ' + str(packet_number))
print('number of flows: ' + str(flow_count))
ndpi.ndpi_twalk(flows_root_ref.contents, guess_walker, None)
print('\nDetected protocols:')
for i in range(0, ndpi.ndpi_get_num_supported_protocols(ndpi_info_mod)):
if count_protocol[i] > 0:
print("{}: {} packets".format(
cast(ndpi.ndpi_get_proto_name(ndpi_info_mod, i), c_char_p).value.decode('utf-8'),
str(count_protocol[i])))
def free(ndpi_struct):
ndpi.ndpi_tdestroy(flows_root, free_walk)
ndpi.ndpi_exit_detection_module(ndpi_struct)
def initialize(ndpi_struct):
all = NDPIProtocolBitMask()
ndpi.ndpi_wrap_NDPI_BITMASK_SET_ALL(pointer(all))
ndpi.ndpi_set_protocol_detection_bitmask2(ndpi_struct, pointer(all))
print('Using nDPI ' + cast(ndpi.ndpi_revision(), c_char_p).value.decode("utf-8"))
initialize(ndpi_info_mod)
if len(sys.argv) != 2:
print("\nUsage: ndpi_example.py <device>")
sys.exit(0)
if "." in sys.argv[1]:
print('Reading pcap from file ' + sys.argv[1] + '...')
scapy_cap = None
def parse_packet(pkt):
""" parse packet and extract 5 tuple and IP bytes """
try:
scapy_cap = rdpcap(sys.argv[1])
except FileNotFoundError:
print("\nFile not found")
except Scapy_Exception:
print("\nBad pcap")
l2 = dpkt.ethernet.Ethernet(pkt)
if isinstance(l2.data, dpkt.ip.IP):
ip_version = 4
elif isinstance(l2.data, dpkt.ip6.IP6):
ip_version = 6
else:
return
except dpkt.dpkt.NeedData:
return
l3 = l2.data
stop_decoding = False
while not stop_decoding:
if isinstance(l3.data, dpkt.tcp.TCP):
l4 = l3.data
proto = "TCP"
stop_decoding = True
elif isinstance(l3.data, dpkt.udp.UDP):
l4 = l3.data
proto = "UDP"
stop_decoding = True
elif isinstance(l3.data, dpkt.ip6.IP6):
l3 = l3.data
else:
return
return ppacket(src_ip=inet_to_str(l3.src), src_port=l4.sport,
dst_ip=inet_to_str(l3.dst), dst_port=l4.dport,
protocol=proto, ip_version=ip_version,
ip_bytes=bytes(l3))
def ppkt_to_flow_key(ppkt):
""" create a consistent direction agnostic flow keyfrom a parsed packet """
if ppkt.src_ip < ppkt.dst_ip:
k = FLOW_KEY.format(ppkt.protocol, ppkt.src_ip, ppkt.src_port, ppkt.dst_ip, ppkt.dst_port)
else:
for packet in scapy_cap:
packetcaptured(packet)
else:
print('Capturing live traffic from device ' + sys.argv[1] + '...')
try:
sniff(iface=sys.argv[1], prn=packetcaptured)
except KeyboardInterrupt:
print('\nInterrupted\n')
except PermissionError:
sys.exit('\nRoot privilege required for live capture on interface: {}\n'.format(sys.argv[1]))
if ppkt.src_ip == ppkt.dst_ip:
if ppkt.src_port <= ppkt.dst_port:
k = FLOW_KEY.format(ppkt.protocol, ppkt.src_ip, ppkt.src_port, ppkt.dst_ip, ppkt.dst_port)
else:
k = FLOW_KEY.format(ppkt.protocol, ppkt.dst_ip, ppkt.dst_port, ppkt.src_ip, ppkt.src_port)
else:
k = FLOW_KEY.format(ppkt.protocol, ppkt.dst_ip, ppkt.dst_port, ppkt.src_ip, ppkt.src_port)
return k
result()
free(ndpi_info_mod)
def parse_arguments():
parser = argparse.ArgumentParser()
parser.add_argument("input", help="input pcap file path")
parser.add_argument('-u', '--include-unknowns', action='store_true')
return parser.parse_args()
if __name__ == "__main__":
nDPI = NDPI() # As simple as that. :)
flow_cache = {} # We store the flows in a dictionary.
flow_count = 0 # Flow counter
print("Using nDPI {}".format(nDPI.revision))
args = parse_arguments()
with open(args.input, 'rb') as pcap_file:
capture = dpkt.pcap.Reader(pcap_file) # We use dpkt pcap capture handler
for time, packet in capture:
time_ms = int(time * 1000) # Convert packet timestamp to milliseconds
ppkt = parse_packet(packet)
if ppkt is not None: # If we succeed to parse the packet
key = ppkt_to_flow_key(ppkt)
try: # Try a Flow update
flow = flow_cache[key]
flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms)
flow.pkts += 1
flow.bytes += len(packet)
except KeyError: # New Flow
flow = Flow()
flow.index = flow_count
flow_count += 1
flow.ndpi_flow = NDPIFlow() # We create an nDPIFlow object per Flow
flow.detected_protocol = nDPI.process_packet(flow.ndpi_flow, ppkt.ip_bytes, time_ms)
flow.pkts += 1
flow.bytes += len(packet)
flow_cache[key] = flow
print(" Detected flows:")
unknown_flows = []
for key, flow in flow_cache.items(): # Iterate over all flows in flow cache
if flow.detected_protocol.app_protocol == PROTOCOL_UNKNWON: # Didn't succeed to identigy it using DPI
flow.detected_protocol = nDPI.giveup(flow.ndpi_flow) # We try to guess it (port matching, LRU, etc.)
FLOW_EXPORT = FLOW_STR.format(flow.index,
key,
nDPI.protocol_name(flow.detected_protocol),
nDPI.protocol_category_name(flow.detected_protocol),
flow.ndpi_flow.confidence.name,
flow.pkts,
flow.bytes)
if flow.detected_protocol.app_protocol != PROTOCOL_UNKNWON:
print(FLOW_EXPORT) # We start by printing detected flows
else:
# Format it for later
unknown_flows.append(FLOW_EXPORT)
if args.include_unknowns:
print(" Unknown flows:")
for unknown_flow in unknown_flows: # Dump unknown flows
print(unknown_flow)
-780
View File
@@ -1,780 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
file: ndpi_typestruct.py
This file is part of nDPI.
Copyright (C) 2011-19 - ntop.org
Copyright (C) 2019 - Zied Aouini <aouinizied@gmail.com> (Incremental improvements)
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License
as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with nDPI.
If not, see <http://www.gnu.org/licenses/>.
"""
from ctypes import CDLL, Structure, c_uint16, c_int, c_ulong, c_uint32, CFUNCTYPE, c_void_p, POINTER, c_char_p, c_uint8
from ctypes import c_char, c_uint, c_int16, c_longlong, c_size_t, Union, c_ubyte, c_uint64, c_int32, c_ushort, cast
from os.path import abspath, dirname
ndpi = CDLL(dirname(abspath(__file__)) + '/ndpi_wrap.so')
# ----------------------------------------------- Structures -----------------------------------------------------------
class NDPIDetectionModuleStruct(Structure):
pass
class NDPIFlowStruct(Structure):
pass
class NDPIProtocol(Structure):
_fields_ = [
("master_protocol", c_uint16),
("app_protocol", c_uint16),
("category", c_int)
]
class TimeVal(Structure):
_fields_ = [("tv_sec", c_ulong), ("tv_usec", c_ulong)]
class PcapPktHdr(Structure):
_fields_ = [("ts", TimeVal), ("caplen", c_uint32), ("len", c_uint32)]
class NDPIMask(Structure):
_fields_ = [("fds_bits", c_uint32)]
class NDPIProtocolBitMask(Structure):
_fields_ = [("fds_bits", NDPIMask * ndpi.ndpi_wrap_ndpi_num_fds_bits())]
class NDPISubprotocolConfStruct(Structure):
_fields_ = [("func", CFUNCTYPE(c_void_p, POINTER(NDPIDetectionModuleStruct), c_char_p, c_char_p, c_int))]
class NDPIAutoma(Structure):
_fields_ = [
("ac_automa", c_void_p),
("ac_automa_finalized", c_uint8)
]
class NDPINode(Structure):
pass
NDPINode._fields_ = [
('key', POINTER(c_char)),
('left', POINTER(NDPINode)),
('right', POINTER(NDPINode)),
]
class NDPICallFunctionStruct(Structure):
_fields_ = [
("detection_bitmask", NDPIProtocolBitMask),
("excluded_protocol_bitmask", NDPIProtocolBitMask),
("ndpi_selection_bitmask", c_uint32),
("func", CFUNCTYPE(None, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
("detection_feature", c_uint8)
]
class NDPIProtoDefaultsT(Structure):
_fields_ = [
("protoName", c_char_p),
("protoCategory", c_uint),
("subprotocols", c_uint16_p),
("subprotocol_count", c_uint32),
("protoId", c_uint16),
("protoIdx", c_uint16),
("protoBreed", c_uint),
("func", CFUNCTYPE(None, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
]
class NDPIDefaultsPortsTreeNodeT(Structure):
_fields_ = [
("proto", NDPIProtoDefaultsT),
("customUserProto", c_uint8),
("default_port", c_int16)
]
class SpinlockT(Structure):
_fields_ = [("val", c_int)]
class AtomicT(Structure):
_fields_ = [("counter", c_int)]
class TimeT(Structure):
_fields_ = [("counter", c_longlong)]
class HashIp4pNode(Structure):
pass
HashIp4pNode._fields_ = [
("next", POINTER(HashIp4pNode)),
("prev", POINTER(HashIp4pNode)),
("lchg", TimeT),
("port", c_uint16),
("count", c_uint16, 12),
("flag", c_uint16, 4),
("ip", c_uint32)
]
class HashIp4p(Structure):
_fields_ = [
("top", POINTER(HashIp4pNode)),
("lock",SpinlockT),
("len", c_size_t)
]
class HashIp4pTable(Structure):
_fields_ = [
("size", c_size_t),
("ipv6", c_int),
("lock", SpinlockT),
("count", AtomicT),
("tbl", HashIp4p)
]
class BtAnnounce(Structure):
_fields_ = [
("hash", c_uint32 * 5),
("ip", c_uint32 * 4),
("time", c_uint32),
("port", c_uint16),
("name_len", c_uint8),
("name", c_uint8 * 149)
]
class NDPILruCacheEntry(Structure):
_fields_ = [
("key", c_uint32),
("is_full", c_uint32, 1),
("value", c_uint32, 16),
("pad", c_uint32, 15)
]
class NDPILruCache(Structure):
_fields_ = [
("num_entries", c_uint32),
("entries", POINTER(NDPILruCacheEntry)),
]
class CacheEntry(Structure):
pass
CacheEntry._fields_ = [
("item", c_void_p),
("item_size", c_uint32),
("prev", POINTER(CacheEntry)),
("next", POINTER(CacheEntry))
]
class CacheEntryMap(Structure):
pass
CacheEntryMap._fields_ = [
("entry", POINTER(CacheEntry)),
("next", POINTER(CacheEntryMap)),
]
class Cache(Structure):
_fields_ = [
("size", c_uint32),
("max_size", c_uint32),
("head", POINTER(CacheEntry)),
("tail", POINTER(CacheEntry)),
("map", POINTER(POINTER(CacheEntryMap)))
]
class CustomCategories(Structure):
_fields_ = [
("hostnames", NDPIAutoma),
("hostnames_shadow", NDPIAutoma),
("ipAddresses", c_void_p),
("ipAddresses_shadow", c_void_p),
("categories_loaded", c_uint8),
]
NDPIDetectionModuleStruct._fields_ = [
("detection_bitmask", NDPIProtocolBitMask),
("current_ts", c_uint32),
("custom_category_labels",
(c_char * ndpi.ndpi_wrap_num_custom_categories()) * ndpi.ndpi_wrap_custom_category_label_len()),
("callback_buffer", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
("callback_buffer_size", c_uint32),
("callback_buffer_tcp_no_payload", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
("callback_buffer_size_tcp_no_payload", c_uint32),
("callback_buffer_tcp_payload", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
("callback_buffer_size_tcp_payload", c_uint32),
("callback_buffer_udp", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
("callback_buffer_size_udp", c_uint32),
("callback_buffer_non_tcp_udp", NDPICallFunctionStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
("callback_buffer_size_non_tcp_udp", c_uint32),
("tcpRoot", POINTER(NDPIDefaultsPortsTreeNodeT)),
("udpRoot", POINTER(NDPIDefaultsPortsTreeNodeT)),
("ndpi_log_level", c_uint),
("tcp_max_retransmission_window_size", c_uint32),
("subprotocol_conf", NDPISubprotocolConfStruct * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() + 1)),
("ndpi_num_supported_protocols", c_uint),
("ndpi_num_custom_protocols", c_uint),
("host_automa", NDPIAutoma),
("bigrams_automa", NDPIAutoma),
("impossible_bigrams_automa", NDPIAutoma),
("custom_categories", CustomCategories),
("protocols_ptree", c_void_p),
("ip_version_limit", c_uint8),
("ookla_cache", POINTER(NDPILruCache)),
("tinc_cache", POINTER(Cache)),
("proto_defaults", NDPIProtoDefaultsT * (ndpi.ndpi_wrap_ndpi_max_supported_protocols() +
ndpi.ndpi_wrap_ndpi_max_num_custom_protocols())),
("direction_detect_disable", c_uint8, 1),
('_pad', c_uint8, 7),
('packet', NDPIPacketStruct),
]
class U6Addr(Union):
_fields_ = [
("u6_addr8", c_uint8 * 16),
("u6_addr16", c_uint16 * 8),
("u6_addr32", c_uint32 * 4),
("u6_addr64", c_uint64 * 2)
]
class NDPIIn6Addr(Structure):
_pack_ = 1
_fields_ = [("u6_addr", U6Addr)]
class NDPIIpAddrT(Union):
_fields_ = [
('ipv4', c_uint32),
('ipv4_u_int8_t', c_uint8 * 4),
('ipv6', NDPIIn6Addr),
]
class NDPIIdStruct(Structure):
_fields_ = [
('detected_protocol_bitmask', NDPIProtocolBitMask),
('irc_port', c_uint16 * 8),
('last_time_port_used', c_uint32 * 8),
('irc_ts', c_uint32),
('gnutella_ts', c_uint32),
('battlefield_ts', c_uint32),
('rtsp_timer', c_uint32),
('jabber_stun_or_ft_ts', c_uint32),
('directconnect_last_safe_access_time', c_uint32),
('detected_directconnect_port', c_uint16),
('detected_directconnect_udp_port', c_uint16),
('detected_directconnect_ssl_port', c_uint16),
('jabber_voice_stun_port', c_uint16 * 6),
('jabber_file_transfer_port', c_uint16 * 2),
('detected_gnutella_udp_port1', c_uint16),
('detected_gnutella_udp_port2', c_uint16),
('irc_number_of_port', c_uint8),
('oscar_ssl_session_id', c_uint8 * 33),
('jabber_voice_stun_used_ports', c_uint8),
]
class NDPIFlowTcpStruct(Structure):
_pack_ = 1
_fields_ = [
('smtp_command_bitmask', c_uint16),
('pop_command_bitmask', c_uint16),
('wa_matched_so_far', c_uint8),
('irc_stage', c_uint8),
('h323_valid_packets', c_uint8),
('gnutella_msg_id', c_uint8 * 3),
('irc_3a_counter', c_uint32, 3),
('irc_stage2', c_uint32, 5),
('irc_direction', c_uint32, 2),
('irc_0x1000_full', c_uint32, 1),
('usenet_stage', c_uint32, 2),
('http_stage', c_uint32, 2),
('http_empty_line_seen', c_uint32, 1),
('gnutella_stage', c_uint32, 2),
('ssh_stage', c_uint32, 3),
('vnc_stage', c_uint32, 2),
('telnet_stage', c_uint32, 2),
('tls_seen_client_cert', c_uint8, 1),
('tls_seen_server_cert', c_uint8, 1),
('tls_seen_certificate', c_uint8, 1),
('tls_srv_cert_fingerprint_found', c_uint8, 1),
('tls_srv_cert_fingerprint_processed', c_uint8, 1),
('tls_stage', c_uint8, 2),
('tls_record_offset', c_int16),
('tls_fingerprint_len', c_int16),
('tls_sha1_certificate_fingerprint', c_uint8 * 20),
('postgres_stage', c_uint32, 3),
('seen_syn', c_uint32, 1),
('seen_syn_ack', c_uint32, 1),
('seen_ack', c_uint32, 1),
('icecast_stage', c_uint32, 1),
('dofus_stage', c_uint32, 1),
('fiesta_stage', c_uint32, 2),
('wow_stage', c_uint32, 2),
('shoutcast_stage', c_uint32, 2),
('rtp_special_packets_seen', c_uint32, 1),
('mail_pop_stage', c_uint32, 2),
('mail_imap_stage', c_uint32, 3),
('mail_imap_starttls', c_uint32, 2),
('skype_packet_id', c_uint8),
('lotus_notes_packet_id', c_uint8),
('teamviewer_stage', c_uint8),
('prev_zmq_pkt_len', c_uint8),
('prev_zmq_pkt', c_char * 10),
('ppstream_stage', c_uint32, 3),
('memcached_matches', c_uint8),
('nest_log_sink_matches', c_uint8),
]
class NDPIFlowUdpStruct(Structure):
_pack_ = 1
_fields_ = [
('ppstream_stage', c_uint32, 3),
('halflife2_stage', c_uint32, 2),
('tftp_stage', c_uint32, 2),
('aimini_stage', c_uint32, 5),
('xbox_stage', c_uint32, 1),
('skype_packet_id', c_uint8),
('teamviewer_stage', c_uint8),
('eaq_pkt_id', c_uint8),
('eaq_sequence', c_uint32),
('rx_conn_epoch', c_uint32),
('rx_conn_id', c_uint32),
('memcached_matches', c_uint8),
('wireguard_stage', c_uint8),
('wireguard_peer_index', c_uint32 * 2),
('quic_reasm_buf', POINTER(c_uint8)),
('quic_reasm_buf_len', c_uint32),
('csgo_strid', c_uint8 * 18),
('csgo_state', c_uint8),
('csgo_s2', c_uint8),
('csgo_id2', c_uint32),
('rdp_to_srv', c_uint8 * 3),
('rdp_from_srv', c_uint8 * 3),
('rdp_to_srv_pkts,', c_uint8),
('rdp_from_srv_pkts', c_uint8),
('imo_last_one_byte_pkt,', c_uint8),
('imo_last_byte', c_uint8),
]
class L4(Union):
_fields_ = [("tcp", NDPIFlowTcpStruct), ("udp", NDPIFlowUdpStruct)]
class Http(Structure):
_fields_ = [
("method", c_int),
("request_version", c_uint8),
("response_status_code", c_uint16),
("url", c_char_p),
("content_type", c_char_p),
("user_agent", c_char_p),
("detected_os", c_char_p),
("nat_ip", c_char_p),
]
class Dns(Structure):
_fields_ = [
("num_queries", c_uint8),
("num_answers", c_uint8),
("reply_code", c_uint8),
("is_query", c_uint8),
("query_type", c_uint16),
("query_class", c_uint16),
("rsp_type", c_uint16),
("rsp_addr", NDPIIpAddrT)
]
class Ntp(Structure):
_fields_ = [("request_code", c_uint8),
("version", c_uint8)]
class Kerberos(Structure):
_fields_ = [("cname", c_char * 24),
("realm", c_char * 24)]
class QuicSsl(Structure):
_fields_ = [
("ssl_version", c_uint16),
("client_certificate", c_char * 64),
("server_certificate", c_char * 64),
("server_organization", c_char * 64),
('notBefore', c_uint32),
('notAfter', c_uint32),
("ja3_client", c_char * 33),
("ja3_server", c_char * 33),
("server_cipher", c_uint16),
("server_unsafe_cipher", c_int)
]
class Stun(Structure):
_fields_ = [
("num_udp_pkts", c_uint8),
("num_processed_pkts", c_uint8),
("num_binding_requests", c_uint8),
]
class Ssh(Structure):
_fields_ = [
("client_signature", c_char * 48),
("server_signature", c_char * 48),
("hassh_client", c_char * 33),
("hassh_server", c_char * 33)
]
class Mdns(Structure):
_fields_ = [("answer", c_char * 96)]
class Ubntac2(Structure):
_fields_ = [("version", c_char * 32)]
class FtpImapPopSmtp(Structure):
_fields_ = [
("auth_found", c_uint8, 1),
("auth_failed", c_uint8, 1),
("auth_tls", c_uint8, 1),
("auth_done", c_uint8, 1),
("_pad", c_uint8, 4),
("username", c_char * 32),
("password", c_char * 16)
]
class Bittorrent(Structure):
_fields_ = [("hash", c_char * 20)]
class Dhcp(Structure):
_fields_ = [
("fingerprint", c_char * 48),
("class_ident", c_char * 48)
]
class Protos(Union):
_fields_ = [
("dns", Dns),
("kerberos", Kerberos),
("quic_ssl", QuicSsl),
("ssh", Ssh),
("mdns", Mdns),
("ubntac2", Ubntac2),
("bittorrent", Bittorrent),
("dhcp", Dhcp)
]
class TincCacheEntry(Structure):
_pack_ = 1
_fields_ = [
('src_address', c_uint32),
('dst_address', c_uint32),
('dst_port', c_uint16),
]
class NDPIIntOneLineStruct(Structure):
_fields_ = [
('ptr', POINTER(c_uint8)),
('len', c_uint16),
]
class NDPIIphdr(Structure):
_pack_ = 1
_fields_ = [
('ihl', c_uint8, 4),
('version', c_uint8, 4),
('tos', c_uint8),
('tot_len', c_uint16),
('id', c_uint16),
('frag_off', c_uint16),
('ttl', c_uint8),
('protocol', c_uint8),
('check', c_uint16),
('saddr', c_uint32),
('daddr', c_uint32)]
class NDPIIp6Hdrctl(Structure):
_pack_ = 1
_fields_ = [
('ip6_un1_flow', c_uint32),
('ip6_un1_plen', c_uint16),
('ip6_un1_nxt', c_uint8),
('ip6_un1_hlim', c_uint8),
]
class NDPIIpv6hdr(Structure):
_pack_ = 1
_fields_ = [
('ip6_hdr', NDPIIp6Hdrctl),
('ip6_src', NDPIIn6Addr),
('ip6_dst', NDPIIn6Addr),
]
class NDPITcpHdr(Structure):
_pack_ = 1
_fields_ = [
('source', c_uint16),
('dest', c_uint16),
('seq', c_uint32),
('ack_seq', c_uint32),
('res1', c_uint16, 4),
('doff', c_uint16, 4),
('fin', c_uint16, 1),
('syn', c_uint16, 1),
('rst', c_uint16, 1),
('psh', c_uint16, 1),
('ack', c_uint16, 1),
('urg', c_uint16, 1),
('ece', c_uint16, 1),
('cwr', c_uint16, 1),
('window', c_uint16),
('check', c_uint16),
('urg_ptr', c_uint16),
]
class NDPIUdpHdr(Structure):
_pack_ = 1
_fields_ = [
('source', c_uint16),
('dest', c_uint16),
('len', c_uint16),
('check', c_uint16),
]
class NDPIPacketStruct(Structure):
_fields_ = [
('iph', POINTER(NDPIIphdr)),
('iphv6', POINTER(NDPIIpv6hdr)),
('tcp', POINTER(NDPITcpHdr)),
('udp', POINTER(NDPIUdpHdr)),
('generic_l4_ptr', POINTER(c_uint8)),
('payload', POINTER(c_uint8)),
('current_timestamp_ms', c_uint64),
('detected_protocol_stack', c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()),
('line', NDPIIntOneLineStruct * 64),
('host_line', NDPIIntOneLineStruct),
('forwarded_line', NDPIIntOneLineStruct),
('referer_line', NDPIIntOneLineStruct),
('content_line', NDPIIntOneLineStruct),
('accept_line', NDPIIntOneLineStruct),
('user_agent_line', NDPIIntOneLineStruct),
('http_url_name', NDPIIntOneLineStruct),
('http_encoding', NDPIIntOneLineStruct),
('http_transfer_encoding', NDPIIntOneLineStruct),
('http_contentlen', NDPIIntOneLineStruct),
('http_cookie', NDPIIntOneLineStruct),
('http_origin', NDPIIntOneLineStruct),
('http_x_session_type', NDPIIntOneLineStruct),
('server_line', NDPIIntOneLineStruct),
('http_method', NDPIIntOneLineStruct),
('http_response', NDPIIntOneLineStruct),
('http_num_headers', c_uint8),
('l3_packet_len', c_uint16),
('payload_packet_len', c_uint16),
('parsed_lines', c_uint16),
('empty_line_position', c_uint16),
('tcp_retransmission', c_uint8),
('packet_lines_parsed_complete', c_uint8, 1),
('packet_direction', c_uint8, 1),
('empty_line_position_set', c_uint8, 1),
('http_check_content', c_uint8, 1),
('pad', c_uint8, 4),
]
class NDPIFlowStructStack(Structure):
_pack_ = 1
_fields_ = [
("detected_protocol_stack", c_uint16 * ndpi.ndpi_wrap_ndpi_procol_size()),
]
NDPIFlowStruct._fields_ = [
("ndpi_flow_stack", NDPIFlowStructStack),
("guessed_protocol_id", c_uint16),
("guessed_host_protocol_id", c_uint16),
("guessed_category", c_uint16),
("guessed_header_category", c_uint16),
("l4_proto", c_uint8),
("protocol_id_already_guessed", c_uint8, 1),
("host_already_guessed", c_uint8, 1),
("init_finished", c_uint8, 1),
("setup_packet_direction", c_uint8, 1),
("packet_direction", c_uint8, 1),
("check_extra_packets", c_uint8, 1),
("next_tcp_seq_nr", c_uint32 * 2),
("max_extra_packets_to_check", c_uint8),
("num_extra_packets_checked", c_uint8),
("num_processed_pkts", c_uint8),
("extra_packets_func", CFUNCTYPE(c_int, POINTER(NDPIDetectionModuleStruct), POINTER(NDPIFlowStruct))),
("l4", L4),
("host_server_name", c_char * 80),
("http", Http),
("stun", Stun),
("ftp_imap_pop_smtp", FtpImapPopSmtp),
("protos", Protos),
("excluded_protocol_bitmask", NDPIProtocolBitMask),
("category", c_int),
('redis_s2d_first_char', c_uint8),
('redis_d2s_first_char', c_uint8),
('packet_counter', c_uint16),
('packet_direction_counter', c_uint16 * 2),
('byte_counter', c_uint16 * 2),
('bittorrent_stage', c_uint8),
('directconnect_stage', c_uint8, 2),
('http_detected', c_uint8, 1),
('http_upper_protocol', c_uint16),
('http_lower_protocol', c_uint16),
('rtsprdt_stage', c_uint8, 2),
('zattoo_stage', c_uint8, 3),
('thunder_stage', c_uint8, 2),
('oscar_ssl_voice_stage', c_uint8, 3),
('oscar_video_voice', c_uint8, 1),
('florensia_stage', c_uint8, 1),
('socks5_stage', c_uint8, 2),
('socks4_stage', c_uint8, 2),
('edonkey_stage', c_uint8, 2),
('ftp_control_stage', c_uint8, 2),
('rtmp_stage', c_uint8, 2),
('pando_stage', c_uint8, 3),
('steam_stage1', c_uint16, 3),
('steam_stage2', c_uint16, 2),
('steam_stage3', c_uint16, 2),
('pplive_stage1', c_uint8, 3),
('pplive_stage2', c_uint8, 2),
('pplive_stage3', c_uint8, 2),
('starcraft_udp_stage', c_uint8, 3),
('ovpn_session_id', c_uint8 * 8),
('ovpn_counter', c_uint8),
('key_mining_cache', c_uint32),
('tinc_state', c_uint8),
('TincCacheEntry', TincCacheEntry),
('src', POINTER(NDPIIdStruct)),
('dst', POINTER(NDPIIdStruct))
]
# ----------------------------------------------- nDPI APIs ------------------------------------------------------------
""" ndpi_detection_giveup: Function to be called before we give up with detection for a given flow.
This function reduces the NDPI_UNKNOWN_PROTOCOL detection. """
ndpi.ndpi_detection_giveup.restype = NDPIProtocol
ndpi.ndpi_detection_giveup.argtypes = [POINTER(NDPIDetectionModuleStruct),
POINTER(NDPIFlowStruct), c_uint8,
POINTER(c_uint8)]
""" ndpi_detection_process_packet: Processes one packet and returns the ID of the detected protocol.
This is the MAIN PACKET PROCESSING FUNCTION. """
ndpi.ndpi_detection_process_packet.restype = NDPIProtocol
ndpi.ndpi_detection_process_packet.argtypes = [POINTER(NDPIDetectionModuleStruct),
POINTER(NDPIFlowStruct),
POINTER(c_ubyte),
c_ushort,
c_uint64,
POINTER(NDPIIdStruct),
POINTER(NDPIIdStruct)]
""" ndpi_ssl_version2str : Converts ssl version to readable string """
ndpi.ndpi_ssl_version2str.restype = c_char_p
ndpi.ndpi_ssl_version2str.argtypes = [c_int16, POINTER(c_uint8)]
""" ndpi_init_detection_module: Returns a new initialized detection module.
Note that before you can use it you can still load hosts and do other things. As soon as you are ready to use
it do not forget to call first ndpi_finalize_initalization() """
ndpi.ndpi_init_detection_module.restype = POINTER(NDPIDetectionModuleStruct)
def ndpi_ndpi_finalize_initalization(detection_module):
""" ndpi_finalize_initalization: Completes the initialization (ndpi_revision >= 3.1)"""
if cast(ndpi.ndpi_revision(), c_char_p).value.decode("utf-8")[:3] >= '3.1':
ndpi.ndpi_finalize_initalization.restype = c_void_p
ndpi.ndpi_finalize_initalization.argtypes = [POINTER(NDPIDetectionModuleStruct)]
return ndpi.ndpi_finalize_initalization(detection_module)
else:
# ignore it
return None
""" ndpi_tfind: find a node, or return 0. """
ndpi.ndpi_tfind.restype = c_void_p
""" ndpi_tsearch: ftp://ftp.cc.uoc.gr/mirrors/OpenBSD/src/lib/libc/stdlib/tsearch.c
find or insert datum into search tree. """
ndpi.ndpi_tsearch.restype = c_void_p
ndpi.ndpi_tsearch.argtypes = [c_void_p, POINTER(c_void_p), CFUNCTYPE(c_int, c_void_p, c_void_p)]
""" ndpi_revision: Get the nDPI version release. """
ndpi.ndpi_revision.restype = c_void_p
""" ndpi_get_proto_name: Get the protocol name associated to the ID."""
ndpi.ndpi_get_proto_name.restype = c_void_p
""" ndpi_category_get_name: Get protocol category as string."""
ndpi.ndpi_category_get_name.restype = c_void_p
""" ndpi_get_num_supported_protocols: Get the total number of the supported protocols."""
ndpi.ndpi_get_num_supported_protocols.restype = c_uint
""" ndpi_wrap_NDPI_BITMASK_SET_ALL: memset((char *)(p), 0xFF, sizeof(*(p)))"""
ndpi.ndpi_wrap_NDPI_BITMASK_SET_ALL.argtypes = [POINTER(NDPIProtocolBitMask)]
""" ndpi_set_protocol_detection_bitmask2: Sets the protocol bitmask2."""
ndpi.ndpi_set_protocol_detection_bitmask2.argtypes = [POINTER(NDPIDetectionModuleStruct),
POINTER(NDPIProtocolBitMask)]
""" ndpi_twalk: Walk the nodes of a tree. """
ndpi.ndpi_twalk.argtypes = [c_void_p, CFUNCTYPE(None, c_void_p, c_int32, c_int, c_void_p), c_void_p]
""" ndpi_tdestroy: node destroy. """
ndpi.ndpi_tdestroy.argtypes = [c_void_p, CFUNCTYPE(None, c_void_p)]
-60
View File
@@ -1,60 +0,0 @@
/*
* ndpi_wrap.c
*
* Copyright (C) 2011-19 - ntop.org
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/
#include "sys/types.h"
#include "ndpi_config.h"
#include "ndpi_main.h"
int ndpi_wrap_get_api_version(){
return NDPI_API_VERSION;
}
int ndpi_wrap_ndpi_num_fds_bits(){
return NDPI_NUM_FDS_BITS;
}
int ndpi_wrap_num_custom_categories(){
return NUM_CUSTOM_CATEGORIES;
}
int ndpi_wrap_custom_category_label_len(){
return CUSTOM_CATEGORY_LABEL_LEN;
}
int ndpi_wrap_ndpi_max_supported_protocols(){
return NDPI_MAX_SUPPORTED_PROTOCOLS;
}
int ndpi_wrap_ndpi_max_num_custom_protocols(){
return NDPI_MAX_NUM_CUSTOM_PROTOCOLS;
}
int ndpi_wrap_ndpi_procol_size(){
return NDPI_PROTOCOL_SIZE;
}
void ndpi_wrap_NDPI_BITMASK_SET_ALL(NDPI_PROTOCOL_BITMASK* bitmask){
NDPI_ONE(bitmask);
}
void dummy() {
/* Dummy call just to cause linker to include the ndpi library */
ndpi_tfind(NULL, NULL, NULL);
}
Binary file not shown.
+1
View File
@@ -0,0 +1 @@
cffi>=1.15.0
+57
View File
@@ -0,0 +1,57 @@
"""
------------------------------------------------------------------------------------------------------------------------
setup.py
Copyright (C) 2011-22 - ntop.org
This file is part of nDPI, an open source deep packet inspection library.
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
version.
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with NFStream.
If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
from setuptools import setup
import os
this_directory = os.path.abspath(os.path.dirname(__file__))
with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
long_description = f.read()
setup(
name="ndpi",
version='4.3.0',
url='https://www.ntop.org/products/deep-packet-inspection/ndpi/',
license='LGPLv3',
description="Open and Extensible LGPLv3 Deep Packet Inspection Library",
long_description=long_description,
long_description_content_type='text/markdown',
author='Zied Aouini',
author_email='aouinizied@gmail.com',
packages=['ndpi'],
setup_requires=["cffi>=1.15.0"],
cffi_modules=["ndpi/ndpi_build.py:ffi_builder"],
install_requires=["cffi>=1.15.0"],
platforms=["Linux",
"Mac OS-X",
"Windows",
"Unix"],
classifiers=[
'Development Status :: 5 - Production/Stable',
'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)',
'Programming Language :: Python :: 3 :: Only',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Topic :: System :: Networking :: Monitoring',
],
project_urls={
'GitHub': 'https://github.com/ntop/nDPI',
}
)
+28
View File
@@ -0,0 +1,28 @@
"""
------------------------------------------------------------------------------------------------------------------------
tests.py
Copyright (C) 2011-22 - ntop.org
This file is part of nDPI, an open source deep packet inspection library.
nDPI is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later
version.
nDPI is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public License along with NFStream.
If not, see <http://www.gnu.org/licenses/>.
------------------------------------------------------------------------------------------------------------------------
"""
from ndpi import NDPI, NDPIFlow
import time
if __name__ == '__main__':
try:
nDPI = NDPI()
ndpi_flow = NDPIFlow()
nDPI.process_packet(ndpi_flow, b'', time.time())
nDPI.giveup(ndpi_flow)
print("nDPI Python bindings: OK")
except Exception:
raise AssertionError("nDPI Python bindings: KO")
+38 -18
View File
@@ -61,7 +61,6 @@ typedef enum {
- nDPI/wireshark/ndpi.lua
- ndpi_risk2str (in ndpi_utils.c)
- doc/flow_risks.rst
- ndpi_risk_enum (in python/ndpi.py)
- ndpi_known_risks (ndpi_main.c)
To make sure the risk is also seen by ntopng:
@@ -170,7 +169,12 @@ typedef struct node_t {
/* NDPI_MASK_SIZE */
typedef u_int32_t ndpi_ndpi_mask;
/* NDPI_PROTO_BITMASK_STRUCT */
#ifdef NDPI_CFFI_PREPROCESSING
#define NDPI_NUM_FDS_BITS 16
#endif
typedef struct ndpi_protocol_bitmask_struct {
ndpi_ndpi_mask fds_bits[NDPI_NUM_FDS_BITS];
} ndpi_protocol_bitmask_struct_t;
@@ -181,6 +185,7 @@ typedef void (*ndpi_debug_function_ptr) (u_int32_t protocol, void *module_struct
const char *func, unsigned line,
const char *format, ...);
#ifndef NDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED
/* ************************************************************ */
/* ******************* NDPI NETWORKS HEADERS ****************** */
/* ************************************************************ */
@@ -198,6 +203,13 @@ typedef void (*ndpi_debug_function_ptr) (u_int32_t protocol, void *module_struct
#define PACK_OFF __attribute__((packed))
#endif
/* PLEASE DO NOT REMOVE OR CHANGE THE ORDER OF WHAT IS DELIMITED BY CFFI.NDPI_PACKED_STRUCTURES FLAG AS IT IS USED FOR
PYTHON BINDINGS AUTO GENERATION */
#ifdef NDPI_CFFI_PREPROCESSING
#define PACK_ON
#define PACK_OFF
#endif
//CFFI.NDPI_PACKED_STRUCTURES
PACK_ON
struct ndpi_chdlc
{
@@ -435,12 +447,6 @@ struct ndpi_dns_packet_header {
u_int16_t additional_rrs;
} PACK_OFF;
typedef union
{
u_int32_t ipv4;
struct ndpi_in6_addr ipv6;
} ndpi_ip_addr_t;
/* +++++++++++++++++++++++ ICMP header +++++++++++++++++++++++ */
@@ -490,6 +496,22 @@ struct ndpi_vxlanhdr {
/* ******************* ********************* ****************** */
/* ************************************************************ */
PACK_ON struct tinc_cache_entry {
u_int32_t src_address;
u_int32_t dst_address;
u_int16_t dst_port;
} PACK_OFF;
//CFFI.NDPI_PACKED_STRUCTURES
#endif // NDPI_CFFI_PREPROCESSING_EXCLUDE_PACKED
typedef union
{
u_int32_t ipv4;
struct ndpi_in6_addr ipv6;
} ndpi_ip_addr_t;
typedef struct message {
u_int8_t *buffer;
u_int buffer_len, buffer_used;
@@ -499,12 +521,6 @@ typedef struct message {
/* NDPI_PROTOCOL_TINC */
#define TINC_CACHE_MAX_SIZE 10
PACK_ON struct tinc_cache_entry {
u_int32_t src_address;
u_int32_t dst_address;
u_int16_t dst_port;
} PACK_OFF;
/*
In case the typedef below is modified, please update
ndpi_http_method2str (ndpi_utils.c)
@@ -968,6 +984,9 @@ typedef struct ndpi_proto {
#define _NDPI_CONFIG_H_
#endif
/* PLEASE DO NOT REMOVE OR CHANGE THE ORDER OF WHAT IS DELIMITED BY CFFI.NDPI_MODULE_STRUCT FLAG AS IT IS USED FOR
PYTHON BINDINGS AUTO GENERATION */
//CFFI.NDPI_MODULE_STRUCT
typedef enum {
ndpi_stun_cache,
ndpi_hangout_cache
@@ -1096,6 +1115,7 @@ struct ndpi_detection_module_struct {
};
#endif /* NDPI_LIB_COMPILATION */
//CFFI.NDPI_MODULE_STRUCT
typedef enum {
ndpi_cipher_safe = NDPI_CIPHER_SAFE,
@@ -1113,11 +1133,7 @@ struct tls_heuristics {
u_int8_t is_safari_tls:1, is_firefox_tls:1, is_chrome_tls:1, notused:5;
};
/*
NOTE
When the struct below is modified don't forget to update
- ndpi_flow_struct (in python/ndpi.py)
*/
struct ndpi_flow_struct {
u_int16_t detected_protocol_stack[NDPI_PROTOCOL_SIZE];
@@ -1473,7 +1489,11 @@ typedef struct {
#define ndpi_private_deserializer ndpi_private_serializer
#ifdef NDPI_CFFI_PREPROCESSING
typedef struct { char c[72]; } ndpi_serializer;
#else
typedef struct { char c[sizeof(ndpi_private_serializer)]; } ndpi_serializer;
#endif
#define ndpi_deserializer ndpi_serializer
+2 -1
View File
@@ -12,12 +12,13 @@
// #define NDPI_ENABLE_DEBUG_INFO_MESSAGES
// #define NDPI_ENABLE_DEBUG_TRACE_MESSAGES
#ifndef NDPI_CFFI_PREPROCESSING
extern void printRawData(const uint8_t *ptr, size_t len);
//extern uint8_t add_segment_to_buffer( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph, uint32_t waited);
//extern uint8_t check_for_sequence( struct ndpi_flow_struct *flow, struct ndpi_tcphdr const * tcph);
extern u_int8_t ndpi_ends_with(char *str, char *ends);
#endif // NDPI_CFFI_PREPROCESSING
/* **************************************** */
/* Can't call libc functions from kernel space, define some stub instead */