Move from PCRE to PCRE2 (#2134)

Move from PCRE to PCRE2. PCRE is EOL and won't receive any security
updates anymore. Convert to PCRE2 by converting any function PCRE2 new
API.

Also update every entry in github workflows and README to point to the
new configure flag. (--with-pcre2)

Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>
This commit is contained in:
Christian Marangi
2023-11-01 14:16:49 +01:00
committed by GitHub
parent 5a2666ce9e
commit d5c9a16a48
9 changed files with 59 additions and 60 deletions
+1 -1
View File
@@ -21,7 +21,7 @@ A clear and concise description of what happening.
* OS version: [e.g. 18.04]
* Architecture: [e.g. arm64]
* nDPI version or commit hash: [e.g. 4.0-stable or 937357e4bc55610f116f66d15a8e0fc1e260c02c].
* nDPI compilation flags used: if you are building from source [e.g. --with-pcre --with-local-libgcrypt].
* nDPI compilation flags used: if you are building from source [e.g. --with-pcre2 --with-local-libgcrypt].
* Attach the `config.log` file generated after `./configure` ran (if you are building from source).
## How to reproduce the reported bug
+16 -16
View File
@@ -93,7 +93,7 @@ jobs:
os: ubuntu-20.04
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
@@ -101,7 +101,7 @@ jobs:
os: ubuntu-22.04
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
@@ -110,7 +110,7 @@ jobs:
os: ubuntu-20.04
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
@@ -120,7 +120,7 @@ jobs:
os: ubuntu-22.04
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
@@ -129,7 +129,7 @@ jobs:
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-thread-sanitizer"
nBPF: ""
@@ -137,7 +137,7 @@ jobs:
os: ubuntu-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: "nBPF"
@@ -145,7 +145,7 @@ jobs:
os: ubuntu-22.04
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-memory-sanitizer"
nBPF: ""
@@ -153,7 +153,7 @@ jobs:
os: macOS-latest
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "" # Disable sanitizer on macos
nBPF: ""
@@ -161,7 +161,7 @@ jobs:
os: macos-12
arch: "x86_64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "" # Disable sanitizer on macos
nBPF: ""
@@ -169,7 +169,7 @@ jobs:
os: ubuntu-latest
arch: "arm64"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "" # Disable sanitizer on arm64
nBPF: ""
@@ -177,7 +177,7 @@ jobs:
os: ubuntu-latest
arch: "armhf"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: "--with-sanitizer"
nBPF: ""
@@ -185,7 +185,7 @@ jobs:
os: ubuntu-latest
arch: "s390x"
gcrypt: ""
pcre: "--with-pcre"
pcre: "--with-pcre2"
maxminddb: "--with-maxminddb"
msan: ""
nBPF: ""
@@ -209,8 +209,8 @@ jobs:
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
run: |
sudo apt-get install libgcrypt20-dev
- name: Install Ubuntu Prerequisites (libpcre)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre')
- name: Install Ubuntu Prerequisites (libpcre2)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2')
run: |
sudo apt-get install libpcre3-dev
- name: Install Ubuntu Prerequisites (maxminddb)
@@ -266,8 +266,8 @@ jobs:
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
run: |
brew install libgcrypt
- name: Install MacOS Prerequisites (libpcre)
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre')
- name: Install MacOS Prerequisites (libpcre2)
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2')
run: |
brew install pcre
- name: Install MacOS Prerequisites (maxminddb)
+2 -2
View File
@@ -23,7 +23,7 @@ jobs:
sudo apt-get install libpcre3-dev libmaxminddb-dev lcov
sudo apt-get install wdiff colordiff
- name: Configure
run: ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre --with-maxminddb --enable-tls-sigs
run: ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre2 --with-maxminddb --enable-tls-sigs
- name: Build
run: make all
- name: Test
@@ -91,7 +91,7 @@ jobs:
pprof -h
- name: Configure nDPI library
run: |
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre --with-maxminddb --enable-tls-sigs
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs
- name: Build nDPI library
run: |
make
+1 -1
View File
@@ -42,7 +42,7 @@ jobs:
pprof -h
- name: Configure nDPI library
run: |
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre --with-maxminddb --enable-tls-sigs --enable-debug-messages
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs --enable-debug-messages
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
+2 -2
View File
@@ -51,7 +51,7 @@ On Windows:
There are three supported ways to build nDPI:
1. MSYS2 (assuming [MSYS2](https://www.msys2.org/) already installed):
- msys2 -c "pacman --noconfirm -S --needed --overwrite '\*' git mingw-w64-x86\_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86\_64-json-c mingw-w64-x86\_64-crt-git mingw-w64-x86\_64-pcre mingw-w64-x86\_64-libpcap"
- msys2 -c "pacman --noconfirm -S --needed --overwrite '\*' git mingw-w64-x86\_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86\_64-json-c mingw-w64-x86\_64-crt-git mingw-w64-x86\_64-pcre2 mingw-w64-x86\_64-libpcap"
2. Mingw-w64
@@ -79,7 +79,7 @@ The entire procedure of adding new protocols in detail:
5. Choose (do not change anything) a selection bitmask from: `src/include/ndpi_define.h`
6. Set protocol default ports in `ndpi_init_protocol_defaults` in: `src/lib/ndpi_main.c`
7. Be sure to have nBPF support, cloning `PF_RING` in the same directory where you cloned `nDPI`: `git clone https://github.com/ntop/PF_RING/ && cd PF_RING/userland/nbpf && ./configure && make`
8. From the `nDPI` root directory, `./autogen.sh --with-pcre` (nBPF and PCRE are usually optional, but they are needed to run/update *all* the unit tests)
8. From the `nDPI` root directory, `./autogen.sh --with-pcre2` (nBPF and PCRE2 are usually optional, but they are needed to run/update *all* the unit tests)
9. `make`
10. `make check`
11. Update the documentation, adding this new protocol to `doc/protocols.rst`
+9 -9
View File
@@ -359,14 +359,14 @@ AS_IF([test "${with_local_libgcrypt+set}" = set],[
AC_DEFINE_UNQUOTED(USE_HOST_LIBGCRYPT, 1, [Use locally installed libgcrypt instead of builtin gcrypt-light])
])
dnl> PCRE
PCRE_ENABLED=0
AC_ARG_WITH(pcre, AS_HELP_STRING([--with-pcre], [Enable nDPI build with libpcre]))
if test "${with_pcre+set}" = set; then :
AC_CHECK_LIB(pcre, pcre_compile, AC_DEFINE_UNQUOTED(HAVE_PCRE, 1, [libpcre(-dev) is present]))
if test "x$ac_cv_lib_pcre_pcre_compile" = xyes; then :
ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lpcre"
PCRE_ENABLED=1
dnl> PCRE2
PCRE2_ENABLED=0
AC_ARG_WITH(pcre2, AS_HELP_STRING([--with-pcre2], [Enable nDPI build with libpcre2]))
if test "${with_pcre2+set}" = set; then :
AC_CHECK_LIB(pcre2-8, pcre2_compile_8, AC_DEFINE_UNQUOTED(HAVE_PCRE2, 1, [libpcre2(-dev) is present]))
if test "x$ac_cv_lib_pcre2_8_pcre2_compile_8" = xyes; then :
ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lpcre2-8"
PCRE2_ENABLED=1
fi
fi
@@ -420,7 +420,7 @@ AC_SUBST(GPROF_CFLAGS)
AC_SUBST(GPROF_LIBS)
AC_SUBST(GPROF_ENABLED)
AC_SUBST(USE_HOST_LIBGCRYPT)
AC_SUBST(PCRE_ENABLED)
AC_SUBST(PCRE2_ENABLED)
AC_SUBST(NBPF_ENABLED)
AC_SUBST(HANDLE_TLS_SIGS)
AC_SUBST(DISABLE_NPCAP)
+23 -24
View File
@@ -62,12 +62,12 @@
// #define DEBUG_REASSEMBLY
#ifdef HAVE_PCRE
#include <pcre.h>
#ifdef HAVE_PCRE2
#define PCRE2_CODE_UNIT_WIDTH 8
#include <pcre2.h>
struct pcre_struct {
pcre *compiled;
pcre_extra *optimized;
struct pcre2_struct {
pcre2_code *compiled;
};
#endif
@@ -1702,18 +1702,19 @@ static int ndpi_is_xss_injection(char* query) {
/* ********************************** */
#ifdef HAVE_PCRE
#ifdef HAVE_PCRE2
static void ndpi_compile_rce_regex() {
const char *pcreErrorStr = NULL;
int pcreErrorOffset;
PCRE2_UCHAR pcreErrorStr[128];
PCRE2_SIZE pcreErrorOffset;
int pcreErrorCode;
for(int i = 0; i < N_RCE_REGEX; i++) {
comp_rx[i] = (struct pcre_struct*)ndpi_malloc(sizeof(struct pcre_struct));
comp_rx[i] = (struct pcre2_struct*)ndpi_malloc(sizeof(struct pcre2_struct));
comp_rx[i]->compiled = pcre_compile(rce_regex[i], 0, &pcreErrorStr,
comp_rx[i]->compiled = pcre2_compile((PCRE2_SPTR)rce_regex[i], PCRE2_ZERO_TERMINATED, 0, &pcreErrorCode,
&pcreErrorOffset, NULL);
pcre2_get_error_message(pcreErrorCode, pcreErrorStr, 128);
if(comp_rx[i]->compiled == NULL) {
#ifdef DEBUG
NDPI_LOG_ERR(ndpi_str, "ERROR: Could not compile '%s': %s\n", rce_regex[i],
@@ -1723,18 +1724,16 @@ static void ndpi_compile_rce_regex() {
continue;
}
comp_rx[i]->optimized = pcre_study(comp_rx[i]->compiled, 0, &pcreErrorStr);
pcreErrorCode = pcre2_jit_compile(comp_rx[i]->compiled, PCRE2_JIT_COMPLETE);
#ifdef DEBUG
if(pcreErrorStr != NULL) {
NDPI_LOG_ERR(ndpi_str, "ERROR: Could not study '%s': %s\n", rce_regex[i],
if(pcreErrorCode < 0) {
pcre2_get_error_message(pcreErrorCode, pcreErrorStr, 128);
NDPI_LOG_ERR(ndpi_str, "ERROR: Could not jit compile '%s': %s\n", rce_regex[i],
pcreErrorStr);
}
#endif
}
if(pcreErrorStr != NULL)
ndpi_free((void *)pcreErrorStr);
}
/* ********************************** */
@@ -1745,17 +1744,17 @@ static int ndpi_is_rce_injection(char* query) {
initialized_comp_rx = 1;
}
pcre2_match_data *pcreMatchData;
int pcreExecRet;
int subStrVec[30];
for(int i = 0; i < N_RCE_REGEX; i++) {
unsigned int length = strlen(query);
pcreExecRet = pcre_exec(comp_rx[i]->compiled,
comp_rx[i]->optimized,
query, length, 0, 0, subStrVec, 30);
if(pcreExecRet >= 0) {
pcreMatchData = pcre2_match_data_create_from_pattern(comp_rx[i]->compiled, NULL);
pcreExecRet = pcre2_match(comp_rx[i]->compiled,
(PCRE2_SPTR)query, length, 0, 0, pcreMatchData, NULL);
pcre2_match_data_free(pcreMatchData);
if(pcreExecRet > 0) {
return 1;
}
#ifdef DEBUG
@@ -1845,7 +1844,7 @@ ndpi_risk_enum ndpi_validate_url(char *url) {
rc = NDPI_URL_POSSIBLE_XSS;
else if(ndpi_is_sql_injection(decoded))
rc = NDPI_URL_POSSIBLE_SQL_INJECTION;
#ifdef HAVE_PCRE
#ifdef HAVE_PCRE2
else if(ndpi_is_rce_injection(decoded))
rc = NDPI_URL_POSSIBLE_RCE_INJECTION;
#endif
+3 -3
View File
@@ -1,4 +1,4 @@
#ifdef HAVE_PCRE
#ifdef HAVE_PCRE2
#ifndef NDPI_RCE_H
#define NDPI_RCE_H
@@ -8,7 +8,7 @@
#define N_RCE_REGEX 7
/* Compiled regex */
static struct pcre_struct *comp_rx[N_RCE_REGEX];
static struct pcre2_struct *comp_rx[N_RCE_REGEX];
static unsigned int initialized_comp_rx = 0;
@@ -615,4 +615,4 @@ static const char *pwsh_commands[] = {
"-PSConsoleFile"
};
#endif //HAVE_PCRE
#endif //HAVE_PCRE2
+2 -2
View File
@@ -26,7 +26,7 @@ CMD_COLORDIFF="$(which colordiff)"
EXE_SUFFIX=@EXE_SUFFIX@
GPROF_ENABLED=@GPROF_ENABLED@
PCRE_ENABLED=@PCRE_ENABLED@
PCRE2_ENABLED=@PCRE2_ENABLED@
PCRE_PCAPS="WebattackRCE.pcap"
NBPF_ENABLED=@NBPF_ENABLED@
NBPF_PCAPS="h323-overflow.pcap"
@@ -84,7 +84,7 @@ check_results() {
[ $SKIP_PCAP = 1 ] && continue
fi
SKIP_PCAP=0
if [ $PCRE_ENABLED -eq 0 ]; then
if [ $PCRE2_ENABLED -eq 0 ]; then
for p in $PCRE_PCAPS; do
if [ $f = $p ]; then
SKIP_PCAP=1