Move from PCRE to PCRE2 (#2134)
Move from PCRE to PCRE2. PCRE is EOL and won't receive any security updates anymore. Convert to PCRE2 by converting any function PCRE2 new API. Also update every entry in github workflows and README to point to the new configure flag. (--with-pcre2) Signed-off-by: Christian Marangi <ansuelsmth@gmail.com>
This commit is contained in:
@@ -21,7 +21,7 @@ A clear and concise description of what happening.
|
||||
* OS version: [e.g. 18.04]
|
||||
* Architecture: [e.g. arm64]
|
||||
* nDPI version or commit hash: [e.g. 4.0-stable or 937357e4bc55610f116f66d15a8e0fc1e260c02c].
|
||||
* nDPI compilation flags used: if you are building from source [e.g. --with-pcre --with-local-libgcrypt].
|
||||
* nDPI compilation flags used: if you are building from source [e.g. --with-pcre2 --with-local-libgcrypt].
|
||||
* Attach the `config.log` file generated after `./configure` ran (if you are building from source).
|
||||
|
||||
## How to reproduce the reported bug
|
||||
|
||||
+16
-16
@@ -93,7 +93,7 @@ jobs:
|
||||
os: ubuntu-20.04
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "--with-sanitizer"
|
||||
nBPF: ""
|
||||
@@ -101,7 +101,7 @@ jobs:
|
||||
os: ubuntu-22.04
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "--with-sanitizer"
|
||||
nBPF: ""
|
||||
@@ -110,7 +110,7 @@ jobs:
|
||||
os: ubuntu-20.04
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "--with-sanitizer"
|
||||
nBPF: ""
|
||||
@@ -120,7 +120,7 @@ jobs:
|
||||
os: ubuntu-22.04
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "--with-sanitizer"
|
||||
nBPF: ""
|
||||
@@ -129,7 +129,7 @@ jobs:
|
||||
os: ubuntu-latest
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "--with-thread-sanitizer"
|
||||
nBPF: ""
|
||||
@@ -137,7 +137,7 @@ jobs:
|
||||
os: ubuntu-latest
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "--with-sanitizer"
|
||||
nBPF: "nBPF"
|
||||
@@ -145,7 +145,7 @@ jobs:
|
||||
os: ubuntu-22.04
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "--with-memory-sanitizer"
|
||||
nBPF: ""
|
||||
@@ -153,7 +153,7 @@ jobs:
|
||||
os: macOS-latest
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "" # Disable sanitizer on macos
|
||||
nBPF: ""
|
||||
@@ -161,7 +161,7 @@ jobs:
|
||||
os: macos-12
|
||||
arch: "x86_64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "" # Disable sanitizer on macos
|
||||
nBPF: ""
|
||||
@@ -169,7 +169,7 @@ jobs:
|
||||
os: ubuntu-latest
|
||||
arch: "arm64"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "" # Disable sanitizer on arm64
|
||||
nBPF: ""
|
||||
@@ -177,7 +177,7 @@ jobs:
|
||||
os: ubuntu-latest
|
||||
arch: "armhf"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: "--with-sanitizer"
|
||||
nBPF: ""
|
||||
@@ -185,7 +185,7 @@ jobs:
|
||||
os: ubuntu-latest
|
||||
arch: "s390x"
|
||||
gcrypt: ""
|
||||
pcre: "--with-pcre"
|
||||
pcre: "--with-pcre2"
|
||||
maxminddb: "--with-maxminddb"
|
||||
msan: ""
|
||||
nBPF: ""
|
||||
@@ -209,8 +209,8 @@ jobs:
|
||||
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
|
||||
run: |
|
||||
sudo apt-get install libgcrypt20-dev
|
||||
- name: Install Ubuntu Prerequisites (libpcre)
|
||||
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre')
|
||||
- name: Install Ubuntu Prerequisites (libpcre2)
|
||||
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2')
|
||||
run: |
|
||||
sudo apt-get install libpcre3-dev
|
||||
- name: Install Ubuntu Prerequisites (maxminddb)
|
||||
@@ -266,8 +266,8 @@ jobs:
|
||||
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
|
||||
run: |
|
||||
brew install libgcrypt
|
||||
- name: Install MacOS Prerequisites (libpcre)
|
||||
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre')
|
||||
- name: Install MacOS Prerequisites (libpcre2)
|
||||
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2')
|
||||
run: |
|
||||
brew install pcre
|
||||
- name: Install MacOS Prerequisites (maxminddb)
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
sudo apt-get install libpcre3-dev libmaxminddb-dev lcov
|
||||
sudo apt-get install wdiff colordiff
|
||||
- name: Configure
|
||||
run: ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre --with-maxminddb --enable-tls-sigs
|
||||
run: ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre2 --with-maxminddb --enable-tls-sigs
|
||||
- name: Build
|
||||
run: make all
|
||||
- name: Test
|
||||
@@ -91,7 +91,7 @@ jobs:
|
||||
pprof -h
|
||||
- name: Configure nDPI library
|
||||
run: |
|
||||
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre --with-maxminddb --enable-tls-sigs
|
||||
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs
|
||||
- name: Build nDPI library
|
||||
run: |
|
||||
make
|
||||
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
pprof -h
|
||||
- name: Configure nDPI library
|
||||
run: |
|
||||
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre --with-maxminddb --enable-tls-sigs --enable-debug-messages
|
||||
./autogen.sh --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb --enable-tls-sigs --enable-debug-messages
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
|
||||
@@ -51,7 +51,7 @@ On Windows:
|
||||
There are three supported ways to build nDPI:
|
||||
|
||||
1. MSYS2 (assuming [MSYS2](https://www.msys2.org/) already installed):
|
||||
- msys2 -c "pacman --noconfirm -S --needed --overwrite '\*' git mingw-w64-x86\_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86\_64-json-c mingw-w64-x86\_64-crt-git mingw-w64-x86\_64-pcre mingw-w64-x86\_64-libpcap"
|
||||
- msys2 -c "pacman --noconfirm -S --needed --overwrite '\*' git mingw-w64-x86\_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86\_64-json-c mingw-w64-x86\_64-crt-git mingw-w64-x86\_64-pcre2 mingw-w64-x86\_64-libpcap"
|
||||
|
||||
2. Mingw-w64
|
||||
|
||||
@@ -79,7 +79,7 @@ The entire procedure of adding new protocols in detail:
|
||||
5. Choose (do not change anything) a selection bitmask from: `src/include/ndpi_define.h`
|
||||
6. Set protocol default ports in `ndpi_init_protocol_defaults` in: `src/lib/ndpi_main.c`
|
||||
7. Be sure to have nBPF support, cloning `PF_RING` in the same directory where you cloned `nDPI`: `git clone https://github.com/ntop/PF_RING/ && cd PF_RING/userland/nbpf && ./configure && make`
|
||||
8. From the `nDPI` root directory, `./autogen.sh --with-pcre` (nBPF and PCRE are usually optional, but they are needed to run/update *all* the unit tests)
|
||||
8. From the `nDPI` root directory, `./autogen.sh --with-pcre2` (nBPF and PCRE2 are usually optional, but they are needed to run/update *all* the unit tests)
|
||||
9. `make`
|
||||
10. `make check`
|
||||
11. Update the documentation, adding this new protocol to `doc/protocols.rst`
|
||||
|
||||
+9
-9
@@ -359,14 +359,14 @@ AS_IF([test "${with_local_libgcrypt+set}" = set],[
|
||||
AC_DEFINE_UNQUOTED(USE_HOST_LIBGCRYPT, 1, [Use locally installed libgcrypt instead of builtin gcrypt-light])
|
||||
])
|
||||
|
||||
dnl> PCRE
|
||||
PCRE_ENABLED=0
|
||||
AC_ARG_WITH(pcre, AS_HELP_STRING([--with-pcre], [Enable nDPI build with libpcre]))
|
||||
if test "${with_pcre+set}" = set; then :
|
||||
AC_CHECK_LIB(pcre, pcre_compile, AC_DEFINE_UNQUOTED(HAVE_PCRE, 1, [libpcre(-dev) is present]))
|
||||
if test "x$ac_cv_lib_pcre_pcre_compile" = xyes; then :
|
||||
ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lpcre"
|
||||
PCRE_ENABLED=1
|
||||
dnl> PCRE2
|
||||
PCRE2_ENABLED=0
|
||||
AC_ARG_WITH(pcre2, AS_HELP_STRING([--with-pcre2], [Enable nDPI build with libpcre2]))
|
||||
if test "${with_pcre2+set}" = set; then :
|
||||
AC_CHECK_LIB(pcre2-8, pcre2_compile_8, AC_DEFINE_UNQUOTED(HAVE_PCRE2, 1, [libpcre2(-dev) is present]))
|
||||
if test "x$ac_cv_lib_pcre2_8_pcre2_compile_8" = xyes; then :
|
||||
ADDITIONAL_LIBS="${ADDITIONAL_LIBS} -lpcre2-8"
|
||||
PCRE2_ENABLED=1
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -420,7 +420,7 @@ AC_SUBST(GPROF_CFLAGS)
|
||||
AC_SUBST(GPROF_LIBS)
|
||||
AC_SUBST(GPROF_ENABLED)
|
||||
AC_SUBST(USE_HOST_LIBGCRYPT)
|
||||
AC_SUBST(PCRE_ENABLED)
|
||||
AC_SUBST(PCRE2_ENABLED)
|
||||
AC_SUBST(NBPF_ENABLED)
|
||||
AC_SUBST(HANDLE_TLS_SIGS)
|
||||
AC_SUBST(DISABLE_NPCAP)
|
||||
|
||||
+23
-24
@@ -62,12 +62,12 @@
|
||||
|
||||
// #define DEBUG_REASSEMBLY
|
||||
|
||||
#ifdef HAVE_PCRE
|
||||
#include <pcre.h>
|
||||
#ifdef HAVE_PCRE2
|
||||
#define PCRE2_CODE_UNIT_WIDTH 8
|
||||
#include <pcre2.h>
|
||||
|
||||
struct pcre_struct {
|
||||
pcre *compiled;
|
||||
pcre_extra *optimized;
|
||||
struct pcre2_struct {
|
||||
pcre2_code *compiled;
|
||||
};
|
||||
#endif
|
||||
|
||||
@@ -1702,18 +1702,19 @@ static int ndpi_is_xss_injection(char* query) {
|
||||
|
||||
/* ********************************** */
|
||||
|
||||
#ifdef HAVE_PCRE
|
||||
#ifdef HAVE_PCRE2
|
||||
|
||||
static void ndpi_compile_rce_regex() {
|
||||
const char *pcreErrorStr = NULL;
|
||||
int pcreErrorOffset;
|
||||
PCRE2_UCHAR pcreErrorStr[128];
|
||||
PCRE2_SIZE pcreErrorOffset;
|
||||
int pcreErrorCode;
|
||||
|
||||
for(int i = 0; i < N_RCE_REGEX; i++) {
|
||||
comp_rx[i] = (struct pcre_struct*)ndpi_malloc(sizeof(struct pcre_struct));
|
||||
comp_rx[i] = (struct pcre2_struct*)ndpi_malloc(sizeof(struct pcre2_struct));
|
||||
|
||||
comp_rx[i]->compiled = pcre_compile(rce_regex[i], 0, &pcreErrorStr,
|
||||
comp_rx[i]->compiled = pcre2_compile((PCRE2_SPTR)rce_regex[i], PCRE2_ZERO_TERMINATED, 0, &pcreErrorCode,
|
||||
&pcreErrorOffset, NULL);
|
||||
|
||||
pcre2_get_error_message(pcreErrorCode, pcreErrorStr, 128);
|
||||
if(comp_rx[i]->compiled == NULL) {
|
||||
#ifdef DEBUG
|
||||
NDPI_LOG_ERR(ndpi_str, "ERROR: Could not compile '%s': %s\n", rce_regex[i],
|
||||
@@ -1723,18 +1724,16 @@ static void ndpi_compile_rce_regex() {
|
||||
continue;
|
||||
}
|
||||
|
||||
comp_rx[i]->optimized = pcre_study(comp_rx[i]->compiled, 0, &pcreErrorStr);
|
||||
pcreErrorCode = pcre2_jit_compile(comp_rx[i]->compiled, PCRE2_JIT_COMPLETE);
|
||||
|
||||
#ifdef DEBUG
|
||||
if(pcreErrorStr != NULL) {
|
||||
NDPI_LOG_ERR(ndpi_str, "ERROR: Could not study '%s': %s\n", rce_regex[i],
|
||||
if(pcreErrorCode < 0) {
|
||||
pcre2_get_error_message(pcreErrorCode, pcreErrorStr, 128);
|
||||
NDPI_LOG_ERR(ndpi_str, "ERROR: Could not jit compile '%s': %s\n", rce_regex[i],
|
||||
pcreErrorStr);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
if(pcreErrorStr != NULL)
|
||||
ndpi_free((void *)pcreErrorStr);
|
||||
}
|
||||
|
||||
/* ********************************** */
|
||||
@@ -1745,17 +1744,17 @@ static int ndpi_is_rce_injection(char* query) {
|
||||
initialized_comp_rx = 1;
|
||||
}
|
||||
|
||||
pcre2_match_data *pcreMatchData;
|
||||
int pcreExecRet;
|
||||
int subStrVec[30];
|
||||
|
||||
for(int i = 0; i < N_RCE_REGEX; i++) {
|
||||
unsigned int length = strlen(query);
|
||||
|
||||
pcreExecRet = pcre_exec(comp_rx[i]->compiled,
|
||||
comp_rx[i]->optimized,
|
||||
query, length, 0, 0, subStrVec, 30);
|
||||
|
||||
if(pcreExecRet >= 0) {
|
||||
pcreMatchData = pcre2_match_data_create_from_pattern(comp_rx[i]->compiled, NULL);
|
||||
pcreExecRet = pcre2_match(comp_rx[i]->compiled,
|
||||
(PCRE2_SPTR)query, length, 0, 0, pcreMatchData, NULL);
|
||||
pcre2_match_data_free(pcreMatchData);
|
||||
if(pcreExecRet > 0) {
|
||||
return 1;
|
||||
}
|
||||
#ifdef DEBUG
|
||||
@@ -1845,7 +1844,7 @@ ndpi_risk_enum ndpi_validate_url(char *url) {
|
||||
rc = NDPI_URL_POSSIBLE_XSS;
|
||||
else if(ndpi_is_sql_injection(decoded))
|
||||
rc = NDPI_URL_POSSIBLE_SQL_INJECTION;
|
||||
#ifdef HAVE_PCRE
|
||||
#ifdef HAVE_PCRE2
|
||||
else if(ndpi_is_rce_injection(decoded))
|
||||
rc = NDPI_URL_POSSIBLE_RCE_INJECTION;
|
||||
#endif
|
||||
|
||||
+3
-3
@@ -1,4 +1,4 @@
|
||||
#ifdef HAVE_PCRE
|
||||
#ifdef HAVE_PCRE2
|
||||
|
||||
#ifndef NDPI_RCE_H
|
||||
#define NDPI_RCE_H
|
||||
@@ -8,7 +8,7 @@
|
||||
#define N_RCE_REGEX 7
|
||||
|
||||
/* Compiled regex */
|
||||
static struct pcre_struct *comp_rx[N_RCE_REGEX];
|
||||
static struct pcre2_struct *comp_rx[N_RCE_REGEX];
|
||||
|
||||
static unsigned int initialized_comp_rx = 0;
|
||||
|
||||
@@ -615,4 +615,4 @@ static const char *pwsh_commands[] = {
|
||||
"-PSConsoleFile"
|
||||
};
|
||||
|
||||
#endif //HAVE_PCRE
|
||||
#endif //HAVE_PCRE2
|
||||
+2
-2
@@ -26,7 +26,7 @@ CMD_COLORDIFF="$(which colordiff)"
|
||||
|
||||
EXE_SUFFIX=@EXE_SUFFIX@
|
||||
GPROF_ENABLED=@GPROF_ENABLED@
|
||||
PCRE_ENABLED=@PCRE_ENABLED@
|
||||
PCRE2_ENABLED=@PCRE2_ENABLED@
|
||||
PCRE_PCAPS="WebattackRCE.pcap"
|
||||
NBPF_ENABLED=@NBPF_ENABLED@
|
||||
NBPF_PCAPS="h323-overflow.pcap"
|
||||
@@ -84,7 +84,7 @@ check_results() {
|
||||
[ $SKIP_PCAP = 1 ] && continue
|
||||
fi
|
||||
SKIP_PCAP=0
|
||||
if [ $PCRE_ENABLED -eq 0 ]; then
|
||||
if [ $PCRE2_ENABLED -eq 0 ]; then
|
||||
for p in $PCRE_PCAPS; do
|
||||
if [ $f = $p ]; then
|
||||
SKIP_PCAP=1
|
||||
|
||||
Reference in New Issue
Block a user