CI: enable parallel tests (for x86_64, at least) (#2444)

TODO: enable parallel tests when using docker with no-x86_64 archs.

When I tried the obviuos solutions:
```
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 make check VERBOSE=1
```
I got:
```
Run configuration "caches_cfg" [--cfg=lru.ookla.size,0 --cfg=lru.msteams.ttl,1]
ookla.pcap	/bin/sh: 1: run_single_pcap: not found
teams.pcap	/bin/sh: 1: run_single_pcap: not found
Run configuration "caches_global" [--cfg=lru.ookla.scope,1 --cfg=lru.bittorrent.scope,1 --cfg=lru.stun.scope,1 --cfg=lru.tls_cert.scope,1 --cfg=lru.mining.scope,1 --cfg=lru.msteams.scope,1 --cfg=lru.stun_zoom.scope,1]
bittorrent.pcap	/bin/sh: 1: run_single_pcap: not found
lru_ipv6_caches.pcapng	/bin/sh: 1: run_single_pcap: not found
mining.pcapng	/bin/sh: 1: run_single_pcap: not found
...
```
This commit is contained in:
Ivan Nardi
2024-05-20 19:07:16 +02:00
committed by GitHub
parent 399be12585
commit bc8bc69beb
2 changed files with 17 additions and 8 deletions
+7 -7
View File
@@ -235,7 +235,7 @@ jobs:
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
sudo apt-get install gcc-mingw-w64 libc6-dev
sudo apt-get install doxygen python3-sphinx python3-sphinx-rtd-theme python3-breathe python3-pip
sudo apt-get install rrdtool librrd-dev
sudo apt-get install rrdtool librrd-dev parallel
- name: Install Ubuntu Prerequisites (libgcrypt)
if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
run: |
@@ -302,7 +302,7 @@ jobs:
find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
sudo rm -rf /Library/Frameworks/Python.framework/
brew install --force python3 && brew unlink python3 && brew unlink python3 && brew link --overwrite python3
brew install coreutils wdiff colordiff autoconf automake libtool pkg-config gettext json-c rrdtool
brew install coreutils wdiff colordiff autoconf automake libtool pkg-config gettext json-c rrdtool parallel
- name: Install MacOS Prerequisites (libgcrypt)
if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt')
run: |
@@ -335,7 +335,7 @@ jobs:
- name: Build nDPI
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.os, 'macos-14')
run: |
make all
make -j all
make -C example ndpiSimpleIntegration
make -C rrdtool
- name: Build nDPI (MacOS M1)
@@ -363,7 +363,7 @@ jobs:
- name: Test nDPI [DIFF]
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows')
run: |
./tests/do.sh
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
- name: Test nDPI [UNIT]
#Some issues with masan + json-c. Disable the test as workaround
if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14')
@@ -417,7 +417,7 @@ jobs:
apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
git config --global --add safe.directory $(realpath .) &&
env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
make all &&
make -j all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make check VERBOSE=1
@@ -441,7 +441,7 @@ jobs:
apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
git config --global --add safe.directory $(realpath .) &&
env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
make all &&
make -j all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make check VERBOSE=1
@@ -465,7 +465,7 @@ jobs:
apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev &&
git config --global --add safe.directory $(realpath .) &&
env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs &&
make all &&
make -j all &&
make -C example ndpiSimpleIntegration &&
make -C rrdtool &&
make check VERBOSE=1
+10 -1
View File
@@ -26,6 +26,11 @@ if [ "${NDPI_FORCE_PARALLEL_UTESTS}" = "1" ]; then
FORCE_PARALLEL_UTESTS=1
fi
SKIP_PARALLEL_BAR=0
if [ "${NDPI_SKIP_PARALLEL_BAR}" = "1" ]; then
SKIP_PARALLEL_BAR=1
fi
#Remember: valgrind and *SAN are incompatible!
CMD_PREFIX="${CMD_PREFIX}"
if [ "${NDPI_TESTS_WINE}" = "1" ]; then
@@ -188,7 +193,11 @@ export -f run_single_pcap
check_results() {
if [ $FORCE_PARALLEL_UTESTS -eq 1 ]; then
parallel --bar --tag "run_single_pcap" ::: $PCAPS
if [ $SKIP_PARALLEL_BAR -eq 1 ]; then
parallel --tag "run_single_pcap" ::: $PCAPS
else
parallel --bar --tag "run_single_pcap" ::: $PCAPS
fi
RET=$? #Number of failed job up to 100
RC=$(( RC + $RET ))
else