Skip to content

Scheduled builds

Scheduled builds #1181

name: Scheduled builds
on:
workflow_dispatch:
schedule:
#At the end of every day
- cron: '0 0 * * *'
jobs:
coverage:
name: Coverage (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
sudo apt-get install rrdtool librrd-dev
sudo apt-get install libpcre3-dev libmaxminddb-dev lcov
sudo apt-get install wdiff colordiff
- name: Configure
run: ./autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages --enable-code-coverage --with-pcre2 --with-maxminddb
- name: Build
run: make all
- name: Test
run: |
make check VERBOSE=1
- name: Generate Coverage Report
run: |
make coverage
- uses: actions/upload-artifact@v7
with:
name: ndpi-coverage-report
path: coverage_report
retention-days: 7
documentation:
name: Documentation (ubuntu-latest)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Set up Python 3.9
uses: actions/setup-python@v6
with:
python-version: "3.9"
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext flex bison doxygen
- name: Configure nDPI library
run: |
./autogen.sh && ./configure --with-only-libndpi --enable-option-checking=fatal --enable-debug-build
- name: Generate Documentation
run: |
pip install --upgrade pip
pip install -r doc/requirements.txt
make doc
mkdir -vp doc/_build/ndpi-documentation-upload/ndpi-documentation
mv -v doc/_build/html doc/_build/ndpi-documentation-upload/ndpi-documentation/html
- uses: actions/upload-artifact@v7
with:
name: ndpi-documentation
path: doc/_build/ndpi-documentation-upload
retention-days: 7
performance:
name: Performance (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install libunwind-dev
sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libpcap-dev parallel
sudo apt-get install libgoogle-perftools-dev graphviz
go install github.com/google/pprof@latest
sudo ln -s ${HOME}/go/bin/pprof /usr/bin/pprof
pprof -h
- name: Configure nDPI library
run: |
./autogen.sh && ./configure --enable-gprof --enable-option-checking=fatal --with-pcre2 --with-maxminddb
- name: Build nDPI library
run: |
make -j
- name: Performance Profiling
run: |
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
mkdir ndpi-performance-upload
for d in $(find ./tests/cfgs/* -type d -maxdepth 0 2>/dev/null) ; do
PROFILE="$(basename $d)"
mv -v tests/cfgs/${PROFILE}/result/cpu_profile.png ndpi-performance-upload/${PROFILE}_cpu_profile.png
mv -v tests/cfgs/${PROFILE}/result/heap_profile.png ndpi-performance-upload/${PROFILE}_heap_profile.png
done
- uses: actions/upload-artifact@v7
with:
name: ndpi-performance
path: ndpi-performance-upload
retention-days: 7
threadsanitizer:
name: Thread Sanitizer (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c-dev libpcap-dev rrdtool librrd-dev parallel
- name: Configure nDPI
run: |
./autogen.sh && ./configure --with-thread-sanitizer
- name: Build nDPI
run: |
make -j $(nproc) all
make -j $(nproc) -C example
make -j $(nproc) -C rrdtool
- name: Tests
run: |
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
./tests/do-unit.sh
./tests/do-dga.sh
nBPF:
name: nBPF (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c-dev libpcap-dev rrdtool librrd-dev parallel
- name: Install Ubuntu Prerequisites (nBPF)
run: |
git clone https://github.com/ntop/PF_RING.git ../PF_RING
cd ../PF_RING/userland/nbpf
./configure
make
cd -
- name: Configure nDPI
run: |
./autogen.sh && ./configure
- name: Build nDPI
run: |
make -j $(nproc) all
make -j $(nproc) -C example
make -j $(nproc) -C rrdtool
- name: Tests
run: |
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
./tests/do-unit.sh
./tests/do-dga.sh
no-dependencies:
name: no-dependencies (global_context and plugin) (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c-dev libpcap-dev rrdtool librrd-dev parallel
- name: Configure nDPI
run: |
./autogen.sh && ./configure --disable-global-context-support --disable-plugin-support
- name: Build nDPI
run: |
make -j $(nproc) all
make -j $(nproc) -C example
make -j $(nproc) -C rrdtool
- name: Tests
run: |
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
./tests/do-unit.sh
./tests/do-dga.sh
builds:
name: ${{ matrix.os }} ${{matrix.gcrypt}}
runs-on: ${{ matrix.os }}
env:
CFLAGS: -g -O2 -Werror -DNDPI_EXTENDED_SANITY_CHECKS
strategy:
fail-fast: true
matrix:
# macos-14,15 and 26 are on **ARM64**
os: ["ubuntu-22.04", "ubuntu-24.04", "macos-15-intel", "macOS-14", "macOS-15", "macOS-26", "ubuntu-22.04-arm", "ubuntu-24.04-arm"]
gcrypt: ["", "--with-local-libgcrypt"]
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Ubuntu Prerequisites
if: startsWith(matrix.os, 'ubuntu')
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c-dev libpcap-dev rrdtool librrd-dev parallel libgcrypt20-dev
- name: Installing MacOS prerequisites
if: startsWith(matrix.os, 'macOS')
run: |
# Avoid (re)installing pkg-config. See: https://github.com/actions/runner-images/issues/10984
# Avoid (re)installing gettext 1.0, to avoid "gettext 1.0 is already installed and up-to-date.
# To reinstall 1.0, run: brew reinstall gettext" warning
# Avoid (re)installing libgcrypt
brew install coreutils wdiff colordiff autoconf automake libtool json-c rrdtool parallel
- name: Configure nDPI
run: |
./autogen.sh && ./configure ${{matrix.gcrypt}}
- name: Build nDPI
run: |
make -j $(nproc) all
make -j $(nproc) -C example
make -j $(nproc) -C rrdtool
- name: Tests
run: |
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh
./tests/do-unit.sh
./tests/do-dga.sh
localgcrypt-windows:
name: Local gcrypt on ${{ matrix.os }} (msys2)
runs-on: ${{ matrix.os }}
env:
CFLAGS: -Werror -DNDPI_EXTENDED_SANITY_CHECKS
strategy:
fail-fast: true
matrix:
os: ["windows-latest"]
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Install Windows msys2 prerequisites
uses: msys2/setup-msys2@v2
with:
msystem: MINGW64
update: true
install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel
- name: Configure nDPI on Windows msys2
run: |
msys2 -c './autogen.sh && ./configure --enable-option-checking=fatal --enable-debug-messages --disable-npcap --with-local-libgcrypt'
- name: Build nDPI on Windows msys2
run: |
msys2 -c 'make -j all'
msys2 -c 'ldd ./example/ndpiReader.exe'
- name: Tests
run: |
# Don't know why but lately the script in parallel mode is stuck...
#msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh'
msys2 -c './tests/do.sh'
msys2 -c './tests/do-unit.sh'
msys2 -c './tests/do-dga.sh'
test-lto-gold:
# Options used by oss-fuzz: we only want to check that everything compile fine; no need to run the tests
name: LTO and Gold Linker ${{ matrix.compiler }}
runs-on: ubuntu-latest
env:
CC: ${{ matrix.compiler }}
CFLAGS: -Werror -DNDPI_EXTENDED_SANITY_CHECKS
strategy:
fail-fast: true
matrix:
compiler: ["gcc-14", "clang-18"] # "Newest" gcc/clang easily available
include:
- compiler: "gcc-14"
ar: "ar"
ranlib: "ranlib"
- compiler: "clang-18"
ar: "llvm-ar-18"
ranlib: "llvm-ranlib-18"
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev parallel
- name: Configure nDPI
run: |
./autogen.sh && AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./configure --enable-option-checking=fatal --enable-debug-messages --with-sanitizer --with-lto-and-gold-linker
- name: Build nDPI
run: |
make -j $(nproc) all
make -j $(nproc) -C example
make -j $(nproc) -C rrdtool
- name: Print nDPI long help
run: |
cd ./example && ./ndpiReader -H
test-out-of-tree-builds:
name: Out-of-tree builds
runs-on: ubuntu-latest
env:
CFLAGS: -Werror -DNDPI_EXTENDED_SANITY_CHECKS -g -O2
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis
- name: Install Ubuntu Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev parallel
- name: Autogen nDPI
run: |
./autogen.sh
- name: Configure and build nDPI (standard)
run: |
mkdir -p build-standard
cd build-standard
../configure
make -j $(nproc) all
make -j $(nproc) -C example
make -j $(nproc) -C rrdtool
cd -
- name: Configure and build nDPI (fuzzing)
run: |
mkdir -p build-fuzzing
cd build-fuzzing
CC=clang CXX=clang++ ../configure --with-sanitizer --enable-fuzztargets
make -j $(nproc) all
make -j $(nproc) -C example
make -j $(nproc) -C rrdtool
cd -
- name: Print nDPI long help
run: |
cd ./build-standard/example && ./ndpiReader -H && cd -
cd ./build-fuzzing/example && ./ndpiReader -H && cd -
- name: Tests #We are only intersting into tests scripts being valid --> less tests than usual
run: |
cd build-standard
NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 ./tests/do.sh
./tests/do-unit.sh
./tests/do-dga.sh
cd -
cd build-fuzzing
MAX_TOTAL_TIME=10 NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_FORCE_PARALLEL_CONFIGS=1 NDPI_SKIP_PARALLEL_BAR=1 NDPI_TEST_ONLY_RECENTLY_UPDATED_PCAPS=1 ./tests/do.sh
./tests/do-unit.sh
./tests/do-dga.sh
cd -
test-usdt:
name: USDT probes (ubuntu-latest)
runs-on: ubuntu-latest
env:
CFLAGS: -Werror -g -O2
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Prerequisites
run: |
sudo apt-get update
sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev
sudo apt-get install systemtap-sdt-dev bpftrace pahole llvm
- name: Download latest bpftrace
run: |
curl -L -O https://github.com/bpftrace/bpftrace/releases/download/v0.24.2/bpftrace
chmod +x bpftrace
- name: Configure nDPI with USDT
run: |
# Workaround: C11 '_Atomic' type (from croaring code) is not fully supported for BPF CO-RE purposes
# This is needed only if you want to dereference `struct ndpi_flow_struct *`
# in the probe actions
./autogen.sh && CFLAGS="-DCROARING_ATOMIC_IMPL=1" ./configure --enable-option-checking=fatal --enable-debug-build --enable-debug-messages --enable-usdt-probes
- name: Build nDPI
run: |
make -j all
- name: Build bpftool
run: |
# Temporary workaround: some 6.14 kernels don't have bpftool (and perf) in their packages
# See: https://bugs.launchpad.net/ubuntu/+source/linux-hwe-6.14/+bug/2117147
# See: https://gist.github.com/karlivory/9111f906f4eb06370b2b237c62a6b00e
curl -L -O https://gist.githubusercontent.com/karlivory/9111f906f4eb06370b2b237c62a6b00e/raw/dbd24228dda79e05815531a1db643d0709301838/build-perf.sh
sudo bash build-perf.sh
- name: Generate header file for bpftrace
run: |
pahole -J src/lib/libndpi.so
pahole -J example/ndpiReader
bpftool btf dump file example/ndpiReader format c > ndpi_types.h
- name: Verify USDT probes are embedded (readelf)
run: |
echo "=== Probes in libndpi.so ==="
readelf -n src/lib/libndpi.so | grep -A4 stapsdt
echo "=== Probes in ndpiReader ==="
readelf -n example/ndpiReader | grep -A4 stapsdt
# Verify both probes are present
readelf -n example/ndpiReader | grep -q 'flow_classified'
readelf -n example/ndpiReader | grep -q 'hostname_set'
readelf -n example/ndpiReader | grep -q 'fragment_ipv4'
readelf -n example/ndpiReader | grep -q 'fragment_ipv6'
echo "All expected USDT probes found."
- name: List probes via bpftrace
run: |
sudo bpftrace -l "usdt:./example/ndpiReader:ndpi:*" | tee /tmp/probe_list.txt
grep -q 'flow_classified' /tmp/probe_list.txt
grep -q 'hostname_set' /tmp/probe_list.txt
grep -q 'fragment_ipv4' /tmp/probe_list.txt
grep -q 'fragment_ipv6' /tmp/probe_list.txt
echo "bpftrace can see all probes."
- name: Test flow_classified probe with ndpiReader
run: |
sudo bpftrace -e '
usdt:./example/ndpiReader:ndpi:flow_classified {
@flows = count();
@proto_master[arg0] = count();
@confidence[arg2] = count();
@category[arg3] = count();
}' -c './example/ndpiReader -q -i tests/pcap/http.pcapng' 2>&1 | tee /tmp/bpf_classified.txt
# Verify we actually traced some flows
grep -q '@flows:' /tmp/bpf_classified.txt
echo "flow_classified probe (scalar args): OK"
- name: Test flow_classified probe with flow pointer (arg4) via header
run: |
# Embedding .BTF section and reading it from bpftrace is still a bit buggy.
# Use explicit header, with latest bpftrace
sudo ./bpftrace -I . --include ndpi_types.h \
-e 'usdt:./example/ndpiReader:ndpi:flow_classified {
$flow = (struct ndpi_flow_struct *)arg4;
@flows = count();
if ($flow->risk != 0) { @risky[arg0] = count(); }
}' -c './example/ndpiReader -q -i tests/pcap/1kxun.pcap' 2>&1 | tee /tmp/bpf_classified_ptr.txt
grep -q '@flows:' /tmp/bpf_classified_ptr.txt
echo "flow_classified probe (struct dereference): OK"
- name: Test hostname_set probe with ndpiReader
run: |
READER=$(realpath example/ndpiReader)
sudo ./bpftrace -I . --include ndpi_types.h \
-e 'usdt:./example/ndpiReader:ndpi:hostname_set {
$flow = (struct ndpi_flow_struct *)arg1;
@hostnames = count();
@top[str(arg0), $flow->detected_protocol_stack[0]] = count();
}' -c './example/ndpiReader -q -i tests/pcap/tls_certificate_too_long.pcap' 2>&1 | tee /tmp/bpf_hostname.txt
# Verify we actually traced some hostnames
grep -q '@hostnames:' /tmp/bpf_hostname.txt
echo "hostname_set probe: OK"
- name: Test both hostname_set and flow_classified simultaneously
run: |
sudo bpftrace -e '
usdt:./example/ndpiReader:ndpi:hostname_set {
@hostnames = count();
}
usdt:./example/ndpiReader:ndpi:flow_classified {
@classified[arg0] = count();
}' -c './example/ndpiReader -q -i tests/pcap/dns.pcap' 2>&1 | tee /tmp/bpf_both.txt
grep -q '@classified' /tmp/bpf_both.txt
grep -q '@hostnames' /tmp/bpf_both.txt
echo "Both probes fired successfully."
- name: Test fragment_ipv4 probe with ndpiReader
run: |
READER=$(realpath example/ndpiReader)
sudo ./bpftrace -I . --include ndpi_types.h \
-e 'usdt:./example/ndpiReader:ndpi:fragment_ipv4 {
$iph = (struct ndpi_iphdr *)arg0;
@frags = count();
@top[ntop($iph->saddr)] = count();
}' -c './example/ndpiReader -q -i tests/pcap/ip_fragmented_garbage.pcap' 2>&1 | tee /tmp/frag_ipv4.txt
# Verify we actually traced some hostnames
grep -q '@frags:' /tmp/frag_ipv4.txt
grep -q '@top' /tmp/frag_ipv4.txt
echo "fragment_ipv4 probe: OK"
- name: Test fragment_ipv6 probe with ndpiReader
run: |
READER=$(realpath example/ndpiReader)
sudo ./bpftrace -I . --include ndpi_types.h \
-e 'usdt:./example/ndpiReader:ndpi:fragment_ipv6 {
$ip6h = (struct ndpi_ipv6hdr *)arg0;
@frags = count();
@top[ntop($ip6h->ip6_src.u6_addr.u6_addr8)] = count();
}' -c './example/ndpiReader -q -i tests/pcap/dns_fragmented.pcap' 2>&1 | tee /tmp/frag_ipv6.txt
# Verify we actually traced some hostnames
grep -q '@frags:' /tmp/frag_ipv6.txt
grep -q '@top' /tmp/frag_ipv6.txt
echo "fragment_ipv6 probe: OK"