Skip to content

Commit 4cb0337

Browse files
rgacognergacogne
authored
Merge pull request PowerDNS#16191 from rgacogne/ddist-test-meson-dist
build-and-test-all: Build dnsdist and recursor via a release tarball
2 parents cf12021 + 2bbb372 commit 4cb0337

8 files changed

Lines changed: 134 additions & 65 deletions

File tree

‎.github/scripts/normalize_paths_in_coverage.py‎

Lines changed: 75 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,68 @@
22

33
import os
44
import sys
5+
from pathlib import Path
56

6-
if __name__ == '__main__':
7+
DEBUG = False
8+
9+
def debug_print(string):
10+
if DEBUG:
11+
print(string)
12+
13+
def get_relative_to_product_source_dir(product, target):
14+
if product == 'auth':
15+
# authoritative or tool
16+
return target
17+
if product == 'recursor':
18+
return os.path.join('pdns', 'recursordist', target)
19+
if product == 'dnsdist':
20+
return os.path.join('pdns', 'dnsdistdist', target)
21+
return None
22+
23+
def remove_meson_dist_path(product, target):
24+
# target looks like this: /tmp/dnsdist-meson-dist-build/meson-dist/dnsdist-0.0.0-git1/xsk.hh
25+
path = Path(target)
26+
index = path.parts.index('meson-dist')
27+
# skip up to meson-dist and the directory below that,
28+
# so we now have: xsk.hh
29+
relevant = str(path.relative_to(path.parents[len(path.parts) - (index + 3)]))
30+
return get_relative_to_product_source_dir(product, relevant)
31+
32+
33+
def remove_dist_dir_path(repositoryRoot, version, target):
34+
# get rid of the distdir path, to get file paths as they are in the repository
35+
# if we are building from meson, it might look like this:
36+
# /__w/pdns/pdns/pdns/dnsdistdist/dnsdist-0.0.0-git1/config.h
37+
if f'pdns-{version}' in target:
38+
# authoritative or tool
39+
authPath = os.path.join(repositoryRoot, f'pdns-{version}')
40+
relativeToAuth = os.path.relpath(target, authPath)
41+
target = get_relative_to_product_source_dir('auth', relativeToAuth)
42+
return target
43+
if f'pdns-recursor-{version}' in target:
44+
recPath = os.path.join(repositoryRoot, 'pdns', 'recursordist', f'pdns-recursor-{version}')
45+
relativeToRec = os.path.relpath(target, recPath)
46+
return get_relative_to_product_source_dir('recursor', relativeToRec)
47+
if f'dnsdist-{version}' in target:
48+
distPath = os.path.join(repositoryRoot, 'pdns', 'dnsdistdist', f'dnsdist-{version}')
49+
relativeToDist = os.path.relpath(target, distPath)
50+
target = get_relative_to_product_source_dir('dnsdist', relativeToDist)
51+
return target
52+
53+
# let's assume we already have a full path to the repository, like
54+
# /__w/pdns/pdns/pdns/auth-catalogzone.hh
55+
distPath = os.path.join(repositoryRoot)
56+
relativeToDist = os.path.relpath(target, distPath)
57+
return relativeToDist
58+
59+
def process():
760
repositoryRoot = os.path.realpath(sys.argv[1])
8-
version = sys.argv[2]
9-
inputFile = sys.argv[3]
10-
outputFile = sys.argv[4]
11-
fromDistDir = sys.argv[5]
12-
with open(inputFile, mode='r') as inputFilePtr:
13-
with open(outputFile, mode='w') as outputFilePtr:
61+
product = sys.argv[2]
62+
version = sys.argv[3]
63+
inputFile = sys.argv[4]
64+
outputFile = sys.argv[5]
65+
with open(inputFile, mode='r', encoding='utf-8') as inputFilePtr:
66+
with open(outputFile, mode='w', encoding='utf-8') as outputFilePtr:
1467
for line in inputFilePtr:
1568
if not line.startswith('SF:'):
1669
outputFilePtr.write(line)
@@ -24,31 +77,30 @@
2477
source_file = parts[1].rstrip()
2578
# get rid of symbolic links
2679
target = os.path.realpath(source_file)
80+
debug_print(f'- Got source_file={source_file}, target={target}')
81+
82+
if '/meson-dist/' in target:
83+
# this is a file that comes from a meson dist tarball
84+
target = remove_meson_dist_path(product, target)
85+
debug_print(f'meson-dist -> target={target}')
86+
else:
87+
target = remove_dist_dir_path(repositoryRoot, version, target)
88+
debug_print(f'dist dir -> target={target}')
2789

28-
# get rid of the distdir path, to get file paths as they are in the repository
29-
if f'pdns-{version}' in target:
30-
# authoritative or tool
31-
authPath = os.path.join(repositoryRoot, f'pdns-{version}')
32-
relativeToAuth = os.path.relpath(target, authPath)
33-
target = relativeToAuth
34-
elif f'pdns-recursor-{version}' in target:
35-
recPath = os.path.join(repositoryRoot, 'pdns', 'recursordist', f'pdns-recursor-{version}')
36-
relativeToRec = os.path.relpath(target, recPath)
37-
target = os.path.join('pdns', 'recursordist', relativeToRec)
38-
elif f'dnsdist-{version}' in target:
39-
distPath = os.path.join(repositoryRoot, 'pdns', 'dnsdistdist', f'dnsdist-{version}')
40-
relativeToDist = os.path.relpath(target, distPath)
41-
target = os.path.join('pdns', 'dnsdistdist', relativeToDist)
42-
elif fromDistDir == '1':
43-
print(f'Ignoring {target} that we could not map to a distdir', file=sys.stderr)
90+
if target is None:
4491
continue
4592

4693
# we need to properly map symbolic links
4794
fullPath = os.path.join(repositoryRoot, target)
95+
debug_print(f'fullPath is {fullPath}')
4896
if os.path.islink(fullPath):
4997
# get the link target
5098
realPath = os.path.realpath(fullPath)
5199
# and make it relative again
52100
target = os.path.relpath(realPath, repositoryRoot)
53101

102+
debug_print(f'=> final target is {target}')
54103
outputFilePtr.write(f"SF:{target}\n")
104+
105+
if __name__ == '__main__':
106+
process()

‎.github/workflows/build-and-test-all.yml‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ jobs:
111111
- run: ${{ env.INV_CMD }} ci-auth-run-unit-tests ${{ matrix.builder == 'meson' && '--meson' || '' }}
112112
env:
113113
PDNS_BUILD_PATH: ../pdns-${{ env.BUILDER_VERSION }}
114-
- run: ${{ env.INV_CMD }} generate-coverage-info ./pdns-auth-testrunner $GITHUB_WORKSPACE
114+
- run: ${{ env.INV_CMD }} generate-coverage-info ./pdns-auth-testrunner 'auth' $GITHUB_WORKSPACE
115115
if: ${{ env.COVERAGE == 'yes' && matrix.builder == 'meson' }}
116116
- name: Coveralls Parallel auth unit
117117
if: ${{ env.COVERAGE == 'yes' && matrix.builder == 'meson' }}
@@ -199,7 +199,7 @@ jobs:
199199
working-directory: ./pdns/recursordist/
200200
- run: ${{ env.INV_CMD }} ci-rec-build ${{ matrix.builder == 'meson' && '--meson' || '' }}
201201
- run: ${{ env.INV_CMD }} ci-rec-run-unit-tests ${{ matrix.builder == 'meson' && '--meson' || '' }}
202-
- run: ${{ env.INV_CMD }} generate-coverage-info ./testrunner $GITHUB_WORKSPACE
202+
- run: ${{ env.INV_CMD }} generate-coverage-info ./testrunner 'recursor' $GITHUB_WORKSPACE
203203
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' && matrix.builder == 'meson' }}
204204
- name: Coveralls Parallel rec unit
205205
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' && matrix.builder == 'meson' }}
@@ -280,7 +280,6 @@ jobs:
280280
- run: ${{ env.INV_CMD }} install-lld-linker-if-needed
281281
working-directory: ./pdns/dnsdistdist/
282282
- run: ${{ env.INV_CMD }} ci-install-rust ${REPO_HOME}
283-
if: ${{ matrix.features != 'least' }}
284283
working-directory: ./pdns/dnsdistdist/
285284
- run: ${{ env.INV_CMD }} ci-build-and-install-quiche ${REPO_HOME}
286285
if: ${{ matrix.features != 'least' }}
@@ -290,14 +289,16 @@ jobs:
290289
working-directory: ./pdns/dnsdistdist/
291290
- run: ${{ env.INV_CMD }} ci-dnsdist-configure ${{ matrix.features }} ${{ matrix.builder }} dnsdist-${{ env.BUILDER_VERSION }}
292291
working-directory: ./pdns/dnsdistdist/
293-
- run: ${{ env.INV_CMD }} ci-make-distdir
292+
if: ${{ matrix.builder != 'autotools' }}
293+
- run: |
294+
mkdir dnsdist-${{ env.BUILDER_VERSION }}
294295
if: ${{ matrix.builder == 'autotools' }}
295296
working-directory: ./pdns/dnsdistdist/
296297
- run: ${{ env.INV_CMD }} ci-dnsdist-configure ${{ matrix.features }} ${{ matrix.builder }} dnsdist-${{ env.BUILDER_VERSION }}
297298
if: ${{ matrix.builder == 'autotools' }}
298299
- run: ${{ env.INV_CMD }} ci-dnsdist-make-bear ${{ matrix.builder }}
299300
- run: ${{ env.INV_CMD }} ci-dnsdist-run-unit-tests ${{ matrix.builder }}
300-
- run: ${{ env.INV_CMD }} generate-coverage-info ./testrunner $GITHUB_WORKSPACE
301+
- run: ${{ env.INV_CMD }} generate-coverage-info ./testrunner 'dnsdist' $GITHUB_WORKSPACE
301302
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' && matrix.builder == 'meson'}}
302303
- name: Coveralls Parallel dnsdist unit
303304
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' && matrix.builder == 'meson' }}
@@ -383,7 +384,7 @@ jobs:
383384
- run: ${{ env.INV_CMD }} install-clang-runtime
384385
- run: ${{ env.INV_CMD }} install-auth-test-deps -b ${{ matrix.backend }}
385386
- run: ${{ env.INV_CMD }} test-api auth -b ${{ matrix.backend }}
386-
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-auth/sbin/pdns-auth $GITHUB_WORKSPACE
387+
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-auth/sbin/pdns-auth 'auth' $GITHUB_WORKSPACE
387388
if: ${{ env.COVERAGE == 'yes' }}
388389
- name: Coveralls Parallel auth API ${{ matrix.backend }}
389390
if: ${{ env.COVERAGE == 'yes' }}
@@ -515,7 +516,7 @@ jobs:
515516
- run: ${{ env.INV_CMD }} install-clang-runtime
516517
- run: ${{ env.INV_CMD }} install-auth-test-deps -b ${{ matrix.backend }}
517518
- run: ${{ env.INV_CMD }} test-auth-backend -b ${{ matrix.backend }}
518-
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-auth/sbin/pdns-auth $GITHUB_WORKSPACE
519+
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-auth/sbin/pdns-auth 'auth' $GITHUB_WORKSPACE
519520
if: ${{ env.COVERAGE == 'yes' }}
520521
- name: Coveralls Parallel auth backend ${{ matrix.backend }}
521522
if: ${{ env.COVERAGE == 'yes' }}
@@ -558,7 +559,7 @@ jobs:
558559
- run: ${{ env.INV_CMD }} install-clang-runtime
559560
- run: ${{ env.INV_CMD }} install-auth-test-deps
560561
- run: ${{ env.INV_CMD }} test-ixfrdist
561-
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-auth/bin/ixfrdist $GITHUB_WORKSPACE
562+
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-auth/bin/ixfrdist 'auth' $GITHUB_WORKSPACE
562563
if: ${{ env.COVERAGE == 'yes' }}
563564
- name: Coveralls Parallel ixfrdist
564565
if: ${{ env.COVERAGE == 'yes' }}
@@ -610,7 +611,7 @@ jobs:
610611
- run: ${{ env.INV_CMD }} install-clang-runtime
611612
- run: ${{ env.INV_CMD }} install-rec-test-deps
612613
- run: ${{ env.INV_CMD }} test-api recursor
613-
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE
614+
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor 'recursor' $GITHUB_WORKSPACE
614615
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
615616
- name: Coveralls Parallel recursor API
616617
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
@@ -664,7 +665,7 @@ jobs:
664665
- run: ${{ env.INV_CMD }} install-clang-runtime
665666
- run: ${{ env.INV_CMD }} install-rec-test-deps
666667
- run: ${{ env.INV_CMD }} test-regression-recursor
667-
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE
668+
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor 'recursor' $GITHUB_WORKSPACE
668669
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
669670
- name: Coveralls Parallel recursor regression
670671
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
@@ -717,7 +718,7 @@ jobs:
717718
- run: ${{ env.INV_CMD }} install-clang-runtime
718719
- run: ${{ env.INV_CMD }} install-rec-bulk-deps
719720
- run: ${{ env.INV_CMD }} test-bulk-recursor 100 ${{ matrix.threads }} ${{ matrix.mthreads }} ${{ matrix.shards }} ${{ matrix.IPv6 }}
720-
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE
721+
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor 'recursor' $GITHUB_WORKSPACE
721722
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
722723
- name: Coveralls Parallel recursor bulk
723724
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
@@ -768,7 +769,7 @@ jobs:
768769
ASAN_OPTIONS: detect_leaks=0
769770
TSAN_OPTIONS: "halt_on_error=1:suppressions=${{ github.workspace }}/pdns/recursordist/recursor-tsan.supp"
770771
# Disabled, it gives us: "/bin/bash: line 1: llvm-profdata-13: command not found" due to mismatch between deb and ubuntu versions
771-
#- run: . ${{ github.workspace }}/.venv/bin/activate && inv generate-coverage-info /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE
772+
#- run: . ${{ github.workspace }}/.venv/bin/activate && inv generate-coverage-info 'recursor' /opt/pdns-recursor/sbin/pdns_recursor $GITHUB_WORKSPACE
772773
# if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
773774
#- name: Coveralls Parallel recursor bulk
774775
# if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
@@ -824,7 +825,7 @@ jobs:
824825
- run: ${{ env.INV_CMD }} install-clang-runtime
825826
- run: ${{ env.INV_CMD }} install-dnsdist-test-deps $([ "$(. /etc/os-release && echo $VERSION_CODENAME)" = "bullseye" ] && echo "--skipXDP=True")
826827
- run: ${{ env.INV_CMD }} test-dnsdist $([ "$(. /etc/os-release && echo $VERSION_CODENAME)" = "bullseye" ] && echo "--skipXDP=True")
827-
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/dnsdist/bin/dnsdist $GITHUB_WORKSPACE
828+
- run: ${{ env.INV_CMD }} generate-coverage-info /opt/dnsdist/bin/dnsdist 'dnsdist' $GITHUB_WORKSPACE
828829
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
829830
- name: Coveralls Parallel dnsdist regression
830831
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}

‎.github/workflows/codeql-analysis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ jobs:
170170
if: matrix.product == 'dnsdist'
171171
working-directory: ./pdns/dnsdistdist/
172172
run: |
173-
inv ci-dnsdist-configure full autotools build-dir
173+
inv ci-dnsdist-configure full autotools ''
174174
- name: Build dnsdist
175175
if: matrix.product == 'dnsdist'
176176
working-directory: ./pdns/dnsdistdist/

‎.github/workflows/coverity.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ jobs:
8686
working-directory: ./pdns/dnsdistdist/
8787
- run: inv ci-build-and-install-quiche $REPO_HOME
8888
working-directory: ./pdns/dnsdistdist/
89-
- run: inv ci-dnsdist-configure full autotools build-dir
89+
- run: inv ci-dnsdist-configure full autotools ''
9090
working-directory: ./pdns/dnsdistdist/
9191
- run: inv coverity-make
9292
working-directory: ./pdns/dnsdistdist/

‎pdns/dnsdistdist/Makefile.am‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -657,12 +657,9 @@ dist_man_MANS=$(MANPAGES)
657657
if HAVE_VENV
658658
if !HAVE_MANPAGES
659659
$(MANPAGES): %: docs/manpages/%.rst .venv
660-
$(AM_V_GEN).venv/bin/python -msphinx -b man docs . $<
660+
$(AM_V_GEN).venv/bin/python -msphinx -b man ${srcdir}/docs . $<
661661
endif # if !HAVE_MANPAGES
662662

663-
docs/reference/yaml-settings.rst: dnsdist-settings-documentation-generator.py dnsdist-settings-definitions.yml dnsdist-actions-definitions.yml dnsdist-response-actions-definitions.yml dnsdist-selectors-definitions.yml
664-
$(PYTHON) dnsdist-settings-documentation-generator.py .
665-
666663
.venv: docs/requirements.txt
667664
$(PYTHON) -m venv .venv
668665
.venv/bin/pip install -U pip setuptools

‎pdns/recursordist/Makefile.am‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,12 @@ CLEANFILES = htmlfiles.h \
4646
recursor.yml-dist
4747

4848
htmlfiles.h: incfiles ${srcdir}/html/* ${srcdir}/html/js/*
49-
$(AM_V_GEN)$(srcdir)/incfiles > $@.tmp
49+
$(AM_V_GEN)$(srcdir)/incfiles ${srcdir} > $@.tmp
5050
@mv $@.tmp $@
5151

5252
# Use patterns to avoid having two instances of generate run simultaneously, a well-known hack for GNU make
5353
rec-metrics-gen%h rec-prometheus-gen%h rec-snmp-gen%h rec-oids-gen%h RECURSOR-MIB%txt: metrics.py metrics_table.py RECURSOR-MIB.in
54-
$(PYTHON) metrics.py
54+
$(PYTHON) ${srcdir}/metrics.py ${srcdir} .
5555

5656
# We explicitly build rec-rust-lib in two steps, as it modifies files in the rec-rust-lib/rust subdir
5757
SUBDIRS=ext rec-rust-lib rec-rust-lib/rust

‎pdns/recursordist/rec-rust-lib/generate.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -842,7 +842,7 @@ def generate():
842842
# with mixed sources both in build and src dir
843843
gen_rust(srcdir, entries)
844844
# Avoid generating doc files in a sdist based build
845-
if os.path.isdir('../docs'):
845+
if os.path.isdir(srcdir + '/../docs'):
846846
gen_oldstyle_docs(srcdir, entries)
847847
gen_newstyle_docs(srcdir, entries)
848848
# Remove cxx generated files, they need to be re-generated after a table change and the rust dependency tracking does

0 commit comments

Comments
 (0)