Skip to content

Commit c8e097a

Browse files
[CI] Regression test jobs now include the C++ standard version and (if applicable) the standard library to link with (#1080)
* Regression test jobs now specify the C++ standard version and (if applicable) the standard library to link with * Add a new job to test clang with libc++ on ubuntu. (All other ubuntu jobs for gcc & clang link with libstdc++.) * Add a new msvc job so that we now test `c++latest` and `c++20`. * Job names now include more information: <short os name> | <compiler> | <c++ std> | <std library> | <full os name> * Reorder jobs so that GCC are first, then clang on ubuntu & macOS, and then msvc on Windows e.g. ubu-24 | g++-14 | c++2b | libstdc++ | ubuntu-24.04 mac-13 | clang++ | c++2b | default | macos-13 win-22 | cl.exe | c++latest | default | windows-2022 * Update msvc test bat files
1 parent b77ac91 commit c8e097a

File tree

2,618 files changed

+1623
-26
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

2,618 files changed

+1623
-26
lines changed

Diff for: .github/workflows/regression-tests.yml

+48-9
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ on:
1111

1212
jobs:
1313
regression-tests:
14-
name: Run on ${{ matrix.os }} using ${{ matrix.compiler }}
14+
name: ${{ matrix.shortosname }} | ${{ matrix.compiler }} | ${{ matrix.cxx_std }} | ${{ matrix.stdlib }} | ${{ matrix.os }}
1515
runs-on: ${{ matrix.os }}
1616
env:
1717
CXX: ${{ matrix.compiler }}
@@ -20,22 +20,61 @@ jobs:
2020
fail-fast: false
2121
matrix:
2222
os: [ubuntu-24.04]
23-
compiler: [g++-13, g++-14, clang++-18]
23+
shortosname: [ubu-24]
24+
compiler: [g++-14, g++-13]
25+
cxx_std: [c++2b]
26+
stdlib: [libstdc++]
2427
include:
28+
- os: ubuntu-20.04
29+
shortosname: ubu-20
30+
compiler: g++-10
31+
cxx_std: c++20
32+
stdlib: libstdc++
33+
- os: ubuntu-24.04
34+
shortosname: ubu-24
35+
compiler: clang++-18
36+
cxx_std: c++20
37+
stdlib: libstdc++
38+
- os: ubuntu-22.04
39+
shortosname: ubu-22
40+
compiler: clang++-15
41+
cxx_std: c++20
42+
stdlib: libstdc++
2543
- os: ubuntu-22.04
44+
shortosname: ubu-22
2645
compiler: clang++-15
46+
cxx_std: c++20
47+
stdlib: libc++-15-dev
2748
- os: ubuntu-20.04
49+
shortosname: ubu-20
2850
compiler: clang++-12
29-
- os: ubuntu-20.04
30-
compiler: g++-10
51+
cxx_std: c++20
52+
stdlib: libstdc++
3153
- os: macos-14
54+
shortosname: mac-14
3255
compiler: clang++
56+
cxx_std: c++2b
57+
stdlib: default
3358
- os: macos-13
59+
shortosname: mac-13
3460
compiler: clang++
61+
cxx_std: c++2b
62+
stdlib: default
3563
- os: macos-13
64+
shortosname: mac-13
3665
compiler: clang++-15
37-
- os: windows-latest
66+
cxx_std: c++2b
67+
stdlib: default
68+
- os: windows-2022
69+
shortosname: win-22
70+
compiler: cl.exe
71+
cxx_std: c++latest
72+
stdlib: default
73+
- os: windows-2022
74+
shortosname: win-22
3875
compiler: cl.exe
76+
cxx_std: c++20
77+
stdlib: default
3978
steps:
4079
- name: Checkout repo
4180
uses: actions/checkout@v4
@@ -50,21 +89,21 @@ jobs:
5089
if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
5190
run: |
5291
cd regression-tests
53-
bash run-tests.sh -c ${{ matrix.compiler }} -l ${{ matrix.os }}
92+
bash run-tests.sh -c ${{ matrix.compiler }} -s ${{ matrix.cxx_std }} -d ${{ matrix.stdlib }} -l ${{ matrix.os }}
5493
5594
- name: Run regression tests - Windows version
5695
if: matrix.os == 'windows-latest'
5796
run: |
5897
"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
5998
git config --local core.autocrlf false && ^
6099
cd regression-tests && ^
61-
bash run-tests.sh -c ${{ matrix.compiler }} -l ${{ matrix.os }}
100+
bash run-tests.sh -c ${{ matrix.compiler }} -s ${{ matrix.cxx_std }} -d ${{ matrix.stdlib }} -l ${{ matrix.os }}
62101
shell: cmd
63102

64103
- name: Upload patch
65104
if: ${{ !cancelled() }}
66105
uses: actions/upload-artifact@v4
67106
with:
68-
name: ${{ matrix.os }}-${{ matrix.compiler }}.patch
69-
path: regression-tests/${{ matrix.os }}-${{ matrix.compiler }}.patch
107+
name: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.cxx_std }}-${{ matrix.stdlib }}.patch
108+
path: regression-tests/${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.cxx_std }}-${{ matrix.stdlib }}.patch
70109
if-no-files-found: ignore

Diff for: regression-tests/run-tests.sh

+46-16
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
usage() {
55
echo "Usage: $0 -c <compiler> [-l <run label>] [-t <tests to run>]"
66
echo " -c <compiler> The compiler to use for the test"
7+
echo " -s <cxx_std> The C++ standard to compile with (e.g. 'c++20', 'c++2b', 'c++latest' depending on the compiler)"
8+
echo " -d <stdlib> Clang-only: the C++ Standard Library to link with ('libstdc++', 'libc++', or 'default' for platform default)"
79
echo " -l <run label> The label to use in output patch file name"
810
echo " -t <tests to run> Runs only the provided, comma-separated tests (filenames including .cpp2)"
911
echo " If the argument is not used all tests are run"
@@ -46,7 +48,7 @@ check_file () {
4648
git ls-files --error-unmatch "$file" > /dev/null 2>&1
4749
untracked=$?
4850

49-
patch_file="$label$cxx_compiler.patch"
51+
patch_file="${label}-${cxx_compiler}-${cxx_std}-${cxx_stdlib}.patch"
5052

5153
if [[ $untracked -eq 1 ]]; then
5254
# Add the file to the index to be able to diff it...
@@ -67,14 +69,20 @@ check_file () {
6769
fi
6870
}
6971

70-
optstring="c:l:t:"
72+
optstring="c:s:d:l:t:"
7173
while getopts ${optstring} arg; do
7274
case "${arg}" in
7375
c)
7476
cxx_compiler="${OPTARG}"
7577
;;
78+
s)
79+
cxx_std="${OPTARG}"
80+
;;
81+
d)
82+
cxx_stdlib="${OPTARG}"
83+
;;
7684
l)
77-
label="${OPTARG}-"
85+
label="${OPTARG}"
7886
;;
7987
t)
8088
# Replace commas with spaces
@@ -125,8 +133,8 @@ expected_results_dir="test-results"
125133
################
126134
# Get the directory with the exec outputs and compilation command
127135
if [[ "$cxx_compiler" == *"cl.exe"* ]]; then
128-
compiler_cmd='cl.exe -nologo -std:c++latest -MD -EHsc -I ..\..\..\include -Fe:'
129-
exec_out_dir="$expected_results_dir/msvc-2022"
136+
compiler_cmd="cl.exe -nologo -std:${cxx_std} -MD -EHsc -I ..\..\..\include -Fe:"
137+
exec_out_dir="$expected_results_dir/msvc-2022-${cxx_std}"
130138
compiler_version=$(cl.exe)
131139
else
132140
# Verify the compiler command
@@ -136,7 +144,6 @@ else
136144
exit 2
137145
fi
138146

139-
cpp_std=c++2b
140147
compiler_version=$("$cxx_compiler" --version)
141148

142149
if [[ "$compiler_version" == *"Apple clang version 14.0"* ||
@@ -148,16 +155,10 @@ else
148155
exec_out_dir="$expected_results_dir/clang-12"
149156
elif [[ "$compiler_version" == *"clang version 15.0"* ]]; then
150157
exec_out_dir="$expected_results_dir/clang-15"
151-
# c++2b causes starge issues on GitHub ubuntu runners
152-
cpp_std="c++20"
153158
elif [[ "$compiler_version" == *"clang version 18.1"* ]]; then
154159
exec_out_dir="$expected_results_dir/clang-18"
155-
# c++2b causes starge issues on GitHub ubuntu runners
156-
cpp_std="c++20"
157160
elif [[ "$compiler_version" == *"g++-10"* ]]; then
158161
exec_out_dir="$expected_results_dir/gcc-10"
159-
# GCC 10 does not support c++2b
160-
cpp_std=c++20
161162
elif [[ "$compiler_version" == *"g++-12"* ||
162163
"$compiler_version" == *"g++-13"*
163164
]]; then
@@ -169,15 +170,44 @@ else
169170
exit 2
170171
fi
171172

172-
compiler_cmd="$cxx_compiler -I../../../include -std=$cpp_std -pthread -o "
173+
# Append the C++ standard (e.g. 'c++20') to the expected output dir name
174+
exec_out_dir="${exec_out_dir}-${cxx_std}"
175+
176+
# Clang can link with either libstdc++ or libc++
177+
# By default clang on ubuntu links with libstdc++ and on macOS links with libc++.
178+
if [[ "$compiler_version" == *"clang"* ]]; then
179+
if [[ "$cxx_stdlib" == "default" || "$cxx_stdlib" == "" ]]; then
180+
cxx_stdlib_link_arg="" # Use compiler/platform default
181+
elif [[ "$cxx_stdlib" == "libstdc++" ]]; then
182+
cxx_stdlib_link_arg="-stdlib=libstdc++"
183+
elif [[ "$cxx_stdlib" == *"libc++"* ]]; then
184+
185+
# Need to install the correct libc++ packages, e.g. `libc++-15-dev` and `libc++abi-15-dev` for clang 15.
186+
# Our `cxx_stdlib` variable contains the `libc++-XX-dev` package name so we need to create the abi version.
187+
cxx_stdlib_abi_package="${cxx_stdlib/libc++/libc++abi}"
188+
printf "Installing packages: $cxx_stdlib $cxx_stdlib_abi_package\n\n"
189+
sudo apt-get install -y $cxx_stdlib $cxx_stdlib_abi_package
190+
191+
cxx_stdlib_link_arg="-stdlib=libc++"
192+
exec_out_dir="${exec_out_dir}-libcpp"
193+
else
194+
printf "Unhandled C++ Standard Library option:\n$cxx_stdlib\n\n"
195+
exit 2
196+
fi
197+
else
198+
cxx_stdlib_link_arg="" # Use compiler/platform default
199+
fi
200+
201+
compiler_cmd="$cxx_compiler -I../../../include -std=$cxx_std $cxx_stdlib_link_arg -pthread -o "
202+
printf "\ncompiler_cmd: $compiler_cmd\n\n"
173203
fi
174204

175205
if [[ -d "$exec_out_dir" ]]; then
176206
printf "Full compiler version for '$cxx_compiler':\n$compiler_version\n\n"
177207

178208
printf "Directory with reference compilation/execution files to use:\n$exec_out_dir\n\n"
179209
else
180-
printf "Directory with reference compilation/execution files not found for compiler: '$cxx_compiler'\n\n"
210+
printf "Directory with reference compilation/execution files not found for compiler: '$cxx_compiler' at $exec_out_dir\n\n"
181211
exit 2
182212
fi
183213

@@ -197,8 +227,8 @@ regression_test_link_obj=""
197227
if [[ "$cxx_compiler" == *"cl.exe"* ]]; then
198228
echo "Building std and std.compat modules"
199229
(cd $exec_out_dir; \
200-
cl.exe -nologo -std:c++latest -MD -EHsc -c "${VCToolsInstallDir}/modules/std.ixx";
201-
cl.exe -nologo -std:c++latest -MD -EHsc -c "${VCToolsInstallDir}/modules/std.compat.ixx")
230+
cl.exe -nologo -std:${cxx_std} -MD -EHsc -c "${VCToolsInstallDir}/modules/std.ixx";
231+
cl.exe -nologo -std:${cxx_std} -MD -EHsc -c "${VCToolsInstallDir}/modules/std.compat.ixx")
202232
regression_test_link_obj="std.obj std.compat.obj"
203233
fi
204234

0 commit comments

Comments
 (0)