diff --git a/.github/actions/run_opencl_cts/action.yml b/.github/actions/run_opencl_cts/action.yml index e1623319f..772964fb3 100644 --- a/.github/actions/run_opencl_cts/action.yml +++ b/.github/actions/run_opencl_cts/action.yml @@ -27,7 +27,6 @@ runs: CTS_FILTER: cts-3.0-online-ignore-linux-host.csv CTS_TIMEOUT: 18:00:00 # OK for github? run: | - cd "$GITHUB_WORKSPACE/OpenCL-CTS" echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER" export QEMU_SETTING="" if [[ "${{inputs.target}}" =~ .*aarch64.* ]] ; then @@ -37,6 +36,11 @@ runs: fi echo QEMU SETTING: $QEMU SETTING set -x + echo > expect_fail.csv + # $CTS_FILTER ignores certain test, so is treated differently to temporary fails. + [ -f .github/opencl_cts/expect_fail_all.csv ] && cat .github/opencl_cts/expect_fail_all.csv >> expect_fail.csv + [ -f .github/opencl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/opencl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv + cat expect_fail.csv "$GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER" > disable.csv # Note: use 'eval' built-in to handle quoting/escaping/splitting reqs RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \ --color=always --timeout $CTS_TIMEOUT \ @@ -47,5 +51,23 @@ runs: -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \ -e CL_PLATFORM_INDEX=0 \ -s $GITHUB_WORKSPACE/test_conformance/$CTS_CSV_FILE \ - -i $GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER" + -i disable.csv" eval $RUN_CITIES + + - name: Run expected failed opencl cts + shell: bash + env: + CTS_TIMEOUT: 18:00:00 + run: | + echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER" + set -x + RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \ + --color=always --timeout $CTS_TIMEOUT \ + $QEMU_SETTING \ + -b $GITHUB_WORKSPACE/test_conformance \ + -L $GITHUB_WORKSPACE/install_icd/lib \ + -e CLC_EXECUTABLE=$GITHUB_WORKSPACE/install_ock/bin/clc \ + -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \ + -e CL_PLATFORM_INDEX=0 \ + -s expect_fail.csv" + eval $RUN_CITIES || echo failed as expected diff --git a/.github/actions/run_sycl_cts/action.yml b/.github/actions/run_sycl_cts/action.yml index 0291fe7c8..780cf221b 100644 --- a/.github/actions/run_sycl_cts/action.yml +++ b/.github/actions/run_sycl_cts/action.yml @@ -62,11 +62,17 @@ runs: SYCL_CTS_TIMEOUT: '02:00:00' SYCL_CTS_FILTER: '' run: | + set -x echo running sycl cts export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib export ONEAPI_DEVICE_SELECTOR=opencl:0 export CTS_CSV_FILE=$GITHUB_WORKSPACE/.github/scripts/sycl-cts.csv - set -x + echo > expect_fail.csv + # $CTS_FILTER ignores certain test, so is treated differently to temporary fails. + [ -f .github/sycl_cts/expect_fail_all.csv ] && cat .github/sycl_cts/expect_fail_all.csv >> expect_fail.csv + [ -f .github/sycl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/sycl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv + cp expect_fail.csv disable.csv + [ -f "$SYCL_CTS_FILTER" ] && cat "$SYCL_CTS_FILTER" >> disable.csv python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \ --color=always \ --timeout $SYCL_CTS_TIMEOUT \ @@ -79,9 +85,30 @@ runs: -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \ -r SYCL-CTS/cts.xml \ -v \ - $SYCL_CTS_FILTER || exitcode=$? + -i disable.csv || exitcode=$? export OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so $GITHUB_WORKSPACE/.github/scripts/create_sycl_cts_test_lists.sh $PREPEND_PATH SYCL-CTS $CTS_CSV_FILE csv.txt cts_all.txt # output a diff of the generated list csv.txt and cts_all.txt diff csv.txt cts_all.txt || echo "WARNING - Missing some tests from sycl cts file based on test_all --list-tests - see > above" exit $exitcode + + - name: run sycl cts expected fails + shell: bash + env: + PREPEND_PATH: '' # TODO: have qemu as input and set up this + SYCL_CTS_TIMEOUT: '02:00:00' + run: | + echo running sycl cts + export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib + export ONEAPI_DEVICE_SELECTOR=opencl:0 + python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \ + --color=always \ + --timeout $SYCL_CTS_TIMEOUT \ + $PREPEND_PATH \ + -p sycl_cts \ + -b SYCL-CTS/bin \ + -L SYCL-CTS/lib \ + -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \ + -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \ + -r SYCL-CTS/cts.xml \ + -s expect_fail.csv || echo failed as expected diff --git a/.github/opencl_cts/expect_fail_all.csv b/.github/opencl_cts/expect_fail_all.csv new file mode 100644 index 000000000..b2098fddd --- /dev/null +++ b/.github/opencl_cts/expect_fail_all.csv @@ -0,0 +1 @@ +API,api/test_api diff --git a/.github/opencl_cts/expect_fail_host_riscv64_linux.csv b/.github/opencl_cts/expect_fail_host_riscv64_linux.csv new file mode 100644 index 000000000..7c78bc659 --- /dev/null +++ b/.github/opencl_cts/expect_fail_host_riscv64_linux.csv @@ -0,0 +1 @@ +Math,math_brute_force/test_bruteforce -w diff --git a/.github/sycl_cts/expect_fail_all.csv b/.github/sycl_cts/expect_fail_all.csv new file mode 100644 index 000000000..d5f5bd439 --- /dev/null +++ b/.github/sycl_cts/expect_fail_all.csv @@ -0,0 +1,2 @@ +SYCL_CTS,test_math_builtin_api "math_builtin_float_base_*" +SYCL_CTS,test_math_builtin_api "math_builtin_float_double_*"