Merge pull request #665 from coldav/colin/ensure_cts_passes_known_fails

Ensure that expected fails on CTS always show as a pass.

Author: coldav

.github/actions/run_opencl_cts/action.yml

@@ -27,7 +27,6 @@ runs:
         CTS_FILTER: cts-3.0-online-ignore-linux-host.csv
         CTS_TIMEOUT: 18:00:00 # OK for github?
       run: |
-        cd "$GITHUB_WORKSPACE/OpenCL-CTS"
         echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER"
         export QEMU_SETTING=""
         if [[ "${{inputs.target}}" =~ .*aarch64.* ]] ; then
@@ -37,6 +36,11 @@ runs:
         fi
         echo QEMU SETTING: $QEMU SETTING
         set -x
+        echo > expect_fail.csv
+        # $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
+        [ -f .github/opencl_cts/expect_fail_all.csv ] && cat .github/opencl_cts/expect_fail_all.csv >> expect_fail.csv
+        [ -f .github/opencl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/opencl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
+        cat expect_fail.csv "$GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER" > disable.csv 
         # Note: use 'eval' built-in to handle quoting/escaping/splitting reqs
         RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
           --color=always --timeout $CTS_TIMEOUT \
@@ -47,5 +51,23 @@ runs:
           -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
           -e CL_PLATFORM_INDEX=0 \
           -s $GITHUB_WORKSPACE/test_conformance/$CTS_CSV_FILE \
-          -i $GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER"
+          -i disable.csv"
         eval $RUN_CITIES
+
+    - name: Run expected failed opencl cts
+      shell: bash
+      env:
+        CTS_TIMEOUT: 18:00:00
+      run: |
+        echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER"
+        set -x
+        RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
+          --color=always --timeout $CTS_TIMEOUT \
+          $QEMU_SETTING \
+          -b $GITHUB_WORKSPACE/test_conformance \
+          -L $GITHUB_WORKSPACE/install_icd/lib \
+          -e CLC_EXECUTABLE=$GITHUB_WORKSPACE/install_ock/bin/clc \
+          -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
+          -e CL_PLATFORM_INDEX=0 \
+          -s expect_fail.csv"
+        eval $RUN_CITIES || echo failed as expected

.github/actions/run_sycl_cts/action.yml

@@ -62,11 +62,17 @@ runs:
         SYCL_CTS_TIMEOUT: '02:00:00'
         SYCL_CTS_FILTER: ''
       run: |
+        set -x
         echo running sycl cts
         export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
         export ONEAPI_DEVICE_SELECTOR=opencl:0
         export CTS_CSV_FILE=$GITHUB_WORKSPACE/.github/scripts/sycl-cts.csv
-        set -x
+        echo > expect_fail.csv
+        # $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
+        [ -f .github/sycl_cts/expect_fail_all.csv ] && cat .github/sycl_cts/expect_fail_all.csv >> expect_fail.csv
+        [ -f .github/sycl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/sycl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
+        cp expect_fail.csv disable.csv
+        [ -f  "$SYCL_CTS_FILTER" ] && cat "$SYCL_CTS_FILTER" >> disable.csv 
         python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
           --color=always \
           --timeout $SYCL_CTS_TIMEOUT \
@@ -79,9 +85,30 @@ runs:
           -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
           -r SYCL-CTS/cts.xml \
           -v \
-          $SYCL_CTS_FILTER || exitcode=$?
+          -i disable.csv || exitcode=$?
         export OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so
         $GITHUB_WORKSPACE/.github/scripts/create_sycl_cts_test_lists.sh $PREPEND_PATH SYCL-CTS $CTS_CSV_FILE csv.txt cts_all.txt
         # output a diff of the generated list csv.txt and cts_all.txt
         diff csv.txt cts_all.txt || echo "WARNING - Missing some tests from sycl cts file based on test_all --list-tests - see > above"
         exit $exitcode
+
+    - name: run sycl cts expected fails
+      shell: bash
+      env:
+        PREPEND_PATH: ''  # TODO: have qemu as input and set up this
+        SYCL_CTS_TIMEOUT: '02:00:00'
+      run: |
+        echo running sycl cts
+        export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
+        export ONEAPI_DEVICE_SELECTOR=opencl:0
+        python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
+          --color=always \
+          --timeout $SYCL_CTS_TIMEOUT \
+          $PREPEND_PATH \
+          -p sycl_cts \
+          -b SYCL-CTS/bin \
+          -L SYCL-CTS/lib \
+          -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
+          -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
+          -r SYCL-CTS/cts.xml \
+          -s expect_fail.csv || echo failed as expected

.github/opencl_cts/expect_fail_all.csv

@@ -0,0 +1 @@
+API,api/test_api

.github/opencl_cts/expect_fail_host_riscv64_linux.csv

@@ -0,0 +1 @@
+Math,math_brute_force/test_bruteforce -w

.github/sycl_cts/expect_fail_all.csv

@@ -0,0 +1,2 @@
+SYCL_CTS,test_math_builtin_api "math_builtin_float_base_*"
+SYCL_CTS,test_math_builtin_api "math_builtin_float_double_*"