Ensure that expected fails on CTS always show as a pass.

When we have regular fails which we do not expect an immediate fix, we
want to show as pass so we can catch be failures more easily.

When we know there is an expected fail, we create files to cover fails
on all targets or fails on a particular target for OpenCL CTS and
SYCL CTS. These will be under .github/opencl_cts and .github/sycl_cts.
Fails files will be of the form expect_fail_all.csv or expect_fail_<target>.csv.
These should match lines in the original csv file being used.

Author: coldav

.github/actions/run_opencl_cts/action.yml

@@ -27,7 +27,6 @@ runs:
         CTS_FILTER: cts-3.0-online-ignore-linux-host.csv
         CTS_TIMEOUT: 18:00:00 # OK for github?
       run: |
-        cd "$GITHUB_WORKSPACE/OpenCL-CTS"
         echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER"
         export QEMU_SETTING=""
         if [[ "${{inputs.target}}" =~ .*aarch64.* ]] ; then
@@ -37,6 +36,11 @@ runs:
         fi
         echo QEMU SETTING: $QEMU SETTING
         set -x
+        echo > expect_fail.csv
+        # $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
+        [ -f .github/opencl_cts/expect_fail_all.csv ] && cat .github/opencl_cts/expect_fail_all.csv >> expect_fail.csv
+        [ -f .github/opencl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/opencl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
+        cat expect_fail.csv "$GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER" > disable.csv 
         # Note: use 'eval' built-in to handle quoting/escaping/splitting reqs
         RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
           --color=always --timeout $CTS_TIMEOUT \
@@ -47,5 +51,23 @@ runs:
           -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
           -e CL_PLATFORM_INDEX=0 \
           -s $GITHUB_WORKSPACE/test_conformance/$CTS_CSV_FILE \
-          -i $GITHUB_WORKSPACE/source/cl/scripts/$CTS_FILTER"
+          -i disable.csv"
         eval $RUN_CITIES
+
+    - name: Run expected failed opencl cts
+      shell: bash
+      env:
+        CTS_TIMEOUT: 18:00:00
+      run: |
+        echo "Running OpenCL CTS tests with CTS file $CTS_CSV_FILE with filter $CTS_FILTER"
+        set -x
+        RUN_CITIES="python3 -u $GITHUB_WORKSPACE/scripts/testing/run_cities.py -v \
+          --color=always --timeout $CTS_TIMEOUT \
+          $QEMU_SETTING \
+          -b $GITHUB_WORKSPACE/test_conformance \
+          -L $GITHUB_WORKSPACE/install_icd/lib \
+          -e CLC_EXECUTABLE=$GITHUB_WORKSPACE/install_ock/bin/clc \
+          -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
+          -e CL_PLATFORM_INDEX=0 \
+          -s expect_fail.csv"
+        eval $RUN_CITIES || echo failed as expected

.github/actions/run_sycl_cts/action.yml

@@ -62,11 +62,17 @@ runs:
         SYCL_CTS_TIMEOUT: '02:00:00'
         SYCL_CTS_FILTER: ''
       run: |
+        set -x
         echo running sycl cts
         export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
         export ONEAPI_DEVICE_SELECTOR=opencl:0
         export CTS_CSV_FILE=$GITHUB_WORKSPACE/.github/scripts/sycl-cts.csv
-        set -x
+        echo > expect_fail.csv
+        # $CTS_FILTER ignores certain test, so is treated differently to temporary fails.
+        [ -f .github/sycl_cts/expect_fail_all.csv ] && cat .github/sycl_cts/expect_fail_all.csv >> expect_fail.csv
+        [ -f .github/sycl_cts/expect_fail_${{ inputs.target }}.csv ] && cat .github/sycl_cts/expect_fail_${{ inputs.target }}.csv >> expect_fail.csv
+        cp expect_fail.csv disable.csv
+        [ -f  "$SYCL_CTS_FILTER" ] && cat "$SYCL_CTS_FILTER" >> disable.csv 
         python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
           --color=always \
           --timeout $SYCL_CTS_TIMEOUT \
@@ -79,9 +85,30 @@ runs:
           -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
           -r SYCL-CTS/cts.xml \
           -v \
-          $SYCL_CTS_FILTER || exitcode=$?
+          -i disable.csv || exitcode=$?
         export OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so
         $GITHUB_WORKSPACE/.github/scripts/create_sycl_cts_test_lists.sh $PREPEND_PATH SYCL-CTS $CTS_CSV_FILE csv.txt cts_all.txt
         # output a diff of the generated list csv.txt and cts_all.txt
         diff csv.txt cts_all.txt || echo "WARNING - Missing some tests from sycl cts file based on test_all --list-tests - see > above"
         exit $exitcode
+
+    - name: run sycl cts expected fails
+      shell: bash
+      env:
+        PREPEND_PATH: ''  # TODO: have qemu as input and set up this
+        SYCL_CTS_TIMEOUT: '02:00:00'
+      run: |
+        echo running sycl cts
+        export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/install_dpcpp/lib:$GITHUB_WORKSPACE/install_ock/lib
+        export ONEAPI_DEVICE_SELECTOR=opencl:0
+        python3 $GITHUB_WORKSPACE/scripts/testing/run_cities.py \
+          --color=always \
+          --timeout $SYCL_CTS_TIMEOUT \
+          $PREPEND_PATH \
+          -p sycl_cts \
+          -b SYCL-CTS/bin \
+          -L SYCL-CTS/lib \
+          -e OCL_ICD_FILENAMES=$GITHUB_WORKSPACE/install_ock/lib/libCL.so \
+          -l SYCL-CTS/cts.log -f SYCL-CTS/cts.fail \
+          -r SYCL-CTS/cts.xml \
+          -s expect_fail.csv || echo failed as expected

.github/opencl_cts/expect_fail_all.csv

@@ -0,0 +1 @@
+API,api/test_api

.github/opencl_cts/expect_fail_host_riscv64_linux.csv

@@ -0,0 +1 @@
+Math,math_brute_force/test_bruteforce -w

.github/sycl_cts/expect_fail_all.csv

@@ -0,0 +1,2 @@
+SYCL_CTS,test_math_builtin_api "math_builtin_float_base_*"
+SYCL_CTS,test_math_builtin_api "math_builtin_float_double_*"