From 769557566fab3cc640eabcf438ef60e9b131781b Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 1 Oct 2024 15:29:18 +0200 Subject: [PATCH 01/10] Update generate_pkgconfig.py --- deploy/pkg-config/generate_pkgconfig.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/pkg-config/generate_pkgconfig.py b/deploy/pkg-config/generate_pkgconfig.py index 323ff7e5138..346b12eac65 100755 --- a/deploy/pkg-config/generate_pkgconfig.py +++ b/deploy/pkg-config/generate_pkgconfig.py @@ -33,8 +33,8 @@ def detect_cpu_architecture(): elif architecture.startswith('arm') or architecture == 'aarch64': return 'aarch64' else: - sys.stderr.write(f"Unknown Architecture {architecture} Detected. " \ - "Only 'x86_64', 'AMD64' and 'aarch64' supported.\n") + sys.stderr.write("Unknown Architecture {} Detected. " \ + "Only 'x86_64', 'AMD64' and 'aarch64' supported.\n".format(architecture)) sys.exit(1) LIBS_PAR_STAT, LIBS_PAR_DYN = [], [] @@ -72,7 +72,7 @@ def detect_cpu_architecture(): elif ARCH == 'aarch64': LIBDIR = 'lib/arm' else: - sys.stderr.write(f"Unknown CPU architecture '{ARCH}'\n") + sys.stderr.write("Unknown CPU architecture '{}'\n".format(ARCH)) SUFF_DYN_LIB = ".so" SUFF_STAT_LIB = ".a" From b2b05efbf8e06aa54f0721459264cda87a92a360 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 1 Oct 2024 15:34:20 +0200 Subject: [PATCH 02/10] Update INSTALL.md --- INSTALL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/INSTALL.md b/INSTALL.md index 9a85701ab81..6cf464f1389 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -22,6 +22,7 @@ Required Software: * [DPC++ Compiler](https://www.intel.com/content/www/us/en/developer/tools/oneapi/dpc-compiler.html) * Microsoft Visual Studio\* (Windows\* only) * [MSYS2](http://msys2.github.io) (Windows\* only) +* Python * `make` and `dos2unix` tools; install these packages using MSYS2 on Windows\* as follows: pacman -S msys/make msys/dos2unix From aacb7579bbbb894fe5b8ed1c0012d372ae5b4f6d Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Tue, 1 Oct 2024 15:42:22 +0200 Subject: [PATCH 03/10] Update generate_pkgconfig.py --- deploy/pkg-config/generate_pkgconfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/pkg-config/generate_pkgconfig.py b/deploy/pkg-config/generate_pkgconfig.py index 346b12eac65..78c3941138e 100755 --- a/deploy/pkg-config/generate_pkgconfig.py +++ b/deploy/pkg-config/generate_pkgconfig.py @@ -33,7 +33,7 @@ def detect_cpu_architecture(): elif architecture.startswith('arm') or architecture == 'aarch64': return 'aarch64' else: - sys.stderr.write("Unknown Architecture {} Detected. " \ + sys.stderr.write("Unknown Architecture {} Detected. " "Only 'x86_64', 'AMD64' and 'aarch64' supported.\n".format(architecture)) sys.exit(1) From 1adca78bf98bdca2d37c6bfed7ce16a73f969efd Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Mon, 7 Oct 2024 07:17:38 +0200 Subject: [PATCH 04/10] remove conformance-scripts --- .../conformance-scripts/algorithms.txt | 11 - .../conformance-scripts/download_tests.sh | 73 ------ .ci/scripts/conformance-scripts/run_tests.py | 49 ---- .../run_tests_with_context.py | 15 -- .../run_xpu_conformance.py | 65 ------ .ci/scripts/conformance-scripts/utils.py | 209 ------------------ 6 files changed, 422 deletions(-) delete mode 100755 .ci/scripts/conformance-scripts/algorithms.txt delete mode 100755 .ci/scripts/conformance-scripts/download_tests.sh delete mode 100755 .ci/scripts/conformance-scripts/run_tests.py delete mode 100644 .ci/scripts/conformance-scripts/run_tests_with_context.py delete mode 100644 .ci/scripts/conformance-scripts/run_xpu_conformance.py delete mode 100755 .ci/scripts/conformance-scripts/utils.py diff --git a/.ci/scripts/conformance-scripts/algorithms.txt b/.ci/scripts/conformance-scripts/algorithms.txt deleted file mode 100755 index 2ef5baf106c..00000000000 --- a/.ci/scripts/conformance-scripts/algorithms.txt +++ /dev/null @@ -1,11 +0,0 @@ -dbscan -elastic_net -kmeans -lin_reg -log_reg -pca -ridge_reg -svm -svm_sparse -forest -knn diff --git a/.ci/scripts/conformance-scripts/download_tests.sh b/.ci/scripts/conformance-scripts/download_tests.sh deleted file mode 100755 index e115f4cd17d..00000000000 --- a/.ci/scripts/conformance-scripts/download_tests.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -#=============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#=============================================================================== - -while [[ $# -gt 0 ]]; do - key="$1" - - case $key in - --alg-name) - ALG_NAME="$2" - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac - shift - shift -done -SKLEARN_PATH="$(pip show scikit-learn | grep Location | cut -d ' ' -f 2)/sklearn/" - -case ${ALG_NAME} in - "dbscan") - cp ${SKLEARN_PATH}cluster/tests/test_dbscan.py test_dbscan.py - ;; - "elastic_net") - cp ${SKLEARN_PATH}linear_model/tests/test_coordinate_descent.py test_elastic_net.py - ;; - "kmeans") - cp ${SKLEARN_PATH}cluster/tests/test_k_means.py test_kmeans.py - ;; - "lin_reg") - cp ${SKLEARN_PATH}linear_model/tests/test_base.py test_lin_reg.py - ;; - "log_reg") - cp ${SKLEARN_PATH}linear_model/tests/test_logistic.py test_log_reg.py - ;; - "pca") - cp ${SKLEARN_PATH}decomposition/tests/test_pca.py test_pca.py - ;; - "ridge_reg") - cp ${SKLEARN_PATH}linear_model/tests/test_ridge.py test_ridge_reg.py - ;; - "svm") - cp ${SKLEARN_PATH}svm/tests/test_svm.py test_svm.py - ;; - "svm_sparse") - cp ${SKLEARN_PATH}svm/tests/test_sparse.py test_svm_sparse.py - ;; - "forest") - cp ${SKLEARN_PATH}ensemble/tests/test_forest.py test_forest.py - ;; - "knn") - cp ${SKLEARN_PATH}neighbors/tests/test_neighbors.py test_knn.py - ;; - *) - echo "Unknown algorithm: ${ALG_NAME}" - exit 1 - ;; -esac diff --git a/.ci/scripts/conformance-scripts/run_tests.py b/.ci/scripts/conformance-scripts/run_tests.py deleted file mode 100755 index 4a5b7cbe594..00000000000 --- a/.ci/scripts/conformance-scripts/run_tests.py +++ /dev/null @@ -1,49 +0,0 @@ -#=============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#=============================================================================== - -import re -import sys -import subprocess -from datetime import datetime -from subprocess import Popen, PIPE -from utils import make_report - -try: - import sklearnex -except Exception: - raise Exception('sklearnex is not installed') - -algs_filename = "algorithms.txt" -report_filename = "report.html" - -if __name__ == "__main__": - with open(algs_filename, "r") as file_algs: - algs = file_algs.read().split("\n") - algs.remove("") - - print("Confromance testing start") - for alg_name in algs: - code = subprocess.call(["./download_tests.sh", "--alg-name", "%s" % (alg_name) ]) - if code: raise Exception('Error while copying test files') - print(alg_name) - - alg_log = open("_log_%s.txt" % (alg_name), "w") - subprocess.call(["python", "-m", "sklearnex", "-m", "pytest", "-s", "--disable-warnings", "-v", "test_%s.py" % (alg_name)], - stdout=alg_log, stderr=alg_log) - alg_log.close() - - make_report(algs_filename=algs_filename, - report_filename = report_filename) diff --git a/.ci/scripts/conformance-scripts/run_tests_with_context.py b/.ci/scripts/conformance-scripts/run_tests_with_context.py deleted file mode 100644 index 9ead91dbd67..00000000000 --- a/.ci/scripts/conformance-scripts/run_tests_with_context.py +++ /dev/null @@ -1,15 +0,0 @@ -import argparse -import pytest - -def get_context(device): - from daal4py.oneapi import sycl_context - return sycl_context(device, host_offload_on_fail=True) - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Script to run scikit-learn tests with device context manager') - parser.add_argument('-a', '--algorithm', type=str, help='datasets which should be downloaded') - parser.add_argument('-d', '--device', type=str, help='device name', choices=['host', 'cpu', 'gpu']) - args = parser.parse_args() - - with get_context(args.device): - pytest.main(["-s", "--disable-warnings", "-v", "test_%s.py" % (args.algorithm)]) diff --git a/.ci/scripts/conformance-scripts/run_xpu_conformance.py b/.ci/scripts/conformance-scripts/run_xpu_conformance.py deleted file mode 100644 index 83bc46ee628..00000000000 --- a/.ci/scripts/conformance-scripts/run_xpu_conformance.py +++ /dev/null @@ -1,65 +0,0 @@ -#=============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#=============================================================================== - -import re -import sys -import subprocess -from datetime import datetime -from subprocess import Popen, PIPE -from utils import make_report -import argparse -import os - -try: - import daal4py -except Exception: - raise Exception('daal4py is not installed') - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Script to make scikit-learn conformance report' - 'based on tests for different device types') - parser.add_argument('-d', '--device', type=str, help='device name', choices=['host', 'cpu', 'gpu']) - parser.add_argument('-f', '--consider_fails', - help='Exclude failed tests from conformance calculation', action="store_true") - args = parser.parse_args() - - os.environ['SKLEARNEX_VERBOSE'] = 'INFO' - - algs_filename = "algorithms.txt" - report_filename = f"report_{args.device}.html" - - with open(algs_filename, "r") as file_algs: - algs = file_algs.read().split("\n") - algs.remove("") - - print("Confromance testing start") - for alg_name in algs: - code = subprocess.call(["./download_tests.sh", "--alg-name", "%s" % (alg_name) ]) - if code: - raise Exception('Error while copying test files') - print(alg_name) - - alg_log = open("_log_%s.txt" % (alg_name), "w") - subprocess.call(["python", "-m", "sklearnex", "run_tests_with_context.py", "-a" "%s" % (alg_name), - "-d" "%s" % (args.device)], - stdout=alg_log) - alg_log.close() - - make_report(algs_filename=algs_filename, - report_filename = report_filename, - device=args.device, - consider_fails=args.consider_fails) diff --git a/.ci/scripts/conformance-scripts/utils.py b/.ci/scripts/conformance-scripts/utils.py deleted file mode 100755 index 63e719a7e6c..00000000000 --- a/.ci/scripts/conformance-scripts/utils.py +++ /dev/null @@ -1,209 +0,0 @@ -#=============================================================================== -# Copyright 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#=============================================================================== - -# -*- coding: utf-8 -*- -from datetime import datetime - -class CallCounter: - def __init__(self): - self.clear() - - def clear(self): - self.sklearnexCalls = 0 - self.sklearnCalls = 0 - self.sklearnexFailCalls = 0 - self.sklearnexDeviceCallsRatio = 0.0 - self.sklearnexDevicePatchedCalls = 0 - - self.sklearnexDeviceOffloadSuccess = 0 - self.sklearnexDeviceOffloadFail = 0 - - def calcDeviceCalls(self): - offloadSum = self.sklearnexDeviceOffloadSuccess + self.sklearnexDeviceOffloadFail - if offloadSum > 0: - self.sklearnexDeviceCallsRatio += self.sklearnexDeviceOffloadSuccess / offloadSum - self.sklearnexDeviceOffloadSuccess = 0 - self.sklearnexDeviceOffloadFail = 0 - - def inc(self, other): - self.sklearnexCalls += other.sklearnexCalls - self.sklearnCalls += other.sklearnCalls - self.sklearnexFailCalls += other.sklearnexFailCalls - self.sklearnexDeviceCallsRatio += other.sklearnexDeviceCallsRatio - self.sklearnexDevicePatchedCalls += other.sklearnexDevicePatchedCalls - self.sklearnexDeviceOffloadSuccess += other.sklearnexDeviceOffloadSuccess - self.sklearnexDeviceOffloadFail += other.sklearnexDeviceOffloadFail - -class LineParser: - - def __init__(self, device=None, consider_fails=False): - self.sklearnexLine = "running accelerated version" - self.sklearnLine = "fallback to original Scikit-learn" - self.sklearnexFailLine = "failed to run accelerated version, fallback to original Scikit-learn" - - if device != 'CPU': - self.sklearnexDeviceOffloadSuccessLine = f"successfully run on {device.lower()}" - self.sklearnexDeviceOffloadFailLine = f"failed to run on {device.lower()}. Fallback to host" - self.device = device - - self.sklearnexDeviceLine = f"{self.sklearnexLine} on {self.device}" - - self.consider_fails = consider_fails - - self.algoCalls = CallCounter() - self._localTestCalls = CallCounter() - - def clearCounters(self): - self.algoCalls.clear() - self._localTestCalls.clear() - - def parseLine(self, line): - test_signal_fail = "FAILED" - test_signal_pass = "PASSED" - - if self.sklearnexLine in line: - self._localTestCalls.sklearnexCalls += 1 - if self.sklearnLine in line: - self._localTestCalls.sklearnCalls += 1 - if self.sklearnexFailLine in line: - self._localTestCalls.sklearnexFailCalls += 1 - if self.sklearnexDeviceLine in line: - self._localTestCalls.sklearnexDevicePatchedCalls += 1 - - if self.device != 'CPU': - if self.sklearnexDeviceOffloadSuccessLine in line: - self._localTestCalls.sklearnexDeviceOffloadSuccess += 1 - elif self.sklearnexDeviceOffloadFailLine in line: - self._localTestCalls.sklearnexDeviceOffloadFail += 1 - else: - self._localTestCalls.calcDeviceCalls() - - if test_signal_fail in line or test_signal_pass in line: - self._localTestCalls.calcDeviceCalls() - if not self.consider_fails or test_signal_pass in line: - self.algoCalls.inc(self._localTestCalls) - self._localTestCalls.clear() - -def make_summory(counter, device): - countAllCalls = counter.sklearnCalls + counter.sklearnexCalls - percentDalCalls = float(counter.sklearnexCalls - counter.sklearnexFailCalls) / (countAllCalls) * 100 if countAllCalls else 0 - - # to calculate deviceOffloadCalls, we try to use sklearnexDeivceCallsRatio as more fine-grained metric - # if it is unavailable, we use sklearnexDevicePatchedCalls count instead - deviceOffloadCalls = counter.sklearnexDeviceCallsRatio if counter.sklearnexDeviceCallsRatio > 0 else counter.sklearnexDevicePatchedCalls - sklearnexOffloadPersent = float(deviceOffloadCalls / counter.sklearnexCalls) * 100 if counter.sklearnexCalls else 0 - totalOffloatPersent = percentDalCalls * sklearnexOffloadPersent / 100 - - reportText = "" - reportText += "Number of Scikit-learn calls: %d
" % counter.sklearnCalls - reportText += "Number of sklearnex calls: %d
" % counter.sklearnexCalls - reportText += "Number of sklearnex fail calls: %d
" % counter.sklearnexFailCalls - reportText += "Percent of using sklearnex: %d %%
" % int(percentDalCalls) - if device != 'CPU': - reportText += "Percent of sklearnex calls offloaded to %s: %d %%
" % (device, int(sklearnexOffloadPersent)) - reportText += "Percent of using sklearnex on %s: %d %%
" % (device, int(totalOffloatPersent)) - - print('Number of Scikit-learn calls: %d' % counter.sklearnCalls) - print('Number of sklearnex calls: %d' % counter.sklearnexCalls) - print('Number of sklearnex fail calls: %d' % counter.sklearnexFailCalls) - print('Percent of using sklearnex: %d %%' % int(percentDalCalls)) - if device != 'CPU': - print("Percent of sklearnex calls offloaded to %s: %d %%" % (device, int(sklearnexOffloadPersent))) - print("Percent of using sklearnex on %s: %d %%" % (device, int(totalOffloatPersent))) - - return reportText - - -def device_from_sycl_terminology(device): - if device == 'cpu' or device == 'host' or device is None: - return 'CPU' - if device == 'gpu': - return 'GPU' - else: - raise ValueError(f"Unexpected device name {device}." - " Supported types are host, cpu and gpu") - - -def make_report(algs_filename, report_filename, device=None, consider_fails=False): - device = device_from_sycl_terminology(device) - - with open(algs_filename, "r") as file_algs: - algs = file_algs.read().split("\n") - algs.remove("") - - report_file = open(report_filename,'wt') - textHTML = """ - - - Report of conformance testing - - - -

- Start of testing in """ + str(datetime.now())+ "
" - report_file.write(textHTML) - - globalCalls = CallCounter() - parser = LineParser(device, consider_fails) - - for alg_name in algs: - parser.clearCounters() - report_file.write("

Testing %s

" % alg_name) - - log_filename = "_log_%s.txt" % alg_name - log_file = open(log_filename, "r") - lines = log_file.readlines() - log_file.close() - - result_str = "" - - for line in lines: - if '====' in line and not 'test session starts' in line: - break - parser.parseLine(line) - - for line in reversed(lines): - if '=====' in line: - if 'test session starts' in line: - raise Exception('Found an error while testing %s' % (alg_name)) - result_str = line - break - - globalCalls.inc(parser.algoCalls) - - if parser.algoCalls.sklearnexCalls == 0 and parser.algoCalls.sklearnCalls == 0 and parser.algoCalls.sklearnexFailCalls == 0: - raise Exception('Algorithm %s has never been called' % (alg_name)) - - print('*********************************************') - print('Algorithm: %s' % alg_name) - reportAlgText = make_summory(parser.algoCalls, device) - report_file.write(reportAlgText) - print(result_str) - report_file.write(result_str + "
") - - print('*********************************************') - print('Summary') - report_file.write("

Summary

") - summaryText = make_summory(globalCalls, device) - report_file.write(summaryText) - - textHTML = """
- Finishing testing in """+str(datetime.now())+"""

- - """ - - report_file.write(textHTML) - report_file.close() From 7e030354a1d9f917e891329ffe4540d31d55fbb8 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Mon, 7 Oct 2024 09:18:34 +0200 Subject: [PATCH 05/10] Update INSTALL.md --- INSTALL.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 3727c7b32bb..d534b89a8a7 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -20,11 +20,10 @@ Required Software: * C/C++ Compiler * [DPC++ Compiler](https://www.intel.com/content/www/us/en/developer/tools/oneapi/dpc-compiler.html) if building with SYCL support -* Python version 3.9 or higher +* Python * TBB library (repository contains script to download it) * Microsoft Visual Studio\* (Windows\* only) * [MSYS2](http://msys2.github.io) (Windows\* only) -* Python * `make` and `dos2unix` tools; install these packages using MSYS2 on Windows\* as follows: pacman -S msys/make msys/dos2unix From d3dc5813c754a5cf25936effd99120702acb46c2 Mon Sep 17 00:00:00 2001 From: "Faust, Ian" Date: Mon, 7 Oct 2024 14:51:45 +0200 Subject: [PATCH 06/10] Revert "remove conformance-scripts" This reverts commit 1adca78bf98bdca2d37c6bfed7ce16a73f969efd. --- .../conformance-scripts/algorithms.txt | 11 + .../conformance-scripts/download_tests.sh | 73 ++++++ .ci/scripts/conformance-scripts/run_tests.py | 49 ++++ .../run_tests_with_context.py | 15 ++ .../run_xpu_conformance.py | 65 ++++++ .ci/scripts/conformance-scripts/utils.py | 209 ++++++++++++++++++ 6 files changed, 422 insertions(+) create mode 100755 .ci/scripts/conformance-scripts/algorithms.txt create mode 100755 .ci/scripts/conformance-scripts/download_tests.sh create mode 100755 .ci/scripts/conformance-scripts/run_tests.py create mode 100644 .ci/scripts/conformance-scripts/run_tests_with_context.py create mode 100644 .ci/scripts/conformance-scripts/run_xpu_conformance.py create mode 100755 .ci/scripts/conformance-scripts/utils.py diff --git a/.ci/scripts/conformance-scripts/algorithms.txt b/.ci/scripts/conformance-scripts/algorithms.txt new file mode 100755 index 00000000000..2ef5baf106c --- /dev/null +++ b/.ci/scripts/conformance-scripts/algorithms.txt @@ -0,0 +1,11 @@ +dbscan +elastic_net +kmeans +lin_reg +log_reg +pca +ridge_reg +svm +svm_sparse +forest +knn diff --git a/.ci/scripts/conformance-scripts/download_tests.sh b/.ci/scripts/conformance-scripts/download_tests.sh new file mode 100755 index 00000000000..e115f4cd17d --- /dev/null +++ b/.ci/scripts/conformance-scripts/download_tests.sh @@ -0,0 +1,73 @@ +#!/bin/bash +#=============================================================================== +# Copyright 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +while [[ $# -gt 0 ]]; do + key="$1" + + case $key in + --alg-name) + ALG_NAME="$2" + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac + shift + shift +done +SKLEARN_PATH="$(pip show scikit-learn | grep Location | cut -d ' ' -f 2)/sklearn/" + +case ${ALG_NAME} in + "dbscan") + cp ${SKLEARN_PATH}cluster/tests/test_dbscan.py test_dbscan.py + ;; + "elastic_net") + cp ${SKLEARN_PATH}linear_model/tests/test_coordinate_descent.py test_elastic_net.py + ;; + "kmeans") + cp ${SKLEARN_PATH}cluster/tests/test_k_means.py test_kmeans.py + ;; + "lin_reg") + cp ${SKLEARN_PATH}linear_model/tests/test_base.py test_lin_reg.py + ;; + "log_reg") + cp ${SKLEARN_PATH}linear_model/tests/test_logistic.py test_log_reg.py + ;; + "pca") + cp ${SKLEARN_PATH}decomposition/tests/test_pca.py test_pca.py + ;; + "ridge_reg") + cp ${SKLEARN_PATH}linear_model/tests/test_ridge.py test_ridge_reg.py + ;; + "svm") + cp ${SKLEARN_PATH}svm/tests/test_svm.py test_svm.py + ;; + "svm_sparse") + cp ${SKLEARN_PATH}svm/tests/test_sparse.py test_svm_sparse.py + ;; + "forest") + cp ${SKLEARN_PATH}ensemble/tests/test_forest.py test_forest.py + ;; + "knn") + cp ${SKLEARN_PATH}neighbors/tests/test_neighbors.py test_knn.py + ;; + *) + echo "Unknown algorithm: ${ALG_NAME}" + exit 1 + ;; +esac diff --git a/.ci/scripts/conformance-scripts/run_tests.py b/.ci/scripts/conformance-scripts/run_tests.py new file mode 100755 index 00000000000..4a5b7cbe594 --- /dev/null +++ b/.ci/scripts/conformance-scripts/run_tests.py @@ -0,0 +1,49 @@ +#=============================================================================== +# Copyright 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +import re +import sys +import subprocess +from datetime import datetime +from subprocess import Popen, PIPE +from utils import make_report + +try: + import sklearnex +except Exception: + raise Exception('sklearnex is not installed') + +algs_filename = "algorithms.txt" +report_filename = "report.html" + +if __name__ == "__main__": + with open(algs_filename, "r") as file_algs: + algs = file_algs.read().split("\n") + algs.remove("") + + print("Confromance testing start") + for alg_name in algs: + code = subprocess.call(["./download_tests.sh", "--alg-name", "%s" % (alg_name) ]) + if code: raise Exception('Error while copying test files') + print(alg_name) + + alg_log = open("_log_%s.txt" % (alg_name), "w") + subprocess.call(["python", "-m", "sklearnex", "-m", "pytest", "-s", "--disable-warnings", "-v", "test_%s.py" % (alg_name)], + stdout=alg_log, stderr=alg_log) + alg_log.close() + + make_report(algs_filename=algs_filename, + report_filename = report_filename) diff --git a/.ci/scripts/conformance-scripts/run_tests_with_context.py b/.ci/scripts/conformance-scripts/run_tests_with_context.py new file mode 100644 index 00000000000..9ead91dbd67 --- /dev/null +++ b/.ci/scripts/conformance-scripts/run_tests_with_context.py @@ -0,0 +1,15 @@ +import argparse +import pytest + +def get_context(device): + from daal4py.oneapi import sycl_context + return sycl_context(device, host_offload_on_fail=True) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Script to run scikit-learn tests with device context manager') + parser.add_argument('-a', '--algorithm', type=str, help='datasets which should be downloaded') + parser.add_argument('-d', '--device', type=str, help='device name', choices=['host', 'cpu', 'gpu']) + args = parser.parse_args() + + with get_context(args.device): + pytest.main(["-s", "--disable-warnings", "-v", "test_%s.py" % (args.algorithm)]) diff --git a/.ci/scripts/conformance-scripts/run_xpu_conformance.py b/.ci/scripts/conformance-scripts/run_xpu_conformance.py new file mode 100644 index 00000000000..83bc46ee628 --- /dev/null +++ b/.ci/scripts/conformance-scripts/run_xpu_conformance.py @@ -0,0 +1,65 @@ +#=============================================================================== +# Copyright 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +import re +import sys +import subprocess +from datetime import datetime +from subprocess import Popen, PIPE +from utils import make_report +import argparse +import os + +try: + import daal4py +except Exception: + raise Exception('daal4py is not installed') + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Script to make scikit-learn conformance report' + 'based on tests for different device types') + parser.add_argument('-d', '--device', type=str, help='device name', choices=['host', 'cpu', 'gpu']) + parser.add_argument('-f', '--consider_fails', + help='Exclude failed tests from conformance calculation', action="store_true") + args = parser.parse_args() + + os.environ['SKLEARNEX_VERBOSE'] = 'INFO' + + algs_filename = "algorithms.txt" + report_filename = f"report_{args.device}.html" + + with open(algs_filename, "r") as file_algs: + algs = file_algs.read().split("\n") + algs.remove("") + + print("Confromance testing start") + for alg_name in algs: + code = subprocess.call(["./download_tests.sh", "--alg-name", "%s" % (alg_name) ]) + if code: + raise Exception('Error while copying test files') + print(alg_name) + + alg_log = open("_log_%s.txt" % (alg_name), "w") + subprocess.call(["python", "-m", "sklearnex", "run_tests_with_context.py", "-a" "%s" % (alg_name), + "-d" "%s" % (args.device)], + stdout=alg_log) + alg_log.close() + + make_report(algs_filename=algs_filename, + report_filename = report_filename, + device=args.device, + consider_fails=args.consider_fails) diff --git a/.ci/scripts/conformance-scripts/utils.py b/.ci/scripts/conformance-scripts/utils.py new file mode 100755 index 00000000000..63e719a7e6c --- /dev/null +++ b/.ci/scripts/conformance-scripts/utils.py @@ -0,0 +1,209 @@ +#=============================================================================== +# Copyright 2021 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#=============================================================================== + +# -*- coding: utf-8 -*- +from datetime import datetime + +class CallCounter: + def __init__(self): + self.clear() + + def clear(self): + self.sklearnexCalls = 0 + self.sklearnCalls = 0 + self.sklearnexFailCalls = 0 + self.sklearnexDeviceCallsRatio = 0.0 + self.sklearnexDevicePatchedCalls = 0 + + self.sklearnexDeviceOffloadSuccess = 0 + self.sklearnexDeviceOffloadFail = 0 + + def calcDeviceCalls(self): + offloadSum = self.sklearnexDeviceOffloadSuccess + self.sklearnexDeviceOffloadFail + if offloadSum > 0: + self.sklearnexDeviceCallsRatio += self.sklearnexDeviceOffloadSuccess / offloadSum + self.sklearnexDeviceOffloadSuccess = 0 + self.sklearnexDeviceOffloadFail = 0 + + def inc(self, other): + self.sklearnexCalls += other.sklearnexCalls + self.sklearnCalls += other.sklearnCalls + self.sklearnexFailCalls += other.sklearnexFailCalls + self.sklearnexDeviceCallsRatio += other.sklearnexDeviceCallsRatio + self.sklearnexDevicePatchedCalls += other.sklearnexDevicePatchedCalls + self.sklearnexDeviceOffloadSuccess += other.sklearnexDeviceOffloadSuccess + self.sklearnexDeviceOffloadFail += other.sklearnexDeviceOffloadFail + +class LineParser: + + def __init__(self, device=None, consider_fails=False): + self.sklearnexLine = "running accelerated version" + self.sklearnLine = "fallback to original Scikit-learn" + self.sklearnexFailLine = "failed to run accelerated version, fallback to original Scikit-learn" + + if device != 'CPU': + self.sklearnexDeviceOffloadSuccessLine = f"successfully run on {device.lower()}" + self.sklearnexDeviceOffloadFailLine = f"failed to run on {device.lower()}. Fallback to host" + self.device = device + + self.sklearnexDeviceLine = f"{self.sklearnexLine} on {self.device}" + + self.consider_fails = consider_fails + + self.algoCalls = CallCounter() + self._localTestCalls = CallCounter() + + def clearCounters(self): + self.algoCalls.clear() + self._localTestCalls.clear() + + def parseLine(self, line): + test_signal_fail = "FAILED" + test_signal_pass = "PASSED" + + if self.sklearnexLine in line: + self._localTestCalls.sklearnexCalls += 1 + if self.sklearnLine in line: + self._localTestCalls.sklearnCalls += 1 + if self.sklearnexFailLine in line: + self._localTestCalls.sklearnexFailCalls += 1 + if self.sklearnexDeviceLine in line: + self._localTestCalls.sklearnexDevicePatchedCalls += 1 + + if self.device != 'CPU': + if self.sklearnexDeviceOffloadSuccessLine in line: + self._localTestCalls.sklearnexDeviceOffloadSuccess += 1 + elif self.sklearnexDeviceOffloadFailLine in line: + self._localTestCalls.sklearnexDeviceOffloadFail += 1 + else: + self._localTestCalls.calcDeviceCalls() + + if test_signal_fail in line or test_signal_pass in line: + self._localTestCalls.calcDeviceCalls() + if not self.consider_fails or test_signal_pass in line: + self.algoCalls.inc(self._localTestCalls) + self._localTestCalls.clear() + +def make_summory(counter, device): + countAllCalls = counter.sklearnCalls + counter.sklearnexCalls + percentDalCalls = float(counter.sklearnexCalls - counter.sklearnexFailCalls) / (countAllCalls) * 100 if countAllCalls else 0 + + # to calculate deviceOffloadCalls, we try to use sklearnexDeivceCallsRatio as more fine-grained metric + # if it is unavailable, we use sklearnexDevicePatchedCalls count instead + deviceOffloadCalls = counter.sklearnexDeviceCallsRatio if counter.sklearnexDeviceCallsRatio > 0 else counter.sklearnexDevicePatchedCalls + sklearnexOffloadPersent = float(deviceOffloadCalls / counter.sklearnexCalls) * 100 if counter.sklearnexCalls else 0 + totalOffloatPersent = percentDalCalls * sklearnexOffloadPersent / 100 + + reportText = "" + reportText += "Number of Scikit-learn calls: %d
" % counter.sklearnCalls + reportText += "Number of sklearnex calls: %d
" % counter.sklearnexCalls + reportText += "Number of sklearnex fail calls: %d
" % counter.sklearnexFailCalls + reportText += "Percent of using sklearnex: %d %%
" % int(percentDalCalls) + if device != 'CPU': + reportText += "Percent of sklearnex calls offloaded to %s: %d %%
" % (device, int(sklearnexOffloadPersent)) + reportText += "Percent of using sklearnex on %s: %d %%
" % (device, int(totalOffloatPersent)) + + print('Number of Scikit-learn calls: %d' % counter.sklearnCalls) + print('Number of sklearnex calls: %d' % counter.sklearnexCalls) + print('Number of sklearnex fail calls: %d' % counter.sklearnexFailCalls) + print('Percent of using sklearnex: %d %%' % int(percentDalCalls)) + if device != 'CPU': + print("Percent of sklearnex calls offloaded to %s: %d %%" % (device, int(sklearnexOffloadPersent))) + print("Percent of using sklearnex on %s: %d %%" % (device, int(totalOffloatPersent))) + + return reportText + + +def device_from_sycl_terminology(device): + if device == 'cpu' or device == 'host' or device is None: + return 'CPU' + if device == 'gpu': + return 'GPU' + else: + raise ValueError(f"Unexpected device name {device}." + " Supported types are host, cpu and gpu") + + +def make_report(algs_filename, report_filename, device=None, consider_fails=False): + device = device_from_sycl_terminology(device) + + with open(algs_filename, "r") as file_algs: + algs = file_algs.read().split("\n") + algs.remove("") + + report_file = open(report_filename,'wt') + textHTML = """ + + + Report of conformance testing + + + +

+ Start of testing in """ + str(datetime.now())+ "
" + report_file.write(textHTML) + + globalCalls = CallCounter() + parser = LineParser(device, consider_fails) + + for alg_name in algs: + parser.clearCounters() + report_file.write("

Testing %s

" % alg_name) + + log_filename = "_log_%s.txt" % alg_name + log_file = open(log_filename, "r") + lines = log_file.readlines() + log_file.close() + + result_str = "" + + for line in lines: + if '====' in line and not 'test session starts' in line: + break + parser.parseLine(line) + + for line in reversed(lines): + if '=====' in line: + if 'test session starts' in line: + raise Exception('Found an error while testing %s' % (alg_name)) + result_str = line + break + + globalCalls.inc(parser.algoCalls) + + if parser.algoCalls.sklearnexCalls == 0 and parser.algoCalls.sklearnCalls == 0 and parser.algoCalls.sklearnexFailCalls == 0: + raise Exception('Algorithm %s has never been called' % (alg_name)) + + print('*********************************************') + print('Algorithm: %s' % alg_name) + reportAlgText = make_summory(parser.algoCalls, device) + report_file.write(reportAlgText) + print(result_str) + report_file.write(result_str + "
") + + print('*********************************************') + print('Summary') + report_file.write("

Summary

") + summaryText = make_summory(globalCalls, device) + report_file.write(summaryText) + + textHTML = """
+ Finishing testing in """+str(datetime.now())+"""

+ + """ + + report_file.write(textHTML) + report_file.close() From 2a13d1830ca0aa5c598bbfbea27c637750b14e37 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 8 Jan 2025 12:29:24 +0100 Subject: [PATCH 07/10] Update ci.yml --- .ci/pipeline/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index fc63e1f9fed..a210d60ea51 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -45,6 +45,7 @@ jobs: vmImage: '$(VM_IMAGE)' steps: - script: | + echo $(CODECOV_TOKEN) .ci/env/apt.sh clang-format .ci/env/editorconfig-checker.sh displayName: 'Install Dependencies' From 9e0e055139e37e9064a302a567e7269b9a8edb76 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 8 Jan 2025 12:34:45 +0100 Subject: [PATCH 08/10] Update ci.yml --- .ci/pipeline/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index a210d60ea51..51269106e99 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -46,9 +46,12 @@ jobs: steps: - script: | echo $(CODECOV_TOKEN) + echo $TOK .ci/env/apt.sh clang-format .ci/env/editorconfig-checker.sh displayName: 'Install Dependencies' + env: + TOK: $(CODECOV_TOKEN) - script: | .ci/scripts/clang-format.sh displayName: 'clang-format check' From 5d48b62ada7d35c226116eefd60ed462b8647891 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 8 Jan 2025 12:36:37 +0100 Subject: [PATCH 09/10] Update ci.yml --- .ci/pipeline/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index 51269106e99..b4ef1a80f2e 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -45,7 +45,6 @@ jobs: vmImage: '$(VM_IMAGE)' steps: - script: | - echo $(CODECOV_TOKEN) echo $TOK .ci/env/apt.sh clang-format .ci/env/editorconfig-checker.sh From 6464aa0a8a31cc08dfd39b4a17474e729bfef1f2 Mon Sep 17 00:00:00 2001 From: Ian Faust Date: Wed, 8 Jan 2025 21:38:36 +0100 Subject: [PATCH 10/10] Update ci.yml --- .ci/pipeline/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml index b4ef1a80f2e..0b6e34df6f3 100755 --- a/.ci/pipeline/ci.yml +++ b/.ci/pipeline/ci.yml @@ -45,12 +45,10 @@ jobs: vmImage: '$(VM_IMAGE)' steps: - script: | - echo $TOK + echo $(Build.SourceBranch) .ci/env/apt.sh clang-format .ci/env/editorconfig-checker.sh displayName: 'Install Dependencies' - env: - TOK: $(CODECOV_TOKEN) - script: | .ci/scripts/clang-format.sh displayName: 'clang-format check'