From df8919fdeb28d411645c2fe81e280322b5e2f1f8 Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 2 Jan 2025 15:38:42 +0200 Subject: [PATCH 01/10] Add test utils for using testbook --- tests/utils_for_testbook.py | 74 +++++++++++++++++++++++++++++++++++++ tests/utils_for_tests.py | 39 ++++++++++++++++--- 2 files changed, 107 insertions(+), 6 deletions(-) create mode 100644 tests/utils_for_testbook.py diff --git a/tests/utils_for_testbook.py b/tests/utils_for_testbook.py new file mode 100644 index 000000000..887ac8827 --- /dev/null +++ b/tests/utils_for_testbook.py @@ -0,0 +1,74 @@ +import json +import os +from typing import Any, Callable +import pytest + +from testbook import testbook +from tests.utils_for_tests import resolve_notebook_path, should_skip_notebook + +from classiq.interface.generator.quantum_program import QuantumProgram + + +def test_notebook(notebook_name: str, timeout_seconds: float = 10) -> Callable: + def inner_decorator(func: Callable) -> Any: + notebook_path = resolve_notebook_path(notebook_name) + + for decorator in [ + testbook(notebook_path, execute=True, timeout=timeout_seconds), + _build_cd_decorator(notebook_path), + _build_skip_decorator(notebook_name), + ]: + func = decorator(func) + return func + + return inner_decorator + + +# The purpose of the `cd_decorator` is to execute the test in the same folder as the `ipynb` file +# so that relative files (images, csv, etc.) will be available +def _build_cd_decorator(file_path: str) -> Callable: + def cd_decorator(func: Callable) -> Any: + def inner(*args: Any, **kwargs: Any) -> Any: + previous_dir = os.getcwd() + os.chdir(os.path.dirname(file_path)) + + func(*args, **kwargs) + + os.chdir(previous_dir) + + return inner + + return cd_decorator + + +def _build_skip_decorator(notebook_name: str) -> Callable: + return pytest.mark.skipif( + should_skip_notebook(notebook_name), + reason="Didn't change", + ) + + +def validate_quantum_model(model: str) -> None: + # currently that's some dummy test - only checking that it's a valid dict + assert isinstance(json.loads(model), dict) + + +def validate_quantum_program_size( + quantum_program: str, + expected_width: int | None = None, + expected_depth: int | None = None, +) -> None: + qp = QuantumProgram.model_validate_json(quantum_program) + + actual_width = qp.data.width + if expected_width is not None: + assert ( + actual_width <= expected_width + ), f"The width of the circuit changed! (for the worse!). From {expected_width} to {actual_width}" + + assert qp.transpiled_circuit is not None # for mypy + actual_depth = qp.transpiled_circuit.depth + if expected_depth is not None: + assert ( + actual_depth <= expected_depth + ), f"The depth of the circuit changed! (for the worse!). From {expected_depth} to {actual_depth}" diff --git a/tests/utils_for_tests.py b/tests/utils_for_tests.py index 44f75fe24..4b9e99e4f 100644 --- a/tests/utils_for_tests.py +++ b/tests/utils_for_tests.py @@ -1,11 +1,11 @@ import os -from collections.abc import Iterable +from functools import lru_cache from pathlib import Path ROOT_DIRECTORY = Path(__file__).parents[1] -def iterate_notebooks() -> Iterable[str]: +def iterate_notebooks() -> list[str]: if os.environ.get("SHOULD_TEST_ALL_FILES", "") == "true": notebooks_to_test = _get_all_notebooks() else: @@ -14,8 +14,35 @@ def iterate_notebooks() -> Iterable[str]: return notebooks_to_test -def _get_all_notebooks(directory: Path = ROOT_DIRECTORY) -> Iterable[str]: - for root, _, files in os.walk(directory): +@lru_cache +def _get_all_notebooks( + directory: Path = ROOT_DIRECTORY, suffix: str = ".ipynb" +) -> list[str]: + return [ + file + for root, _, files in os.walk(directory) + for file in files + if file.endswith(suffix) + ] + + +def should_run_notebook(notebook_name: str) -> bool: + notebook_path = resolve_notebook_path(notebook_name) + return notebook_path in iterate_notebooks() + + +def should_skip_notebook(notebook_name: str) -> bool: + return not should_run_notebook(notebook_name) + + +@lru_cache +def resolve_notebook_path(notebook_name: str) -> str: + notebook_name_lower = notebook_name.lower() + if not notebook_name_lower.endswith(".ipynb"): + notebook_name_lower += ".ipynb" + + for root, _, files in os.walk(ROOT_DIRECTORY): for file in files: - if file.endswith(".ipynb"): - yield os.path.join(root, file) + if file.lower() == notebook_name_lower: + return os.path.join(root, file) + raise LookupError(f"Notebook not found: {notebook_name}") From 01508bfe2284a2fdcd57c97b42077283f9769d61 Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 2 Jan 2025 15:39:00 +0200 Subject: [PATCH 02/10] Add test for notebook bernstein_vazirani --- tests/notebooks/test_bernstein_vazirani.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/notebooks/test_bernstein_vazirani.py diff --git a/tests/notebooks/test_bernstein_vazirani.py b/tests/notebooks/test_bernstein_vazirani.py new file mode 100644 index 000000000..34609ea0f --- /dev/null +++ b/tests/notebooks/test_bernstein_vazirani.py @@ -0,0 +1,17 @@ +from tests.utils_for_testbook import ( + validate_quantum_program_size, + validate_quantum_model, + test_notebook, +) +from testbook.client import TestbookNotebookClient + + +@test_notebook("bernstein_vazirani", timeout_seconds=20) +def test_bernstein_vazirani(tb: TestbookNotebookClient) -> None: + # test models + validate_quantum_model(tb.ref("qmod")) + # test quantum programs + validate_quantum_program_size(tb.ref("qprog"), expected_width=6, expected_depth=5) + + # test notebook content + assert int(tb.ref("secret_integer_q")) == tb.ref("SECRET_INT") From cfabe3824e16df3049355fb19c04f9424357b55e Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 2 Jan 2025 15:58:44 +0200 Subject: [PATCH 03/10] Add test for notebook hidden_shift --- tests/notebooks/test_hidden_shift.py | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 tests/notebooks/test_hidden_shift.py diff --git a/tests/notebooks/test_hidden_shift.py b/tests/notebooks/test_hidden_shift.py new file mode 100644 index 000000000..46c0c8f7b --- /dev/null +++ b/tests/notebooks/test_hidden_shift.py @@ -0,0 +1,35 @@ +from tests.utils_for_testbook import ( + validate_quantum_program_size, + validate_quantum_model, + test_notebook, +) +from testbook.client import TestbookNotebookClient + + +@test_notebook( + "hidden_shift", + timeout_seconds=272, # we may lower this value +) +def test_hidden_shift(tb: TestbookNotebookClient) -> None: + # test models + validate_quantum_model(tb.ref("qmod_simple")) + validate_quantum_model(tb.ref("qmod_complex")) + validate_quantum_model(tb.ref("qmod_no_dual")) + # test quantum programs + validate_quantum_program_size( + tb.ref("qprog_simple"), + expected_width=7, # actual width: 7 + expected_depth=50, # actual depth: 47 + ) + validate_quantum_program_size( + tb.ref("qprog_complex"), + expected_width=20, # actual width: 20 + expected_depth=1700, # actual depth: 1656 + ) + validate_quantum_program_size( + tb.ref("qprog_no_dual"), + expected_width=20, # actual width: 20 + expected_depth=1700, # actual depth: 1685 + ) + # test notebook content + pass # TODO From 25bc649c39c926c14da8838935a4020204e94039 Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 2 Jan 2025 15:49:18 +0200 Subject: [PATCH 04/10] delete me --- algorithms/bernstein_vazirani/bernstein_vazirani.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algorithms/bernstein_vazirani/bernstein_vazirani.ipynb b/algorithms/bernstein_vazirani/bernstein_vazirani.ipynb index b0d779520..6e45379ad 100644 --- a/algorithms/bernstein_vazirani/bernstein_vazirani.ipynb +++ b/algorithms/bernstein_vazirani/bernstein_vazirani.ipynb @@ -13,7 +13,7 @@ "id": "22ec3e91-cc89-43be-88d0-1a36639e5e4e", "metadata": {}, "source": [ - "The Bernstein-Vazirani (BV) algorithm [[1](#BVWiki)], named after Ethan Bernstein and Umesh Vazirani, is a basic quantum algorithm. It gives a linear speedup compared to its classical counterpart, in the oracle complexity setting.\n", + "The Bernstein-Vazirani (BV) algorithm [[1](#BVWiki)], named after Ethan Bernstein and Umesh Vazirani, is a basic quantum algorithm. It gives a linear speedup compared to its classical counterpart, in the oracle complexity setting.\n", "\n", "The algorithm treats the following problem:\n", "\n", From 22c7745e3bc714d6ee672bd728c065cbbbe9d93f Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 2 Jan 2025 15:55:26 +0200 Subject: [PATCH 05/10] debug --- tests/utils_for_testbook.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/utils_for_testbook.py b/tests/utils_for_testbook.py index 887ac8827..400697dbb 100644 --- a/tests/utils_for_testbook.py +++ b/tests/utils_for_testbook.py @@ -42,6 +42,12 @@ def inner(*args: Any, **kwargs: Any) -> Any: def _build_skip_decorator(notebook_name: str) -> Callable: + import logging + + logger = logging.getLogger(__name__) + logger.error( + f"{notebook_name=} ; should skip: {should_skip_notebook(notebook_name)}" + ) return pytest.mark.skipif( should_skip_notebook(notebook_name), reason="Didn't change", From 8a962acf2ac1af1d86240b60d51bbfac4c971c14 Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 2 Jan 2025 17:27:00 +0200 Subject: [PATCH 06/10] rename test_notebook -> wrap_testbook --- tests/notebooks/test_bernstein_vazirani.py | 4 ++-- tests/notebooks/test_hidden_shift.py | 4 ++-- tests/utils_for_testbook.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/notebooks/test_bernstein_vazirani.py b/tests/notebooks/test_bernstein_vazirani.py index 34609ea0f..601200a66 100644 --- a/tests/notebooks/test_bernstein_vazirani.py +++ b/tests/notebooks/test_bernstein_vazirani.py @@ -1,12 +1,12 @@ from tests.utils_for_testbook import ( validate_quantum_program_size, validate_quantum_model, - test_notebook, + wrap_testbook, ) from testbook.client import TestbookNotebookClient -@test_notebook("bernstein_vazirani", timeout_seconds=20) +@wrap_testbook("bernstein_vazirani", timeout_seconds=20) def test_bernstein_vazirani(tb: TestbookNotebookClient) -> None: # test models validate_quantum_model(tb.ref("qmod")) diff --git a/tests/notebooks/test_hidden_shift.py b/tests/notebooks/test_hidden_shift.py index 46c0c8f7b..a1424fa2b 100644 --- a/tests/notebooks/test_hidden_shift.py +++ b/tests/notebooks/test_hidden_shift.py @@ -1,12 +1,12 @@ from tests.utils_for_testbook import ( validate_quantum_program_size, validate_quantum_model, - test_notebook, + wrap_testbook, ) from testbook.client import TestbookNotebookClient -@test_notebook( +@wrap_testbook( "hidden_shift", timeout_seconds=272, # we may lower this value ) diff --git a/tests/utils_for_testbook.py b/tests/utils_for_testbook.py index 400697dbb..f78670477 100644 --- a/tests/utils_for_testbook.py +++ b/tests/utils_for_testbook.py @@ -9,7 +9,7 @@ from classiq.interface.generator.quantum_program import QuantumProgram -def test_notebook(notebook_name: str, timeout_seconds: float = 10) -> Callable: +def wrap_testbook(notebook_name: str, timeout_seconds: float = 10) -> Callable: def inner_decorator(func: Callable) -> Any: notebook_path = resolve_notebook_path(notebook_name) From 5d232cb6e4ba7ecce788d77896a1088f8eaa0f9e Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 2 Jan 2025 17:27:51 +0200 Subject: [PATCH 07/10] more debug --- tests/utils_for_testbook.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/utils_for_testbook.py b/tests/utils_for_testbook.py index f78670477..6bfbeef72 100644 --- a/tests/utils_for_testbook.py +++ b/tests/utils_for_testbook.py @@ -43,10 +43,11 @@ def inner(*args: Any, **kwargs: Any) -> Any: def _build_skip_decorator(notebook_name: str) -> Callable: import logging + from tests.utils_for_tests import iterate_notebooks logger = logging.getLogger(__name__) logger.error( - f"{notebook_name=} ; should skip: {should_skip_notebook(notebook_name)}" + f"{notebook_name=} ; should skip: {should_skip_notebook(notebook_name)} ; notebooks: {iterate_notebooks()}" ) return pytest.mark.skipif( should_skip_notebook(notebook_name), From ffcf8652f61d7be60803ef4067ae1460a5c32e56 Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 2 Jan 2025 17:35:49 +0200 Subject: [PATCH 08/10] fix skip decorator --- tests/utils_for_testbook.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/utils_for_testbook.py b/tests/utils_for_testbook.py index 6bfbeef72..c65de4cc8 100644 --- a/tests/utils_for_testbook.py +++ b/tests/utils_for_testbook.py @@ -16,7 +16,7 @@ def inner_decorator(func: Callable) -> Any: for decorator in [ testbook(notebook_path, execute=True, timeout=timeout_seconds), _build_cd_decorator(notebook_path), - _build_skip_decorator(notebook_name), + _build_skip_decorator(notebook_path), ]: func = decorator(func) return func @@ -41,16 +41,16 @@ def inner(*args: Any, **kwargs: Any) -> Any: return cd_decorator -def _build_skip_decorator(notebook_name: str) -> Callable: +def _build_skip_decorator(notebook_path: str) -> Callable: import logging from tests.utils_for_tests import iterate_notebooks logger = logging.getLogger(__name__) logger.error( - f"{notebook_name=} ; should skip: {should_skip_notebook(notebook_name)} ; notebooks: {iterate_notebooks()}" + f"{notebook_path=} ; should skip: {should_skip_notebook(notebook_path)} ; notebooks: {iterate_notebooks()}" ) return pytest.mark.skipif( - should_skip_notebook(notebook_name), + should_skip_notebook(notebook_path), reason="Didn't change", ) From ef19fe80dc48d1ef89f86ded3fbdbff4a9c084c9 Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 9 Jan 2025 14:21:40 +0200 Subject: [PATCH 09/10] fix skip decorator --- tests/utils_for_testbook.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/utils_for_testbook.py b/tests/utils_for_testbook.py index c65de4cc8..192a0dc7f 100644 --- a/tests/utils_for_testbook.py +++ b/tests/utils_for_testbook.py @@ -45,12 +45,14 @@ def _build_skip_decorator(notebook_path: str) -> Callable: import logging from tests.utils_for_tests import iterate_notebooks + notebook_name = os.path.basename(notebook_path) + logger = logging.getLogger(__name__) logger.error( - f"{notebook_path=} ; should skip: {should_skip_notebook(notebook_path)} ; notebooks: {iterate_notebooks()}" + f"{notebook_path=} ; should skip: {should_skip_notebook(notebook_name)} ; notebooks: {iterate_notebooks()}" ) return pytest.mark.skipif( - should_skip_notebook(notebook_path), + should_skip_notebook(notebook_name), reason="Didn't change", ) From 639fa70f784d8c13e15f006665f566ca4becbb7a Mon Sep 17 00:00:00 2001 From: Dor Harpaz Date: Thu, 9 Jan 2025 14:29:38 +0200 Subject: [PATCH 10/10] work with file name rather than file path --- tests/utils_for_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/utils_for_tests.py b/tests/utils_for_tests.py index 4b9e99e4f..e2feef69b 100644 --- a/tests/utils_for_tests.py +++ b/tests/utils_for_tests.py @@ -14,6 +14,11 @@ def iterate_notebooks() -> list[str]: return notebooks_to_test +@lru_cache +def iterate_notebook_names() -> list[str]: + return list(map(os.path.basename, iterate_notebooks())) + + @lru_cache def _get_all_notebooks( directory: Path = ROOT_DIRECTORY, suffix: str = ".ipynb" @@ -27,8 +32,7 @@ def _get_all_notebooks( def should_run_notebook(notebook_name: str) -> bool: - notebook_path = resolve_notebook_path(notebook_name) - return notebook_path in iterate_notebooks() + return notebook_name in iterate_notebook_names() def should_skip_notebook(notebook_name: str) -> bool: