From 106d8beee33a0e24c03bad022e97b2b38a980858 Mon Sep 17 00:00:00 2001 From: Athroniaeth Date: Tue, 21 Nov 2023 22:49:12 +0100 Subject: [PATCH 1/4] change init, BenchmarkFixture for get best import and typing for IDEs --- src/pytest_benchmark/__init__.py | 3 ++ src/pytest_benchmark/fixture.py | 10 +++++-- tests/test_acceptation.py | 49 ++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 tests/test_acceptation.py diff --git a/src/pytest_benchmark/__init__.py b/src/pytest_benchmark/__init__.py index d6497a8..2858644 100644 --- a/src/pytest_benchmark/__init__.py +++ b/src/pytest_benchmark/__init__.py @@ -1 +1,4 @@ +from pytest_benchmark.fixture import BenchmarkFixture +from pytest_benchmark.stats import Stats + __version__ = '4.0.0' diff --git a/src/pytest_benchmark/fixture.py b/src/pytest_benchmark/fixture.py index b40377c..ae38b1a 100644 --- a/src/pytest_benchmark/fixture.py +++ b/src/pytest_benchmark/fixture.py @@ -8,9 +8,10 @@ import time import traceback from math import ceil +from typing import Any from .timers import compute_timer_precision -from .utils import NameWrapper +from .utils import NameWrapper, cached_property from .utils import format_time try: @@ -20,7 +21,7 @@ statistics = None else: statistics_error = None - from .stats import Metadata + from .stats import Metadata, Stats class FixtureAlreadyUsed(Exception): @@ -66,6 +67,11 @@ def __init__(self, node, disable_gc, timer, min_rounds, min_time, max_time, warm def enabled(self): return not self.disabled + @property + def statistics(self): + """Make statistics of the benchmarked function easier to access.""" + return self.stats.stats + def _get_precision(self, timer): if timer in self._precisions: timer_precision = self._precisions[timer] diff --git a/tests/test_acceptation.py b/tests/test_acceptation.py new file mode 100644 index 0000000..7420a23 --- /dev/null +++ b/tests/test_acceptation.py @@ -0,0 +1,49 @@ +""" Testing to ensure that __init__ makes typing import simple and accessible to IDEs. """ +from pytest_benchmark.fixture import BenchmarkFixture +from pytest_benchmark.stats import Stats + + +def test_import_benchmark_fixture(): + """ Test to ensure that __init__ makes importing for typing simple and accessible to IDEs. """ + Module = None + + # Check that the import is successful + try: + from pytest_benchmark import BenchmarkFixture as Module + except ImportError: + pass + finally: + assert Module is not None + + +def test_import_stats(): + """ Test to ensure that __init__ makes importing for typing simple and accessible to IDEs. """ + Module = None + + # Check that the import is successful + try: + from pytest_benchmark import Stats as Module + except ImportError: + pass + finally: + assert Module is not None + + +def test_benchmark_fixture_access_stats(benchmark: BenchmarkFixture): + """ + Test to ensure that __init__ makes accessing stats simple and accessible to IDEs. + + Sometimes it can be useful to modify the statistics to take into account + something that the benchmark cannot natively (a benchmark on a function which + takes care of a hundred files, while wishing to have the OPS per file) + """ + + benchmark.pedantic(lambda: 1 + 1, rounds=1, iterations=1, warmup_rounds=0) + + assert hasattr(benchmark, 'statistics') + assert isinstance(benchmark.statistics, Stats) + + old_ops = benchmark.statistics.ops + benchmark.statistics.ops *= 2 + assert benchmark.statistics.ops == old_ops * 2 + From e1e55681127239a21db5d2f766194aafab810caa Mon Sep 17 00:00:00 2001 From: Athroniaeth Date: Wed, 22 Nov 2023 00:22:05 +0100 Subject: [PATCH 2/4] correction of bad typing for benchmark.statistics.attributes (can't set) --- src/pytest_benchmark/fixture.py | 2 +- src/pytest_benchmark/stats.py | 257 +++++++++++++++++++++++++++----- tests/test_acceptation.py | 85 +++++++++-- 3 files changed, 290 insertions(+), 54 deletions(-) diff --git a/src/pytest_benchmark/fixture.py b/src/pytest_benchmark/fixture.py index ae38b1a..7750e04 100644 --- a/src/pytest_benchmark/fixture.py +++ b/src/pytest_benchmark/fixture.py @@ -68,7 +68,7 @@ def enabled(self): return not self.disabled @property - def statistics(self): + def statistics(self) -> Stats: """Make statistics of the benchmarked function easier to access.""" return self.stats.stats diff --git a/src/pytest_benchmark/stats.py b/src/pytest_benchmark/stats.py index ddb5202..e2684da 100644 --- a/src/pytest_benchmark/stats.py +++ b/src/pytest_benchmark/stats.py @@ -1,3 +1,9 @@ +""" +Don't use cached_property because we want a typing for getter and setter, +and we can't use iqr_outliers.setter with cached_property because Python apply the +decorator on cached_property and not on the property. +""" + from __future__ import division from __future__ import print_function @@ -6,7 +12,6 @@ from bisect import bisect_left from bisect import bisect_right -from .utils import cached_property from .utils import funcname from .utils import get_cprofile_functions @@ -19,6 +24,7 @@ class Stats(object): def __init__(self): self.data = [] + self.cache = {} def __bool__(self): return bool(self.data) @@ -35,38 +41,101 @@ def as_dict(self): def update(self, duration): self.data.append(duration) - @cached_property + @property def sorted_data(self): return sorted(self.data) - @cached_property - def total(self): + @property + def total(self) -> float or int: + """ Return the total time of round / iterations.""" + cache_value = self.cache.get('total') + + if cache_value is not None: + return cache_value + return sum(self.data) - @cached_property - def min(self): + @total.setter + def total(self, value: float or int) -> None: + """ Set the total time of round / iterations.""" + self.cache['total'] = value + + @property + def min(self) -> float or int: + """ Return the minimum observed time of round / iterations.""" + cache_value = self.cache.get('min') + + if cache_value is not None: + return cache_value + return min(self.data) - @cached_property - def max(self): + @min.setter + def min(self, value: float or int) -> None: + """ Set the minimum observed time of round / iterations.""" + self.cache['min'] = value + + @property + def max(self) -> float or int: + """ Return the maximum observed time of round / iterations""" + cache_value = self.cache.get('max') + + if cache_value is not None: + return cache_value + return max(self.data) - @cached_property - def mean(self): + @max.setter + def max(self, value: float or int) -> None: + """ Set the maximum observed time of round / iterations.""" + self.cache['max'] = value + + @property + def mean(self) -> float or int: + """ Return the mean of time of round / iterations.""" + cache_value = self.cache.get('mean') + + if cache_value is not None: + return cache_value + return statistics.mean(self.data) - @cached_property - def stddev(self): + @mean.setter + def mean(self, value: float or int) -> None: + """ Set the mean. """ + self.cache['mean'] = value + + @property + def stddev(self) -> float or int: + """ Return the standard deviation. """ + cache_value = self.cache.get('stddev') + + if cache_value is not None: + return cache_value + if len(self.data) > 1: return statistics.stdev(self.data) else: return 0 + @stddev.setter + def stddev(self, value: float or int) -> None: + """ Set the standard deviation. """ + self.cache['stddev'] = value + @property - def stddev_outliers(self): + def stddev_outliers(self) -> float or int: """ - Count of StdDev outliers: what's beyond (Mean - StdDev, Mean - StdDev) + Return the number of outliers (StdDev-style). + + Notes: + Count of StdDev outliers: what's beyond (Mean - StdDev, Mean - StdDev) """ + cache_value = self.cache.get('stddev_outliers') + + if cache_value is not None: + return cache_value + count = 0 q0 = self.mean - self.stddev q4 = self.mean + self.stddev @@ -75,29 +144,57 @@ def stddev_outliers(self): count += 1 return count - @cached_property - def rounds(self): + @stddev_outliers.setter + def stddev_outliers(self, value: str) -> None: + """ Set the number of outliers. """ + self.cache['stddev_outliers'] = value + + @property + def rounds(self) -> float or int: + """ Return the number of rounds, can't be changed by setter.""" return len(self.data) - @cached_property - def median(self): + @property + def median(self) -> float or int: + """ Return the median of time of round / iterations. """ + cache_value = self.cache.get('median') + + if cache_value is not None: + return cache_value + return statistics.median(self.data) - @cached_property - def ld15iqr(self): - """ - Tukey-style Lowest Datum within 1.5 IQR under Q1. - """ + @median.setter + def median(self, value: float or int) -> None: + """ Set the median of time of round / iterations.""" + self.cache['median'] = value + + @property + def ld15iqr(self) -> float or int: + """ Return the lowest datum within 1.5 IQR under Q1 (Tukey-style). """ + cache_value = self.cache.get('ld15iqr') + + if cache_value is not None: + return cache_value + if len(self.data) == 1: return self.data[0] else: return self.sorted_data[bisect_left(self.sorted_data, self.q1 - 1.5 * self.iqr)] - @cached_property - def hd15iqr(self): - """ - Tukey-style Highest Datum within 1.5 IQR over Q3. - """ + @ld15iqr.setter + def ld15iqr(self, value: int or float) -> None: + """ Set the lowest datum within 1.5 IQR under Q1 (Tukey-style). """ + self.cache['ld15iqr'] = value + + @property + def hd15iqr(self) -> float or int: + """ Return the highest datum within 1.5 IQR over Q3 (Tukey-style). """ + cache_value = self.cache.get('hd15iqr') + + if cache_value is not None: + return cache_value + if len(self.data) == 1: return self.data[0] else: @@ -107,8 +204,19 @@ def hd15iqr(self): else: return self.sorted_data[pos] - @cached_property - def q1(self): + @hd15iqr.setter + def hd15iqr(self, value: int or float) -> None: + """ Set the highest datum within 1.5 IQR over Q3 (Tukey-style). """ + self.cache['hd15iqr'] = value + + @property + def q1(self) -> float or int: + """ Return the first quartile. """ + cache_value = self.cache.get('q1') + + if cache_value is not None: + return cache_value + rounds = self.rounds data = self.sorted_data @@ -124,8 +232,19 @@ def q1(self): else: # Method 2 return statistics.median(data[:rounds // 2]) - @cached_property - def q3(self): + @q1.setter + def q1(self, value: int or float) -> None: + """ Set the first quartile. """ + self.cache['q1'] = value + + @property + def q3(self) -> float or int: + """ Return the third quartile. """ + cache_value = self.cache.get('q3') + + if cache_value is not None: + return cache_value + rounds = self.rounds data = self.sorted_data @@ -141,15 +260,39 @@ def q3(self): else: # Method 2 return statistics.median(data[rounds // 2:]) - @cached_property - def iqr(self): + @q3.setter + def q3(self, value: int or float) -> None: + """ Set the third quartile. """ + self.cache['q3'] = value + + @property + def iqr(self) -> float or int: + """ Return the interquartile range. """ + cache_value = self.cache.get('iqr') + + if cache_value is not None: + return cache_value + return self.q3 - self.q1 + @iqr.setter + def iqr(self, value) -> None: + """ Set the interquartile range. """ + self.cache['iqr'] = value + @property - def iqr_outliers(self): + def iqr_outliers(self) -> float or int: """ - Count of Tukey outliers: what's beyond (Q1 - 1.5IQR, Q3 + 1.5IQR) + Return the number of outliers (Tukey-style). + + Notes: + Count of Tukey outliers: what's beyond (Q1 - 1.5IQR, Q3 + 1.5IQR) """ + cache_value = self.cache.get('iqr_outliers') + + if cache_value is not None: + return cache_value + count = 0 q0 = self.q1 - 1.5 * self.iqr q4 = self.q3 + 1.5 * self.iqr @@ -158,16 +301,50 @@ def iqr_outliers(self): count += 1 return count - @cached_property - def outliers(self): + @iqr_outliers.setter + def iqr_outliers(self, value: str) -> None: + """ Set the number of outliers. """ + self.cache['iqr_outliers'] = value + + @property + def outliers(self) -> str: + """ + Return the number of outliers. + + Notes: + This is a string because it is used in a template. + The separator is a semicolon ';' because it is used in a template. + """ + + cache_value = self.cache.get('outliers') + + if cache_value is not None: + return cache_value + return "%s;%s" % (self.stddev_outliers, self.iqr_outliers) - @cached_property - def ops(self): + @outliers.setter + def outliers(self, value: str) -> None: + """ Set the number of outliers. """ + self.cache['outliers'] = value + + @property + def ops(self) -> float or int: + """ Return the average of operations per second of round / iterations.""" + cache_value = self.cache.get('ops') + + if cache_value is not None: + return cache_value + if self.total: return self.rounds / self.total return 0 + @ops.setter + def ops(self, value: float or int) -> None: + """Set the average of operations per second of round / iterations.""" + self.cache['ops'] = value + class Metadata(object): def __init__(self, fixture, iterations, options): diff --git a/tests/test_acceptation.py b/tests/test_acceptation.py index 7420a23..82f9641 100644 --- a/tests/test_acceptation.py +++ b/tests/test_acceptation.py @@ -1,7 +1,38 @@ -""" Testing to ensure that __init__ makes typing import simple and accessible to IDEs. """ +""" +Sometimes it can be useful to modify the statistics to take into account +something that the benchmark cannot natively (a benchmark on a function which +takes care of a hundred files, while wishing to have the OPS per file) +""" + +import pytest + from pytest_benchmark.fixture import BenchmarkFixture from pytest_benchmark.stats import Stats +# List of modifiable attributes of the statistics attribute of the benchmark fixture +list_modifiable_statistics_attributes = [ + 'mean', + 'stddev', + 'stddev_outliers', + 'median', + 'min', + 'max', + 'q1', + 'q3', + 'iqr', + 'iqr_outliers', + 'ld15iqr', + 'hd15iqr', + 'outliers', + 'ops', + 'total', +] + +# List of unmodifiable attributes of the statistics attribute of the benchmark fixture +list_unmodifiable_statistics_attributes = [ + 'rounds', +] + def test_import_benchmark_fixture(): """ Test to ensure that __init__ makes importing for typing simple and accessible to IDEs. """ @@ -29,21 +60,49 @@ def test_import_stats(): assert Module is not None -def test_benchmark_fixture_access_stats(benchmark: BenchmarkFixture): - """ - Test to ensure that __init__ makes accessing stats simple and accessible to IDEs. +@pytest.mark.parametrize('attribute', list_modifiable_statistics_attributes) +def test_benchmark_stats_can_be_modified(benchmark: BenchmarkFixture, attribute: str): + """ Test to ensure that attributes of statistics can be modified. """ + + # Run a benchmark to have some statistics + benchmark.pedantic(lambda: 1 + 1, rounds=1, iterations=1, warmup_rounds=0) + + # Check that the attribute exists + assert hasattr(benchmark.statistics, attribute) + + # Check that the value is not None + old_value = benchmark.statistics.__getattribute__(attribute) + + # Can matter whether it's a string or a number it will work + benchmark.statistics.__setattr__(attribute, old_value * 2) # like "benchmark.statistics.mean *= 2" + + # Check that the value has been modified + assert benchmark.statistics.__getattribute__(attribute) == old_value * 2 - Sometimes it can be useful to modify the statistics to take into account - something that the benchmark cannot natively (a benchmark on a function which - takes care of a hundred files, while wishing to have the OPS per file) - """ +@pytest.mark.parametrize('attribute', list_unmodifiable_statistics_attributes) +def test_benchmark_stats_cant_be_modified(benchmark: BenchmarkFixture, attribute: str): + """ Test to ensure that attributes of statistics can't be modified. """ + + # Run a benchmark to have some statistics benchmark.pedantic(lambda: 1 + 1, rounds=1, iterations=1, warmup_rounds=0) - assert hasattr(benchmark, 'statistics') - assert isinstance(benchmark.statistics, Stats) + # Check that the attribute exists + assert hasattr(benchmark.statistics, attribute) - old_ops = benchmark.statistics.ops - benchmark.statistics.ops *= 2 - assert benchmark.statistics.ops == old_ops * 2 + # Check that the value is not None + old_value = benchmark.statistics.__getattribute__(attribute) + # AttributeError + with pytest.raises(AttributeError): + benchmark.statistics.__setattr__(attribute, old_value * 2) # like "benchmark.statistics.mean *= 2" + + +def test_benchmark_fixture_access_stats(benchmark: BenchmarkFixture): + """ Test to ensure that the benchmark fixture has a statistics attribute. """ + + # Delete warning "Benchmark fixture was not used at all in this test!" + benchmark.pedantic(lambda: 1 + 1, rounds=1, iterations=1, warmup_rounds=0) + + assert hasattr(benchmark, 'statistics') + assert isinstance(benchmark.statistics, Stats) From 19b21bbced03e7e751893e534fad62024ae01f9d Mon Sep 17 00:00:00 2001 From: Athroniaeth <76869761+Athroniaeth@users.noreply.github.com> Date: Fri, 24 Nov 2023 18:19:25 +0100 Subject: [PATCH 3/4] Update src/pytest_benchmark/stats.py --- src/pytest_benchmark/stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pytest_benchmark/stats.py b/src/pytest_benchmark/stats.py index e2684da..f525887 100644 --- a/src/pytest_benchmark/stats.py +++ b/src/pytest_benchmark/stats.py @@ -14,6 +14,7 @@ from .utils import funcname from .utils import get_cprofile_functions +from .utils import cached_property class Stats(object): From 60f79f52b0660c231d3e746b01cb5e5af1bc1884 Mon Sep 17 00:00:00 2001 From: Athroniaeth <76869761+Athroniaeth@users.noreply.github.com> Date: Fri, 24 Nov 2023 18:20:18 +0100 Subject: [PATCH 4/4] Update src/pytest_benchmark/stats.py --- src/pytest_benchmark/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest_benchmark/stats.py b/src/pytest_benchmark/stats.py index f525887..b6aedd4 100644 --- a/src/pytest_benchmark/stats.py +++ b/src/pytest_benchmark/stats.py @@ -42,7 +42,7 @@ def as_dict(self): def update(self, duration): self.data.append(duration) - @property + @cached_property def sorted_data(self): return sorted(self.data)