From ef262b17056d1bcf9713e98469eda611093aa161 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 3 Jun 2026 05:57:20 +0000 Subject: [PATCH 01/14] test: Demonstrate exponential custom-option id growth on re-customization Re-customizing a retained, already-customized object via .opts() mints a new id of old_id + max(all_ids) + 2. When the object already holds the store's largest id this doubles the maximum id on every call, so ids grow exponentially over a long-running session. This test asserts per-call growth of the maximum custom-option id stays bounded; it currently fails (growth follows 2^(i+1)), pinning the defect before the fix. --- holoviews/tests/core/test_storeoptions.py | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/holoviews/tests/core/test_storeoptions.py b/holoviews/tests/core/test_storeoptions.py index f55155bcf4..80f8a2ccc7 100644 --- a/holoviews/tests/core/test_storeoptions.py +++ b/holoviews/tests/core/test_storeoptions.py @@ -5,6 +5,8 @@ from __future__ import annotations +import itertools + import numpy as np import holoviews as hv @@ -111,3 +113,35 @@ def test_holomap_options(self): def test_holomap_options_empty_no_exception(self): hv.HoloMap({0: hv.Image(np.random.rand(10, 10))}).options() + + +@mpl_skip +class TestStoreOptionsCustomIdGrowth: + """ + Re-customizing a retained, already-customized object must not grow the + custom-option ids exponentially. The buggy behaviour minted a new id of + ``old_id + max(all_ids) + 2``, which doubles the maximum id on every call + when the object already holds the store's largest id. + """ + + def setup_method(self): + hv.Store.current_backend = "matplotlib" + + @staticmethod + def _max_custom_id(): + keys = list(hv.Store._custom_options.get("matplotlib", {}).keys()) + return max(keys) if keys else 0 + + def test_recustomization_id_growth_is_not_exponential(self): + el = hv.Curve([1, 2, 3]) + max_ids = [] + for _ in range(20): + el = el.opts(color="red") + max_ids.append(self._max_custom_id()) + + # The object carries a single custom-option id, so a correct + # relocation grows the maximum by a small constant per call (linear). + # The exponential bug instead grows it by roughly the current maximum + # each call, so per-step growth quickly exceeds any small bound. + growth = [b - a for a, b in itertools.pairwise(max_ids)] + assert max(growth) <= 4, f"id growth per call is not bounded: {max_ids}" From 68956c33be429ffb7c08b7c96d8e7b527cca8760 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 3 Jun 2026 06:12:36 +0000 Subject: [PATCH 02/14] test: Cover cross-backend child-id collision in create_custom_trees When an object's children carry custom-option ids that are absent from the target backend's store (e.g. children customized in bokeh, then the container re-customized in matplotlib), every such id takes the fresh-tree branch of create_custom_trees, which assigns them all the same id (offset). The children are thereby aliased onto a single option tree and lose their distinct styles. This regression test asserts the children keep distinct ids and styles. It fails against the current old_id + offset + 1 / clones[offset] logic, with both children collapsed onto one id. --- holoviews/tests/core/test_storeoptions.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/holoviews/tests/core/test_storeoptions.py b/holoviews/tests/core/test_storeoptions.py index 80f8a2ccc7..26e357b4c2 100644 --- a/holoviews/tests/core/test_storeoptions.py +++ b/holoviews/tests/core/test_storeoptions.py @@ -145,3 +145,18 @@ def test_recustomization_id_growth_is_not_exponential(self): # each call, so per-step growth quickly exceeds any small bound. growth = [b - a for a, b in itertools.pairwise(max_ids)] assert max(growth) <= 4, f"id growth per call is not bounded: {max_ids}" + + def test_cross_backend_recustomization_keeps_children_distinct(self): + # Children customized in bokeh hold distinct ids that are absent from + # the matplotlib store. Re-customizing the overlay in matplotlib with a + # spec targeting the children must relocate each child id to a distinct + # new id; collapsing them onto a single id aliases the children and + # loses their distinct bokeh styles. + c1 = hv.Curve([1, 2, 3]).opts(color="red", backend="bokeh") + c2 = hv.Curve([3, 2, 1]).opts(color="green", backend="bokeh") + ov = (c1 * c2).opts({"Curve": {"linewidth": 3}}, backend="matplotlib") + + first, second = ov.Curve.I, ov.Curve.II + assert first.id != second.id + assert hv.Store.lookup_options("bokeh", first, "style").kwargs["color"] == "red" + assert hv.Store.lookup_options("bokeh", second, "style").kwargs["color"] == "green" From cb933077a85dedc62765a946cf9a1815f906b168 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 3 Jun 2026 07:15:34 +0000 Subject: [PATCH 03/14] test: Cover exponential id growth from jittered overlay updates A streaming overlay that re-styles itself each cycle while only a subset of its curves receives fresh data (e.g. message-arrival jitter) retains the non-updating curves across cycles. Restyling relocates their ids, and deriving the new id from the old value doubled them every cycle -- the production trigger, with no static/reference element involved. This test drives round-robin partial updates and asserts the maximum id stays polynomial; it grows exponentially (doubling per cycle) before the fix. --- holoviews/tests/core/test_storeoptions.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/holoviews/tests/core/test_storeoptions.py b/holoviews/tests/core/test_storeoptions.py index 26e357b4c2..945a7e2f09 100644 --- a/holoviews/tests/core/test_storeoptions.py +++ b/holoviews/tests/core/test_storeoptions.py @@ -160,3 +160,25 @@ def test_cross_backend_recustomization_keeps_children_distinct(self): assert first.id != second.id assert hv.Store.lookup_options("bokeh", first, "style").kwargs["color"] == "red" assert hv.Store.lookup_options("bokeh", second, "style").kwargs["color"] == "green" + + def test_overlay_subset_update_keeps_ids_bounded(self): + # A dashboard overlay where only a subset of curves receives fresh data + # each restyle cycle (e.g. jittered streaming updates where some curves + # have no new message). Curves that skip an update are the same retained + # objects carrying their previous ids; restyling the overlay relocates + # them, and deriving the new id from the old value doubled them every + # cycle. No static/reference curve is required -- the retention comes + # purely from the partial updates. + names = list("abc") + curves = {k: hv.Curve([(0, 0)], label=k) for k in names} + base = self._max_custom_id() + max_ids = [] + for cycle in range(15): + k = names[cycle % len(names)] # only one curve gets fresh data + curves[k] = hv.Curve([(x, x + cycle) for x in range(cycle + 2)], label=k) + hv.Overlay(list(curves.values())).opts({"Curve": {"linewidth": 2}}) + max_ids.append(self._max_custom_id() - base) + + # Polynomial growth after the fix; exponential (doubling per cycle) + # before it, so the final value blows far past any polynomial bound. + assert max_ids[-1] < 500, f"id growth is not polynomial: {max_ids}" From a375344e752428b7f23cfba26d9cd555b79daf90 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 3 Jun 2026 06:01:09 +0000 Subject: [PATCH 04/14] fix: Relocate custom-option ids to a contiguous block, not by old-id value create_custom_trees relocated an object's existing custom-option trees to new ids computed as old_id + offset + 1, where offset is one above the store's current maximum id. When the object already held the store's largest id -- the normal case when re-styling the retained result of a previous .opts() call -- the new id was roughly twice the maximum, so the maximum id doubled on every re-customization and grew exponentially. Ids are opaque keys into Store._custom_options; their absolute value carries no meaning. The relocation only needs new ids disjoint from existing keys and a bijection over the object's own id set. Assigning a fresh contiguous block above the offset (indexed by position) satisfies both and keeps the maximum id linear in the number of live customizations, avoiding the unbounded bignum growth and the Python 3.11 4300-digit integer-to-string ValueError it eventually triggered. --- holoviews/core/options.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/holoviews/core/options.py b/holoviews/core/options.py index 4639792e78..96ad73e12e 100644 --- a/holoviews/core/options.py +++ b/holoviews/core/options.py @@ -1754,18 +1754,23 @@ def create_custom_trees(cls, obj, options, backend=None): } custom_options = Store.custom_options(backend=backend) - for tree_id in obj_ids: + # Relocate the object's trees to a fresh contiguous block above the + # offset. Ids are opaque keys into Store._custom_options; the only + # requirements are that the new ids are disjoint from existing keys + # (guaranteed by the offset) and that distinct old ids map to distinct + # new ids. Indexing by position keeps the maximum id linear in the + # number of live customizations; deriving it from the old id's value + # (tree_id + offset + 1) doubles the maximum on every re-customization. + for new_id, tree_id in enumerate(obj_ids, start=offset): if tree_id is not None and tree_id in custom_options: original = custom_options[tree_id] clone = OptionTree( items=original.items(), groups=original.groups, backend=original.backend ) - clones[tree_id + offset + 1] = clone - id_mapping.append((tree_id, tree_id + offset + 1)) else: clone = OptionTree(groups=available_options.groups, backend=backend) - clones[offset] = clone - id_mapping.append((tree_id, offset)) + clones[new_id] = clone + id_mapping.append((tree_id, new_id)) # Nodes needed to ensure allowed_keywords is respected for obj_type, opts in used_options.items(): From 1ba07211fdc0f44e66cdee4ad35d403b726fff97 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 3 Jun 2026 08:17:53 +0000 Subject: [PATCH 05/14] test: Cover thread-safety of concurrent custom-option id allocation Customizing HoloViews objects from multiple threads concurrently must not collide on relocated custom-option ids. The test drives eight threads re-customizing their own elements and asserts none raises. It guards against an id-relocation scheme that recomputes a shared max(store)+1 offset per call: concurrent calls receive the same block, one thread's freshly stored tree is reclaimed by another, and the id is gone by the time it is propagated ("New option id ... does not match any option trees"). --- holoviews/tests/core/test_storeoptions.py | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/holoviews/tests/core/test_storeoptions.py b/holoviews/tests/core/test_storeoptions.py index 945a7e2f09..91cdd7155a 100644 --- a/holoviews/tests/core/test_storeoptions.py +++ b/holoviews/tests/core/test_storeoptions.py @@ -182,3 +182,32 @@ def test_overlay_subset_update_keeps_ids_bounded(self): # Polynomial growth after the fix; exponential (doubling per cycle) # before it, so the final value blows far past any polynomial bound. assert max_ids[-1] < 500, f"id growth is not polynomial: {max_ids}" + + def test_concurrent_recustomization_is_thread_safe(self): + # Customizing from multiple threads must not collide on relocated ids. + # Allocating ids from a recomputed max(store)+1 offset gives concurrent + # calls the same block, so one thread's freshly stored tree is reused + # by another and the id is gone by the time it is propagated, raising + # "New option id ... does not match any option trees". + import threading + + errors = [] + barrier = threading.Barrier(8) + + def worker(w): + try: + barrier.wait() + for i in range(60): + el = hv.Curve([1, 2, 3], label=f"w{w}") + for _ in range(4): + el = el.opts(linewidth=(i % 5) + 1) + except Exception as exc: + errors.append(exc) + + threads = [threading.Thread(target=worker, args=(w,)) for w in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert not errors, f"concurrent customization raised: {errors[:3]}" From c97f7141196120267da45611ce7af817b6137182 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 3 Jun 2026 08:25:03 +0000 Subject: [PATCH 06/14] Revert "fix: Relocate custom-option ids to a contiguous block, not by old-id value" This reverts commit a375344e7. The contiguous-block relocation assigned new ids from a freshly recomputed `max(store) + 1` offset. That fixes the exponential growth, but the offset is not concurrency-safe: two threads customizing at the same time compute the same offset and receive overlapping id blocks, so one thread's just-stored tree is reclaimed by the other and the id is gone by the time it is propagated -- raising "New option id ... does not match any option trees" in multi-threaded apps (e.g. a live dashboard). The preceding thread-safety test demonstrates the failure. Reverting to the original logic before applying a thread-safe replacement. --- holoviews/core/options.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/holoviews/core/options.py b/holoviews/core/options.py index 96ad73e12e..4639792e78 100644 --- a/holoviews/core/options.py +++ b/holoviews/core/options.py @@ -1754,23 +1754,18 @@ def create_custom_trees(cls, obj, options, backend=None): } custom_options = Store.custom_options(backend=backend) - # Relocate the object's trees to a fresh contiguous block above the - # offset. Ids are opaque keys into Store._custom_options; the only - # requirements are that the new ids are disjoint from existing keys - # (guaranteed by the offset) and that distinct old ids map to distinct - # new ids. Indexing by position keeps the maximum id linear in the - # number of live customizations; deriving it from the old id's value - # (tree_id + offset + 1) doubles the maximum on every re-customization. - for new_id, tree_id in enumerate(obj_ids, start=offset): + for tree_id in obj_ids: if tree_id is not None and tree_id in custom_options: original = custom_options[tree_id] clone = OptionTree( items=original.items(), groups=original.groups, backend=original.backend ) + clones[tree_id + offset + 1] = clone + id_mapping.append((tree_id, tree_id + offset + 1)) else: clone = OptionTree(groups=available_options.groups, backend=backend) - clones[new_id] = clone - id_mapping.append((tree_id, new_id)) + clones[offset] = clone + id_mapping.append((tree_id, offset)) # Nodes needed to ensure allowed_keywords is respected for obj_type, opts in used_options.items(): From 7b3015edff0407c57f6f20847285c1b4f6fd9a34 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 3 Jun 2026 08:25:38 +0000 Subject: [PATCH 07/14] fix: Relocate custom-option ids via a monotonic, thread-safe counter create_custom_trees relocated an object's existing option trees to new ids computed as old_id + offset + 1, where offset is one above the store's current maximum id. When the object already held the store's largest id -- the normal case when re-styling the retained result of a previous .opts() call, including streaming overlays whose curves update only partially each cycle -- the new id was ~2x the maximum, so it doubled on every re-customization and grew exponentially. Over a long-running session this pins a CPU core on bignum arithmetic and, on Python 3.11+, raises the 4300-digit integer-to-string ValueError from any repr()/str() of the object. Ids are opaque keys into Store._custom_options; their value carries no meaning. Allocate relocated ids from a process-wide monotonic counter instead, handing each customization a fresh contiguous block. The counter never decreases, so ids are never reused and concurrent customizations receive disjoint blocks rather than colliding on a recomputed offset (the failure the reverted contiguous-block attempt introduced). Allocation is guarded by a lock. The maximum id now grows linearly with the number of re-customizations rather than exponentially. This also fixes a latent aliasing bug: when a container was re-customized in a different backend (children's ids absent from the target backend's store), the previous logic mapped them all to a single offset, collapsing their per-backend options together. --- holoviews/core/options.py | 42 +++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/holoviews/core/options.py b/holoviews/core/options.py index 4639792e78..304bb6b0bf 100644 --- a/holoviews/core/options.py +++ b/holoviews/core/options.py @@ -37,6 +37,7 @@ import difflib import inspect import pickle +import threading import traceback import typing as t from collections import defaultdict @@ -1213,6 +1214,12 @@ class Store: _weakrefs = {} _options_context = False + # Monotonic high-water mark for custom-option ids and the lock guarding + # its allocation. Ids are never reused, so concurrent customizations + # receive disjoint id blocks instead of colliding on a recomputed offset. + _id_counter = 0 + _id_lock = threading.Lock() + # Backend option caches _lookup_cache = {} @@ -1735,8 +1742,8 @@ def create_custom_trees(cls, obj, options, backend=None): """ clones, id_mapping = {}, [] obj_ids = cls.get_object_ids(obj) - offset = cls.id_offset() obj_ids = [None] if len(obj_ids) == 0 else obj_ids + offset = cls.reserve_ids(len(obj_ids)) used_obj_types = [(opt.split(".")[0],) for opt in options] backend = backend or Store.current_backend @@ -1754,18 +1761,23 @@ def create_custom_trees(cls, obj, options, backend=None): } custom_options = Store.custom_options(backend=backend) - for tree_id in obj_ids: + # Relocate the object's trees to a fresh contiguous block above the + # offset. Ids are opaque keys into Store._custom_options; the only + # requirements are that the new ids are disjoint from existing keys + # (guaranteed by the offset) and that distinct old ids map to distinct + # new ids. Indexing by position keeps the maximum id linear in the + # number of live customizations; deriving it from the old id's value + # (tree_id + offset + 1) doubles the maximum on every re-customization. + for new_id, tree_id in enumerate(obj_ids, start=offset): if tree_id is not None and tree_id in custom_options: original = custom_options[tree_id] clone = OptionTree( items=original.items(), groups=original.groups, backend=original.backend ) - clones[tree_id + offset + 1] = clone - id_mapping.append((tree_id, tree_id + offset + 1)) else: clone = OptionTree(groups=available_options.groups, backend=backend) - clones[offset] = clone - id_mapping.append((tree_id, offset)) + clones[new_id] = clone + id_mapping.append((tree_id, new_id)) # Nodes needed to ensure allowed_keywords is respected for obj_type, opts in used_options.items(): @@ -1871,6 +1883,24 @@ def id_offset(cls): # If no backends defined (e.g. plotting not imported) return zero return max(max_ids) if max_ids else 0 + @classmethod + def reserve_ids(cls, n): + """Atomically reserve a contiguous block of ``n`` fresh custom-option + ids and return its first id. + + Ids are drawn from a monotonic counter that never decreases, so a + block is never reused even as store entries are reclaimed, and + concurrent customizations receive disjoint blocks rather than + colliding on a recomputed offset. The counter is floored by + ``id_offset`` so it also clears any ids already present in the store + (e.g. restored by unpickling). + + """ + with Store._id_lock: + base = max(Store._id_counter, cls.id_offset()) + Store._id_counter = base + n + return base + @classmethod def update_backends(cls, id_mapping, custom_trees, backend=None): """Given the id_mapping from previous ids to new ids and the new From 2a4a4a94b2242a55945cb5050f9f64af1007893a Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Wed, 3 Jun 2026 11:55:15 +0000 Subject: [PATCH 08/14] test: Re-raise worker errors via futures in thread-safety test Use ThreadPoolExecutor and future.result() instead of collecting worker exceptions into a list. A failing worker now re-raises in the main thread (surfacing the actual error directly), and the defensive except branch -- which never ran on a passing build -- is gone, so the test no longer leaves uncovered lines. --- holoviews/tests/core/test_storeoptions.py | 34 ++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/holoviews/tests/core/test_storeoptions.py b/holoviews/tests/core/test_storeoptions.py index 91cdd7155a..bf4b85eee0 100644 --- a/holoviews/tests/core/test_storeoptions.py +++ b/holoviews/tests/core/test_storeoptions.py @@ -188,26 +188,22 @@ def test_concurrent_recustomization_is_thread_safe(self): # Allocating ids from a recomputed max(store)+1 offset gives concurrent # calls the same block, so one thread's freshly stored tree is reused # by another and the id is gone by the time it is propagated, raising - # "New option id ... does not match any option trees". + # "New option id ... does not match any option trees". future.result() + # re-raises any such error from the worker threads, failing the test. import threading + from concurrent.futures import ThreadPoolExecutor - errors = [] - barrier = threading.Barrier(8) + n_workers = 8 + barrier = threading.Barrier(n_workers) def worker(w): - try: - barrier.wait() - for i in range(60): - el = hv.Curve([1, 2, 3], label=f"w{w}") - for _ in range(4): - el = el.opts(linewidth=(i % 5) + 1) - except Exception as exc: - errors.append(exc) - - threads = [threading.Thread(target=worker, args=(w,)) for w in range(8)] - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - assert not errors, f"concurrent customization raised: {errors[:3]}" + barrier.wait() + for i in range(60): + el = hv.Curve([1, 2, 3], label=f"w{w}") + for _ in range(4): + el = el.opts(linewidth=(i % 5) + 1) + + with ThreadPoolExecutor(max_workers=n_workers) as executor: + futures = [executor.submit(worker, w) for w in range(n_workers)] + for future in futures: + future.result() From 2a4f8ef24f9a94cb776c160c745fd9b4f497153d Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 4 Jun 2026 07:35:25 +0000 Subject: [PATCH 09/14] Revert "fix: Relocate custom-option ids via a monotonic, thread-safe counter" This reverts commit 7b3015edff0407c57f6f20847285c1b4f6fd9a34. --- holoviews/core/options.py | 42 ++++++--------------------------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/holoviews/core/options.py b/holoviews/core/options.py index 304bb6b0bf..4639792e78 100644 --- a/holoviews/core/options.py +++ b/holoviews/core/options.py @@ -37,7 +37,6 @@ import difflib import inspect import pickle -import threading import traceback import typing as t from collections import defaultdict @@ -1214,12 +1213,6 @@ class Store: _weakrefs = {} _options_context = False - # Monotonic high-water mark for custom-option ids and the lock guarding - # its allocation. Ids are never reused, so concurrent customizations - # receive disjoint id blocks instead of colliding on a recomputed offset. - _id_counter = 0 - _id_lock = threading.Lock() - # Backend option caches _lookup_cache = {} @@ -1742,8 +1735,8 @@ def create_custom_trees(cls, obj, options, backend=None): """ clones, id_mapping = {}, [] obj_ids = cls.get_object_ids(obj) + offset = cls.id_offset() obj_ids = [None] if len(obj_ids) == 0 else obj_ids - offset = cls.reserve_ids(len(obj_ids)) used_obj_types = [(opt.split(".")[0],) for opt in options] backend = backend or Store.current_backend @@ -1761,23 +1754,18 @@ def create_custom_trees(cls, obj, options, backend=None): } custom_options = Store.custom_options(backend=backend) - # Relocate the object's trees to a fresh contiguous block above the - # offset. Ids are opaque keys into Store._custom_options; the only - # requirements are that the new ids are disjoint from existing keys - # (guaranteed by the offset) and that distinct old ids map to distinct - # new ids. Indexing by position keeps the maximum id linear in the - # number of live customizations; deriving it from the old id's value - # (tree_id + offset + 1) doubles the maximum on every re-customization. - for new_id, tree_id in enumerate(obj_ids, start=offset): + for tree_id in obj_ids: if tree_id is not None and tree_id in custom_options: original = custom_options[tree_id] clone = OptionTree( items=original.items(), groups=original.groups, backend=original.backend ) + clones[tree_id + offset + 1] = clone + id_mapping.append((tree_id, tree_id + offset + 1)) else: clone = OptionTree(groups=available_options.groups, backend=backend) - clones[new_id] = clone - id_mapping.append((tree_id, new_id)) + clones[offset] = clone + id_mapping.append((tree_id, offset)) # Nodes needed to ensure allowed_keywords is respected for obj_type, opts in used_options.items(): @@ -1883,24 +1871,6 @@ def id_offset(cls): # If no backends defined (e.g. plotting not imported) return zero return max(max_ids) if max_ids else 0 - @classmethod - def reserve_ids(cls, n): - """Atomically reserve a contiguous block of ``n`` fresh custom-option - ids and return its first id. - - Ids are drawn from a monotonic counter that never decreases, so a - block is never reused even as store entries are reclaimed, and - concurrent customizations receive disjoint blocks rather than - colliding on a recomputed offset. The counter is floored by - ``id_offset`` so it also clears any ids already present in the store - (e.g. restored by unpickling). - - """ - with Store._id_lock: - base = max(Store._id_counter, cls.id_offset()) - Store._id_counter = base + n - return base - @classmethod def update_backends(cls, id_mapping, custom_trees, backend=None): """Given the id_mapping from previous ids to new ids and the new From 08006b8fd14cf10c14384243d6a0102461ad10c5 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 4 Jun 2026 09:05:35 +0000 Subject: [PATCH 10/14] test: Tidy custom-option id tests and make the concurrency check deterministic Restore the current backend in teardown, lift the threading imports to module scope, and drop comments that merely restate the test body. Replace the concurrency test's reliance on a rare propagation error with a check that concurrent customizations each keep their own style. A tiny thread-switch interval forces the id race to surface instead of hiding within a single scheduling quantum, so the test fails reliably on the unfixed code. Co-Authored-By: Claude Opus 4.8 --- holoviews/tests/core/test_storeoptions.py | 71 +++++++++-------------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/holoviews/tests/core/test_storeoptions.py b/holoviews/tests/core/test_storeoptions.py index bf4b85eee0..94bf128d6c 100644 --- a/holoviews/tests/core/test_storeoptions.py +++ b/holoviews/tests/core/test_storeoptions.py @@ -6,6 +6,9 @@ from __future__ import annotations import itertools +import sys +import threading +from concurrent.futures import ThreadPoolExecutor import numpy as np @@ -118,14 +121,16 @@ def test_holomap_options_empty_no_exception(self): @mpl_skip class TestStoreOptionsCustomIdGrowth: """ - Re-customizing a retained, already-customized object must not grow the - custom-option ids exponentially. The buggy behaviour minted a new id of - ``old_id + max(all_ids) + 2``, which doubles the maximum id on every call - when the object already holds the store's largest id. + Re-customizing retained, already-customized objects must keep custom-option + ids bounded and must not let concurrent customizations collide on an id. """ def setup_method(self): - hv.Store.current_backend = "matplotlib" + self._backend = hv.Store.current_backend + hv.Store.set_current_backend("matplotlib") + + def teardown_method(self): + hv.Store.current_backend = self._backend @staticmethod def _max_custom_id(): @@ -139,19 +144,10 @@ def test_recustomization_id_growth_is_not_exponential(self): el = el.opts(color="red") max_ids.append(self._max_custom_id()) - # The object carries a single custom-option id, so a correct - # relocation grows the maximum by a small constant per call (linear). - # The exponential bug instead grows it by roughly the current maximum - # each call, so per-step growth quickly exceeds any small bound. growth = [b - a for a, b in itertools.pairwise(max_ids)] assert max(growth) <= 4, f"id growth per call is not bounded: {max_ids}" def test_cross_backend_recustomization_keeps_children_distinct(self): - # Children customized in bokeh hold distinct ids that are absent from - # the matplotlib store. Re-customizing the overlay in matplotlib with a - # spec targeting the children must relocate each child id to a distinct - # new id; collapsing them onto a single id aliases the children and - # loses their distinct bokeh styles. c1 = hv.Curve([1, 2, 3]).opts(color="red", backend="bokeh") c2 = hv.Curve([3, 2, 1]).opts(color="green", backend="bokeh") ov = (c1 * c2).opts({"Curve": {"linewidth": 3}}, backend="matplotlib") @@ -162,13 +158,6 @@ def test_cross_backend_recustomization_keeps_children_distinct(self): assert hv.Store.lookup_options("bokeh", second, "style").kwargs["color"] == "green" def test_overlay_subset_update_keeps_ids_bounded(self): - # A dashboard overlay where only a subset of curves receives fresh data - # each restyle cycle (e.g. jittered streaming updates where some curves - # have no new message). Curves that skip an update are the same retained - # objects carrying their previous ids; restyling the overlay relocates - # them, and deriving the new id from the old value doubled them every - # cycle. No static/reference curve is required -- the retention comes - # purely from the partial updates. names = list("abc") curves = {k: hv.Curve([(0, 0)], label=k) for k in names} base = self._max_custom_id() @@ -179,31 +168,29 @@ def test_overlay_subset_update_keeps_ids_bounded(self): hv.Overlay(list(curves.values())).opts({"Curve": {"linewidth": 2}}) max_ids.append(self._max_custom_id() - base) - # Polynomial growth after the fix; exponential (doubling per cycle) - # before it, so the final value blows far past any polynomial bound. assert max_ids[-1] < 500, f"id growth is not polynomial: {max_ids}" - def test_concurrent_recustomization_is_thread_safe(self): - # Customizing from multiple threads must not collide on relocated ids. - # Allocating ids from a recomputed max(store)+1 offset gives concurrent - # calls the same block, so one thread's freshly stored tree is reused - # by another and the id is gone by the time it is propagated, raising - # "New option id ... does not match any option trees". future.result() - # re-raises any such error from the worker threads, failing the test. - import threading - from concurrent.futures import ThreadPoolExecutor - + def test_concurrent_recustomization_keeps_styles_distinct(self): n_workers = 8 barrier = threading.Barrier(n_workers) def worker(w): barrier.wait() - for i in range(60): - el = hv.Curve([1, 2, 3], label=f"w{w}") - for _ in range(4): - el = el.opts(linewidth=(i % 5) + 1) - - with ThreadPoolExecutor(max_workers=n_workers) as executor: - futures = [executor.submit(worker, w) for w in range(n_workers)] - for future in futures: - future.result() + mismatches = 0 + for _ in range(120): + el = hv.Curve([1, 2, 3], label=f"w{w}").opts(linewidth=w + 1) + got = hv.Store.lookup_options("matplotlib", el, "style").kwargs.get("linewidth") + mismatches += got != w + 1 + return mismatches + + # A normal opts() runs within one thread-scheduling quantum, so the id + # race stays hidden; a tiny switch interval forces interleaving. + old = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + try: + with ThreadPoolExecutor(max_workers=n_workers) as executor: + mismatches = sum(executor.map(worker, range(n_workers))) + finally: + sys.setswitchinterval(old) + + assert mismatches == 0, f"concurrent customizations collided on ids: {mismatches}" From 8480d3329a26c634ccbbab28cd77ceb0f55770ff Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 4 Jun 2026 09:05:48 +0000 Subject: [PATCH 11/14] Reapply "fix: Relocate custom-option ids via a monotonic, thread-safe counter" This reverts commit 2a4f8ef24f9a94cb776c160c745fd9b4f497153d. --- holoviews/core/options.py | 42 +++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/holoviews/core/options.py b/holoviews/core/options.py index 4639792e78..304bb6b0bf 100644 --- a/holoviews/core/options.py +++ b/holoviews/core/options.py @@ -37,6 +37,7 @@ import difflib import inspect import pickle +import threading import traceback import typing as t from collections import defaultdict @@ -1213,6 +1214,12 @@ class Store: _weakrefs = {} _options_context = False + # Monotonic high-water mark for custom-option ids and the lock guarding + # its allocation. Ids are never reused, so concurrent customizations + # receive disjoint id blocks instead of colliding on a recomputed offset. + _id_counter = 0 + _id_lock = threading.Lock() + # Backend option caches _lookup_cache = {} @@ -1735,8 +1742,8 @@ def create_custom_trees(cls, obj, options, backend=None): """ clones, id_mapping = {}, [] obj_ids = cls.get_object_ids(obj) - offset = cls.id_offset() obj_ids = [None] if len(obj_ids) == 0 else obj_ids + offset = cls.reserve_ids(len(obj_ids)) used_obj_types = [(opt.split(".")[0],) for opt in options] backend = backend or Store.current_backend @@ -1754,18 +1761,23 @@ def create_custom_trees(cls, obj, options, backend=None): } custom_options = Store.custom_options(backend=backend) - for tree_id in obj_ids: + # Relocate the object's trees to a fresh contiguous block above the + # offset. Ids are opaque keys into Store._custom_options; the only + # requirements are that the new ids are disjoint from existing keys + # (guaranteed by the offset) and that distinct old ids map to distinct + # new ids. Indexing by position keeps the maximum id linear in the + # number of live customizations; deriving it from the old id's value + # (tree_id + offset + 1) doubles the maximum on every re-customization. + for new_id, tree_id in enumerate(obj_ids, start=offset): if tree_id is not None and tree_id in custom_options: original = custom_options[tree_id] clone = OptionTree( items=original.items(), groups=original.groups, backend=original.backend ) - clones[tree_id + offset + 1] = clone - id_mapping.append((tree_id, tree_id + offset + 1)) else: clone = OptionTree(groups=available_options.groups, backend=backend) - clones[offset] = clone - id_mapping.append((tree_id, offset)) + clones[new_id] = clone + id_mapping.append((tree_id, new_id)) # Nodes needed to ensure allowed_keywords is respected for obj_type, opts in used_options.items(): @@ -1871,6 +1883,24 @@ def id_offset(cls): # If no backends defined (e.g. plotting not imported) return zero return max(max_ids) if max_ids else 0 + @classmethod + def reserve_ids(cls, n): + """Atomically reserve a contiguous block of ``n`` fresh custom-option + ids and return its first id. + + Ids are drawn from a monotonic counter that never decreases, so a + block is never reused even as store entries are reclaimed, and + concurrent customizations receive disjoint blocks rather than + colliding on a recomputed offset. The counter is floored by + ``id_offset`` so it also clears any ids already present in the store + (e.g. restored by unpickling). + + """ + with Store._id_lock: + base = max(Store._id_counter, cls.id_offset()) + Store._id_counter = base + n + return base + @classmethod def update_backends(cls, id_mapping, custom_trees, backend=None): """Given the id_mapping from previous ids to new ids and the new From 8aeab01094178e8d1067fc29f9b5e4f7273306dc Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 4 Jun 2026 09:07:09 +0000 Subject: [PATCH 12/14] test: Add focused check that concurrent id reservation is disjoint Reserving id blocks from many threads must never hand the same id to two callers. This exercises reserve_ids directly with a tiny thread-switch interval, so the read-modify-write race surfaces deterministically and fails fast when the lock is absent, complementing the slower end-to-end concurrency test. Co-Authored-By: Claude Opus 4.8 --- holoviews/tests/core/test_storeoptions.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/holoviews/tests/core/test_storeoptions.py b/holoviews/tests/core/test_storeoptions.py index 94bf128d6c..aad8d8b5de 100644 --- a/holoviews/tests/core/test_storeoptions.py +++ b/holoviews/tests/core/test_storeoptions.py @@ -194,3 +194,25 @@ def worker(w): sys.setswitchinterval(old) assert mismatches == 0, f"concurrent customizations collided on ids: {mismatches}" + + def test_concurrent_id_reservation_yields_disjoint_blocks(self): + n_workers = 8 + barrier = threading.Barrier(n_workers) + + def worker(_): + barrier.wait() + ids = [] + for n in range(1, 120): + start = hv.StoreOptions.reserve_ids(n) + ids.extend(range(start, start + n)) + return ids + + old = sys.getswitchinterval() + sys.setswitchinterval(1e-6) # force interleaving of the read-modify-write + try: + with ThreadPoolExecutor(max_workers=n_workers) as executor: + reserved = [i for block in executor.map(worker, range(n_workers)) for i in block] + finally: + sys.setswitchinterval(old) + + assert len(reserved) == len(set(reserved)), "reserved id blocks overlap" From b44cbcb92279461c306b4859e6ee8a8d3c446d12 Mon Sep 17 00:00:00 2001 From: Simon Heybrock Date: Thu, 4 Jun 2026 09:08:28 +0000 Subject: [PATCH 13/14] Drop redundant comments around custom-option id relocation The relocation assigns each tree a fresh id by position from a reserved offset (enumerate), keeping the maximum id linear in the number of live customizations. Deriving the new id from the old id's value (tree_id + offset + 1) instead doubled the maximum on every re-customization; that history belongs here, not in the code. The remaining code is self-explanatory given reserve_ids, which already documents the monotonic, disjoint allocation. Co-Authored-By: Claude Opus 4.8 --- holoviews/core/options.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/holoviews/core/options.py b/holoviews/core/options.py index 304bb6b0bf..7df0ca20f7 100644 --- a/holoviews/core/options.py +++ b/holoviews/core/options.py @@ -1214,9 +1214,7 @@ class Store: _weakrefs = {} _options_context = False - # Monotonic high-water mark for custom-option ids and the lock guarding - # its allocation. Ids are never reused, so concurrent customizations - # receive disjoint id blocks instead of colliding on a recomputed offset. + # Allocator for custom-option ids; see StoreOptions.reserve_ids. _id_counter = 0 _id_lock = threading.Lock() @@ -1761,13 +1759,6 @@ def create_custom_trees(cls, obj, options, backend=None): } custom_options = Store.custom_options(backend=backend) - # Relocate the object's trees to a fresh contiguous block above the - # offset. Ids are opaque keys into Store._custom_options; the only - # requirements are that the new ids are disjoint from existing keys - # (guaranteed by the offset) and that distinct old ids map to distinct - # new ids. Indexing by position keeps the maximum id linear in the - # number of live customizations; deriving it from the old id's value - # (tree_id + offset + 1) doubles the maximum on every re-customization. for new_id, tree_id in enumerate(obj_ids, start=offset): if tree_id is not None and tree_id in custom_options: original = custom_options[tree_id] From c69b05772e3159b0fb047776545b3a27ddc9978f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20H=C3=B8xbro=20Hansen?= Date: Thu, 4 Jun 2026 12:36:54 +0200 Subject: [PATCH 14/14] small updates --- holoviews/tests/core/test_storeoptions.py | 53 ++++++++++++++--------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/holoviews/tests/core/test_storeoptions.py b/holoviews/tests/core/test_storeoptions.py index aad8d8b5de..dfce4bf83d 100644 --- a/holoviews/tests/core/test_storeoptions.py +++ b/holoviews/tests/core/test_storeoptions.py @@ -9,6 +9,7 @@ import sys import threading from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager import numpy as np @@ -21,12 +22,30 @@ import holoviews.plotting.mpl +@contextmanager +def fast_switch(): + """ + Sets sys.getswitchinterval to 1 microsecond to force thread interleaving in + tests that check for id collisions with multiple threads + """ + old = sys.getswitchinterval() + sys.setswitchinterval(1e-6) + try: + yield + finally: + sys.setswitchinterval(old) + + @mpl_skip class TestStoreOptionsMerge: def setup_method(self): + self._backend = hv.Store.current_backend hv.Store.current_backend = "matplotlib" self.expected = {"Image": {"plot": {"fig_size": 150}, "style": {"cmap": "Blues"}}} + def teardown_method(self): + hv.Store.current_backend = self._backend + def test_full_spec_format(self): out = hv.StoreOptions.merge_options( ["plot", "style"], @@ -58,7 +77,11 @@ class TestStoreOptsMethod: """ def setup_method(self): - hv.Store.current_backend = "matplotlib" + self._backend = hv.Store.current_backend + hv.Store.set_current_backend("matplotlib") + + def teardown_method(self): + hv.Store.current_backend = self._backend def test_overlay_options_partitioned(self): """ @@ -171,48 +194,36 @@ def test_overlay_subset_update_keeps_ids_bounded(self): assert max_ids[-1] < 500, f"id growth is not polynomial: {max_ids}" def test_concurrent_recustomization_keeps_styles_distinct(self): - n_workers = 8 + n_workers = 4 barrier = threading.Barrier(n_workers) def worker(w): barrier.wait() mismatches = 0 - for _ in range(120): + for _ in range(60): el = hv.Curve([1, 2, 3], label=f"w{w}").opts(linewidth=w + 1) got = hv.Store.lookup_options("matplotlib", el, "style").kwargs.get("linewidth") mismatches += got != w + 1 return mismatches - # A normal opts() runs within one thread-scheduling quantum, so the id - # race stays hidden; a tiny switch interval forces interleaving. - old = sys.getswitchinterval() - sys.setswitchinterval(1e-6) - try: - with ThreadPoolExecutor(max_workers=n_workers) as executor: - mismatches = sum(executor.map(worker, range(n_workers))) - finally: - sys.setswitchinterval(old) + with fast_switch(), ThreadPoolExecutor(max_workers=n_workers) as executor: + mismatches = sum(executor.map(worker, range(n_workers))) assert mismatches == 0, f"concurrent customizations collided on ids: {mismatches}" def test_concurrent_id_reservation_yields_disjoint_blocks(self): - n_workers = 8 + n_workers = 4 barrier = threading.Barrier(n_workers) def worker(_): barrier.wait() ids = [] - for n in range(1, 120): + for n in range(10, 30): start = hv.StoreOptions.reserve_ids(n) ids.extend(range(start, start + n)) return ids - old = sys.getswitchinterval() - sys.setswitchinterval(1e-6) # force interleaving of the read-modify-write - try: - with ThreadPoolExecutor(max_workers=n_workers) as executor: - reserved = [i for block in executor.map(worker, range(n_workers)) for i in block] - finally: - sys.setswitchinterval(old) + with fast_switch(), ThreadPoolExecutor(max_workers=n_workers) as executor: + reserved = [i for block in executor.map(worker, range(n_workers)) for i in block] assert len(reserved) == len(set(reserved)), "reserved id blocks overlap"