Thread safety with Dask? #594
Description
Vector Version
1.6.2
Python Version
3.12.0
OS / Environment
Alma linux 9:
Linux thinkbook 5.14.0-503.38.1.el9_5.x86_64 #1 SMP PREEMPT_DYNAMIC Fri Apr 18 08:52:10 EDT 2025 x86_64 x86_64 x86_64 GNU/Linux
Describe the bug
TL;DR: There seems to be a crash caused by combining multithreading of dask dataframes and vector.
I am trying to run a multithreaded operation with dask dataframes that involve vector objects and I seem to see stuff like what is below. The code I am running is here and here.
The problem seems to go away when using multiprocessing though. Unfortunately I do not have more time to provide further assistance/details.
Cheers.
tests/test_preprocessor.py::test_flat_bias[0.9] Tcl_AsyncDelete: async handler deleted by the wrong thread
Fatal Python error: Aborted
Thread 0x00007ff7493fa640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff749bfb640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff74a7fc640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff74affd640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff74b7fe640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff74bfff640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff758bfe640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff7618a9640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Current thread 0x00007ff7624a8640 (most recent call first):
Garbage-collecting
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/vector/_methods.py", line 4484 in _flavor_of
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/vector/_compute/spatial/mag.py", line 82 in dispatch
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/vector/_methods.py", line 3591 in mag
File "/home/acampove/Packages/ecal_calibration/src/ecal_calibration/preprocessor.py", line 61 in _get_normal
File "/home/acampove/Packages/ecal_calibration/src/ecal_calibration/preprocessor.py", line 71 in _get_correction
File "/home/acampove/Packages/ecal_calibration/src/ecal_calibration/preprocessor.py", line 126 in _build_features
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pandas/core/apply.py", line 1081 in apply_series_generator
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pandas/core/apply.py", line 1063 in apply_standard
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pandas/core/apply.py", line 916 in apply
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pandas/core/frame.py", line 10374 in apply
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/utils.py", line 1232 in __call__
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/_task_spec.py", line 763 in __call__
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/local.py", line 258 in execute_task
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/local.py", line 272 in batch_execute_tasks
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 58 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 92 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff762ca9640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff7634aa640 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/concurrent/futures/thread.py", line 89 in _worker
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 989 in run
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1052 in _bootstrap_inner
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 1009 in _bootstrap
Thread 0x00007ff7ded5b740 (most recent call first):
File "/home/acampove/micromamba/envs/rx/lib/python3.12/threading.py", line 334 in wait
File "/home/acampove/micromamba/envs/rx/lib/python3.12/queue.py", line 171 in get
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/local.py", line 141 in queue_get
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/local.py", line 538 in get_async
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/threaded.py", line 91 in get
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/base.py", line 681 in compute
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/base.py", line 373 in compute
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/dask/dataframe/dask_expr/_collection.py", line 692 in head
File "/home/acampove/Packages/ecal_calibration/tests/test_preprocessor.py", line 96 in test_flat_bias
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/python.py", line 159 in pytest_pyfunc_call
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/python.py", line 1627 in runtest
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/runner.py", line 174 in pytest_runtest_call
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/runner.py", line 242 in <lambda>
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/runner.py", line 341 in from_call
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/runner.py", line 241 in call_and_report
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/runner.py", line 132 in runtestprotocol
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/runner.py", line 113 in pytest_runtest_protocol
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/main.py", line 362 in pytest_runtestloop
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/main.py", line 337 in _main
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/main.py", line 283 in wrap_session
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/main.py", line 330 in pytest_cmdline_main
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_callers.py", line 103 in _multicall
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_manager.py", line 120 in _hookexec
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/pluggy/_hooks.py", line 513 in __call__
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/config/__init__.py", line 175 in main
File "/home/acampove/micromamba/envs/rx/lib/python3.12/site-packages/_pytest/config/__init__.py", line 201 in console_main
File "/home/acampove/micromamba/envs/rx/bin/pytest", line 8 in <module>
Extension modules: numpy._core._multiarray_umath, numpy.linalg._umath_linalg, yaml._yaml, pyarrow.lib, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, pandas._libs.tslibs.ccalendar, pandas._libs.tslibs.np_datetime, pandas._libs.tslibs.dtypes, pandas._libs.tslibs.base, pandas._libs.tslibs.nattype, pandas._libs.tslibs.timezones, pandas._libs.tslibs.fields, pandas._libs.tslibs.timedeltas, pandas._libs.tslibs.tzconversion, pandas._libs.tslibs.timestamps, pandas._libs.properties, pandas._libs.tslibs.offsets, pandas._libs.tslibs.strptime, pandas._libs.tslibs.parsing, pandas._libs.tslibs.conversion, pandas._libs.tslibs.period, pandas._libs.tslibs.vectorized, pandas._libs.ops_dispatch, pandas._libs.missing, pandas._libs.hashtable, pandas._libs.algos, pandas._libs.interval, pandas._libs.lib, pyarrow._compute, pandas._libs.ops, numexpr.interpreter, pandas._libs.hashing, pandas._libs.arrays, pandas._libs.tslib, pandas._libs.sparse, pandas._libs.internals, pandas._libs.indexing, pandas._libs.index, pandas._libs.writers, pandas._libs.join, pandas._libs.window.aggregations, pandas._libs.window.indexers, pandas._libs.reshape, pandas._libs.groupby, pandas._libs.json, pandas._libs.parsers, pandas._libs.testing, PIL._imaging, kiwisolver._cext, zstandard.backend_c, xxhash._xxhash, psutil._psutil_linux, psutil._psutil_posix, markupsafe._speedups, scipy._lib._ccallback_c, scipy.sparse._sparsetools, _csparsetools, scipy.sparse._csparsetools, scipy.linalg._fblas, scipy.linalg._flapack, scipy.linalg.cython_lapack, scipy.linalg._cythonized_array_utils, scipy.linalg._solve_toeplitz, scipy.linalg._decomp_lu_cython, scipy.linalg._matfuncs_sqrtm_triu, scipy.linalg._matfuncs_expm, scipy.linalg._linalg_pythran, scipy.linalg.cython_blas, scipy.linalg._decomp_update, scipy.sparse.linalg._dsolve._superlu, scipy.sparse.linalg._eigen.arpack._arpack, scipy.sparse.linalg._propack._spropack, scipy.sparse.linalg._propack._dpropack, scipy.sparse.linalg._propack._cpropack, scipy.sparse.linalg._propack._zpropack, scipy.sparse.csgraph._tools, scipy.sparse.csgraph._shortest_path, scipy.sparse.csgraph._traversal, scipy.sparse.csgraph._min_spanning_tree, scipy.sparse.csgraph._flow, scipy.sparse.csgraph._matching, scipy.sparse.csgraph._reordering, scipy._lib._uarray._uarray, scipy.special._ufuncs_cxx, scipy.special._ufuncs, scipy.special._specfun, scipy.special._comb, scipy.special._ellip_harm_2, scipy.fftpack.convolve, pyarrow._fs, pyarrow._azurefs, pyarrow._hdfs, pyarrow._gcsfs, pyarrow._s3fs, pyarrow._acero, pyarrow._csv, pyarrow._json, pyarrow._substrait, pyarrow._dataset, pyarrow._dataset_orc, pyarrow._parquet, pyarrow._parquet_encryption, pyarrow._dataset_parquet_encryption, pyarrow._dataset_parquet, tornado.speedups, msgpack._cmsgpack, PIL._imagingmath, PIL._imagingtk (total: 116)
Aborted (core dumped)
Any additional but relevant log output
No response