Blackhole post-commit tests #14975
Annotations
10 errors and 1 warning
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-256-4-64-14336-H2DMode.HOST_PUSH]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2ae017c000 with size 0x1d000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/ven
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-128-2-512-1792-H2DMode.DEVICE_PULL]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2b52a05000 with size 0x2000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/ve
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-128-2-512-1792-H2DMode.HOST_PUSH]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2b50000000 with size 0x2000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/venv
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L50
test_host_io_loopback[blackhole-H2DMode.HOST_PUSH-32768-65536-128]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2ae0188000 with size 0x11000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/venv/bin/python3(
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-128-2-256-3584-H2DMode.DEVICE_PULL]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2b4aa46000 with size 0x4000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/ve
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-128-2-256-3584-H2DMode.HOST_PUSH]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2b4aa46000 with size 0x4000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/venv
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-128-2-128-7168-H2DMode.DEVICE_PULL]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2b4aa42000 with size 0x8000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/ve
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-128-2-128-7168-H2DMode.HOST_PUSH]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2b4aa42000 with size 0x8000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/venv
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-128-2-64-14336-H2DMode.DEVICE_PULL]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2ae018a000 with size 0xf000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/ve
|
|
Sockets fast unit test tests:
models/demos/deepseek_v3_b1/tests/unit_tests/test_host_io.py#L139
test_host_io_loopback_with_embedding[blackhole-128-2-64-14336-H2DMode.HOST_PUSH]
RuntimeError: TT_THROW @ /project/tt_metal/third_party/umd/device/pcie/pci_device.cpp:668: tt::exception
info:
Failed to pin pages for DMA buffer at virtual address 0x7f2ae018a000 with size 0xf000 and flags 0x1: Invalid argument
backtrace:
--- tt::umd::PCIDevice::map_for_dma(void*, unsigned long)
--- tt::umd::SysmemBuffer::SysmemBuffer(tt::umd::TLBManager*, void*, unsigned long, bool)
--- tt::umd::SiliconSysmemManager::map_sysmem_buffer(void*, unsigned long, bool)
--- tt::Cluster::map_sysmem_buffer(int, void*, unsigned long, bool) const
--- tt::tt_metal::experimental::PinnedMemoryImpl::initialize_from_devices(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemoryImpl::PinnedMemoryImpl(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::PinnedMemory(std::vector<tt::tt_metal::IDevice*, std::allocator<tt::tt_metal::IDevice*> > const&, void*, unsigned long, bool)
--- tt::tt_metal::experimental::PinnedMemory::Create(tt::tt_metal::distributed::MeshDevice&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, tt::tt_metal::HostBuffer&, bool)
--- tt::tt_metal::distributed::D2HSocket::init_host_buffer(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoordinateRangeSet const&, unsigned int)
--- tt::tt_metal::distributed::D2HSocket::D2HSocket(std::shared_ptr<tt::tt_metal::distributed::MeshDevice> const&, tt::tt_metal::distributed::MeshCoreCoord const&, unsigned int)
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x33794b) [0x7f2b744cc94b]
--- /opt/venv/lib/python3.10/site-packages/ttnn/_ttnn.cpython-310-x86_64-linux-gnu.so(+0x23ee65) [0x7f2b743d3e65]
--- /opt/venv/bin/python3(+0x2a5b30) [0x56274b5afb30]
--- /opt/venv/bin/python3(+0x2a7125) [0x56274b5b1125]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x12f3e) [0x56274b5e364e]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x36749) [0x56274b606e59]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(+0x2fd2a2) [0x56274b6072a2]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x3eef) [0x56274b5d45ff]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8924) [0x56274b5d9034]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0x8507) [0x56274b5d8c17]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(+0x273350) [0x56274b57d350]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa3df) [0x56274b5daaef]
--- /opt/venv/bin/python3(_PyFunction_Vectorcall+0x202) [0x56274b57b352]
--- /opt/venv/bin/python3(_PyObject_Call_Prepend+0x115) [0x56274b678c65]
--- /opt/venv/bin/python3(+0x3a2b46) [0x56274b6acb46]
--- /opt/venv/bin/python3(_PyEval_EvalFrameDefault+0xa16c) [0x56274b5da87c]
--- /opt/venv
|
|
Set up runner
MLPerf mount service not found.
|
Loading