illegal memory access error was encountered with broadcasting

[Alexander-Barth]

Describe the bug

If I run the following code using include it works fine. But if I repeat the last function call foo.(dt_gpu) in the REPL (or call again the script with include), I get an CUDA error: an illegal memory access was encountered (code 700, ERROR_ILLEGAL_ADDRESS). Maybe some memory gets freed during the brief pause at the REPL.

Does somebody has an idea why this could be the case?

To reproduce

The Minimal Working Example (MWE) for this bug:

using CUDA

@inline function _division(::Type{T},factor,exponent) where T
    TIME_DIVISION = (
    (:day,         24*60*60,      0),
    (:hour,           60*60,      0),
    (:minute,            60,      0),
    (:second,             1,      0),
    (:millisecond,        1,     -3),
    (:microsecond,        1,     -6),
    (:nanosecond,         1,     -9),
)
    ntuple(length(TIME_DIVISION)) do i
        (T(10)^(-exponent) * TIME_DIVISION[i][2]) ÷
            (T(10)^(-TIME_DIVISION[i][3]) * factor)
    end
end

function foo(dt)
    factor,exponent = (86400,0)
    divi = _division(Int64,factor,exponent)
    return divi[1] * dt
end

dt_cpu = [1,2,3]
dt_gpu = cu(dt_cpu)


foo.(dt_gpu) # fine
foo.(dt_gpu) # fine too

Here is the session with the full stack trace:

julia> include("/home/abarth/.julia/dev/CFTime/test/test_gpu_rep.jl");
julia> foo.(dt_gpu)
ERROR: CUDA error: an illegal memory access was encountered (code 700, ERROR_ILLEGAL_ADDRESS)
Stacktrace:
  [1] throw_api_error(res::CUDA.cudaError_enum)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:30
  [2] check(f::CUDA.var"#7#8"{Base.RefValue{UInt64}, Base.RefValue{UInt64}})
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:37
  [3] cuMemGetInfo_v2(free::Base.RefValue{UInt64}, total::Base.RefValue{UInt64})
    @ CUDA ~/.julia/packages/GPUToolbox/cZlg7/src/ccalls.jl:33
  [4] memory_info
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/memory.jl:869 [inlined]
  [5] free_memory
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/memory.jl:878 [inlined]
  [6] maybe_collect(will_block::Bool)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/src/memory.jl:80
  [7] maybe_collect
    @ ~/.julia/packages/CUDA/ja0IX/src/memory.jl:65 [inlined]
  [8] pool_alloc
    @ ~/.julia/packages/CUDA/ja0IX/src/memory.jl:621 [inlined]
  [9] (::CUDA.var"#1139#1141"{CUDA.DeviceMemory, Int64})()
    @ CUDA ~/.julia/packages/CUDA/ja0IX/src/array.jl:92
 [10] cached_alloc(f::CUDA.var"#1139#1141"{CUDA.DeviceMemory, Int64}, key::Tuple{UnionAll, CuDevice, DataType, Int64})                                                             
    @ GPUArrays ~/.julia/packages/GPUArrays/uiVyU/src/host/alloc_cache.jl:36
 [11] CuArray{Int64, 1, CUDA.DeviceMemory}(::UndefInitializer, dims::Tuple{Int64})
    @ CUDA ~/.julia/packages/CUDA/ja0IX/src/array.jl:91
 [12] similar
    @ ./abstractarray.jl:868 [inlined]
 [13] similar
    @ ./abstractarray.jl:867 [inlined]
 [14] similar
    @ ~/.julia/packages/CUDA/ja0IX/src/broadcast.jl:25 [inlined]
 [15] similar
    @ ./broadcast.jl:223 [inlined]
 [16] copy
    @ ~/.julia/packages/GPUArrays/uiVyU/src/host/broadcast.jl:29 [inlined]
 [17] materialize(bc::Base.Broadcast.Broadcasted{CUDA.CuArrayStyle{…}, Nothing, typeof(foo), Tuple{…}})                                            
    @ Base.Broadcast ./broadcast.jl:872
 [18] top-level scope
    @ REPL[1]:1
Some type information was truncated. Use `show(err)` to see complete types.

Manifest.toml

# This file is machine-generated - editing it directly is not advised

julia_version = "1.11.5"
manifest_format = "2.0"
project_hash = "c15941d90173f1d3bf4de798ab6b7eefbdea14a3"

[[deps.AbstractFFTs]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef"
uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
version = "1.5.0"

    [deps.AbstractFFTs.extensions]
    AbstractFFTsChainRulesCoreExt = "ChainRulesCore"
    AbstractFFTsTestExt = "Test"

    [deps.AbstractFFTs.weakdeps]
    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
    Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[deps.Adapt]]
deps = ["LinearAlgebra", "Requires"]
git-tree-sha1 = "f7817e2e585aa6d924fd714df1e2a84be7896c60"
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
version = "4.3.0"
weakdeps = ["SparseArrays", "StaticArrays"]

    [deps.Adapt.extensions]
    AdaptSparseArraysExt = "SparseArrays"
    AdaptStaticArraysExt = "StaticArrays"

[[deps.ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
version = "1.1.2"

[[deps.Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
version = "1.11.0"

[[deps.Atomix]]
deps = ["UnsafeAtomics"]
git-tree-sha1 = "b5bb4dc6248fde467be2a863eb8452993e74d402"
uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458"
version = "1.1.1"

    [deps.Atomix.extensions]
    AtomixCUDAExt = "CUDA"
    AtomixMetalExt = "Metal"
    AtomixOpenCLExt = "OpenCL"
    AtomixoneAPIExt = "oneAPI"

    [deps.Atomix.weakdeps]
    CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
    Metal = "dde4c033-4e86-420c-a63e-0dd931031962"
    OpenCL = "08131aa3-fb12-5dee-8b74-c09406e224a2"
    oneAPI = "8f75cd03-7ff8-4ecb-9b8f-daf728133b1b"

[[deps.BFloat16s]]
deps = ["LinearAlgebra", "Printf", "Random"]
git-tree-sha1 = "3b642331600250f592719140c60cf12372b82d66"
uuid = "ab4f0b2a-ad5b-11e8-123f-65d77653426b"
version = "0.5.1"

[[deps.Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
version = "1.11.0"

[[deps.CEnum]]
git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc"
uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
version = "0.5.0"

[[deps.CFTime]]
deps = ["Dates", "Printf"]
path = "/home/abarth/.julia/dev/CFTime"
uuid = "179af706-886a-5703-950a-314cd64e0468"
version = "0.2.1"

[[deps.CUDA]]
deps = ["AbstractFFTs", "Adapt", "BFloat16s", "CEnum", "CUDA_Driver_jll", "CUDA_Runtime_Discovery", "CUDA_Runtime_jll", "Crayons", "DataFrames", "ExprTools", "GPUArrays", "GPUCompiler", "GPUToolbox", "KernelAbstractions", "LLVM", "LLVMLoopInfo", "LazyArtifacts", "Libdl", "LinearAlgebra", "Logging", "NVTX", "Preferences", "PrettyTables", "Printf", "Random", "Random123", "RandomNumbers", "Reexport", "Requires", "SparseArrays", "StaticArrays", "Statistics", "demumble_jll"]
git-tree-sha1 = "b8ae59258f3d96ce75a00f9229e719356eb929d6"
uuid = "052768ef-5323-5732-b1bb-66c8b64840ba"
version = "5.8.2"

    [deps.CUDA.extensions]
    ChainRulesCoreExt = "ChainRulesCore"
    EnzymeCoreExt = "EnzymeCore"
    SparseMatricesCSRExt = "SparseMatricesCSR"
    SpecialFunctionsExt = "SpecialFunctions"

    [deps.CUDA.weakdeps]
    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
    EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
    SparseMatricesCSR = "a0a7dd2c-ebf4-11e9-1f05-cf50bc540ca1"
    SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"

[[deps.CUDA_Driver_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "18afa851ed10552e6df25dfaa7ef450104ae73d4"
uuid = "4ee394cb-3365-5eb0-8335-949819d2adfc"
version = "0.13.1+0"

[[deps.CUDA_Runtime_Discovery]]
deps = ["Libdl"]
git-tree-sha1 = "33576c7c1b2500f8e7e6baa082e04563203b3a45"
uuid = "1af6417a-86b4-443c-805f-a4643ffb695f"
version = "0.3.5"

[[deps.CUDA_Runtime_jll]]
deps = ["Artifacts", "CUDA_Driver_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"]
git-tree-sha1 = "b5c173a64f9f4224a82fdc26fda8614cb2ecfa27"
uuid = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
version = "0.17.1+0"

[[deps.ColorTypes]]
deps = ["FixedPointNumbers", "Random"]
git-tree-sha1 = "67e11ee83a43eb71ddc950302c53bf33f0690dfe"
uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
version = "0.12.1"

    [deps.ColorTypes.extensions]
    StyledStringsExt = "StyledStrings"

    [deps.ColorTypes.weakdeps]
    StyledStrings = "f489334b-da3d-4c2e-b8f0-e476e12c162b"

[[deps.Colors]]
deps = ["ColorTypes", "FixedPointNumbers", "Reexport"]
git-tree-sha1 = "37ea44092930b1811e666c3bc38065d7d87fcc74"
uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
version = "0.13.1"

[[deps.Compat]]
deps = ["TOML", "UUIDs"]
git-tree-sha1 = "8ae8d32e09f0dcf42a36b90d4e17f5dd2e4c4215"
uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
version = "4.16.0"
weakdeps = ["Dates", "LinearAlgebra"]

    [deps.Compat.extensions]
    CompatLinearAlgebraExt = "LinearAlgebra"

[[deps.CompilerSupportLibraries_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
version = "1.1.1+0"

[[deps.Crayons]]
git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
version = "4.1.1"

[[deps.DataAPI]]
git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
version = "1.16.0"

[[deps.DataFrames]]
deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"]
git-tree-sha1 = "fb61b4812c49343d7ef0b533ba982c46021938a6"
uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
version = "1.7.0"

[[deps.DataStructures]]
deps = ["Compat", "InteractiveUtils", "OrderedCollections"]
git-tree-sha1 = "4e1fe97fdaed23e9dc21d4d664bea76b65fc50a0"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
version = "0.18.22"

[[deps.DataValueInterfaces]]
git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
version = "1.0.0"

[[deps.Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
version = "1.11.0"

[[deps.Downloads]]
deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
version = "1.6.0"

[[deps.ExprTools]]
git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec"
uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04"
version = "0.1.10"

[[deps.FileWatching]]
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
version = "1.11.0"

[[deps.FixedPointNumbers]]
deps = ["Statistics"]
git-tree-sha1 = "05882d6995ae5c12bb5f36dd2ed3f61c98cbb172"
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
version = "0.8.5"

[[deps.Future]]
deps = ["Random"]
uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
version = "1.11.0"

[[deps.GPUArrays]]
deps = ["Adapt", "GPUArraysCore", "KernelAbstractions", "LLVM", "LinearAlgebra", "Printf", "Random", "Reexport", "ScopedValues", "Serialization", "Statistics"]
git-tree-sha1 = "eea7b3a1964b4de269bb380462a9da604be7fcdb"
uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
version = "11.2.2"

[[deps.GPUArraysCore]]
deps = ["Adapt"]
git-tree-sha1 = "83cf05ab16a73219e5f6bd1bdfa9848fa24ac627"
uuid = "46192b85-c4d5-4398-a991-12ede77f4527"
version = "0.2.0"

[[deps.GPUCompiler]]
deps = ["ExprTools", "InteractiveUtils", "LLVM", "Libdl", "Logging", "PrecompileTools", "Preferences", "Scratch", "Serialization", "TOML", "Tracy", "UUIDs"]
git-tree-sha1 = "38e96dd44b8b1db92c81c6eb11ac39d207aa83d5"
uuid = "61eb1bfa-7361-4325-ad38-22787b887f55"
version = "1.5.2"

[[deps.GPUToolbox]]
git-tree-sha1 = "15d8b0f5a6dca9bf8c02eeaf6687660dafa638d0"
uuid = "096a3bc2-3ced-46d0-87f4-dd12716f4bfc"
version = "0.2.0"

[[deps.HashArrayMappedTries]]
git-tree-sha1 = "2eaa69a7cab70a52b9687c8bf950a5a93ec895ae"
uuid = "076d061b-32b6-4027-95e0-9a2c6f6d7e74"
version = "0.2.0"

[[deps.InlineStrings]]
git-tree-sha1 = "8594fac023c5ce1ef78260f24d1ad18b4327b420"
uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
version = "1.4.4"

    [deps.InlineStrings.extensions]
    ArrowTypesExt = "ArrowTypes"
    ParsersExt = "Parsers"

    [deps.InlineStrings.weakdeps]
    ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
    Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"

[[deps.InteractiveUtils]]
deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
version = "1.11.0"

[[deps.InvertedIndices]]
git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe"
uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f"
version = "1.3.1"

[[deps.IteratorInterfaceExtensions]]
git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
uuid = "82899510-4779-5014-852e-03e436cf321d"
version = "1.0.0"

[[deps.JLLWrappers]]
deps = ["Artifacts", "Preferences"]
git-tree-sha1 = "a007feb38b422fbdab534406aeca1b86823cb4d6"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.7.0"

[[deps.JuliaNVTXCallbacks_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "af433a10f3942e882d3c671aacb203e006a5808f"
uuid = "9c1d0b0a-7046-5b2e-a33f-ea22f176ac7e"
version = "0.2.1+0"

[[deps.KernelAbstractions]]
deps = ["Adapt", "Atomix", "InteractiveUtils", "MacroTools", "PrecompileTools", "Requires", "StaticArrays", "UUIDs"]
git-tree-sha1 = "602c0e9efadafb8abfe8281c3fbf9cf6f406fc03"
uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
version = "0.9.35"

    [deps.KernelAbstractions.extensions]
    EnzymeExt = "EnzymeCore"
    LinearAlgebraExt = "LinearAlgebra"
    SparseArraysExt = "SparseArrays"

    [deps.KernelAbstractions.weakdeps]
    EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869"
    LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
    SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"

[[deps.LLVM]]
deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Unicode"]
git-tree-sha1 = "cfedf80c59000507cc8115a0281931253f4a33cd"
uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
version = "9.4.1"
weakdeps = ["BFloat16s"]

    [deps.LLVM.extensions]
    BFloat16sExt = "BFloat16s"

[[deps.LLVMExtra_jll]]
deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"]
git-tree-sha1 = "2ea068aac1e7f0337d381b0eae3110581e3f3216"
uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab"
version = "0.0.37+2"

[[deps.LLVMLoopInfo]]
git-tree-sha1 = "2e5c102cfc41f48ae4740c7eca7743cc7e7b75ea"
uuid = "8b046642-f1f6-4319-8d3c-209ddc03c586"
version = "1.0.0"

[[deps.LaTeXStrings]]
git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c"
uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
version = "1.4.0"

[[deps.LazyArtifacts]]
deps = ["Artifacts", "Pkg"]
uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
version = "1.11.0"

[[deps.LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
version = "0.6.4"

[[deps.LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
version = "8.6.0+0"

[[deps.LibGit2]]
deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
version = "1.11.0"

[[deps.LibGit2_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
version = "1.7.2+0"

[[deps.LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
version = "1.11.0+1"

[[deps.LibTracyClient_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
git-tree-sha1 = "d2bc4e1034b2d43076b50f0e34ea094c2cb0a717"
uuid = "ad6e5548-8b26-5c9f-8ef3-ef0ad883f3a5"
version = "0.9.1+6"

[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
version = "1.11.0"

[[deps.LinearAlgebra]]
deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
version = "1.11.0"

[[deps.Logging]]
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
version = "1.11.0"

[[deps.MacroTools]]
git-tree-sha1 = "1e0228a030642014fe5cfe68c2c0a818f9e3f522"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
version = "0.5.16"

[[deps.Markdown]]
deps = ["Base64"]
uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
version = "1.11.0"

[[deps.MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
version = "2.28.6+0"

[[deps.Missings]]
deps = ["DataAPI"]
git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d"
uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
version = "1.2.0"

[[deps.MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
version = "2023.12.12"

[[deps.NVTX]]
deps = ["Colors", "JuliaNVTXCallbacks_jll", "Libdl", "NVTX_jll"]
git-tree-sha1 = "1a24c3430fa2ef3317c4c97fa7e431ef45793bd2"
uuid = "5da4648a-3479-48b8-97b9-01cb529c0a1f"
version = "1.0.0"

[[deps.NVTX_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
git-tree-sha1 = "cd475b587ff77910789a18e68da789fc446a2a05"
uuid = "e98f9f5b-d649-5603-91fd-7774390e6439"
version = "3.2.1+0"

[[deps.NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
version = "1.2.0"

[[deps.OpenBLAS_jll]]
deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
version = "0.3.27+1"

[[deps.OrderedCollections]]
git-tree-sha1 = "05868e21324cede2207c6f0f466b4bfef6d5e7ee"
uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.8.1"

[[deps.Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
version = "1.11.0"

    [deps.Pkg.extensions]
    REPLExt = "REPL"

    [deps.Pkg.weakdeps]
    REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[deps.PooledArrays]]
deps = ["DataAPI", "Future"]
git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3"
uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
version = "1.4.3"

[[deps.PrecompileTools]]
deps = ["Preferences"]
git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
version = "1.2.1"

[[deps.Preferences]]
deps = ["TOML"]
git-tree-sha1 = "9306f6085165d270f7e3db02af26a400d580f5c6"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.4.3"

[[deps.PrettyTables]]
deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"]
git-tree-sha1 = "1101cd475833706e4d0e7b122218257178f48f34"
uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
version = "2.4.0"

[[deps.Printf]]
deps = ["Unicode"]
uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
version = "1.11.0"

[[deps.Random]]
deps = ["SHA"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
version = "1.11.0"

[[deps.Random123]]
deps = ["Random", "RandomNumbers"]
git-tree-sha1 = "dbe5fd0b334694e905cb9fda73cd8554333c46e2"
uuid = "74087812-796a-5b5d-8853-05524746bad3"
version = "1.7.1"

[[deps.RandomNumbers]]
deps = ["Random"]
git-tree-sha1 = "c6ec94d2aaba1ab2ff983052cf6a606ca5985902"
uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143"
version = "1.6.0"

[[deps.Reexport]]
git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
uuid = "189a3867-3050-52da-a836-e630ba90ab69"
version = "1.2.2"

[[deps.Requires]]
deps = ["UUIDs"]
git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
version = "1.3.1"

[[deps.SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"

[[deps.ScopedValues]]
deps = ["HashArrayMappedTries", "Logging"]
git-tree-sha1 = "1147f140b4c8ddab224c94efa9569fc23d63ab44"
uuid = "7e506255-f358-4e82-b7e4-beb19740aa63"
version = "1.3.0"

[[deps.Scratch]]
deps = ["Dates"]
git-tree-sha1 = "9b81b8393e50b7d4e6d0a9f14e192294d3b7c109"
uuid = "6c6a2e73-6563-6170-7368-637461726353"
version = "1.3.0"

[[deps.SentinelArrays]]
deps = ["Dates", "Random"]
git-tree-sha1 = "712fb0231ee6f9120e005ccd56297abbc053e7e0"
uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
version = "1.4.8"

[[deps.Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
version = "1.11.0"

[[deps.SortingAlgorithms]]
deps = ["DataStructures"]
git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085"
uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
version = "1.2.1"

[[deps.SparseArrays]]
deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"]
uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
version = "1.11.0"

[[deps.StaticArrays]]
deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"]
git-tree-sha1 = "0feb6b9031bd5c51f9072393eb5ab3efd31bf9e4"
uuid = "90137ffa-7385-5640-81b9-e52037218182"
version = "1.9.13"

    [deps.StaticArrays.extensions]
    StaticArraysChainRulesCoreExt = "ChainRulesCore"
    StaticArraysStatisticsExt = "Statistics"

    [deps.StaticArrays.weakdeps]
    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
    Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

[[deps.StaticArraysCore]]
git-tree-sha1 = "192954ef1208c7019899fbf8049e717f92959682"
uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
version = "1.4.3"

[[deps.Statistics]]
deps = ["LinearAlgebra"]
git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0"
uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
version = "1.11.1"
weakdeps = ["SparseArrays"]

    [deps.Statistics.extensions]
    SparseArraysExt = ["SparseArrays"]

[[deps.StringManipulation]]
deps = ["PrecompileTools"]
git-tree-sha1 = "725421ae8e530ec29bcbdddbe91ff8053421d023"
uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e"
version = "0.4.1"

[[deps.SuiteSparse_jll]]
deps = ["Artifacts", "Libdl", "libblastrampoline_jll"]
uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c"
version = "7.7.0+0"

[[deps.TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
version = "1.0.3"

[[deps.TableTraits]]
deps = ["IteratorInterfaceExtensions"]
git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
version = "1.0.1"

[[deps.Tables]]
deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"]
git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344"
uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
version = "1.12.1"

[[deps.Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
version = "1.10.0"

[[deps.Tracy]]
deps = ["ExprTools", "LibTracyClient_jll", "Libdl"]
git-tree-sha1 = "16439d004690d4086da35528f0c6b4d7006d6dae"
uuid = "e689c965-62c8-4b79-b2c5-8359227902fd"
version = "0.1.4"

    [deps.Tracy.extensions]
    TracyProfilerExt = "TracyProfiler_jll"

    [deps.Tracy.weakdeps]
    TracyProfiler_jll = "0c351ed6-8a68-550e-8b79-de6f926da83c"

[[deps.UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
version = "1.11.0"

[[deps.Unicode]]
uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
version = "1.11.0"

[[deps.UnsafeAtomics]]
git-tree-sha1 = "b13c4edda90890e5b04ba24e20a310fbe6f249ff"
uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f"
version = "0.3.0"
weakdeps = ["LLVM"]

    [deps.UnsafeAtomics.extensions]
    UnsafeAtomicsLLVM = ["LLVM"]

[[deps.Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
version = "1.2.13+1"

[[deps.demumble_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl"]
git-tree-sha1 = "6498e3581023f8e530f34760d18f75a69e3a4ea8"
uuid = "1e29f10c-031c-5a83-9565-69cddfc27673"
version = "1.3.0+0"

[[deps.libblastrampoline_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
version = "5.11.0+0"

[[deps.nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
version = "1.59.0+0"

[[deps.p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
version = "17.4.0+2"

Expected behavior

The call in the REPL should not fail.

Version info

Details on Julia:

Julia Version 1.11.5
Commit 760b2e5b739 (2025-04-14 06:53 UTC)
Build Info:
  Official https://julialang.org/ release
Platform Info:
  OS: Linux (x86_64-linux-gnu)
  CPU: 8 × Intel(R) Core(TM) i7-7700 CPU @ 3.60GHz
  WORD_SIZE: 64
  LLVM: libLLVM-16.0.6 (ORCJIT, skylake)
Threads: 1 default, 0 interactive, 1 GC (on 8 virtual cores)
Environment:
  JULIA_ERROR_COLOR = red
  JULIA_PKG_PRESERVE_TIERED_INSTALLED = true

Details on CUDA:

CUDA runtime 12.9, artifact installation
CUDA driver 12.6
NVIDIA driver 560.35.5

CUDA libraries: 
- CUBLAS: 12.9.1
- CURAND: 10.3.10
- CUFFT: 11.4.1
- CUSOLVER: 11.7.5
- CUSPARSE: 12.5.10
- CUPTI: 2025.2.1 (API 28.0.0)
- NVML: 12.0.0+560.35.5

Julia packages: 
- CUDA: 5.8.2
- CUDA_Driver_jll: 0.13.1+0
- CUDA_Runtime_jll: 0.17.1+0

Toolchain:
- Julia: 1.11.5
- LLVM: 16.0.6

1 device:
  0: NVIDIA GeForce GTX 1080 (sm_61, 7.338 GiB / 8.000 GiB available)

Is there anything wrong with my code?

Thanks for this great package!

[maleadt]

I can't reproduce. Is this a plain REPL or the VSCode one? Any startup scripts?

[Alexander-Barth]

I am using the REPL via emacs. But on my side I get the error also with:

julia --project=@CUDA-CFTime  --startup-file=no test_gpu_rep.jl

where I updated test_gpu_rep.jl:

using CUDA

@inline function _division(::Type{T},factor,exponent) where T
    TIME_DIVISION = (
    (:day,         24*60*60,      0),
    (:hour,           60*60,      0),
    (:minute,            60,      0),
    (:second,             1,      0),
    (:millisecond,        1,     -3),
    (:microsecond,        1,     -6),
    (:nanosecond,         1,     -9),
)
    ntuple(length(TIME_DIVISION)) do i
        (T(10)^(-exponent) * TIME_DIVISION[i][2]) ÷
            (T(10)^(-TIME_DIVISION[i][3]) * factor)
    end
end

function foo(dt)
    factor,exponent = (86400,0)
    divi = _division(Int64,factor,exponent)
    return divi[1] * dt
end

dt_cpu = [1,2,3]
dt_gpu = cu(dt_cpu)

a_d = foo.(dt_gpu) # fine
a = Array(a_d) # error

Rather than reruning the last line in REPL, it seems sufficient to transfert the data to CPU to trigger the error for me.

Full error

$ julia --project=@CUDA-CFTime  --startup-file=no test_gpu_rep.jl
ERROR: LoadError: CUDA error: an illegal memory access was encountered (code 700, ERROR_ILLEGAL_ADDRESS)
Stacktrace:
  [1] throw_api_error(res::CUDA.cudaError_enum)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:30
  [2] isdone
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/stream.jl:134 [inlined]
  [3] spinning_synchronization(f::typeof(CUDA.isdone), obj::CuStream)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/synchronization.jl:79
  [4] synchronize(stream::CuStream; blocking::Bool, spin::Bool)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/synchronization.jl:196
  [5] synchronize
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/synchronization.jl:194 [inlined]
  [6] synchronize
    @ ~/.julia/packages/CUDA/ja0IX/src/memory.jl:527 [inlined]
  [7] synchronize(x::CuArray{Int64, 1, CUDA.DeviceMemory})
    @ CUDA ~/.julia/packages/CUDA/ja0IX/src/array.jl:493
  [8] (::CUDA.var"#1161#1162"{Int64, Vector{Int64}, Int64, CuArray{Int64, 1, CUDA.DeviceMemory}, Int64, Int64})()
    @ CUDA ~/.julia/packages/CUDA/ja0IX/src/array.jl:613
  [9] #context!#1024
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:168 [inlined]
 [10] context!
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:163 [inlined]
 [11] unsafe_copyto!(dest::Vector{Int64}, doffs::Int64, src::CuArray{Int64, 1, CUDA.DeviceMemory}, soffs::Int64, n::Int64)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/src/array.jl:608
 [12] copyto!
    @ ~/.julia/packages/CUDA/ja0IX/src/array.jl:560 [inlined]
 [13] copyto!
    @ ~/.julia/packages/CUDA/ja0IX/src/array.jl:564 [inlined]
 [14] copyto_axcheck!
    @ ./abstractarray.jl:1167 [inlined]
 [15] Array
    @ ./array.jl:626 [inlined]
 [16] Array(A::CuArray{Int64, 1, CUDA.DeviceMemory})
    @ Core ./boot.jl:605
 [17] top-level scope
    @ /mnt/data1/abarth/.julia/dev/CFTime/test/test_gpu_rep.jl:33
in expression starting at /mnt/data1/abarth/.julia/dev/CFTime/test/test_gpu_rep.jl:33
error in running finalizer: CUDA.CuError(code=CUDA.cudaError_enum(0x000002bc))
throw_api_error at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:30
check at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:37 [inlined]
cuModuleUnload at /home/abarth/.julia/packages/GPUToolbox/cZlg7/src/ccalls.jl:33 [inlined]
#988 at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/module.jl:92 [inlined]
#context!#1024 at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:168 [inlined]
context! at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:163 [inlined]
unsafe_unload! at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/module.jl:91
jfptr_unsafe_unloadNOT._13868 at /home/abarth/.julia/compiled/v1.11/CUDA/oWw5k_4oemg.so (unknown line)
run_finalizer at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/gc.c:303
jl_gc_run_finalizers_in_list at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/gc.c:393
run_finalizers at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/gc.c:439
ijl_atexit_hook at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/init.c:299
ijl_exit at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/init.c:207
jlplt_ijl_exit_69296.1 at /mnt/data1/abarth/.julia/juliaup/julia-1.11.5+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
exit at ./initdefs.jl:28
exec_options at ./client.jl:328
_start at ./client.jl:531
jfptr__start_73430.1 at /mnt/data1/abarth/.julia/juliaup/julia-1.11.5+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/julia.h:2157 [inlined]
true_main at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/jlapi.c:900
jl_repl_entrypoint at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/jlapi.c:1059
main at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/cli/loader_exe.c:58
unknown function (ip: 0x7c63f0a2a1c9)
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
unknown function (ip: 0x4010b8)
WARNING: Error while freeing DeviceMemory(24 bytes at 0x0000000302000200):
CUDA.CuError(code=CUDA.cudaError_enum(0x000002bc))

Stacktrace:
  [1] throw_api_error(res::CUDA.cudaError_enum)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:30
  [2] check
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:37 [inlined]
  [3] cuMemFreeAsync
    @ ~/.julia/packages/GPUToolbox/cZlg7/src/ccalls.jl:33 [inlined]
  [4] free(mem::CUDA.DeviceMemory; stream::CuStream)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/memory.jl:87
  [5] free
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/memory.jl:82 [inlined]
  [6] #1134
    @ ~/.julia/packages/CUDA/ja0IX/src/memory.jl:714 [inlined]
  [7] #context!#1024
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:168 [inlined]
  [8] context!
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:163 [inlined]
  [9] _pool_free
    @ ~/.julia/packages/CUDA/ja0IX/src/memory.jl:713 [inlined]
 [10] macro expansion
    @ ./timing.jl:421 [inlined]
 [11] pool_free(managed::CUDA.Managed{CUDA.DeviceMemory})
    @ CUDA ~/.julia/packages/CUDA/ja0IX/src/memory.jl:695
 [12] release(::GPUArrays.RefCounted{CUDA.Managed{CUDA.DeviceMemory}})
    @ GPUArrays ~/.julia/packages/GPUArrays/uiVyU/src/host/abstractarray.jl:42
 [13] unsafe_free!
    @ ~/.julia/packages/GPUArrays/uiVyU/src/host/abstractarray.jl:100 [inlined]
 [14] unsafe_free!(x::CuArray{Int64, 1, CUDA.DeviceMemory})
    @ GPUArrays ~/.julia/packages/GPUArrays/uiVyU/src/host/abstractarray.jl:115
 [15] exit(n::Int64)
    @ Base ./initdefs.jl:28
 [16] exec_options(opts::Base.JLOptions)
    @ Base ./client.jl:328
 [17] _start()
    @ Base ./client.jl:531
WARNING: Error while freeing DeviceMemory(24 bytes at 0x0000000302000000):
CUDA.CuError(code=CUDA.cudaError_enum(0x000002bc))

Stacktrace:
  [1] throw_api_error(res::CUDA.cudaError_enum)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:30
  [2] check
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:37 [inlined]
  [3] cuMemFreeAsync
    @ ~/.julia/packages/GPUToolbox/cZlg7/src/ccalls.jl:33 [inlined]
  [4] free(mem::CUDA.DeviceMemory; stream::CuStream)
    @ CUDA ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/memory.jl:87
  [5] free
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/memory.jl:82 [inlined]
  [6] #1134
    @ ~/.julia/packages/CUDA/ja0IX/src/memory.jl:714 [inlined]
  [7] #context!#1024
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:168 [inlined]
  [8] context!
    @ ~/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:163 [inlined]
  [9] _pool_free
    @ ~/.julia/packages/CUDA/ja0IX/src/memory.jl:713 [inlined]
 [10] macro expansion
    @ ./timing.jl:421 [inlined]
 [11] pool_free(managed::CUDA.Managed{CUDA.DeviceMemory})
    @ CUDA ~/.julia/packages/CUDA/ja0IX/src/memory.jl:695
 [12] release(::GPUArrays.RefCounted{CUDA.Managed{CUDA.DeviceMemory}})
    @ GPUArrays ~/.julia/packages/GPUArrays/uiVyU/src/host/abstractarray.jl:42
 [13] unsafe_free!
    @ ~/.julia/packages/GPUArrays/uiVyU/src/host/abstractarray.jl:100 [inlined]
 [14] unsafe_free!(x::CuArray{Int64, 1, CUDA.DeviceMemory})
    @ GPUArrays ~/.julia/packages/GPUArrays/uiVyU/src/host/abstractarray.jl:115
 [15] exit(n::Int64)
    @ Base ./initdefs.jl:28
 [16] exec_options(opts::Base.JLOptions)
    @ Base ./client.jl:328
 [17] _start()
    @ Base ./client.jl:531
error in running finalizer: CUDA.CuError(code=CUDA.cudaError_enum(0x000002bc))
throw_api_error at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:30
check at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:37 [inlined]
cuStreamDestroy_v2 at /home/abarth/.julia/packages/GPUToolbox/cZlg7/src/ccalls.jl:33 [inlined]
#956 at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/stream.jl:89
#context!#1024 at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:168
context! at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/state.jl:163
unsafe_destroy! at /home/abarth/.julia/packages/CUDA/ja0IX/lib/cudadrv/stream.jl:88
jfptr_unsafe_destroyNOT._13573 at /home/abarth/.julia/compiled/v1.11/CUDA/oWw5k_4oemg.so (unknown line)
run_finalizer at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/gc.c:303
jl_gc_run_finalizers_in_list at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/gc.c:393
run_finalizers at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/gc.c:439
ijl_atexit_hook at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/init.c:299
ijl_exit at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/init.c:207
jlplt_ijl_exit_69296.1 at /mnt/data1/abarth/.julia/juliaup/julia-1.11.5+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
exit at ./initdefs.jl:28
exec_options at ./client.jl:328
_start at ./client.jl:531
jfptr__start_73430.1 at /mnt/data1/abarth/.julia/juliaup/julia-1.11.5+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
jl_apply at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/julia.h:2157 [inlined]
true_main at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/jlapi.c:900
jl_repl_entrypoint at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/src/jlapi.c:1059
main at /cache/build/tester-amdci5-12/julialang/julia-release-1-dot-11/cli/loader_exe.c:58
unknown function (ip: 0x7c63f0a2a1c9)
__libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line)
unknown function (ip: 0x4010b8)

The environement CUDA-CFTime, containts CUDA and my package CFTime (which I try to make GPU compatible)

[maleadt]

Broadcast is asynchronous, so it's expected that you have to synchronize (e.g. call synchronize which is done by copying back to the CPU) or re-run to trigger the error.

[Alexander-Barth]

Indeed, CUDA.synchronize() is sufficient to trigger the error for me. I now also see additional error messages (Error while freeing DeviceMemory(24 bytes at 0x0000000302000200) see full error above, in case this is helpful).

[Alexander-Barth]

I am seeing this error also on a newer system with a GeForce RTX 4090 (with CUDA v5.8.2). Here are the steps:

rm -R ~/.julia/environments/CUDA-new;  julia --project=@CUDA-new --startup-file=no --eval 'using Pkg; Pkg.add("CUDA");'
wget https://gist.githubusercontent.com/Alexander-Barth/adcffa5074ae44b803523ab2fedddf54/raw/fda392eb4539a0ee9798a3a73e687400cb5c6f6d/test_gpu_rep.jl
julia --project=@CUDA-new --startup-file=no test_gpu_rep.jl

[maleadt]

You're not using the REPL here anymore, does that mean you can reproduce this regardless of the setting?

[Alexander-Barth]

Yes, I do not need the REPL anymore to reproduce the issue. An explicit call to CUDA.synchronize() is sufficient.

[maleadt]

❯ export JULIA_DEPOT_PATH=$(mktemp -d)
❯ julia --startup-file=no --eval 'using Pkg; Pkg.add("CUDA");'
❯ wget https://gist.githubusercontent.com/Alexander-Barth/adcffa5074ae44b803523ab2fedddf54/raw/fda392eb4539a0ee9798a3a73e687400cb5c6f6d/test_gpu_rep.jl
❯ julia --startup-file=no test_gpu_rep.jl

Works fine here.

[Alexander-Barth]

For me it also fails on a 3rd machine with a A100-SXM4-40GB with julia 1.11.4. Strange...

Full session

$ export JULIA_DEPOT_PATH=$HOME/tmp/julia-depot
abarth@cna050:~ 
$ julia --startup-file=no --eval 'using Pkg; Pkg.add("CUDA");'
The latest version of Julia in the `release` channel is 1.11.5+0.x64.linux.gnu. You currently have `1.11.4+0.x64.linux.gnu` installed. Run:

  juliaup update

in your terminal shell to install Julia 1.11.5+0.x64.linux.gnu and update the `release` channel to that version.
  Installing known registries into `~/tmp/julia-depot`
       Added `General` registry to `~/tmp/julia-depot/registries`
    Updating registry at `~/tmp/julia-depot/registries/General.toml`
   Resolving package versions...
   Installed GPUArraysCore ─────────────── v0.2.0
   Installed demumble_jll ──────────────── v1.3.0+0
   Installed Adapt ─────────────────────── v4.3.0
   Installed ColorTypes ────────────────── v0.12.1
   Installed Crayons ───────────────────── v4.1.1
   Installed Scratch ───────────────────── v1.3.0
   Installed Preferences ───────────────── v1.4.3
   Installed PrettyTables ──────────────── v2.4.0
   Installed TableTraits ───────────────── v1.0.1
   Installed CUDA_Driver_jll ───────────── v0.13.1+0
   Installed SentinelArrays ────────────── v1.4.8
   Installed Tables ────────────────────── v1.12.1
   Installed LLVMLoopInfo ──────────────── v1.0.0
   Installed DataAPI ───────────────────── v1.16.0
   Installed GPUCompiler ───────────────── v1.5.3
   Installed HashArrayMappedTries ──────── v0.2.0
   Installed FixedPointNumbers ─────────── v0.8.5
   Installed PooledArrays ──────────────── v1.4.3
   Installed InlineStrings ─────────────── v1.4.4
   Installed IteratorInterfaceExtensions ─ v1.0.0
   Installed JLLWrappers ───────────────── v1.7.0
   Installed AbstractFFTs ──────────────── v1.5.0
   Installed Tracy ─────────────────────── v0.1.4
   Installed StaticArrays ──────────────── v1.9.13
   Installed PrecompileTools ───────────── v1.2.1
   Installed CUDA_Runtime_Discovery ────── v0.3.5
   Installed DataValueInterfaces ───────── v1.0.0
   Installed NVTX_jll ──────────────────── v3.2.1+0
   Installed LLVMExtra_jll ─────────────── v0.0.37+2
   Installed GPUToolbox ────────────────── v0.2.0
   Installed OrderedCollections ────────── v1.8.1
   Installed NVTX ──────────────────────── v1.0.0
   Installed StaticArraysCore ──────────── v1.4.3
   Installed LaTeXStrings ──────────────── v1.4.0
   Installed ScopedValues ──────────────── v1.3.0
   Installed BFloat16s ─────────────────── v0.5.1
   Installed CEnum ─────────────────────── v0.5.0
   Installed InvertedIndices ───────────── v1.3.1
   Installed Random123 ─────────────────── v1.7.1
   Installed Reexport ──────────────────── v1.2.2
   Installed GPUArrays ─────────────────── v11.2.3
   Installed JuliaNVTXCallbacks_jll ────── v0.2.1+0
   Installed LibTracyClient_jll ────────── v0.9.1+6
   Installed CUDA_Runtime_jll ──────────── v0.17.1+0
   Installed Statistics ────────────────── v1.11.1
   Installed DataFrames ────────────────── v1.7.0
   Installed DataStructures ────────────── v0.18.22
   Installed Colors ────────────────────── v0.13.1
   Installed RandomNumbers ─────────────── v1.6.0
   Installed Requires ──────────────────── v1.3.1
   Installed Compat ────────────────────── v4.16.0
   Installed ExprTools ─────────────────── v0.1.10
   Installed MacroTools ────────────────── v0.5.16
   Installed UnsafeAtomics ─────────────── v0.3.0
   Installed KernelAbstractions ────────── v0.9.35
   Installed SortingAlgorithms ─────────── v1.2.1
   Installed StringManipulation ────────── v0.4.1
   Installed Missings ──────────────────── v1.2.0
   Installed Atomix ────────────────────── v1.1.1
   Installed CUDA ──────────────────────── v5.8.2
   Installed LLVM ──────────────────────── v9.4.2
  Downloaded artifact: demumble
  Downloaded artifact: CUDA_Driver
  Downloaded artifact: LLVMExtra
  Downloaded artifact: NVTX
  Downloaded artifact: JuliaNVTXCallbacks
  Downloaded artifact: LibTracyClient
    Updating `~/tmp/julia-depot/environments/v1.11/Project.toml`
  [052768ef] + CUDA v5.8.2
    Updating `~/tmp/julia-depot/environments/v1.11/Manifest.toml`
Precompiling CUDA_Driver_jll...
  5 dependencies successfully precompiled in 2 seconds. 21 already precompiled.
   [621f4979] + AbstractFFTs v1.5.0
   [79e6a3ab] + Adapt v4.3.0
   [a9b6321e] + Atomix v1.1.1
   [ab4f0b2a] + BFloat16s v0.5.1
   [fa961155] + CEnum v0.5.0
   [052768ef] + CUDA v5.8.2
   [1af6417a] + CUDA_Runtime_Discovery v0.3.5
   [3da002f7] + ColorTypes v0.12.1
   [5ae59095] + Colors v0.13.1
   [34da2185] + Compat v4.16.0
   [a8cc5b0e] + Crayons v4.1.1
   [9a962f9c] + DataAPI v1.16.0
   [a93c6f00] + DataFrames v1.7.0
   [864edb3b] + DataStructures v0.18.22
   [e2d170a0] + DataValueInterfaces v1.0.0
   [e2ba6199] + ExprTools v0.1.10
   [53c48c17] + FixedPointNumbers v0.8.5
   [0c68f7d7] + GPUArrays v11.2.3
   [46192b85] + GPUArraysCore v0.2.0
   [61eb1bfa] + GPUCompiler v1.5.3
   [096a3bc2] + GPUToolbox v0.2.0
   [076d061b] + HashArrayMappedTries v0.2.0
   [842dd82b] + InlineStrings v1.4.4
   [41ab1584] + InvertedIndices v1.3.1
   [82899510] + IteratorInterfaceExtensions v1.0.0
   [692b3bcd] + JLLWrappers v1.7.0
   [63c18a36] + KernelAbstractions v0.9.35
   [929cbde3] + LLVM v9.4.2
   [8b046642] + LLVMLoopInfo v1.0.0
   [b964fa9f] + LaTeXStrings v1.4.0
   [1914dd2f] + MacroTools v0.5.16
   [e1d29d7a] + Missings v1.2.0
   [5da4648a] + NVTX v1.0.0
   [bac558e1] + OrderedCollections v1.8.1
   [2dfb63ee] + PooledArrays v1.4.3
 ⌅ [aea7be01] + PrecompileTools v1.2.1
   [21216c6a] + Preferences v1.4.3
   [08abe8d2] + PrettyTables v2.4.0
   [74087812] + Random123 v1.7.1
   [e6cf234a] + RandomNumbers v1.6.0
   [189a3867] + Reexport v1.2.2
   [ae029012] + Requires v1.3.1
   [7e506255] + ScopedValues v1.3.0
   [6c6a2e73] + Scratch v1.3.0
   [91c51154] + SentinelArrays v1.4.8
   [a2af1166] + SortingAlgorithms v1.2.1
   [90137ffa] + StaticArrays v1.9.13
   [1e83bf80] + StaticArraysCore v1.4.3
   [10745b16] + Statistics v1.11.1
   [892a3eda] + StringManipulation v0.4.1
   [3783bdb8] + TableTraits v1.0.1
   [bd369af6] + Tables v1.12.1
   [e689c965] + Tracy v0.1.4
   [013be700] + UnsafeAtomics v0.3.0
 ⌅ [4ee394cb] + CUDA_Driver_jll v0.13.1+0
→⌅ [76a88914] + CUDA_Runtime_jll v0.17.1+0
   [9c1d0b0a] + JuliaNVTXCallbacks_jll v0.2.1+0
   [dad2f222] + LLVMExtra_jll v0.0.37+2
   [ad6e5548] + LibTracyClient_jll v0.9.1+6
   [e98f9f5b] + NVTX_jll v3.2.1+0
   [1e29f10c] + demumble_jll v1.3.0+0
   [0dad84c5] + ArgTools v1.1.2
   [56f22d72] + Artifacts v1.11.0
   [2a0f44e3] + Base64 v1.11.0
   [ade2ca70] + Dates v1.11.0
   [f43a241f] + Downloads v1.6.0
   [7b1f6079] + FileWatching v1.11.0
   [9fa8497b] + Future v1.11.0
   [b77e0a4c] + InteractiveUtils v1.11.0
   [4af54fe1] + LazyArtifacts v1.11.0
   [b27032c2] + LibCURL v0.6.4
   [76f85450] + LibGit2 v1.11.0
   [8f399da3] + Libdl v1.11.0
   [37e2e46d] + LinearAlgebra v1.11.0
   [56ddb016] + Logging v1.11.0
   [d6f4376e] + Markdown v1.11.0
   [ca575930] + NetworkOptions v1.2.0
   [44cfe95a] + Pkg v1.11.0
   [de0858da] + Printf v1.11.0
   [9a3f8284] + Random v1.11.0
   [ea8e919c] + SHA v0.7.0
   [9e88b42a] + Serialization v1.11.0
   [2f01184e] + SparseArrays v1.11.0
   [fa267f1f] + TOML v1.0.3
   [a4e569a6] + Tar v1.10.0
   [cf7118a7] + UUIDs v1.11.0
   [4ec0a83e] + Unicode v1.11.0
   [e66e0078] + CompilerSupportLibraries_jll v1.1.1+0
   [deac9b47] + LibCURL_jll v8.6.0+0
   [e37daf67] + LibGit2_jll v1.7.2+0
   [29816b5a] + LibSSH2_jll v1.11.0+1
   [c8ffd9c3] + MbedTLS_jll v2.28.6+0
   [14a3606d] + MozillaCACerts_jll v2023.12.12
   [4536629a] + OpenBLAS_jll v0.3.27+1
   [bea87d4a] + SuiteSparse_jll v7.7.0+0
   [83775a58] + Zlib_jll v1.2.13+1
   [8e850b90] + libblastrampoline_jll v5.11.0+0
   [8e850ede] + nghttp2_jll v1.59.0+0
   [3f19e933] + p7zip_jll v17.4.0+2
        Info Packages marked with → are not downloaded, use `instantiate` to download
        Info Packages marked with ⌅ have new versions available but compatibility constraints restrict them from upgrading. To see why use `status --outdated -m`
  Downloaded artifact: CUDA_Runtime
Precompiling project...
  80 dependencies successfully precompiled in 243 seconds. 21 already precompiled.
abarth@cna050:~ 
$ wget https://gist.githubusercontent.com/Alexander-Barth/adcffa5074ae44b803523ab2fedddf54/raw/fda392eb4539a0ee9798a3a73e687400cb5c6f6d/test_gpu_rep.jl
--2025-06-27 17:54:54--  https://gist.githubusercontent.com/Alexander-Barth/adcffa5074ae44b803523ab2fedddf54/raw/fda392eb4539a0ee9798a3a73e687400cb5c6f6d/test_gpu_rep.jl
Resolving gist.githubusercontent.com (gist.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.108.133, ...
Connecting to gist.githubusercontent.com (gist.githubusercontent.com)|185.199.110.133|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 727 [text/plain]
Saving to: 'test_gpu_rep.jl'

test_gpu_rep.jl                               100%[=================================================================================================>]     727  --.-KB/s    in 0s      

2025-06-27 17:54:54 (15.5 MB/s) - 'test_gpu_rep.jl' saved [727/727]

abarth@cna050:~ 
$ julia --startup-file=no test_gpu_rep.jl
The latest version of Julia in the `release` channel is 1.11.5+0.x64.linux.gnu. You currently have `1.11.4+0.x64.linux.gnu` installed. Run:

  juliaup update

in your terminal shell to install Julia 1.11.5+0.x64.linux.gnu and update the `release` channel to that version.
ERROR: LoadError: CUDA error: an illegal memory access was encountered (code 700, ERROR_ILLEGAL_ADDRESS)
Stacktrace:
 [1] throw_api_error(res::CUDA.cudaError_enum)
   @ CUDA ~/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:30
 [2] nonblocking_synchronize(val::CuStream)
   @ CUDA ~/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/synchronization.jl:172
 [3] synchronize(stream::CuStream; blocking::Bool, spin::Bool)
   @ CUDA ~/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/synchronization.jl:200
 [4] synchronize
   @ ~/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/synchronization.jl:194 [inlined]
 [5] synchronize()
   @ CUDA ~/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/synchronization.jl:194
 [6] top-level scope
   @ ~/test_gpu_rep.jl:30
in expression starting at /gpfs/home/acad/ulg-gher/abarth/test_gpu_rep.jl:30
error in running finalizer: CUDA.CuError(code=CUDA.cudaError_enum(0x000002bc))
throw_api_error at /gpfs/home/acad/ulg-gher/abarth/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:30
check at /gpfs/home/acad/ulg-gher/abarth/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/libcuda.jl:37 [inlined]
cuModuleUnload at /gpfs/home/acad/ulg-gher/abarth/tmp/julia-depot/packages/GPUToolbox/cZlg7/src/ccalls.jl:33 [inlined]
#988 at /gpfs/home/acad/ulg-gher/abarth/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/module.jl:92 [inlined]
#context!#1024 at /gpfs/home/acad/ulg-gher/abarth/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/state.jl:168 [inlined]
context! at /gpfs/home/acad/ulg-gher/abarth/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/state.jl:163 [inlined]
unsafe_unload! at /gpfs/home/acad/ulg-gher/abarth/tmp/julia-depot/packages/CUDA/ja0IX/lib/cudadrv/module.jl:91
jfptr_unsafe_unloadNOT._13840 at /gpfs/home/acad/ulg-gher/abarth/tmp/julia-depot/compiled/v1.11/CUDA/oWw5k_0s5hg.so (unknown line)
run_finalizer at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/gc.c:299
jl_gc_run_finalizers_in_list at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/gc.c:389
run_finalizers at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/gc.c:435
ijl_atexit_hook at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/init.c:299
ijl_exit at /cache/build/builder-amdci5-5/julialang/julia-release-1-dot-11/src/init.c:207
jlplt_ijl_exit_69429.1 at /gpfs/home/acad/ulg-gher/abarth/.julia/juliaup/julia-1.11.4+0.x64.linux.gnu/lib/julia/sys.so (unknown line)
exit at ./initdefs.jl:28

I get also the same failure with current master 740e888 or with a newly created user account to exclude the possibility that there are some environment variables that create this issue.

Also on Google Colab which currently uses julia 1.10 I have this issue:

Here is the link to the notebook.