Skip to content

Fea/atomic nvvm backend #7993

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wmaxey wants to merge 9 commits into
base: main
Choose a base branch
from
+1,027 −94
Draft

Fea/atomic nvvm backend #7993

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
cc6a657
Move atomic header 'functions' to 'backends' for clarity.
wmaxey Oct 21, 2025
084ca5c
Move __atomic/functions -> __atomic/backends and add intrinsic suppor…
wmaxey Nov 6, 2025
84baf4e
Use atomic order conversion instead of static cast.
wmaxey Nov 6, 2025
7344702
Reformat atomic order functions.
wmaxey Mar 4, 2026
dc7af88
Use special tactics to dispatch to NVVM functions (they only accept l…
wmaxey Mar 4, 2026
4acb8fa
Inject cluster support
wmaxey Mar 4, 2026
028a8b1
Fix handling of atomic fetch ops values
wmaxey Mar 4, 2026
b7a10f2
Work in progress for NVVM fallbacks
wmaxey Mar 11, 2026
e66cb80
Merge remote-tracking branch 'origin/main' into fea/atomic_nvvm_backend
wmaxey Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions libcudacxx/codegen/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,12 @@ add_executable(codegen EXCLUDE_FROM_ALL codegen.cpp)

target_compile_features(codegen PRIVATE cxx_std_20)

set(
atomic_generated_output
"${libcudacxx_BINARY_DIR}/codegen/cuda_ptx_generated.h"
)
set(
atomic_install_location
"${libcudacxx_SOURCE_DIR}/include/cuda/std/__atomic/functions"
)
set(atomic_generated_output "${libcudacxx_BINARY_DIR}/codegen/cuda_ptx_generated.h")
set(atomic_install_location "${libcudacxx_SOURCE_DIR}/include/cuda/std/__atomic/backends/")

target_link_libraries(codegen PRIVATE fmt)

set_property(TARGET codegen PROPERTY CXX_STANDARD 17)

add_custom_target(
libcudacxx.atomics.codegen
Expand Down
12 changes: 6 additions & 6 deletions libcudacxx/codegen/generators/header.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ inline void FormatHeader(std::ostream& out)
// This is an autogenerated file, we want to ensure that it contains exactly the contents we want to generate
// clang-format off

#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
#ifndef _CUDA_STD___ATOMIC_BACKENDS_CUDA_PTX_GENERATED_H
#define _CUDA_STD___ATOMIC_BACKENDS_CUDA_PTX_GENERATED_H

#include <cuda/std/detail/__config>

Expand All @@ -50,9 +50,9 @@ inline void FormatHeader(std::ostream& out)

#include <cuda/std/__atomic/scopes.h>
#include <cuda/std/__atomic/order.h>
#include <cuda/std/__atomic/functions/common.h>
#include <cuda/std/__atomic/functions/cuda_ptx_generated_helper.h>
#include <cuda/std/__atomic/functions/cuda_local.h>
#include <cuda/std/__atomic/backends/common.h>
#include <cuda/std/__atomic/backends/cuda_supported_atomics_helper.h>
#include <cuda/std/__atomic/backends/cuda_local.h>

#include <cuda/std/__cccl/prologue.h>

Expand All @@ -77,7 +77,7 @@ _CCCL_END_NAMESPACE_CUDA_STD

#include <cuda/std/__cccl/epilogue.h>

#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
#endif // _CUDA_STD___ATOMIC_BACKENDS_CUDA_PTX_GENERATED_H

// clang-format on
)XXX";
Expand Down
2 changes: 1 addition & 1 deletion libcudacxx/codegen/generators/ld_st.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ static inline _CCCL_DEVICE void __atomic_store_cuda(_Type* __ptr, _Type& __val,
__cuda_atomic_store_memory_order_dispatch(__bound_store, __memorder, _Sco{});
}
template <class _Type, class _Sco>
static inline _CCCL_DEVICE void __atomic_store_cuda(volatile _Type* __ptr, _Type& __val, int __memorder, _Sco)
static inline _CCCL_DEVICE void __atomic_store_cuda(_Type volatile* __ptr, _Type& __val, int __memorder, _Sco)
{
using __proxy_t = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
using __proxy_tag = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
Expand Down
17 changes: 10 additions & 7 deletions ...cxx/include/cuda/std/__atomic/functions.h → ...acxx/include/cuda/std/__atomic/backends.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
//
//===----------------------------------------------------------------------===//

#ifndef __CUDA_STD___ATOMIC_FUNCTIONS_H
#define __CUDA_STD___ATOMIC_FUNCTIONS_H
#ifndef __CUDA_STD___ATOMIC_BACKENDS_H
#define __CUDA_STD___ATOMIC_BACKENDS_H

#include <cuda/std/detail/__config>

Expand All @@ -22,12 +22,15 @@
#endif // no system header

#include <cuda/std/__atomic/platform.h>

#if _CCCL_CUDA_COMPILER(NVCC, >, 12, 8)
# include <cuda/std/__atomic/backends/cuda_nvvm.h>
#else
// Device atomics
#include <cuda/std/__atomic/functions/cuda_ptx_derived.h>
#include <cuda/std/__atomic/functions/cuda_ptx_generated.h>
# include <cuda/std/__atomic/backends/cuda_ptx_generated.h>
#endif
#include <cuda/std/__atomic/backends/cuda_ptx_derived.h>

// Host atomics
#include <cuda/std/__atomic/functions/host.h>
#include <cuda/std/__atomic/backends/host.h>

#endif // __CUDA_STD___ATOMIC_FUNCTIONS_H
#endif // __CUDA_STD___ATOMIC_BACKENDS_H
6 changes: 3 additions & 3 deletions ...lude/cuda/std/__atomic/functions/common.h → ...clude/cuda/std/__atomic/backends/common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
//
//===----------------------------------------------------------------------===//

#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_COMMON_H
#define _CUDA_STD___ATOMIC_FUNCTIONS_COMMON_H
#ifndef _CUDA_STD___ATOMIC_BACKENDS_COMMON_H
#define _CUDA_STD___ATOMIC_BACKENDS_COMMON_H

#include <cuda/std/detail/__config>

Expand Down Expand Up @@ -55,4 +55,4 @@ _CCCL_END_NAMESPACE_CUDA_STD

#include <cuda/std/__cccl/epilogue.h>

#endif // _CUDA_STD___ATOMIC_FUNCTIONS_COMMON_H
#endif // _CUDA_STD___ATOMIC_BACKENDS_COMMON_H
6 changes: 3 additions & 3 deletions .../cuda/std/__atomic/functions/cuda_local.h → ...e/cuda/std/__atomic/backends/cuda_local.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#ifndef __CUDA_STD___ATOMIC_FUNCTIONS_CUDA_LOCAL_H
#define __CUDA_STD___ATOMIC_FUNCTIONS_CUDA_LOCAL_H
#ifndef __CUDA_STD___ATOMIC_BACKENDS_CUDA_LOCAL_H
#define __CUDA_STD___ATOMIC_BACKENDS_CUDA_LOCAL_H

#include <cuda/std/detail/__config>

Expand Down Expand Up @@ -205,4 +205,4 @@ _CCCL_END_NAMESPACE_CUDA_STD

#include <cuda/std/__cccl/epilogue.h>

#endif // __CUDA_STD___ATOMIC_FUNCTIONS_CUDA_LOCAL_H
#endif // __CUDA_STD___ATOMIC_BACKENDS_CUDA_LOCAL_H
Loading
Loading