reduce_kernel.cu

// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "paddle/phi/kernels/reduce_kernel.h"
#include "paddle/phi/kernels/reduce_nansum_grad_kernel.h"

#include "paddle/phi/kernels/funcs/for_range.h"
#include "paddle/phi/kernels/gpu/reduce.h"
#include "paddle/phi/kernels/gpu/reduce_amin_amax_common.h"
#include "paddle/phi/kernels/reduce_amin_grad_kernel.h"
#include "paddle/phi/kernels/reduce_max_grad_kernel.h"
#include "paddle/phi/kernels/reduce_mean_grad_kernel.h"
#include "paddle/phi/kernels/reduce_min_grad_kernel.h"
#include "paddle/phi/kernels/reduce_sum_grad_kernel.h"

#include "paddle/phi/backends/gpu/gpu_context.h"
#include "paddle/phi/kernels/elementwise_multiply_kernel.h"
#include "paddle/phi/kernels/funcs/broadcast_function.h"
#include "paddle/phi/kernels/funcs/compare_functors.h"
#include "paddle/phi/kernels/funcs/elementwise_functor.h"
#include "paddle/phi/kernels/funcs/reduce_function.h"
#include "paddle/phi/kernels/gpu/reduce_grad.h"

#include "paddle/phi/backends/all_context.h"
#include "paddle/phi/core/kernel_registry.h"

#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
#include "paddle/phi/core/distributed/nccl_comm_context.h"
#endif

namespace phi {

template <typename T, typename Context>
void ReduceSumGradKernel(const Context& dev_ctx,
                         const DenseTensor& x,
                         const DenseTensor& out_grad,
                         const IntArray& dims,
                         bool keep_dim,
                         bool reduce_all,
                         DenseTensor* x_grad) {
  reduce_all = recompute_reduce_all(x, dims, reduce_all);
  if (x_grad && x_grad->numel() == 0) {
    dev_ctx.template Alloc<T>(x_grad);
    return;
  }
  // get reduce_dim for reduce_mean_grad
  int dim_size = x.dims().size();
  std::vector<int> reduce_dims =
      funcs::details::GetReduceDim(dims.GetData(), dim_size, reduce_all);

  auto update_dims = vectorize(x.dims());
  for (auto i : reduce_dims) {
    update_dims[i] = 1;
  }

  // make new tensor
  DenseTensor new_out_grad(out_grad.dtype());
  new_out_grad.ShareDataWith(out_grad);
  new_out_grad.Resize(make_ddim(update_dims));

  // call ReduceGrad
  dev_ctx.Alloc(x_grad, x.dtype());
  using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
  phi::ReduceGrad<kps::IdentityFunctor<T, MPType>>(
      dev_ctx,
      &new_out_grad,
      x_grad,
      x.dtype(),
      kps::IdentityFunctor<T, MPType>());
}

template <typename T, typename Context>
void ReduceMeanGradKernel(const Context& dev_ctx,
                          const DenseTensor& x,
                          const DenseTensor& out_grad,
                          const IntArray& dims,
                          bool keep_dim,
                          bool reduce_all,
                          DenseTensor* x_grad) {
  if (x_grad && x_grad->numel() == 0) {
    dev_ctx.template Alloc<T>(x_grad);
    return;
  }

  reduce_all = recompute_reduce_all(x, dims, reduce_all);
  // get reduce_dim and reduce_num for reduce_mean_grad
  int dim_size = x.dims().size();
  std::vector<int> reduce_dims =
      funcs::details::GetReduceDim(dims.GetData(), dim_size, reduce_all);

  auto update_dims = vectorize(x.dims());
  int64_t reduce_num = 1;
  for (auto i : reduce_dims) {
    reduce_num *= (x.dims())[i];
    update_dims[i] = 1;
  }

  // make new tensor
  DenseTensor new_out_grad(out_grad.dtype());
  new_out_grad.ShareDataWith(out_grad);
  new_out_grad.Resize(make_ddim(update_dims));

  // call BroadcastKernel
  dev_ctx.Alloc(x_grad, x.dtype());
  std::vector<const DenseTensor*> inputs = {&new_out_grad};
  std::vector<DenseTensor*> outputs = {x_grad};

  using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
  funcs::BroadcastKernel<T>(
      dev_ctx, inputs, &outputs, kps::DivideFunctor<T, MPType>(reduce_num), 0);
}

template <typename T, typename Context>
void ReduceAMinGradKernel(const Context& dev_ctx,
                          const DenseTensor& x,
                          const DenseTensor& out,
                          const DenseTensor& out_grad,
                          const std::vector<int64_t>& dims,
                          bool keep_dim,
                          bool reduce_all,
                          DenseTensor* x_grad) {
  reduce_all = recompute_reduce_all(x, dims, reduce_all);
  ReduceCudaAMaxAMinGrad<T, Context>(
      dev_ctx, x, out, out_grad, dims, keep_dim, reduce_all, x_grad);
}

template <typename T, typename Context>
void ReduceMinGradKernel(const Context& dev_ctx,
                         const DenseTensor& x,
                         const DenseTensor& out,
                         const DenseTensor& out_grad,
                         const IntArray& dims,
                         bool keep_dim,
                         bool reduce_all,
                         DenseTensor* x_grad) {
  ReduceAMinGradKernel<T, Context>(
      dev_ctx, x, out, out_grad, dims.GetData(), keep_dim, reduce_all, x_grad);
}

template <typename T, typename Context>
void ReduceAMaxGradKernel(const Context& dev_ctx,
                          const DenseTensor& x,
                          const DenseTensor& out,
                          const DenseTensor& out_grad,
                          const std::vector<int64_t>& dims,
                          bool keep_dim,
                          bool reduce_all,
                          DenseTensor* x_grad) {
  reduce_all = recompute_reduce_all(x, dims, reduce_all);
  ReduceCudaAMaxAMinGrad<T, Context>(
      dev_ctx, x, out, out_grad, dims, keep_dim, reduce_all, x_grad);
}

template <typename T, typename Context>
void ReduceMaxGradKernel(const Context& dev_ctx,
                         const DenseTensor& x,
                         const DenseTensor& out,
                         const DenseTensor& out_grad,
                         const IntArray& dims,
                         bool keep_dim,
                         bool reduce_all,
                         DenseTensor* x_grad) {
  ReduceAMaxGradKernel<T, Context>(
      dev_ctx, x, out, out_grad, dims.GetData(), keep_dim, reduce_all, x_grad);
}

template <typename T, typename Context>
void ReduceKernel(const Context& dev_ctx,
                  const DenseTensor& x,
                  int root,
                  int reduce_type,
                  DenseTensor* out) {
  PADDLE_ENFORCE_GT(x.numel(),
                    0,
                    common::errors::InvalidArgument(
                        "Tensor need be reduced must not empty."));
#if defined(PADDLE_WITH_NCCL) || defined(PADDLE_WITH_RCCL)
  out->Resize(x.dims());
  dev_ctx.template Alloc<T>(out);

  auto comm_ctx =
      static_cast<distributed::NCCLCommContext*>(dev_ctx.GetCommContext());
  PADDLE_ENFORCE_NE(
      comm_ctx,
      nullptr,
      errors::Unavailable("NCCLCommContext is nullptr, collective op should "
                          "has ring_id attr."));
  gpuStream_t stream = dev_ctx.stream();
  PADDLE_ENFORCE_NOT_NULL(stream,
                          errors::NotFound("Should initialize NCCL firstly."));

  ncclRedOp_t red_type = ncclSum;
  switch (static_cast<ReduceType>(reduce_type)) {
    case ReduceType::kRedSum:
      red_type = ncclSum;
      break;
    case ReduceType::kRedMax:
      red_type = ncclMax;
      break;
    case ReduceType::kRedMin:
      red_type = ncclMin;
      break;
    case ReduceType::kRedProd:
      red_type = ncclProd;
      break;
#if NCCL_VERSION_CODE >= 21000
    case ReduceType::kRedAvg:
      red_type = ncclAvg;
      break;
#endif
  }
  comm_ctx->Reduce(out, x, red_type, root, stream);
#else
  PADDLE_THROW(
      errors::PreconditionNotMet("PaddlePaddle should compile with GPU."));
#endif
}

template <typename T>
struct NanMaskFunctor {
  const T* x_data;
  T* x_grad_data;

  NanMaskFunctor(const T* x_data, T* x_grad_data)
      : x_data(x_data), x_grad_data(x_grad_data) {}

  HOSTDEVICE void operator()(size_t idx) const {
    // NaN != NaN for floating-point; always false for integral types
    if (x_data[idx] != x_data[idx]) {
      x_grad_data[idx] = static_cast<T>(0);
    }
  }
};

template <typename T, typename Context>
void NansumGradKernel(const Context& dev_ctx,
                      const DenseTensor& x,
                      const DenseTensor& out_grad,
                      const IntArray& dims,
                      bool keep_dim,
                      bool reduce_all,
                      DenseTensor* x_grad) {
  reduce_all = recompute_reduce_all(x, dims, reduce_all);
  if (x_grad && x_grad->numel() == 0) {
    dev_ctx.template Alloc<T>(x_grad);
    return;
  }

  // Step 1: broadcast out_grad to x_grad shape (same as sum_grad)
  int dim_size = x.dims().size();
  std::vector<int> reduce_dims =
      funcs::details::GetReduceDim(dims.GetData(), dim_size, reduce_all);

  auto update_dims = vectorize(x.dims());
  for (auto i : reduce_dims) {
    update_dims[i] = 1;
  }

  DenseTensor new_out_grad(out_grad.dtype());
  new_out_grad.ShareDataWith(out_grad);
  new_out_grad.Resize(make_ddim(update_dims));

  dev_ctx.Alloc(x_grad, x.dtype());
  using MPType = typename phi::dtype::MPTypeTrait<T>::Type;
  phi::ReduceGrad<kps::IdentityFunctor<T, MPType>>(
      dev_ctx,
      &new_out_grad,
      x_grad,
      x.dtype(),
      kps::IdentityFunctor<T, MPType>());

  // Step 2: zero out gradient where x is NaN
  const T* x_data = x.data<T>();
  T* x_grad_data = x_grad->data<T>();
  int64_t numel = x.numel();
  phi::funcs::ForRange<Context> for_range(dev_ctx, numel);
  for_range(NanMaskFunctor<T>(x_data, x_grad_data));
}

}  // namespace phi

#if NCCL_VERSION_CODE >= 21000
PD_REGISTER_KERNEL(reduce,
                   GPU,
                   ALL_LAYOUT,
                   phi::ReduceKernel,
                   float,
                   double,
                   int,
                   bool,
                   int8_t,
                   uint8_t,
                   int64_t,
                   phi::bfloat16,
                   phi::float16) {}
#else
PD_REGISTER_KERNEL(reduce,
                   GPU,
                   ALL_LAYOUT,
                   phi::ReduceKernel,
                   float,
                   double,
                   int,
                   bool,
                   int8_t,
                   uint8_t,
                   int64_t,
                   phi::float16) {}
#endif

PD_REGISTER_KERNEL(amax_grad,
                   GPU,
                   ALL_LAYOUT,
                   phi::ReduceAMaxGradKernel,
                   float,
                   double,
                   int,
                   int64_t,
                   phi::float16,
                   phi::bfloat16) {}

PD_REGISTER_KERNEL(amin_grad,
                   GPU,
                   ALL_LAYOUT,
                   phi::ReduceAMinGradKernel,
                   float,
                   double,
                   int,
                   int64_t) {}

PD_REGISTER_KERNEL(max_grad,
                   GPU,
                   ALL_LAYOUT,
                   phi::ReduceMaxGradKernel,
                   float,
                   double,
                   int,
                   int64_t,
                   phi::float16,
                   phi::bfloat16) {}

PD_REGISTER_KERNEL(mean_grad,
                   GPU,
                   ALL_LAYOUT,
                   phi::ReduceMeanGradKernel,
                   bool,
                   float,
                   double,
                   phi::float8_e4m3fn,
                   phi::float16,
                   phi::bfloat16,
                   phi::complex64,
                   phi::complex128,
                   int,
                   int64_t) {}

PD_REGISTER_KERNEL(min_grad,
                   GPU,
                   ALL_LAYOUT,
                   phi::ReduceMinGradKernel,
                   float,
                   double,
                   int,
                   int64_t,
                   phi::float16,
                   phi::bfloat16) {}

PD_REGISTER_KERNEL(sum_grad,
                   GPU,
                   ALL_LAYOUT,
                   phi::ReduceSumGradKernel,
                   bool,
                   float,
                   double,
                   phi::float16,
                   phi::bfloat16,
                   int8_t,
                   uint8_t,
                   int16_t,
                   int,
                   int64_t,
                   phi::complex64,
                   phi::complex128) {
  kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}

PD_REGISTER_KERNEL(nansum_grad,
                   GPU,
                   ALL_LAYOUT,
                   phi::NansumGradKernel,
                   bool,
                   float,
                   double,
                   phi::float16,
                   phi::bfloat16,
                   int8_t,
                   uint8_t,
                   int16_t,
                   int,
                   int64_t,
                   phi::complex64,
                   phi::complex128) {
  kernel->OutputAt(0).SetDataType(phi::DataType::UNDEFINED);
}