fix(cudf): Fix complex data type name in format conversion and add tests(Part1) (facebookincubator#16818)

Summary:
Fix facebookincubator#16786
And map is not supported.

Pull Request resolved: facebookincubator#16818

Reviewed By: bikramSingh91

Differential Revision: D98168787

Pulled By: kgpai

fbshipit-source-id: 9b35087baef05b5790d462fe08c286a4fbea40c8

Author: jinchengchenghh

velox/experimental/cudf/connectors/hive/CudfHiveDataSource.cpp

@@ -335,11 +335,7 @@ std::optional<RowVectorPtr> CudfHiveDataSource::next(
       ? std::make_shared<CudfVector>(
             pool_, outputType_, nRows, std::move(cudfTable), stream_)
       : with_arrow::toVeloxColumn(
-            cudfTable->view(),
-            pool_,
-            outputType_->names(),
-            stream_,
-            get_temp_mr());
+            cudfTable->view(), pool_, outputType_, stream_, get_temp_mr());
   stream_.synchronize();
 
   // Check if conversion yielded a nullptr

velox/experimental/cudf/exec/VeloxCudfInterop.cpp

@@ -246,6 +246,38 @@ getMetadata(Iterator begin, Iterator end, const std::string& namePrefix) {
   return metadata;
 }
 
+// Recursively generate metadata using exact names from Velox RowType.
+cudf::column_metadata getMetadataWithName(
+    const facebook::velox::TypePtr& type,
+    const std::string& name) {
+  cudf::column_metadata meta(name);
+  if (type->kind() == facebook::velox::TypeKind::ROW) {
+    auto rowType =
+        std::dynamic_pointer_cast<const facebook::velox::RowType>(type);
+    for (size_t i = 0; i < rowType->size(); ++i) {
+      meta.children_meta.push_back(
+          getMetadataWithName(rowType->childAt(i), rowType->nameOf(i)));
+    }
+  } else if (type->kind() == facebook::velox::TypeKind::ARRAY) {
+    // cudf::lists_column_view::child_column_index is 1, the first metadata is
+    // offsets
+    meta.children_meta.emplace_back(cudf::column_metadata(name + "_offsets"));
+    meta.children_meta.push_back(
+        getMetadataWithName(type->childAt(0), "element"));
+  }
+  return meta;
+}
+
+std::vector<cudf::column_metadata> getMetadataWithName(
+    const RowTypePtr& rowType) {
+  std::vector<cudf::column_metadata> metadata;
+  for (size_t i = 0; i < rowType->size(); ++i) {
+    metadata.push_back(
+        getMetadataWithName(rowType->childAt(i), rowType->nameOf(i)));
+  }
+  return metadata;
+}
+
 } // namespace
 
 facebook::velox::RowVectorPtr toVeloxColumn(
@@ -269,18 +301,23 @@ facebook::velox::RowVectorPtr toVeloxColumn(
   return toVeloxColumn(table, pool, metadata, &outputType, stream, mr);
 }
 
+// New overload: Accepts a Velox TypePtr for recursive metadata construction.
 RowVectorPtr toVeloxColumn(
     const cudf::table_view& table,
     memory::MemoryPool* pool,
-    const std::vector<std::string>& columnNames,
+    const TypePtr& type,
     rmm::cuda_stream_view stream,
     rmm::device_async_resource_ref mr) {
-  std::vector<cudf::column_metadata> metadata;
-  for (auto name : columnNames) {
-    metadata.emplace_back(cudf::column_metadata(name));
-  }
-  return toVeloxColumn(table, pool, metadata, nullptr, stream, mr);
+  // Recursively generate metadata using Velox type names for all columns.
+  // This assumes 'type' is a RowType and its children match the cudf table
+  // columns.
+  auto rowType =
+      std::dynamic_pointer_cast<const facebook::velox::RowType>(type);
+  VELOX_CHECK_NOT_NULL(rowType);
+  auto metadata = getMetadataWithName(rowType);
+  return toVeloxColumn(table, pool, metadata, &rowType, stream, mr);
 }
 
 } // namespace with_arrow
+
 } // namespace facebook::velox::cudf_velox

velox/experimental/cudf/exec/VeloxCudfInterop.h

@@ -45,6 +45,7 @@ facebook::velox::RowVectorPtr toVeloxColumn(
     rmm::cuda_stream_view stream,
     rmm::device_async_resource_ref mr);
 
+// Accepts a Velox TypePtr for recursive metadata construction.
 facebook::velox::RowVectorPtr toVeloxColumn(
     const cudf::table_view& table,
     facebook::velox::memory::MemoryPool* pool,
@@ -56,7 +57,7 @@ facebook::velox::RowVectorPtr toVeloxColumn(
 facebook::velox::RowVectorPtr toVeloxColumn(
     const cudf::table_view& table,
     facebook::velox::memory::MemoryPool* pool,
-    const std::vector<std::string>& columnNames,
+    const TypePtr& type,
     rmm::cuda_stream_view stream,
     rmm::device_async_resource_ref mr);
 

velox/experimental/cudf/tests/CMakeLists.txt

@@ -38,6 +38,7 @@ add_executable(
 # add_executable(velox_cudf_table_write_test Main.cpp TableWriteTest.cpp)
 add_executable(velox_cudf_topn_test Main.cpp TopNTest.cpp)
 add_executable(velox_cudf_batch_concat_test Main.cpp BatchConcatTest.cpp)
+add_executable(velox_cudf_interop_test Main.cpp InteropTest.cpp)
 
 add_test(
   NAME velox_cudf_aggregation_test
@@ -130,6 +131,12 @@ add_test(
   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
 )
 
+add_test(
+  NAME velox_cudf_interop_test
+  COMMAND velox_cudf_interop_test
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
 add_test(
   NAME velox_cudf_aggregation_selection_test
   COMMAND velox_cudf_aggregation_selection_test
@@ -179,6 +186,7 @@ set_tests_properties(
   PROPERTIES LABELS cuda_driver TIMEOUT 3000
 )
 set_tests_properties(velox_cudf_batch_concat_test PROPERTIES LABELS cuda_driver TIMEOUT 3000)
+set_tests_properties(velox_cudf_interop_test PROPERTIES LABELS cuda_driver TIMEOUT 3000)
 
 target_link_libraries(
   velox_cudf_aggregation_test
@@ -376,6 +384,16 @@ target_link_libraries(
   velox_cudf_exec
 )
 
+target_link_libraries(
+  velox_cudf_interop_test
+  velox_cudf_exec
+  velox_exec
+  velox_exec_test_lib
+  velox_test_util
+  gtest
+  gtest_main
+)
+
 add_subdirectory(utils)
 
 if(${VELOX_ENABLE_SPARK_FUNCTIONS})