Add the function of comparing results between Paddle and Torch (#22)

Author: BingooYang

CMakeLists.txt

@@ -58,7 +58,7 @@ set(COMMAND_TO_RUN
 include_directories(${Python3_INCLUDE_DIRS})
 link_directories("${Python3_LIBRARY_DIRS}")
 
-set(COMMON_INCLUDES ${PROJECT_SOURCE_DIR}/include)
+set(COMMON_INCLUDES ${PROJECT_SOURCE_DIR}/include ${PROJECT_SOURCE_DIR}/src)
 
 enable_testing()
 include_directories(${COMMON_INCLUDES})

cmake/build.cmake

@@ -10,13 +10,16 @@ function(
   foreach(_test_file ${TEST_SRC_FILES})
     get_filename_component(_file_name ${_test_file} NAME_WE)
     set(_test_name ${BIN_PREFIX}${_file_name})
-    add_executable(${_test_name} ${_test_file} ${TEST_BASE_FILES})
+    add_executable(${_test_name} ${_test_file} ${TEST_BASE_FILES}
+                                 ${PROJECT_SOURCE_DIR}/src/file_manager.cpp)
     add_dependencies(${_test_name} "googletest.git")
     target_link_libraries(
       ${_test_name} gtest gtest_main ${CMAKE_THREAD_LIBS_INIT}
       ${DEPS_LIBRARIES} ${Python3_LIBRARIES})
     target_include_directories(${_test_name} PRIVATE ${Python3_INCLUDE_DIRS})
-    target_include_directories(${_test_name} PRIVATE ${INCLUDE_DIR})
+    target_include_directories(${_test_name} PRIVATE ${INCLUDE_DIR}
+                                                     ${PROJECT_SOURCE_DIR}/src)
+    target_include_directories(${_test_name} PRIVATE ${PROJECT_SOURCE_DIR}/src)
     message(STATUS "include dir: ${INCLUDE_DIR}")
     target_compile_definitions(${_test_name}
                                PRIVATE USE_PADDLE_API=${USE_PADDLE_API})

src/file_manager.cpp

@@ -0,0 +1,56 @@
+#include "src/file_manager.h"
+
+#include <filesystem>
+#include <iostream>
+
+namespace paddle_api_test {
+
+void FileManerger::createFile() {
+  std::unique_lock<std::shared_mutex> lock(mutex_);
+
+  std::error_code ec;
+  if (!std::filesystem::create_directories(basic_path_, ec) && ec) {
+    throw std::runtime_error("Failed to create directory: " + basic_path_ +
+                             ", error: " + ec.message());
+  }
+
+  std::string full_path = basic_path_ + file_name_;
+
+  if (std::filesystem::exists(full_path)) {
+    std::filesystem::remove(full_path);
+  }
+
+  file_stream_.open(full_path, std::ios::out | std::ios::trunc);
+  if (!file_stream_.is_open()) {
+    throw std::runtime_error("Failed to create file: " + full_path);
+  }
+}
+
+void FileManerger::writeString(const std::string& str) {
+  std::shared_lock<std::shared_mutex> lock(mutex_);
+  if (file_stream_.is_open()) {
+    file_stream_ << str;
+  } else {
+    throw std::runtime_error(
+        "File stream is not open. Call createFile() first.");
+  }
+}
+
+FileManerger& FileManerger::operator<<(const std::string& str) {
+  writeString(str);
+  return *this;
+}
+
+void FileManerger::saveFile() {
+  std::unique_lock<std::shared_mutex> lock(mutex_);
+  if (file_stream_.is_open()) {
+    file_stream_.flush();
+    file_stream_.close();
+  }
+}
+
+void FileManerger::setFileName(const std::string& value) {
+  std::unique_lock<std::shared_mutex> lock(mutex_);
+  file_name_ = value;
+}
+}  // namespace paddle_api_test

src/file_manager.h

@@ -0,0 +1,42 @@
+#pragma once
+#include <fstream>
+#include <mutex>
+#include <shared_mutex>
+#include <string>
+
+namespace paddle_api_test {
+class FileManerger {
+ public:
+  FileManerger() = default;
+  explicit FileManerger(const std::string& first) : file_name_(first) {}
+
+  void setFileName(const std::string& value);
+  void createFile();
+  void writeString(const std::string& str);
+  FileManerger& operator<<(const std::string& str);
+  void saveFile();
+
+ private:
+  mutable std::shared_mutex mutex_;
+  std::string basic_path_ = "/tmp/paddle_cpp_api_test/";
+  std::string file_name_ = "";
+  std::ofstream file_stream_;
+};
+
+class ThreadSafeParam {
+ private:
+  std::string param_;
+  mutable std::mutex mutex_;
+
+ public:
+  void set(const std::string& value) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    param_ = value;
+  }
+
+  std::string get() const {
+    std::lock_guard<std::mutex> lock(mutex_);
+    return param_;
+  }
+};
+}  // namespace paddle_api_test

src/main.cpp

@@ -1,11 +1,30 @@
+#include <cstdlib>
+#include <iostream>
+#include <mutex>
+
 #include "gtest/gtest.h"
 #if USE_PADDLE_API
 #include "paddle/extension.h"
 #endif
 
+#include "src/file_manager.h"
+
+paddle_api_test::ThreadSafeParam g_custom_param;
+
+std::string extract_filename(const std::string& path) {
+  size_t last_slash = path.find_last_of('/');
+  if (last_slash != std::string::npos) {
+    return path.substr(last_slash + 1);
+  }
+  return path;
+}
+
 int main(int argc, char** argv) {  // NOLINT
   testing::InitGoogleTest(&argc, argv);
 
+  auto exe_cmd = std::string(argv[0]);
+  g_custom_param.set(extract_filename(exe_cmd) + ".txt");
+
   int ret = RUN_ALL_TESTS();
 
   return ret;

test/TensorTest.cpp

@@ -6,9 +6,11 @@
 
 #include <vector>
 
+#include "../src/file_manager.h"
 namespace at {
 namespace test {
 
+using paddle_api_test::FileManerger;
 class TensorTest : public ::testing::Test {
  protected:
   void SetUp() override {

test/TensorTest_compare.cpp

@@ -0,0 +1,39 @@
+#include <ATen/ATen.h>
+#include <ATen/core/Tensor.h>
+#include <ATen/ops/ones.h>
+#include <gtest/gtest.h>
+#include <torch/all.h>
+
+#include <string>
+#include <vector>
+
+#include "../src/file_manager.h"
+
+extern paddle_api_test::ThreadSafeParam g_custom_param;
+
+namespace at {
+namespace test {
+
+using paddle_api_test::FileManerger;
+using paddle_api_test::ThreadSafeParam;
+class TensorTest : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    std::vector<int64_t> shape = {2, 3, 4};
+    tensor = at::ones(shape, at::kFloat);
+  }
+
+  at::Tensor tensor;
+};
+
+TEST_F(TensorTest, test) {
+  auto file_name = g_custom_param.get();
+  FileManerger file(file_name);
+  file.createFile();
+  file << std::to_string(tensor.dim()) << " ";
+  file << std::to_string(tensor.numel()) << " ";
+  file.saveFile();
+}
+
+}  // namespace test
+}  // namespace at

test/result_cmp.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -e  # 出错时退出
+
+# using guide: ./result_cmp.sh <BUILD_PATH>
+BUILD_PATH=$1
+
+PADDLE_PATH=${BUILD_PATH}/paddle/
+TORCH_PATH=${BUILD_PATH}/torch/
+RESULT_FILE_PATH="/tmp/paddle_cpp_api_test/"
+
+# 记录PADDLE_PATH下所有可执行文件到列表
+echo "Collecting and executing Paddle executables..."
+PADDLE_EXECUTABLES=()
+for test_file in ${PADDLE_PATH}/*; do
+    if [[ -x "$test_file" && -f "$test_file" ]]; then
+        filename=$(basename $test_file)
+        ${PADDLE_PATH}${filename}
+        PADDLE_EXECUTABLES+=("$filename")
+        echo "Executing Paddle test: $filename"
+        $test_file
+    fi
+done
+
+# 记录并执行TORCH_PATH下所有可执行文件
+echo "Collecting and executing Torch executables..."
+TORCH_EXECUTABLES=()
+for test_file in ${TORCH_PATH}/*; do
+    if [[ -x "$test_file" && -f "$test_file" ]]; then
+        filename=$(basename $test_file)
+        ${TORCH_PATH}${filename}
+        TORCH_EXECUTABLES+=("$filename")
+        echo "Executing Torch test: $filename"
+        $test_file
+    fi
+done
+
+# 比较结果文件
+echo "Comparing result files..."
+for ((i=0; i<${#PADDLE_EXECUTABLES[@]}; i++)); do
+    paddle_file="${RESULT_FILE_PATH}/${PADDLE_EXECUTABLES[i]}.txt"
+    torch_file="${RESULT_FILE_PATH}/${TORCH_EXECUTABLES[i]}.txt"
+
+    if [[ -f "$paddle_file" && -f "$torch_file" ]]; then
+        if diff -q "$paddle_file" "$torch_file" >/dev/null; then
+            echo "MATCH: ${PADDLE_EXECUTABLES[i]} and ${TORCH_EXECUTABLES[i]}"
+        else
+            echo "DIFFER: ${PADDLE_EXECUTABLES[i]} and ${TORCH_EXECUTABLES[i]}"
+            diff "$paddle_file" "$torch_file"
+        fi
+    else
+        echo "MISSING: ${paddle_file} or ${torch_file}"
+    fi
+done