[REFACTOR] Introduce and modernize FFI system (#17920)

This PR modernizes the FFI foundation of the project and introduce
a new minimal and lightweight module [tvm ffi](https://github.com/apache/tvm/tree/refactor-s3/ffi)
based on our lessons in the past few years. It implements a modern
version of the [Unified Packed and Object RFC](https://github.com/apache/tvm-rfcs/blob/main/rfcs/0097-unify-packed-and-object.md)
that unifies the packed function call and object systems.

Summary of the change:
- A dedicated clean Any/AnyView that can store strong and weak
references of items
- Function(previously PackedFunc) system built on top of the Any/AnyView
- A minimal C API that backs the overall calls. We are stabilizing the
API with a goal to bring clean, stable FFI conventions for both compiled
and registered code
- A rewrite of core python binding and generated code based on the module
- Update existing code and test cases to the new module
- Latest dlpack support
 
The new module brings many benefits thanks to the cleaner design,
to name a few:
- Any can support both POD types(int) and object types.
- Containers (e.g. Array) can now also contain Any value, e.g. now
`Array<int>` is supported, no need for boxed types
- Error handling now upgrades to object-based, allowing cleaner
traceback across languages
- Map now preserves insertion orders
- Path toward isolated stabilize minimum core ABI/API foundation module
- Type traits based design that cleanly defines how values interact
with Any system
- Automatic conversion of different types based on traits if needed 

Because FFI upgrade is at heart of the project, the change touches every
component of the system. Importantly, this is an upgrade of the ABI so the
change is not backward compatible.  The code compiled under the old
FFI won't work under the new one. We did provide example ABI translation
(e.g. LegacyTVMArgValueToFFIAny) functions for compatibility. 
The PR tries to leave files in their old places while creating redirections.
The goal is to have the first milestone landed and infrastructure in place,
so we can do further refactors to complete features and cleanup legacy code
as trackable PRs. As of now, python binding and compiled code are under the
new convention while RPC and some  other bindings still relies on legacy ABI
translation. We will work on upgrades in the coming PRs, including areas such
as reflection, phasing out legacy redirections etc.

Author: tqchen

.github/workflows/main.yml

@@ -49,22 +49,22 @@ jobs:
         run: >-
           conda build --output-folder=conda/pkg  conda/recipe &&
           conda install tvm -c ./conda/pkg
-      - name: Build iOS RPC
-        run: |
-          IOS_VERSION="14.0"
-          CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
-                       -DCMAKE_SYSTEM_NAME=iOS \
-                       -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
-                       -DCMAKE_OSX_SYSROOT=iphonesimulator \
-                       -DCMAKE_OSX_ARCHITECTURES=x86_64 \
-                       -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
-                       -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
-                       -DUSE_IOS_RPC=ON"
-
-          mkdir build-ios-simulator
-          cd build-ios-simulator
-          cmake .. ${CMAKE_FLAGS}
-          cmake --build . --target ios_rpc
+#      - name: Build iOS RPC
+#        run: |
+#          IOS_VERSION="14.0"
+#          CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
+#                       -DCMAKE_SYSTEM_NAME=iOS \
+#                       -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
+#                       -DCMAKE_OSX_SYSROOT=iphonesimulator \
+#                       -DCMAKE_OSX_ARCHITECTURES=x86_64 \
+#                       -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
+#                       -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
+#                       -DUSE_IOS_RPC=ON"
+#
+#          mkdir build-ios-simulator
+#          cd build-ios-simulator
+#          cmake .. ${CMAKE_FLAGS}
+#          cmake --build . --target ios_rpc
       - name: Test
         shell: bash -l {0}
         run: >-
@@ -108,74 +108,39 @@ jobs:
         run: >-
           python -m pytest -v tests/python/all-platform-minimal-test
 
-  # Disabled due to https://github.com/apache/tvm/issues/13950
-  # Windows-Static-Runtime:
-  #   if: ${{ github.repository == 'apache/tvm' }}
-  #   runs-on: windows-2019
-  #   steps:
-  #     - uses: actions/checkout@v2
-  #       with:
-  #         submodules: 'recursive'
-  #     - name: Set up environment
-  #       uses: ./.github/actions/setup
-  #     - name: Build static TVM runtime
-  #       shell: bash -l {0}
-  #       run: |
-  #         tests/scripts/task_config_build_static.sh build
-  #         cd build
-  #         cmake .. -A x64 -DCMAKE_CONFIGURATION_TYPES="Release"
-  #         cmake --build . --config Release --target runtime
-
-  Linux-Static-Runtime:
-    if: ${{ github.repository == 'apache/tvm' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-      - name: Set up environment
-        uses: ./.github/actions/setup
-      - name: Build static TVM runtime
-        shell: bash -l {0}
-        run: |
-          tests/scripts/task_config_build_static.sh build
-          cd build
-          cmake ..
-          cmake --build . --config Release --target runtime
-
-  Android:
-    if: ${{ github.repository == 'apache/tvm' }}
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-      - name: Set up environment
-        uses: ./.github/actions/setup
-      - name: Set up java
-        uses: actions/setup-java@v3
-        with:
-          distribution: 'zulu'
-          java-version: '11'
-      - name: Build TVM
-        shell: bash -l {0}
-        run: |
-          mkdir build
-          cd build
-          ../tests/scripts/task_config_build_jvm.sh .
-          cmake ..
-          make
-      - name: Build TVM4J
-        run: |
-          make jvmpkg
-      - name: Build android_rpc
-        working-directory: apps/android_rpc
-        run: |
-          set -eux
-          export PATH="${ANDROID_NDK_LATEST_HOME}:$PATH"
-          gradle clean build
-      - name: Upload android_rpc APK
-        uses: actions/upload-artifact@v4
-        with:
-          name: android_rpc-debug.apk
-          path: ./apps/android_rpc/app/build/outputs/apk/debug/app-debug.apk
+ # Android:
+ #   if: ${{ github.repository == 'apache/tvm' }}
+ #   runs-on: ubuntu-22.04
+ #   steps:
+ #     - uses: actions/checkout@v2
+ #       with:
+ #         submodules: 'recursive'
+ #     - name: Set up environment
+ #       uses: ./.github/actions/setup
+ #     - name: Set up java
+ #       uses: actions/setup-java@v3
+ #       with:
+ #         distribution: 'zulu'
+ #         java-version: '11'
+ #     - name: Build TVM
+ #       shell: bash -l {0}
+ #       run: |
+ #         mkdir build
+ #         cd build
+ #         ../tests/scripts/task_config_build_jvm.sh .
+ #         cmake ..
+ #         make
+ #     - name: Build TVM4J
+ #       run: |
+ #         make jvmpkg
+ #     - name: Build android_rpc
+ #       working-directory: apps/android_rpc
+ #       run: |
+ #         set -eux
+ #         export PATH="${ANDROID_NDK_LATEST_HOME}:$PATH"
+ #         gradle clean build
+ #     - name: Upload android_rpc APK
+ #       uses: actions/upload-artifact@v4
+ #       with:
+ #         name: android_rpc-debug.apk
+ #         path: ./apps/android_rpc/app/build/outputs/apk/debug/app-debug.apk

.gitmodules

@@ -31,3 +31,6 @@
 [submodule "3rdparty/zlib"]
 	path = 3rdparty/zlib
 	url = https://github.com/madler/zlib.git
+[submodule "ffi/3rdparty/dlpack"]
+	path = ffi/3rdparty/dlpack
+	url = https://github.com/dmlc/dlpack.git

3rdparty/cutlass_fpA_intB_gemm

@@ -159,7 +159,6 @@ if(MSVC)
   add_definitions(-D_SCL_SECURE_NO_WARNINGS)
   add_definitions(-D_ENABLE_EXTENDED_ALIGNED_STORAGE)
   add_definitions(-DNOMINMAX)
-
   # regeneration does not work well with msbuild custom rules.
   set(CMAKE_SUPPRESS_REGENERATION ON)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc")
@@ -496,6 +495,9 @@ list(REMOVE_ITEM COMPILER_SRCS ${LIBINFO_FILE})
 add_library(tvm_objs OBJECT ${COMPILER_SRCS})
 add_library(tvm_runtime_objs OBJECT ${RUNTIME_SRCS})
 add_library(tvm_libinfo_objs OBJECT ${LIBINFO_FILE})
+target_link_libraries(tvm_objs PUBLIC tvm_ffi_header)
+target_link_libraries(tvm_runtime_objs PUBLIC tvm_ffi_header)
+target_link_libraries(tvm_libinfo_objs PUBLIC tvm_ffi_header)
 
 include(GNUInstallDirs)
 if(NOT BUILD_DUMMY_LIBTVM)
@@ -567,6 +569,9 @@ if(USE_IOS_RPC)
   add_subdirectory("apps/ios_rpc")
 endif()
 
+add_subdirectory(ffi)
+
+
 if(TVM_DEBUG_WITH_ABI_CHANGE)
   message(STATUS "Building with debug code that may cause ABI changes...")
   target_compile_definitions(tvm_objs PRIVATE "TVM_DEBUG_WITH_ABI_CHANGE")
@@ -602,6 +607,10 @@ endif()
 target_link_libraries(tvm PRIVATE ${TVM_RUNTIME_LINKER_LIBS})
 target_link_libraries(tvm_runtime PRIVATE ${TVM_RUNTIME_LINKER_LIBS})
 
+target_link_libraries(tvm PUBLIC tvm_ffi_objs)
+target_link_libraries(tvm_runtime PUBLIC tvm_ffi_objs)
+
+
 if(BUILD_FOR_HEXAGON AND DEFINED USE_HEXAGON_GTEST AND EXISTS ${USE_HEXAGON_GTEST})
   include(FetchContent)
   FetchContent_Declare(googletest SOURCE_DIR "${USE_HEXAGON_GTEST}")
@@ -639,6 +648,7 @@ if (HIDE_PRIVATE_SYMBOLS AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   target_link_libraries(tvm_runtime PRIVATE ${HIDE_SYMBOLS_LINKER_FLAGS})
   target_compile_definitions(tvm_allvisible PUBLIC $<TARGET_PROPERTY:tvm,INTERFACE_COMPILE_DEFINITONS>)
   target_compile_definitions(tvm_allvisible PRIVATE $<TARGET_PROPERTY:tvm,COMPILE_DEFINITONS>)
+  target_link_libraries(tvm_allvisible PUBLIC tvm_ffi_objs)
 endif()
 
 # Create the `cpptest` target if we can find GTest.  If not, we create dummy
@@ -696,10 +706,10 @@ if(NOT DEFINED ENV{CONDA_BUILD})
 endif()
 
 # Installation rules
-install(TARGETS tvm EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
-install(TARGETS tvm_runtime EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
+install(TARGETS tvm DESTINATION lib${LIB_SUFFIX})
+install(TARGETS tvm_runtime DESTINATION lib${LIB_SUFFIX})
 if(BUILD_FOR_HEXAGON AND DEFINED USE_HEXAGON_GTEST AND EXISTS ${USE_HEXAGON_GTEST})
-  install(TARGETS gtest EXPORT ${PROJECT_NAME}Targets DESTINATION lib${LIB_SUFFIX})
+  install(TARGETS gtest DESTINATION lib${LIB_SUFFIX})
 endif()
 
 if (INSTALL_DEV)
@@ -734,9 +744,9 @@ string(APPEND PROJECT_CONFIG_CONTENT
        "include(\"\${CMAKE_CURRENT_LIST_DIR}/${PROJECT_NAME}Targets.cmake\")")
 file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/temp_config_file.cmake" ${PROJECT_CONFIG_CONTENT})
 
-install(EXPORT ${PROJECT_NAME}Targets
-  NAMESPACE ${PROJECT_NAME}::
-  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
+# install(EXPORT ${PROJECT_NAME}Targets
+#   NAMESPACE ${PROJECT_NAME}::
+#   DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME})
 
 # Create config for find_package()
 configure_package_config_file(
@@ -750,9 +760,9 @@ install(
 
 # More target definitions
 if(MSVC)
-  target_compile_definitions(tvm_objs PRIVATE -DTVM_EXPORTS)
-  target_compile_definitions(tvm_libinfo_objs PRIVATE -DTVM_EXPORTS)
-  target_compile_definitions(tvm_runtime_objs PRIVATE -DTVM_EXPORTS)
+  target_compile_definitions(tvm_objs PRIVATE -DTVM_EXPORTS -DTVM_FFI_EXPORTS)
+  target_compile_definitions(tvm_libinfo_objs PRIVATE -DTVM_EXPORTS -DTVM_FFI_EXPORTS)
+  target_compile_definitions(tvm_runtime_objs PRIVATE -DTVM_EXPORTS -DTVM_FFI_EXPORTS)
 endif()
 
 set(TVM_IS_DEBUG_BUILD OFF)
@@ -774,17 +784,8 @@ if(TVM_IS_DEBUG_BUILD)
   endif()
 endif()
 
-# Run dsymutil to generate debugging symbols for backtraces
-if(APPLE AND TVM_IS_DEBUG_BUILD)
-  find_program(DSYMUTIL dsymutil)
-  mark_as_advanced(DSYMUTIL)
-  add_custom_command(TARGET tvm
-      POST_BUILD
-      COMMAND ${DSYMUTIL} ARGS $<TARGET_FILE:tvm>
-      COMMENT "Running dsymutil"
-      VERBATIM
-		  )
-endif()
+add_dsymutil(tvm)
+add_dsymutil(tvm_runtime)
 
 if(BUILD_FOR_HEXAGON)
   # Wrap pthread_create to allow setting custom stack size.

CMakeLists.txt

@@ -37,7 +37,8 @@ LOCAL_SRC_FILES := org_apache_tvm_native_c_api.cc
 LOCAL_LDFLAGS := -L$(SYSROOT)/usr/lib/ -llog
 
 LOCAL_C_INCLUDES := $(ROOT_PATH)/include \
-                    $(ROOT_PATH)/3rdparty/dlpack/include \
+                    $(ROOT_PATH)/ffi/include \
+                    $(ROOT_PATH)/ffi/3rdparty/dlpack/include \
                     $(ROOT_PATH)/3rdparty/dmlc-core/include \
                     $(ROOT_PATH)/3rdparty/OpenCL-Headers
 

apps/android_rpc/app/src/main/jni/Android.mk

@@ -32,7 +32,16 @@
  * Android logcat.
  */
 #define TVM_LOG_CUSTOMIZE 1
+#define TVM_FFI_USE_LIBBACKTRACE 0
 
+#include "../ffi/src/ffi/container.cc"
+#include "../ffi/src/ffi/dtype.cc"
+#include "../ffi/src/ffi/error.cc"
+#include "../ffi/src/ffi/function.cc"
+#include "../ffi/src/ffi/ndarray.cc"
+#include "../ffi/src/ffi/object.cc"
+#include "../ffi/src/ffi/testing.cc"
+#include "../ffi/src/ffi/traceback.cc"
 #include "../src/runtime/c_runtime_api.cc"
 #include "../src/runtime/container.cc"
 #include "../src/runtime/cpu_device_api.cc"

apps/android_rpc/app/src/main/jni/tvm_runtime.h

@@ -126,46 +126,44 @@ RPCEnv::RPCEnv(const std::string& wd) {
     mkdir(base_.c_str(), 0777);
   }
 
-  TVM_REGISTER_GLOBAL("tvm.rpc.server.workpath").set_body([this](TVMArgs args, TVMRetValue* rv) {
-    *rv = this->GetPath(args[0]);
-  });
+  ffi::Function::SetGlobal(
+      "tvm.rpc.server.workpath",
+      ffi::Function::FromUnpacked([this](const std::string& path) { return this->GetPath(path); }));
 
-  TVM_REGISTER_GLOBAL("tvm.rpc.server.listdir").set_body([this](TVMArgs args, TVMRetValue* rv) {
-    std::string dir = this->GetPath(args[0]);
-    std::ostringstream os;
-    for (auto d : ListDir(dir)) {
-      os << d << ",";
-    }
-    *rv = os.str();
-  });
-
-  TVM_REGISTER_GLOBAL("tvm.rpc.server.load_module").set_body([this](TVMArgs args, TVMRetValue* rv) {
-    std::string file_name = this->GetPath(args[0]);
-    file_name = BuildSharedLibrary(file_name);
-    *rv = Module::LoadFromFile(file_name, "");
-    LOG(INFO) << "Load module from " << file_name << " ...";
-  });
+  ffi::Function::SetGlobal("tvm.rpc.server.listdir",
+                           ffi::Function::FromUnpacked([this](const std::string& path) {
+                             std::string dir = this->GetPath(path);
+                             std::ostringstream os;
+                             for (auto d : ListDir(dir)) {
+                               os << d << ",";
+                             }
+                             return os.str();
+                           }));
 
-  TVM_REGISTER_GLOBAL("tvm.rpc.server.download_linked_module")
-      .set_body([this](TVMArgs args, TVMRetValue* rv) {
-        std::string file_name = this->GetPath(args[0]);
-        file_name = BuildSharedLibrary(file_name);
-        std::string bin;
+  ffi::Function::SetGlobal("tvm.rpc.server.load_module",
+                           ffi::Function::FromUnpacked([this](const std::string& path) {
+                             std::string file_name = this->GetPath(path);
+                             file_name = BuildSharedLibrary(file_name);
+                             LOG(INFO) << "Load module from " << file_name << " ...";
+                             return Module::LoadFromFile(file_name, "");
+                           }));
 
-        std::ifstream fs(file_name, std::ios::in | std::ios::binary);
-        ICHECK(!fs.fail()) << "Cannot open " << file_name;
-        fs.seekg(0, std::ios::end);
-        size_t size = static_cast<size_t>(fs.tellg());
-        fs.seekg(0, std::ios::beg);
-        bin.resize(size);
-        fs.read(dmlc::BeginPtr(bin), size);
+  ffi::Function::SetGlobal("tvm.rpc.server.download_linked_module",
+                           ffi::Function::FromUnpacked([this](const std::string& path) {
+                             std::string file_name = this->GetPath(path);
+                             file_name = BuildSharedLibrary(file_name);
+                             std::string bin;
 
-        TVMByteArray binarr;
-        binarr.data = bin.data();
-        binarr.size = bin.length();
-        *rv = binarr;
-        LOG(INFO) << "Send linked module " << file_name << " to client";
-      });
+                             std::ifstream fs(file_name, std::ios::in | std::ios::binary);
+                             ICHECK(!fs.fail()) << "Cannot open " << file_name;
+                             fs.seekg(0, std::ios::end);
+                             size_t size = static_cast<size_t>(fs.tellg());
+                             fs.seekg(0, std::ios::beg);
+                             bin.resize(size);
+                             fs.read(dmlc::BeginPtr(bin), size);
+                             LOG(INFO) << "Send linked module " << file_name << " to client";
+                             return ffi::Bytes(bin);
+                           }));
 }
 /*!
  * \brief GetPath To get the work path from packed function

apps/cpp_rpc/rpc_env.cc

@@ -398,8 +398,6 @@ void RPCServerCreate(std::string host, int port, int port_end, std::string track
   rpc.Start();
 }
 
-TVM_REGISTER_GLOBAL("rpc.ServerCreate").set_body([](TVMArgs args, TVMRetValue* rv) {
-  RPCServerCreate(args[0], args[1], args[2], args[3], args[4], args[5], args[6], args[7]);
-});
+TVM_REGISTER_GLOBAL("rpc.ServerCreate").set_body_typed(RPCServerCreate);
 }  // namespace runtime
 }  // namespace tvm