feat: update v1.1 codes

Author: byshiue

.gitignore

@@ -6,4 +6,4 @@
 compile_commands.json
 ../all_models/fastertransformer/1/*
 *.pyc
-*.bin
+*.bin

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -36,15 +36,18 @@ option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON)
 set(TRITON_PYTORCH_INCLUDE_PATHS "" CACHE PATH "Paths to Torch includes")
 set(TRITON_PYTORCH_LIB_PATHS "" CACHE PATH "Paths to Torch libraries")
 
-set(TRITON_BACKEND_REPO_TAG "r21.02" CACHE STRING "Tag for triton-inference-server/backend repo")
-set(TRITON_CORE_REPO_TAG "r21.02" CACHE STRING "Tag for triton-inference-server/core repo")
-set(TRITON_COMMON_REPO_TAG "r21.02" CACHE STRING "Tag for triton-inference-server/common repo")
+set(TRITON_BACKEND_REPO_TAG "r22.03" CACHE STRING "Tag for triton-inference-server/backend repo")
+set(TRITON_CORE_REPO_TAG "r22.03" CACHE STRING "Tag for triton-inference-server/core repo")
+set(TRITON_COMMON_REPO_TAG "r22.03" CACHE STRING "Tag for triton-inference-server/common repo")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release)
 endif()
 
-set(BUILD_GPT "ON")
+set(BUILD_MULTI_GPU "ON")
+message("-- Enable BUILD_MULTI_GPU")
+set(USE_TRITONSERVER_DATATYPE "ON")
+message("-- Enable USE_TRITONSERVER_DATATYPE")
 
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
 
@@ -63,6 +66,11 @@ if (${CUDA_VERSION} GREATER_EQUAL 11.0)
   add_definitions("-DCUDA11_MODE")
 endif()
 
+if(${CUDA_VERSION_MAJOR} VERSION_GREATER_EQUAL "11")
+  add_definitions("-DENABLE_BF16")
+  message("CUDA_VERSION ${CUDA_VERSION_MAJOR} is greater or equal than 11, enable -DENABLE_BF16 flag")
+endif()
+
 #
 # Dependencies
 #
@@ -91,8 +99,8 @@ FetchContent_Declare(
 )
 FetchContent_Declare(
   repo-ft
-  GIT_REPOSITORY https://github.com/NVIDIA/FasterTransformer
-  GIT_TAG dev/v5.0_beta
+  GIT_REPOSITORY https://github.com/NVIDIA/FasterTransformer.git
+  GIT_TAG main
   GIT_SHALLOW ON
 )
 FetchContent_MakeAvailable(repo-common repo-core repo-backend repo-ft)
@@ -132,17 +140,17 @@ set(CUDA_PATH ${CUDA_TOOLKIT_ROOT_DIR})
 target_compile_definitions(triton-fastertransformer-backend
   PUBLIC
   USE_TRITONSERVER_DATATYPE
-  BUILD_GPT)
+  BUILD_MULTI_GPU)
 
 target_include_directories(
   triton-fastertransformer-backend
   PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/src
-#  ${CMAKE_CURRENT_SOURCE_DIR}/test
   ${TRITON_PYTORCH_INCLUDE_PATHS}
   ${Python3_INCLUDE_DIRS}
   ${MPI_INCLUDE_PATH}
   ${repo-ft_SOURCE_DIR}
+  ${repo-core_SOURCE_DIR}/include
   )
 
 target_link_directories(

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
+Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions

README.md

@@ -10,6 +10,71 @@ input [
   {
     name: "INPUT_1"
     data_type: TYPE_UINT32
+    dims: [ -1 ]
+  },
+  {
+   name: "INPUT_2"
+   data_type: TYPE_STRING
+   dims: [ -1 ]
+  },
+  {
+   name: "INPUT_3"
+   data_type: TYPE_STRING
+   dims: [ -1 ]
+  },
+  {
+    name: "runtime_top_k"
+    data_type: TYPE_UINT32
+    dims: [ 1 ]
+  },
+  {
+    name: "runtime_top_p"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  },
+  {
+    name: "beam_search_diversity_rate"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  },
+  {
+    name: "temperature"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  },
+  {
+    name: "len_penalty"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  },
+  {
+    name: "repetition_penalty"
+    data_type: TYPE_FP32
+    dims: [ 1 ]
+  },
+  {
+    name: "random_seed"
+    data_type: TYPE_INT32
+    dims: [ 1 ]
+  },
+  {
+    name: "is_return_log_probs"
+    data_type: TYPE_BOOL
+    dims: [ 1 ]
+  },
+  {
+    name: "beam_width"
+    data_type: TYPE_UINT32
+    dims: [ 1 ]
+  },
+  {
+    name: "start_id"
+    data_type: TYPE_UINT32
+    dims: [ 1 ]
+  },
+  {
+    name: "end_id"
+    data_type: TYPE_UINT32
     dims: [ 1 ]
   }
 ]
@@ -18,6 +83,21 @@ output [
     name: "OUTPUT_0"
     data_type: TYPE_STRING
     dims: [ -1, -1 ]
+  },
+  {
+    name: "sequence_length"
+    data_type: TYPE_UINT32
+    dims: [ -1 ]
+  },
+  {
+    name: "cum_log_probs"
+    data_type: TYPE_FP32
+    dims: [ -1 ]
+  },
+  {
+    name: "output_log_probs"
+    data_type: TYPE_FP32
+    dims: [ -1, -1 ]
   }
 ]
 ensemble_scheduling {
@@ -33,10 +113,26 @@ ensemble_scheduling {
         key: "REQUEST_OUTPUT_LEN"
         value: "INPUT_1"
       }
+      input_map {
+        key: "BAD_WORDS_DICT"
+        value: "INPUT_2"
+      }
+      input_map {
+        key: "STOP_WORDS_DICT"
+        value: "INPUT_3"
+      }
       output_map {
         key: "INPUT_ID"
         value: "_INPUT_ID"
       }
+      output_map {
+        key: "BAD_WORDS_IDS"
+        value: "_BAD_WORDS_IDS"
+      }
+      output_map {
+        key: "STOP_WORDS_IDS"
+        value: "_STOP_WORDS_IDS"
+      }
       output_map {
         key: "REQUEST_INPUT_LEN"
         value: "_REQUEST_INPUT_LEN"
@@ -50,21 +146,85 @@ ensemble_scheduling {
       model_name: "fastertransformer"
       model_version: -1
       input_map {
-        key: "INPUT_ID"
+        key: "input_ids"
         value: "_INPUT_ID"
       }
       input_map {
-        key: "REQUEST_INPUT_LEN"
+        key: "input_lengths"
         value: "_REQUEST_INPUT_LEN"
       }
       input_map {
-        key: "REQUEST_OUTPUT_LEN"
+        key: "request_output_len"
         value: "_REQUEST_OUTPUT_LEN"
       }
+      input_map {
+          key: "runtime_top_k"
+          value: "runtime_top_k"
+      }
+      input_map {
+          key: "runtime_top_p"
+          value: "runtime_top_p"
+      }
+      input_map {
+          key: "beam_search_diversity_rate"
+          value: "beam_search_diversity_rate"
+      }
+      input_map {
+          key: "temperature"
+          value: "temperature"
+      }
+      input_map {
+          key: "len_penalty"
+          value: "len_penalty"
+      }
+      input_map {
+          key: "repetition_penalty"
+          value: "repetition_penalty"
+      }
+      input_map {
+          key: "random_seed"
+          value: "random_seed"
+      }
+      input_map {
+          key: "is_return_log_probs"
+          value: "is_return_log_probs"
+      }
+      input_map {
+          key: "beam_width"
+          value: "beam_width"
+      }
+      input_map {
+          key: "start_id"
+          value: "start_id"
+      }
+      input_map {
+          key: "end_id"
+          value: "end_id"
+      }
+      input_map {
+          key: "stop_words_list"
+          value: "_STOP_WORDS_IDS"
+      }
+      input_map {
+          key: "bad_words_list"
+          value: "_BAD_WORDS_IDS"
+      }
       output_map {
-        key: "OUTPUT0"
+        key: "output_ids"
         value: "_TOKENS_BATCH"
       }
+      output_map {
+        key: "sequence_length"
+        value: "sequence_length"
+      }
+      output_map {
+        key: "cum_log_probs"
+        value: "cum_log_probs"
+      }
+      output_map {
+        key: "output_log_probs"
+        value: "output_log_probs"
+      }
     },
     {
       model_name: "postprocessing"