Cherry pick round 3 (#1245)

Author: baijumeswani

.github/workflows/android-build.yml

@@ -15,6 +15,7 @@ env:
   ORT_NIGHTLY_SOURCE: "https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json"
   # ANDROID_ABI: "arm64-v8a"
   ANDROID_ABI: "x86_64"
+  ANDROID_NDK_VERSION: "27.2.12479018"  # LTS version
 jobs:
   android_x64:
     # Note: linux is the only good option for the Android emulator currently.
@@ -49,8 +50,11 @@ jobs:
           ls -l $ANDROID_HOME
 
           echo "ANDROID_SDK_ROOT=$ANDROID_SDK_ROOT"
+          "${ANDROID_SDK_ROOT}/cmdline-tools/latest/bin/sdkmanager" --install "ndk;${{ env.ANDROID_NDK_VERSION }}"
+
+          echo "ANDROID_NDK_HOME=${ANDROID_SDK_ROOT}/ndk/${{ env.ANDROID_NDK_VERSION }}" >> $GITHUB_ENV
+          echo "ANDROID_AVD_HOME=${{ runner.temp }}" >> $GITHUB_ENV
 
-          echo "ANDROID_NDK_HOME=$ANDROID_NDK_HOME"
           echo "ANDROID_NDK_LATEST_HOME=$ANDROID_NDK_LATEST_HOME"
           ls -l $ANDROID_HOME/ndk
 
@@ -76,12 +80,12 @@ jobs:
         run: |
           set -e -x
           rm -rf build
-          ./build.sh --android --android_api=27 --android_ndk_path=${ANDROID_NDK_LATEST_HOME} --config=RelWithDebInfo --android_abi=${{ env.ANDROID_ABI }} --parallel --build_java --update
+          ./build.sh --android --android_api=27 --android_ndk_path=${ANDROID_NDK_HOME} --config=RelWithDebInfo --android_abi=${{ env.ANDROID_ABI }} --parallel --build_java --update
 
       - name: Run Android build
         run: |
           set -e -x
-          ./build.sh --android --android_api=27 --android_ndk_path=${ANDROID_NDK_LATEST_HOME} --config=RelWithDebInfo --android_abi=${{ env.ANDROID_ABI }} --parallel --build_java --build
+          ./build.sh --android --android_api=27 --android_ndk_path=${ANDROID_NDK_HOME} --config=RelWithDebInfo --android_abi=${{ env.ANDROID_ABI }} --parallel --build_java --build
 
       - name: Enable KVM group perms so Android emulator can run
         run: |
@@ -92,4 +96,4 @@ jobs:
       - name: Run Android tests
         run: |
           set -e -x
-          ./build.sh --android --android_api=27 --android_ndk_path=${ANDROID_NDK_LATEST_HOME} --config=RelWithDebInfo --android_abi=${{ env.ANDROID_ABI }} --parallel --build_java --android_run_emulator --test
+          ./build.sh --android --android_api=27 --android_ndk_path=${ANDROID_NDK_HOME} --config=RelWithDebInfo --android_abi=${{ env.ANDROID_ABI }} --parallel --build_java --android_run_emulator --test

.pipelines/stages/jobs/py-packaging-job.yml

@@ -60,6 +60,9 @@ jobs:
         Python312:
           PyDotVer: '3.12'
           PyNoDotVer: '312'
+        Python313:
+          PyDotVer: '3.13'
+          PyNoDotVer: '313'
 
   timeoutInMinutes: 240
   workspace:

README.md

@@ -15,7 +15,7 @@ See documentation at https://onnxruntime.ai/docs/genai.
 
 |Support matrix|Supported now|Under development|On the roadmap|
 | -------------- | ------------- | ----------------- | -------------- |
-| Model architectures | Gemma <br/> Llama * <br/> Mistral + <br/> Phi (language + vision) <br/> Qwen <br/> Nemotron <br/> Granite <br/> AMD OLMo | Whisper | Stable diffusion |
+| Model architectures | DeepSeek <br/> Gemma <br/> Llama * <br/> Mistral + <br/> Phi (language + vision) <br/> Qwen <br/> Nemotron <br/> Granite <br/> AMD OLMo | Whisper | Stable diffusion |
 |API| Python <br/>C# <br/>C/C++ <br/> Java ^ |Objective-C||
 |Platform| Linux <br/> Windows <br/>Mac ^ <br/>Android ^  ||iOS |||
 |Architecture|x86 <br/> x64 <br/> Arm64 ~ ||||

VERSION_INFO

@@ -1 +1 @@
-0.6.0-rc4
+0.6.0

build.py

@@ -77,7 +77,7 @@ class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescript
     parser.add_argument("--build", action="store_true", help="Build.")
     parser.add_argument("--test", action="store_true", help="Run tests.")
     parser.add_argument(
-        "--clean", action="store_true", help="Run 'cmake --build --target clean' for the selected config/s."
+        "--clean", action="store_true", help="Run 'cmake --build --target clean' for the selected config."
     )
 
     parser.add_argument("--skip_tests", action="store_true", help="Skip all tests. Overrides --test.")
@@ -320,7 +320,7 @@ def _validate_cmake_args(args: argparse.Namespace):
 
 def _validate_args(args: argparse.Namespace):
     # default to all 3 stages
-    if not args.update and not args.build and not args.test:
+    if not any((args.update, args.clean, args.build, args.test)):
         args.update = True
         args.build = True
         args.test = True
@@ -639,7 +639,7 @@ def clean(args: argparse.Namespace, env: dict[str, str]):
     Clean the build output.
     """
     log.info("Cleaning targets")
-    cmd_args = [str(args.cmake), "--build", str(args.build_dir), "--config", args.config, "--target", "clean"]
+    cmd_args = [str(args.cmake_path), "--build", str(args.build_dir), "--config", args.config, "--target", "clean"]
     util.run(cmd_args, env=env)
 
 
@@ -655,6 +655,9 @@ def clean(args: argparse.Namespace, env: dict[str, str]):
     if arguments.update:
         update(arguments, environment)
 
+    if arguments.clean:
+        clean(arguments, environment)
+
     if arguments.build:
         build(arguments, environment)
 

cmake/check_cuda.cmake

@@ -46,11 +46,6 @@ if(USE_CUDA AND CMAKE_CUDA_COMPILER)
     "${GENERATORS_ROOT}/cuda/*.cuh"
   )
 
-  file(GLOB test_cuda_srcs CONFIGURE_DEPENDS
-    "${TESTS_ROOT}/*.cu"
-    "${TESTS_ROOT}/*.cuh"
-  )
-  list(APPEND test_srcs ${test_cuda_srcs})
   add_compile_definitions(USE_CUDA=1)
   include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}")
 elseif(USE_CUDA)

examples/c/CMakeLists.txt

@@ -6,6 +6,7 @@ set(CMAKE_CXX_STANDARD 20)
 option(USE_CUDA "Build with CUDA support" OFF)
 option(USE_CXX "Invoke the C++ example" ON)
 option(PHI3 "Build the Phi example" OFF)
+option(PHI3_QA "Build the Phi Q&A example without multi-turn prompting" OFF)
 option(PHI3V "Build the Phi3v example" OFF)
 option(WHISPER "Build the Whisper example" OFF)
 
@@ -53,17 +54,24 @@ function(prepare_executable executable)
   endforeach()
 endfunction()
 
+set(EXAMPLES_SOURCE_DIR ${CMAKE_SOURCE_DIR}/src)
+
 if(PHI3)
-  add_executable(phi3 ${CMAKE_SOURCE_DIR}/src/phi3.cpp)
+  add_executable(phi3 ${EXAMPLES_SOURCE_DIR}/phi3.cpp  ${EXAMPLES_SOURCE_DIR}/common.cpp)
   prepare_executable(phi3)
 endif()
 
+if(PHI3_QA)
+  add_executable(phi3_qa ${EXAMPLES_SOURCE_DIR}/phi3_qa.cpp ${EXAMPLES_SOURCE_DIR}/common.cpp)
+  prepare_executable(phi3_qa)
+endif()
+
 if(PHI3V)
-  add_executable(phi3v ${CMAKE_SOURCE_DIR}/src/phi3v.cpp)
+  add_executable(phi3v ${EXAMPLES_SOURCE_DIR}/phi3v.cpp ${EXAMPLES_SOURCE_DIR}/common.cpp)
   prepare_executable(phi3v)
 endif()
 
 if(WHISPER)
-  add_executable(whisper ${CMAKE_SOURCE_DIR}/src/whisper.cpp)
+  add_executable(whisper ${CMAKE_SOURCE_DIR}/src/whisper.cpp ${EXAMPLES_SOURCE_DIR}/common.cpp)
   prepare_executable(whisper)
 endif()

examples/c/README.md

@@ -27,9 +27,13 @@ This example uses the [Phi-3.5 mini model](https://huggingface.co/microsoft/Phi-
 You can clone this entire model repository or download individual model variants. To download individual variants, you need to install the Hugging Face CLI.
 
 ```bash
-huggingface-cli download microsoft/Phi-3.5-mini-instruct-onnx --include cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4/* --local-dir .
+huggingface-cli download microsoft/Phi-3.5-mini-instruct-onnx --include cpu_and_mobile/cpu-int4-awq-block-128-acc-level-4/* --local-dir .
 ```
 
+### Chat vs Question Answering
+
+The below examples are for scenarios where you can have chat with the model (i.e. model retains the previous conversation), but if you want to run just Question Answering (i.e. model does not remember past conversation) use `-DPHI3_QA` instead of `-DPHI3`. Also, the executable name would be `phi3_qa` instead of `phi3`.
+
 ### Windows x64 CPU
 
 #### Install the onnxruntime and onnxruntime-genai binaries
@@ -69,7 +73,7 @@ cmake --build . --config Release
 
 ```bash
 cd Release
-.\phi3.exe path_to_model
+.\phi3.exe <path_to_model> <execution_provider>
 ```
 
 ### Windows x64 DirectML
@@ -114,7 +118,7 @@ cmake --build . --config Release
 
 ```bash
 cd Release
-.\phi3.exe path_to_model
+.\phi3.exe <path_to_model> <execution_provider>
 ```
 
 ### Windows arm64 CPU
@@ -156,7 +160,7 @@ cmake --build . --config Release
 
 ```bash
 cd Release
-.\phi3.exe path_to_model
+.\phi3.exe <path_to_model> <execution_provider>
 ```
 
 ### Windows arm64 DirectML
@@ -201,7 +205,7 @@ cmake --build . --config Release
 
 ```bash
 cd Release
-.\phi3.exe path_to_model
+.\phi3.exe <path_to_model> <execution_provider>
 ```
 
 ### Linux
@@ -216,8 +220,8 @@ Change into the onnxruntime-genai directory.
    cd examples/c
    curl -L https://github.com/microsoft/onnxruntime/releases/download/v1.20.1/onnxruntime-linux-x64-1.20.1.tgz -o onnxruntime-linux-x64-1.20.1.tgz
    tar xvzf onnxruntime-linux-x64-1.20.1.tgz
-   cp onnxruntime-linux-x64-1.20.1/include/* include
-   cp onnxruntime-linux-x64-1.20.1/lib/* lib
+   cp -r onnxruntime-linux-x64-1.20.1/include/* include
+   cp -r onnxruntime-linux-x64-1.20.1/lib/* lib
    cd ../..
    ```
 
@@ -257,7 +261,8 @@ cmake --build . --config Release
 #### Run the sample
 
 ```bash
-./phi3 path_to_model
+./phi3 <path_to_model> <execution_provider>
+# Example for CPU: ./phi3 ../cpu_and_mobile/cpu-int4-awq-block-128-acc-level-4/ cpu
 ```
 
 ## Phi-3.5 vision
@@ -314,7 +319,7 @@ cmake --build . --config Release
 
 ```bash
 cd Release
-.\phi3v.exe path_to_model
+.\phi3v.exe <path_to_model> <execution_provider>
 ```
 
 ### Run on Linux
@@ -329,8 +334,8 @@ Change into the onnxruntime-genai directory.
    cd examples/c
    curl -L https://github.com/microsoft/onnxruntime/releases/download/v1.20.1/onnxruntime-linux-x64-1.20.1.tgz -o onnxruntime-linux-x64-1.20.1.tgz
    tar xvzf onnxruntime-linux-x64-1.20.1.tgz
-   cp onnxruntime-linux-x64-1.20.1/include/* include
-   cp onnxruntime-linux-x64-1.20.1/lib/* lib
+   cp -r onnxruntime-linux-x64-1.20.1/include/* include
+   cp -r onnxruntime-linux-x64-1.20.1/lib/* lib
    cd ../..
    ```
 
@@ -369,5 +374,5 @@ cmake --build . --config Release
 
 ```bash
 cd build/Release
-./phi3v path_to_model
+./phi3v <path_to_model> <execution_provider>
 ```

examples/c/src/common.cpp

@@ -0,0 +1,94 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "common.h"
+#include <cassert>
+
+void Timing::RecordStartTimestamp() {
+  assert(start_timestamp_.time_since_epoch().count() == 0);
+  start_timestamp_ = Clock::now();
+}
+
+void Timing::RecordFirstTokenTimestamp() {
+  assert(first_token_timestamp_.time_since_epoch().count() == 0);
+  first_token_timestamp_ = Clock::now();
+}
+
+void Timing::RecordEndTimestamp() {
+  assert(end_timestamp_.time_since_epoch().count() == 0);
+  end_timestamp_ = Clock::now();
+}
+
+void Timing::Log(const int prompt_tokens_length, const int new_tokens_length) {
+  assert(start_timestamp_.time_since_epoch().count() != 0);
+  assert(first_token_timestamp_.time_since_epoch().count() != 0);
+  assert(end_timestamp_.time_since_epoch().count() != 0);
+
+  Duration prompt_time = (first_token_timestamp_ - start_timestamp_);
+  Duration run_time = (end_timestamp_ - first_token_timestamp_);
+
+  const auto default_precision{std::cout.precision()};
+  std::cout << std::endl;
+  std::cout << "-------------" << std::endl;
+  std::cout << std::fixed << std::showpoint << std::setprecision(2)
+            << "Prompt length: " << prompt_tokens_length << ", New tokens: " << new_tokens_length
+            << ", Time to first: " << prompt_time.count() << "s"
+            << ", Prompt tokens per second: " << prompt_tokens_length / prompt_time.count() << " tps"
+            << ", New tokens per second: " << new_tokens_length / run_time.count() << " tps"
+            << std::setprecision(default_precision) << std::endl;
+  std::cout << "-------------" << std::endl;
+}
+
+void TerminateSession::signalHandler(int signum) {
+  std::cout << "Interrupt signal received. Terminating current session...\n";
+  std::unique_lock<std::mutex> lock(mtx);
+  stopFlag = true;
+  cv.notify_one();
+}
+
+void TerminateSession::Generator_SetTerminate_Call(OgaGenerator* generator) {
+  std::unique_lock<std::mutex> lock(mtx);
+  while (!generator->IsDone()) {
+    if (stopFlag) {
+      generator->SetRuntimeOption("terminate_session", "1");
+      stopFlag = false;
+      break;
+    }
+    // Wait for stopflag to become true or it will timeout after 1000 ms
+    auto timeout = std::chrono::milliseconds(1000);
+    cv.wait_for(lock, timeout, [this] { return stopFlag; });
+  }
+}
+
+void TerminateSession::Generator_SetTerminate_Call_C(OgaGenerator* generator) {
+  std::unique_lock<std::mutex> lock(mtx);
+  while (!OgaGenerator_IsDone(generator)) {
+    if (stopFlag) {
+      OgaGenerator_SetRuntimeOption(generator, "terminate_session", "1");
+      stopFlag = false;
+      break;
+    }
+    // Wait for stopflag to become true or it will timeout after 1000 ms
+    auto timeout = std::chrono::milliseconds(1000);
+    cv.wait_for(lock, timeout, [this] { return stopFlag; });
+  }
+}
+
+bool FileExists(const char* path) {
+  return static_cast<bool>(std::ifstream(path));
+}
+
+std::string trim(const std::string& str) {
+  const size_t first = str.find_first_not_of(' ');
+  if (std::string::npos == first) {
+    return str;
+  }
+  const size_t last = str.find_last_not_of(' ');
+  return str.substr(first, (last - first + 1));
+}
+
+void print_usage(int /*argc*/, char** argv) {
+  std::cerr << "usage: " << argv[0] << std::endl;
+  std::cerr << "model_path = " << argv[1] << std::endl;
+  std::cerr << "execution_provider = " << argv[2] << std::endl;
+}