Merge remote-tracking branch 'one/master' into pyproject_draft

Author: tomdol

packaging/SENTENCEPIECE.tar.gz

@@ -11,6 +11,7 @@ Source1001: CPUINFO.tar.gz
 Source1002: EIGEN.tar.gz
 Source1003: GEMMLOWP.tar.gz
 Source1004: RUY.tar.gz
+Source1005: SENTENCEPIECE.tar.gz
 Source2001: EXTERNALS_FOR_ODC.tar.gz
 
 %{!?build_type:     %define build_type      Release}
@@ -96,16 +97,6 @@ Summary: GGMA generative AI framework
 
 %description ggma
 GGMA package for on-device generative AI framework
-
-%files ggma
-%manifest %{name}.manifest
-%defattr(-,root,root,-)
-%ifarch arm armv7l armv7hl aarch64 x86_64 %ix86 riscv64
-%{_libdir}/ggma/libggma_api.so
-%{_libdir}/ggma/libggma_tokenize.so
-%{_includedir}/ggma/*
-%{_libdir}/pkgconfig/ggma.pc
-%endif
 %endif # ggma_build
 
 %if %{test_build} == 1
@@ -183,6 +174,10 @@ mkdir ./externals
 tar -xf %{SOURCE2001} -C ./externals
 %endif # odc_build
 
+%if %{ggma_build} == 1
+tar -xf %{SOURCE1005} -C ./runtime/externals
+%endif # ggma_build
+
 %build
 %ifarch arm armv7l armv7hl aarch64 x86_64 %ix86 riscv64
 # nncc build
@@ -331,6 +326,19 @@ install -m 644 %{overlay_path}/lib/libloco*.so %{buildroot}%{_libdir}/nnfw/odc
 %endif # arm armv7l armv7hl aarch64 x86_64 %ix86
 %endif # odc_build
 
+%if %{ggma_build} == 1
+%files ggma
+%manifest %{name}.manifest
+%defattr(-,root,root,-)
+%ifarch arm armv7l armv7hl aarch64 x86_64 %ix86 riscv64
+%dir %{_libdir}/ggma
+%dir %{_includedir}/ggma
+%{_libdir}/ggma/lib*
+%{_includedir}/ggma/*
+%{_libdir}/pkgconfig/ggma.pc
+%endif
+%endif # ggma_build
+
 %changelog
 * Thu Mar 15 2018 Chunseok Lee <chunseok.lee@samsung.com>
 - Initial spec file for nnfw

packaging/nnfw.spec

@@ -76,10 +76,11 @@ size_t SentencePieceTokenizer::tokenize(const char *text, size_t text_len, int32
   int bos_id = _processor->bos_id();
   size_t bos_offset = 0;
 
+  // TODO: Make BOS token prepending configurable
   if (bos_id >= 0 && max_tokens > 0)
   {
-    tokens[0] = 1;  // Add BOS token
-    bos_offset = 1; // Start actual tokens from index 1
+    tokens[0] = bos_id; // Add BOS token
+    bos_offset = 1;     // Start actual tokens from index 1
   }
 
   size_t available_space = max_tokens - bos_offset;

runtime/ggma/src/tokenize/TokenizerSentencePiece.cc

@@ -29,6 +29,7 @@ option(BUILD_RUNTIME_NNAPI_TEST "Build Runtime NN API Generated Test" ON)
 option(BUILD_RUNTIME_NNFW_API_TEST "Build Runtime NNFW API Tests" ON)
 option(BUILD_TFLITE_RUN "Build tflite_run test driver" ON)
 option(BUILD_ONERT_RUN "Build onert_run test driver" ON)
+option(BUILD_GGMA_RUN "Build ggma_run test driver" ON)
 option(BUILD_ONERT_TRAIN "Build onert_train test driver" ON)
 option(BUILD_TFLITE_COMPARATOR_TEST_TOOL "Build testing tool to compare runtime result with TFLite" ON)
 option(BUILD_WITH_HDF5 "Build test tool with HDF5 library" ON)

runtime/infra/cmake/CfgOptionFlags.cmake

@@ -8,3 +8,7 @@ option(DOWNLOAD_GTEST "Download Google Test source and build Google Test" OFF)
 option(ENVVAR_ONERT_CONFIG "Use environment variable for onert configuration" OFF)
 
 option(BUILD_XNNPACK "Build XNNPACK" OFF)
+
+option(BUILD_GGMA_API "Build GGMA API for Generative AI" OFF)
+option(DOWNLOAD_SENTENCEPIECE "Download SentencePiece source" OFF)
+option(BUILD_SENTENCEPIECE "Build SentencePiece library from the source" OFF)

runtime/infra/cmake/options/options_aarch64-tizen.cmake

@@ -8,3 +8,7 @@ option(DOWNLOAD_GTEST "Download Google Test source and build Google Test" OFF)
 option(ENVVAR_ONERT_CONFIG "Use environment variable for onert configuration" OFF)
 
 option(BUILD_XNNPACK "Build XNNPACK" OFF)
+
+option(BUILD_GGMA_API "Build GGMA API for Generative AI" OFF)
+option(DOWNLOAD_SENTENCEPIECE "Download SentencePiece source" OFF)
+option(BUILD_SENTENCEPIECE "Build SentencePiece library from the source" OFF)

runtime/infra/cmake/options/options_armv7hl-tizen.cmake

@@ -9,6 +9,3 @@ option(ENVVAR_ONERT_CONFIG "Use environment variable for onert configuration" OF
 
 option(BUILD_XNNPACK "Build XNNPACK" OFF)
 
-option(BUILD_GGMA_API "Build GGMA API for Generative AI" OFF)
-option(DOWNLOAD_SENTENCEPIECE "Download SentencePiece source" OFF)
-option(BUILD_SENTENCEPIECE "Build SentencePiece library from the source" OFF)

runtime/infra/cmake/options/options_armv7l-tizen.cmake

@@ -8,6 +8,3 @@ option(ENVVAR_ONERT_CONFIG "Use environment variable for onert configuration" OF
 
 option(BUILD_XNNPACK "Build XNNPACK" OFF)
 
-option(BUILD_GGMA_API "Build GGMA API for Generative AI" OFF)
-option(DOWNLOAD_SENTENCEPIECE "Download SentencePiece source" OFF)
-option(BUILD_SENTENCEPIECE "Build SentencePiece library from the source" OFF)

runtime/infra/cmake/options/options_i686-tizen.cmake

@@ -7,3 +7,7 @@ option(DOWNLOAD_GTEST "Download Google Test source and build Google Test" OFF)
 option(ENVVAR_ONERT_CONFIG "Use environment variable for onert configuration" OFF)
 
 option(BUILD_XNNPACK "Build XNNPACK" OFF)
+
+option(BUILD_GGMA_API "Build GGMA API for Generative AI" OFF)
+option(DOWNLOAD_SENTENCEPIECE "Download SentencePiece source" OFF)
+option(BUILD_SENTENCEPIECE "Build SentencePiece library from the source" OFF)

runtime/infra/cmake/options/options_x86_64-tizen.cmake

@@ -92,6 +92,6 @@ if(Flatbuffers_FOUND)
   target_link_libraries(tensorflow-lite INTERFACE flatbuffers::flatbuffers)
 endif(Flatbuffers_FOUND)
 
-target_link_libraries(tensorflow-lite INTERFACE Threads::Threads)
+target_link_libraries(tensorflow-lite INTERFACE Threads::Threads ${CMAKE_DL_LIBS})
 
 set(TensorFlowLite_FOUND TRUE)