Merge pull request #1237 from google/abseil-switch

Abseil switch

Author: taku910

.github/workflows/cmake.yml

@@ -20,26 +20,13 @@ jobs:
       matrix:
         os: [ ubuntu-latest, macos-latest ]
         arch: [ x64, arm64 ]
-        absl_provider: [ module, internal ]
         include:
           - os: windows-latest
             arch: Win32
-            absl_provider: internal
           - os: windows-latest
             arch: x64
-            absl_provider: internal
           - os: windows-11-arm
             arch: ARM64
-            absl_provider: internal
-          - os: windows-latest
-            arch: Win32
-            absl_provider: module
-          - os: windows-latest
-            arch: x64
-            absl_provider: module
-          - os: windows-11-arm
-            arch: ARM64
-            absl_provider: module
 
     runs-on: ${{ matrix.os }}
 
@@ -51,11 +38,10 @@ jobs:
 
     - name: Config for Windows
       if: runner.os == 'Windows'
-      run: cmake -A ${{ matrix.arch }} -B build -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=build/root -DSPM_ABSL_PROVIDER=${{ matrix.absl_provider }}
-
+      run: cmake -A ${{ matrix.arch }} -B build -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_INSTALL_PREFIX=build/root
     - name: Config for Linux/MacOSX
       if: runner.os != 'Windows'
-      run: cmake -B build -DSPM_BUILD_TEST=ON -DCMAKE_INSTALL_PREFIX=build/root -DSPM_ABSL_PROVIDER=${{ matrix.absl_provider }}
+      run: cmake -B build -DSPM_BUILD_TEST=ON -DCMAKE_INSTALL_PREFIX=build/root
       env:
         CMAKE_OSX_ARCHITECTURES: arm64;x86_64
 
@@ -73,7 +59,7 @@ jobs:
     - name: Upload artifacts
       uses: actions/upload-artifact@v7
       with:
-        name: artifacts-${{ matrix.os }}-${{ matrix.arch }}-absl_${{ matrix.absl_provider }}
+        name: artifacts-${{ matrix.os }}-${{ matrix.arch }}
         path: ./build/*.7z
         overwrite: false
 

.github/workflows/cross_build.yml

@@ -21,7 +21,6 @@ jobs:
       fail-fast: false
       matrix:
         arch: [ i686, arm, aarch64, riscv64, powerpc, powerpc64, powerpc64le, s390x, sparc64, m68k, sh4, alpha ]
-        absl_provider: [ module, internal ]
 
     steps:
     - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
@@ -38,7 +37,7 @@ jobs:
       run: |
         mkdir -p build
         cd build
-        env CXX=/usr/bin/${{matrix.arch}}-linux-gnu-g++-10 CC=/usr/bin/${{matrix.arch}}-linux-gnu-gcc-10 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${{matrix.arch}}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${{matrix.arch}} -DSPM_ABSL_PROVIDER=${{ matrix.absl_provider }}
+        env CXX=/usr/bin/${{matrix.arch}}-linux-gnu-g++-10 CC=/usr/bin/${{matrix.arch}}-linux-gnu-gcc-10 cmake .. -DSPM_BUILD_TEST=ON -DSPM_ENABLE_SHARED=OFF -DCMAKE_FIND_ROOT_PATH=/usr/${{matrix.arch}}-linux-gnu -DSPM_CROSS_SYSTEM_PROCESSOR=${{matrix.arch}}
         make -j$(nproc)
 
     - name: Test on QEMU

.github/workflows/wheel.yml

@@ -38,21 +38,21 @@ jobs:
       - name: Build for Windows (Win32/x64)
         if: runner.os == 'Windows' && runner.arch != 'ARM64'
         run: |
-          cmake -A Win32 -B build_win32 -DSPM_ENABLE_SHARED=OFF -DSPM_DISABLE_EMBEDDED_DATA=ON -DSPM_ABSL_PROVIDER=module -DCMAKE_INSTALL_PREFIX=build_win32/root
+          cmake -A Win32 -B build_win32 -DSPM_ENABLE_SHARED=OFF -DSPM_DISABLE_EMBEDDED_DATA=ON -DCMAKE_INSTALL_PREFIX=build_win32/root
           cmake --build build_win32 --config Release --target install --parallel 8
-          cmake -A x64   -B build_amd64 -DSPM_ENABLE_SHARED=OFF -DSPM_DISABLE_EMBEDDED_DATA=ON -DSPM_ABSL_PROVIDER=module -DCMAKE_INSTALL_PREFIX=build_amd64/root
+          cmake -A x64   -B build_amd64 -DSPM_ENABLE_SHARED=OFF -DSPM_DISABLE_EMBEDDED_DATA=ON -DCMAKE_INSTALL_PREFIX=build_amd64/root
           cmake --build build_amd64 --config Release --target install --parallel 8
 
       - name: Build for Windows (ARM64)
         if: runner.os == 'Windows' && runner.arch == 'ARM64'
         run: |
-          cmake -A arm64 -B build_arm64 -DSPM_ENABLE_SHARED=OFF -DSPM_DISABLE_EMBEDDED_DATA=ON -DSPM_ABSL_PROVIDER=module -DCMAKE_INSTALL_PREFIX=build_arm64/root
+          cmake -A arm64 -B build_arm64 -DSPM_ENABLE_SHARED=OFF -DSPM_DISABLE_EMBEDDED_DATA=ON -DCMAKE_INSTALL_PREFIX=build_arm64/root
           cmake --build build_arm64 --config Release --target install --parallel 8
 
       - name: Build for Mac
         if: runner.os == 'macOS'
         run: |
-          cmake -B build -DSPM_ENABLE_SHARED=OFF -DSPM_DISABLE_EMBEDDED_DATA=ON -DSPM_ABSL_PROVIDER=module -DCMAKE_INSTALL_PREFIX=build/root
+          cmake -B build -DSPM_ENABLE_SHARED=OFF -DSPM_DISABLE_EMBEDDED_DATA=ON -DCMAKE_INSTALL_PREFIX=build/root
           cmake --build build --config Release --target install --parallel 8
         env:
           MACOSX_DEPLOYMENT_TARGET: 10.13

CMakeLists.txt

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.!
 
-cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
 file(STRINGS "VERSION.txt" SPM_VERSION)
 message(STATUS "VERSION: ${SPM_VERSION}")
 
@@ -36,8 +36,8 @@ option(SPM_NLCODEC_BPE
 
 set(SPM_PROTOBUF_PROVIDER "internal" CACHE STRING "Provider of protobuf library")
 set_property(CACHE SPM_PROTOBUF_PROVIDER PROPERTY STRINGS "internal" "package")
-set(SPM_ABSL_PROVIDER "internal" CACHE STRING "Provider of absl library")
-set_property(CACHE SPM_ABSL_PROVIDER PROPERTY STRINGS "internal" "module" "package")
+set(SPM_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library")
+set_property(CACHE SPM_ABSL_PROVIDER PROPERTY STRINGS "module" "package")
 
 if(NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
@@ -51,7 +51,7 @@ endif()
 
 # Includes processor name to avoid conflicts.
 set(CPACK_PACKAGE_FILE_NAME
-    "sentencepiece-${SPM_VERSION}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}-absl_${SPM_ABSL_PROVIDER}")
+    "sentencepiece-${SPM_VERSION}-${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}")
 
 if (SPM_CROSS_SYSTEM_PROCESSOR)
  set(CMAKE_SYSTEM_PROCESSOR ${SPM_CROSS_SYSTEM_PROCESSOR})
@@ -184,29 +184,28 @@ if (SPM_DISABLE_EMBEDDED_DATA)
  install(FILES ${EMBEDDED_DATA_FILES} DESTINATION ${INSTALL_DATADIR})
 endif()
 
-if (SPM_ABSL_PROVIDER STREQUAL "internal")
-  include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl)
-elseif (SPM_ABSL_PROVIDER STREQUAL "module")
+if (SPM_ABSL_PROVIDER STREQUAL "module")
   include(FetchContent)
   FetchContent_Populate(abseil-cpp
         GIT_REPOSITORY  https://github.com/abseil/abseil-cpp.git
         SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp
         GIT_PROGRESS TRUE
         GIT_TAG 20260107.1)
   add_subdirectory(third_party/abseil-cpp)
-  if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl.org)
-    file(RENAME ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl.org)
-    execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
-      ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp/absl
-      ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl)
+  if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl)
+    file(CREATE_LINK
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp/absl
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl
+         SYMBOLIC)
   endif()
 elseif (SPM_ABSL_PROVIDER STREQUAL "package")
   find_package(absl REQUIRED)
   get_target_property(ABSL_INCLUDE_DIRS absl::base INTERFACE_INCLUDE_DIRECTORIES)
-  if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl.org)
-    file(RENAME ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl.org)
-    execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
-        ${ABSL_INCLUDE_DIRS}/absl ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl)
+  if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl)
+    file(CREATE_LINK
+         ${ABSL_INCLUDE_DIRS}/absl
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/absl
+         SYMBOLIC)
   endif()
   include_directories(${ABSL_INCLUDE_DIRS})
 endif()

python/build_bundled.sh

@@ -22,7 +22,6 @@ cmake ${SRC_DIR} -B ${BUILD_DIR} \
   -DSPM_ENABLE_SHARED=OFF \
   -DSPM_DISABLE_EMBEDDED_DATA=ON \
   -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-  -DSPM_ABSL_PROVIDER=module \
   -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
   -DCMAKE_CXX_FLAGS="-fPIC -fvisibility=default -ffunction-sections -fdata-sections"
 cmake --build ${BUILD_DIR} --config Release --target install --parallel $(nproc)

python/setup.py

@@ -160,8 +160,7 @@ def build_extension(self, ext):
           '-B',
           'build',
           '-DSPM_ENABLE_SHARED=OFF',
-          '-DSPM_ABSL_PROVIDER=module',
-          '-DCMAKE_SHARED_LINKER_FLAGS="/OPT:REF /OPT:ICF /LTCG"',
+          #          '-DCMAKE_SHARED_LINKER_FLAGS="/OPT:REF /OPT:ICF /LTCG"',
           '-DCMAKE_INSTALL_PREFIX=build\\root',
       ])
       subprocess.check_call([

src/CMakeLists.txt

@@ -12,20 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.!
 
-if (SPM_ABSL_PROVIDER STREQUAL "module" OR SPM_ABSL_PROVIDER STREQUAL "package")
-  set(ABSL_FLAGS_SRCS "")
-  set(ABSL_STRINGS_SRCS "")
-  list(APPEND SPM_LIBS absl::strings)
-  list(APPEND SPM_LIBS absl::flags)
-  list(APPEND SPM_LIBS absl::flags_parse)
-  list(APPEND SPM_LIBS absl::log)
-  list(APPEND SPM_LIBS absl::check)
-  add_compile_definitions(_USE_EXTERNAL_ABSL)
-elseif (SPM_ABSL_PROVIDER STREQUAL "internal")
- set(ABSL_FLAGS_SRCS
-    ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/absl/flags/flag.cc
-    ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/absl/log/log.cc)
-endif()
+list(APPEND SPM_LIBS absl::strings)
+list(APPEND SPM_LIBS absl::flags)
+list(APPEND SPM_LIBS absl::flags_parse)
+list(APPEND SPM_LIBS absl::log)
+list(APPEND SPM_LIBS absl::check)
 
 if (SPM_PROTOBUF_PROVIDER STREQUAL "internal")
   set(SPM_PROTO_HDRS builtin_pb/sentencepiece.pb.h)
@@ -111,9 +102,7 @@ set(SPM_SRCS
   sentencepiece_processor.cc
   unigram_model.cc
   util.cc
-  word_model.cc
-  ${ABSL_STRINGS_SRCS}
-  ${ABSL_FLAGS_SRCS})
+  word_model.cc)
 
 set(SPM_TRAIN_SRCS
   ${SPM_PROTO_HDRS}
@@ -151,7 +140,6 @@ set(SPM_TEST_SRCS
   char_model_test.cc
   char_model_trainer_test.cc
   filesystem_test.cc
-  init_test.cc
   model_factory_test.cc
   model_interface_test.cc
   normalizer_test.cc

src/common.h

@@ -63,6 +63,10 @@ static constexpr uint32_t kUnicodeError = 0xFFFD;
     CHECK(_status.ok()) << _status.ToString(); \
   } while (0)
 
-#define QCHECK_OK CHECK_OK
+#define QCHECK_OK(expr)                         \
+  do {                                          \
+    const auto _status = expr;                  \
+    QCHECK(_status.ok()) << _status.ToString(); \
+  } while (0)
 
 #endif  // COMMON_H_

src/compile_charsmap_main.cc

@@ -127,11 +127,12 @@ struct BinaryBlob {
   os << kHeader;
 
   os << "#if defined(_WIN32) && !defined(__CYGWIN__)\n";
-  os << "constexpr unsigned long long int kNormalizationRules_blob_uint64_t[] = "
+  os << "constexpr unsigned long long int kNormalizationRules_blob_uint64_t[] "
+        "= "
         "{\n";
   std::vector<size_t> offset;
   os << ToHexUInt64Array(data, &offset);
-  CHECK_EQ(offset.size(), data.size());
+  QCHECK_EQ(offset.size(), data.size());
   os << "};\n\n";
   os << "const BinaryBlob kNormalizationRules_blob[] = {\n";
   for (size_t i = 0; i < data.size(); ++i) {
@@ -179,14 +180,15 @@ int main(int argc, char **argv) {
   std::vector<std::pair<std::string, std::string>> data;
   for (const auto &[name, func] : kRuleList) {
     Builder::CharsMap normalized_map;
-    CHECK_OK(func(&normalized_map));
+    QCHECK_OK(func(&normalized_map));
 
     // Write Header.
     std::string index;
-    CHECK_OK(Builder::CompileCharsMap(normalized_map, &index));
+    QCHECK_OK(Builder::CompileCharsMap(normalized_map, &index));
 
     // Write TSV file.
-    CHECK_OK(Builder::SaveCharsMap(absl::StrCat(name, ".tsv"), normalized_map));
+    QCHECK_OK(
+        Builder::SaveCharsMap(absl::StrCat(name, ".tsv"), normalized_map));
 
     // Do not make NFKD map as it is optionally created.
     if (name == "nfkd" || name == "nfd" || name == "nfc" || name == "nfkd_cf" ||
@@ -201,7 +203,7 @@ int main(int argc, char **argv) {
     constexpr char kPrecompiledHeaderFileName[] = "normalization_rule.h";
     auto output =
         sentencepiece::filesystem::NewWritableFile(kPrecompiledHeaderFileName);
-    CHECK_OK(output->status());
+    QCHECK_OK(output->status());
     output->Write(sentencepiece::MakeHeader(data));
   }