Refactoring for new models and new kernels

.github/workflows/arm64_centos7.yml

@@ -34,7 +34,7 @@ jobs:
           - name: Test
             run: |
               ./build/test/kiwi-test
-              mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
+              mkdir eval_results && ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t knlm -o eval_results/ && ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t sbg -o eval_results/
               cp -r build /artifacts/
               cp -r eval_results /artifacts/
           - name: Benchmark

.github/workflows/centos7.yml

@@ -40,8 +40,8 @@ jobs:
           - name: Run Evaluator
             run: |
               mkdir eval_results
-              ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
-              ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
+              ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t knlm -o eval_results/
+              ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t sbg -o eval_results/
           - run: tar -zcvf arts.tgz build/*kiwi* build/test/*kiwi* eval_results/*.txt build/bindings/java/*.jar
     - name: Archive binaries
       uses: actions/upload-artifact@v4

.github/workflows/macos.yml

@@ -60,8 +60,8 @@ jobs:
     - name: Run Evaluator
       run: |
         mkdir eval_results
-        ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
-        ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
+        ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t knlm -o eval_results/
+        ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t sbg -o eval_results/
     - name: Run Benchmark
       run: |
         curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt

.github/workflows/ppc64le_centos7.yml

@@ -28,7 +28,7 @@ jobs:
           mkdir build && pushd build && cmake -DCMAKE_BUILD_TYPE=Release -DKIWI_USE_MIMALLOC=0 -DKIWI_JAVA_BINDING=1 ..
           make -j2 && popd
           ./build/test/kiwi-test
-          mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/ && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
+          mkdir eval_results && ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t knlm -o eval_results/ && ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t sbg -o eval_results/
           cp -r build /artifacts/
           cp -r eval_results /artifacts/
     - name: Archive binaries

.github/workflows/release.yml

@@ -220,7 +220,7 @@ jobs:
           - name: Test
             run: |
               ./build/test/kiwi-test
-              mkdir eval_results && ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
+              mkdir eval_results && ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -o eval_results/
           - name: Release
             run: |
               cd build

.github/workflows/ubuntu.yml

@@ -60,8 +60,8 @@ jobs:
     - name: Run Evaluator
       run: |
         mkdir eval_results
-        ./build/kiwi-evaluator -m ./models/base eval_data/*.txt -o eval_results/
-        ./build/kiwi-evaluator -m ./models/base eval_data/*.txt --sbg -o eval_results/
+        ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t knlm -o eval_results/
+        ./build/kiwi-evaluator -m ./models/base --morph eval_data/*.txt -t sbg -o eval_results/
     - name: Run Benchmark
       run: |
         curl -OL https://latina.bab2min.pe.kr/_data/kowiki1000.txt

.github/workflows/windows.yml

@@ -35,8 +35,8 @@ jobs:
     - name: Run Evaluator
       run: |
         mkdir eval_results
-        .\build\Release\kiwi-evaluator.exe -m .\models\base (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
-        .\build\Release\kiwi-evaluator.exe -m .\models\base --sbg (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
+        .\build\Release\kiwi-evaluator.exe -m .\models\base -t knlm --morph (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
+        .\build\Release\kiwi-evaluator.exe -m .\models\base -t sbg --morph (Get-ChildItem eval_data\*.txt | Select-Object -Expand FullName) -o eval_results\
     - name: Archive binaries
       uses: actions/upload-artifact@v4
       with:

.gitmodules

@@ -15,12 +15,12 @@
 [submodule "third_party/cpuinfo"]
 	path = third_party/cpuinfo
 	url = https://github.com/pytorch/cpuinfo
-[submodule "third_party/variant"]
-	path = third_party/variant
-	url = https://github.com/mapbox/variant
 [submodule "third_party/eigen"]
 	path = third_party/eigen
 	url = https://gitlab.com/libeigen/eigen
 [submodule "third_party/json"]
 	path = third_party/json
 	url = https://github.com/nlohmann/json
+[submodule "third_party/streamvbyte"]
+	path = third_party/streamvbyte
+	url = https://github.com/fast-pack/streamvbyte

CMakeLists.txt

@@ -1,8 +1,8 @@
 cmake_minimum_required(VERSION 3.12)
 
-project(kiwi VERSION 0.20.4 DESCRIPTION "Kiwi, Korean Intelligent Word Identifier")
+project(kiwi VERSION 0.21.0 DESCRIPTION "Kiwi, Korean Intelligent Word Identifier")
 
-set ( CMAKE_CXX_STANDARD 14 )
+set ( CMAKE_CXX_STANDARD 17 )
 set ( CMAKE_VERBOSE_MAKEFILE true )
 
 option(KIWI_USE_MIMALLOC  "Use mimalloc for faster memory allocation" ON)
@@ -38,17 +38,29 @@ if(NOT KIWI_CPU_ARCH)
   set(KIWI_CPU_ARCH "${KIWI_CPU_ARCH}" PARENT_SCOPE)
 endif()
 
+
+if (KIWI_USE_CPUINFO AND 
+  (MSVC OR 
+   ((CMAKE_CXX_COMPILER_ID STREQUAL "GNU" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang") AND CMAKE_CXX_COMPILER_VERSION GREATER_EQUAL 11)
+  )
+)
+  set ( AVX_VNNI_SUPPORTED ON )
+else()
+  set ( AVX_VNNI_SUPPORTED OFF )
+endif()
+
 if(APPLE)
   set(CMAKE_OSX_ARCHITECTURES "${KIWI_CPU_ARCH}")
 endif()
 
 set ( CORE_SRCS
   src/ArchUtils.cpp
   src/Combiner.cpp
+  src/CoNgramModel.cpp
+  src/Dataset.cpp
   src/Form.cpp
   src/FeatureTestor.cpp
   src/FileUtils.cpp
-  src/Dataset.cpp
   src/Joiner.cpp
   src/Kiwi.cpp
   src/KiwiBuilder.cpp
@@ -57,6 +69,7 @@ set ( CORE_SRCS
   src/PatternMatcher.cpp
   src/search.cpp
   src/ScriptType.cpp
+  src/SkipBigramModel.cpp
   src/SubstringExtractor.cpp
   src/SwTokenizer.cpp
   src/TagUtils.cpp
@@ -81,9 +94,13 @@ endif()
 include_directories( include/ )
 include_directories( third_party/tclap/include )
 include_directories( third_party/cpp-btree )
-include_directories( third_party/variant/include )
 include_directories( third_party/eigen )
 include_directories( third_party/json/include )
+include_directories( third_party/streamvbyte/include )
+add_subdirectory( third_party/streamvbyte )
+set ( STREAMVBYTE_OBJECTS 
+  $<TARGET_OBJECTS:streamvbyte>
+)
 if(KIWI_USE_CPUINFO)
   message(STATUS "Use cpuinfo")
   include_directories( third_party/cpuinfo/include )
@@ -98,9 +115,6 @@ if(KIWI_USE_CPUINFO)
   set ( ADDITIONAL_FLAGS ${ADDITIONAL_FLAGS} "-DKIWI_USE_CPUINFO" )
 
   if(MSVC)
-    target_compile_options("clog" PUBLIC
-      /MT
-    )
     target_compile_options("cpuinfo" PUBLIC
       /MT
     )
@@ -110,15 +124,18 @@ if(KIWI_USE_CPUINFO)
   endif()
 
   set ( CPUINFO_OBJECTS_STATIC 
-    $<TARGET_OBJECTS:clog>
     $<TARGET_OBJECTS:cpuinfo_internals>
   )
   set ( CPUINFO_OBJECTS_SHARED 
-    $<TARGET_OBJECTS:clog>
     $<TARGET_OBJECTS:cpuinfo>
   )
 endif()
 
+if (AVX_VNNI_SUPPORTED)
+  message(STATUS "AVX-VNNI is supported")
+  set ( ADDITIONAL_FLAGS ${ADDITIONAL_FLAGS} "-DKIWI_AVX_VNNI_SUPPORTED" )
+endif()
+
 if(MSVC)
   set ( CMAKE_C_FLAGS_DEBUG "-DDEBUG -DC_FLAGS -Zi -Od /utf-8 /bigobj" )
   set ( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}" )
@@ -143,6 +160,12 @@ else()
   set ( CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELEASE} -g3")
   set ( CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
   set ( CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELEASE}" )
+
+  if (APPLE)
+    set ( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wno-unqualified-std-cast-call" )
+    set ( CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Wno-unqualified-std-cast-call" )
+    set ( CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -Wno-unqualified-std-cast-call" )
+  endif()
 endif()
 
 if (KIWI_CPU_ARCH MATCHES "x86_64")
@@ -157,21 +180,36 @@ if (KIWI_CPU_ARCH MATCHES "x86_64")
       ${CORE_SRCS}
       src/archImpl/avx2.cpp
       src/archImpl/avx512bw.cpp
+      src/archImpl/avx512vnni.cpp
     )
+    # If AVX-VNNI is supported (MSVC, GCC 11+ or Clang 11+)
+    if (AVX_VNNI_SUPPORTED)
+      set( CORE_SRCS 
+        ${CORE_SRCS}
+        src/archImpl/avx_vnni.cpp
+      )
+    endif()
   endif()
+
   if(MSVC)
     set_source_files_properties(src/archImpl/sse2.cpp PROPERTIES COMPILE_FLAGS "/arch:SSE2")
     set_source_files_properties(src/archImpl/sse4_1.cpp PROPERTIES COMPILE_FLAGS "/arch:SSE2")
     if (KIWI_USE_CPUINFO)
       set_source_files_properties(src/archImpl/avx2.cpp PROPERTIES COMPILE_FLAGS "/arch:AVX2")
+      set_source_files_properties(src/archImpl/avx_vnni.cpp PROPERTIES COMPILE_FLAGS "/arch:AVX2")
       set_source_files_properties(src/archImpl/avx512bw.cpp PROPERTIES COMPILE_FLAGS "/arch:AVX512")
+      set_source_files_properties(src/archImpl/avx512vnni.cpp PROPERTIES COMPILE_FLAGS "/arch:AVX512")
     endif()
   else()
     set_source_files_properties(src/archImpl/sse2.cpp PROPERTIES COMPILE_FLAGS "-msse2")
     set_source_files_properties(src/archImpl/sse4_1.cpp PROPERTIES COMPILE_FLAGS "-msse2 -msse4.1")
     if (KIWI_USE_CPUINFO)
       set_source_files_properties(src/archImpl/avx2.cpp PROPERTIES COMPILE_FLAGS "-mavx -mavx2 -mfma")
-      set_source_files_properties(src/archImpl/avx512bw.cpp PROPERTIES COMPILE_FLAGS "-mavx512f -mavx512bw")
+      set_source_files_properties(src/archImpl/avx512bw.cpp PROPERTIES COMPILE_FLAGS "-mavx -mavx2 -mfma -mavx512f -mavx512vl -mavx512dq -mavx512bw")
+      set_source_files_properties(src/archImpl/avx512vnni.cpp PROPERTIES COMPILE_FLAGS "-mavx -mavx2 -mfma -mavx512f -mavx512vl -mavx512dq -mavx512bw -mavx512vnni")
+      if (AVX_VNNI_SUPPORTED)
+        set_source_files_properties(src/archImpl/avx_vnni.cpp PROPERTIES COMPILE_FLAGS "-mavx -mavx2 -mfma -mavxvnni")
+      endif()
     endif()
   endif()
 elseif (KIWI_CPU_ARCH MATCHES "arm64")
@@ -191,12 +229,14 @@ add_library( "${PROJECT_NAME}_static" STATIC
   ${CORE_SRCS}
   src/capi/kiwi_c.cpp
   ${CPUINFO_OBJECTS_STATIC}
+  ${STREAMVBYTE_OBJECTS}
 )
 
 add_library( "${PROJECT_NAME}" SHARED
   ${CORE_SRCS}
   src/capi/kiwi_c.cpp
   ${CPUINFO_OBJECTS_SHARED}
+  ${STREAMVBYTE_OBJECTS}
 )
 
 # Install the kiwi library as well as header files to (`include/kiwi` directory)
@@ -265,6 +305,9 @@ if(MSVC)
     target_compile_options("${PROJECT_NAME}_static" PUBLIC
       /MT
     )
+    target_compile_options("streamvbyte" PUBLIC
+      /MT
+    )
   endif()
 
   target_compile_options("${PROJECT_NAME}" PUBLIC

bindings/java/CMakeLists.txt

@@ -9,8 +9,8 @@ set(CMAKE_JAVA_COMPILE_FLAGS -source 8 -target 8 -encoding utf-8)
 set(pkg_name "KiwiJava-${PROJECT_VERSION}")
 add_library (${pkg_name} SHARED kiwi_java.cpp
     $<TARGET_OBJECTS:${PROJECT_NAME}>
-    $<TARGET_OBJECTS:clog>
     $<TARGET_OBJECTS:cpuinfo>
+    $<TARGET_OBJECTS:streamvbyte>
 )
 if(UNIX AND NOT APPLE)
     target_link_libraries( ${pkg_name}

bindings/java/JniUtils.hpp

@@ -6,6 +6,7 @@
 #include <vector>
 #include <optional>
 #include <iostream>
+#include <cstdint>
 
 #include <jni.h>
 

bindings/java/kiwi_java.cpp

@@ -95,6 +95,23 @@ namespace jni
 		}
 	};
 
+	template<>
+	struct ValueBuilder<kiwi::ModelType> : public ValueBuilder<uint32_t>
+	{
+		using CppType = kiwi::ModelType;
+		using JniType = jint;
+
+		CppType fromJava(JNIEnv* env, JniType v)
+		{
+			return (CppType)v;
+		}
+
+		JniType toJava(JNIEnv* env, CppType v)
+		{
+			return (JniType)v;
+		}
+	};
+
 	template<>
 	struct ValueBuilder<kiwi::Match> : public ValueBuilder<uint32_t>
 	{
@@ -564,7 +581,7 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved)
 			.template method<&JTypoTransformer::scaleCost>("_scaleCost"),
 
 		jni::define<JKiwiBuilder>()
-			.template ctor<std::string, size_t, kiwi::BuildOption, bool>()
+			.template ctor<std::string, size_t, kiwi::BuildOption, kiwi::ModelType>()
 			.template method<&JKiwiBuilder::addWord>("addWord")
 			.template method<&JKiwiBuilder::addWord2>("addWord")
 			.template method<&JKiwiBuilder::addPreAnalyzedWord>("addPreAnalyzedWord")

bindings/java/kr/pe/bab2min/Kiwi.java

@@ -12,7 +12,7 @@
 
 public class Kiwi implements AutoCloseable  {
 	private long _inst;
-	final private static String _version = "0.20.4";
+	final private static String _version = "0.21.0";
 
 	public static class Match {
 		final static public int none = 0,
@@ -345,8 +345,8 @@ public Kiwi(long _inst) {
 		this._inst = _inst;
 	}
 
-	public static Kiwi init(String modelPath, int numWorkers, int buildOptions, boolean useSBG) throws Exception {
-		try(KiwiBuilder b = new KiwiBuilder(modelPath, numWorkers, buildOptions, useSBG)) {
+	public static Kiwi init(String modelPath, int numWorkers, int buildOptions, int modelType) throws Exception {
+		try(KiwiBuilder b = new KiwiBuilder(modelPath, numWorkers, buildOptions, modelType)) {
 			return b.build();
 		}
 	}

bindings/java/kr/pe/bab2min/KiwiBuilder.java

@@ -12,6 +12,14 @@ public static class BuildOption	{
 		default_ = integrateAllomorph | loadDefaultDict | loadTypoDict | loadMultiDict;
 	}
 
+	public static class ModelType {
+		final static public int none = 0,
+		knlm = 1,
+		sbg = 2,
+		cong = 3,
+		congGlobal = 4;
+	}
+
 	public static class AnalyzedMorph {
 		public String form;
 		public byte tag = Kiwi.POSTag.nng;
@@ -113,20 +121,20 @@ public KiwiBuilder(long _inst) {
 		this._inst = _inst;
 	}
 
-	public KiwiBuilder(String modelPath, int numWorkers, int buildOptions, boolean useSBG) {
-		ctor(modelPath, numWorkers, buildOptions, useSBG);
+	public KiwiBuilder(String modelPath, int numWorkers, int buildOptions, int modelType) {
+		ctor(modelPath, numWorkers, buildOptions, modelType);
 	}
 
 	public KiwiBuilder(String modelPath, int numWorkers, int buildOptions) {
-		ctor(modelPath, numWorkers, buildOptions, false);
+		ctor(modelPath, numWorkers, buildOptions, ModelType.none);
 	}
 
 	public KiwiBuilder(String modelPath, int numWorkers) {
-		ctor(modelPath, numWorkers, BuildOption.default_, false);
+		ctor(modelPath, numWorkers, BuildOption.default_, ModelType.none);
 	}
 
 	public KiwiBuilder(String modelPath) {
-		ctor(modelPath, 1, BuildOption.default_, false);
+		ctor(modelPath, 1, BuildOption.default_, ModelType.none);
 	}
 
 	protected void finalize() throws Exception {
@@ -137,7 +145,7 @@ public boolean isAlive() {
 		return _inst != 0;
 	}
 
-	private native void ctor(String modelPath, int numWorkers, int buildOptions, boolean useSBG);
+	private native void ctor(String modelPath, int numWorkers, int buildOptions, int modelType);
 
 	@Override
 	public native void close() throws Exception;