Merge pull request #157 from bab2min/dev_0170

Dev 0.17.0

Author: bab2min

Diff for: CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.12)
 
-project(kiwi VERSION 0.16.1 DESCRIPTION "Kiwi, Korean Intelligent Word Identifier")
+project(kiwi VERSION 0.17.0 DESCRIPTION "Kiwi, Korean Intelligent Word Identifier")
 
 set ( CMAKE_CXX_STANDARD 14 )
 set ( CMAKE_VERBOSE_MAKEFILE true )

Diff for: ModelGenerator/multi.dict

@@ -12,7 +12,7 @@
 
 public class Kiwi implements AutoCloseable  {
 	private long _inst;
-	final private static String _version = "0.16.1";
+	final private static String _version = "0.17.0";
 
 	public static class Match {
 		final static public int none = 0,

Diff for: bindings/java/kr/pe/bab2min/Kiwi.java

@@ -2,7 +2,7 @@
  * @file Form.h
  * @author bab2min ([email protected])
  * @brief 형태 및 형태소에 관한 정보를 담는 구조체들이 선언된 헤더
- * @version 0.16.1
+ * @version 0.17.0
  * @date 2022-09-01
  * 
  * 

Diff for: include/kiwi/Form.h

@@ -2,7 +2,7 @@
  * @file Kiwi.h
  * @author bab2min ([email protected])
  * @brief Kiwi C++ API를 담고 있는 헤더 파일
- * @version 0.16.1
+ * @version 0.17.0
  * @date 2022-09-01
  * 
  * 

Diff for: include/kiwi/Kiwi.h

@@ -2,7 +2,7 @@
  * @file Types.h
  * @author bab2min ([email protected])
  * @brief Kiwi C++ API에 쓰이는 주요 타입들을 모아놓은 헤더 파일
- * @version 0.16.1
+ * @version 0.17.0
  * @date 2022-09-01
  * 
  * 

Diff for: include/kiwi/Types.h

@@ -160,7 +160,23 @@ namespace kiwi
 		case u'\r':
 		case u'\t':
 		case u'\v':
+		case u'\xA0':
+		case u'\u1680':
+		case u'\u2000':
+		case u'\u2001':
+		case u'\u2002':
+		case u'\u2003':
+		case u'\u2004':
+		case u'\u2005':
+		case u'\u2006':
+		case u'\u2007':
+		case u'\u2008':
+		case u'\u2009':
+		case u'\u200A':
+		case u'\u202F':
+		case u'\u205F':
 		case u'\u2800':
+		case u'\u3000':
 			return true;
 		}
 		return false;

Diff for: include/kiwi/Utils.h

@@ -2,7 +2,7 @@
  * @file capi.h
  * @author bab2min ([email protected])
  * @brief Kiwi C API를 담고 있는 헤더 파일
- * @version 0.16.1
+ * @version 0.17.0
  * @date 2022-09-01
  * 
  * 

Diff for: include/kiwi/capi.h

@@ -779,7 +779,7 @@ void RuleSet::loadRules(istream& istr)
 	while (getline(istr, line))
 	{
 		if (line[0] == '#') continue;
-		while (!line.empty() && isSpace(line.back())) line.pop_back();
+		while (!line.empty() && line.back() < 0x80 && isSpace(line.back())) line.pop_back();
 		if (line.empty()) continue;
 
 		auto fields = split(line, '\t');

Diff for: src/Combiner.cpp

@@ -240,10 +240,15 @@ size_t PatternMatcherImpl::testSerial(const char16_t* first, const char16_t* las
 	{
 		++b;
 		if (b != last && *b == ' ') ++b;
-		if (b == last || !isDigit(*b)) return b - first;
+		if (b == last || !isDigit(*b))
+		{
+			if (b[-1] == ' ') --b;
+			return b - first;
+		}
 		++b;
 		while (b != last && isDigit(*b)) ++b;
 	}
+	if (b[-1] == ' ') --b;
 	return b - first;
 }
 

Diff for: src/PatternMatcher.cpp

@@ -531,13 +531,13 @@ TEST(KiwiCpp, SpaceTolerant)
 TEST(KiwiCpp, MultiWordDictionary)
 {
 	auto& kiwi = reuseKiwiInstance();
-	const auto text = u"밀리언 달러 베이비랑 바람과 함께 사라지다랑 뭐가 더 재밌었어?";
+	const auto text = u"밀리언 달러 베이비랑 더 웨이 백 중 뭐가 더 재밌었어?";
 
 	auto res = kiwi.analyze(text, Match::allWithNormalizing).first;
 	EXPECT_EQ(res[0].str, u"밀리언 달러 베이비");
 	EXPECT_EQ(res[0].tag, POSTag::nnp);
 
-	EXPECT_EQ(res[2].str, u"바람과 함께 사라지다");
+	EXPECT_EQ(res[2].str, u"더 웨이 백");
 	EXPECT_EQ(res[2].tag, POSTag::nnp);
 
 	auto kiwi2 = KiwiBuilder{ MODEL_PATH, 0, BuildOption::default_ & ~BuildOption::loadMultiDict, }.build();
@@ -547,7 +547,7 @@ TEST(KiwiCpp, MultiWordDictionary)
 
 TEST(KiwiCpp, WordsWithSpaces)
 {
-	KiwiBuilder kw{ MODEL_PATH, 0, BuildOption::default_, };
+	KiwiBuilder kw{ MODEL_PATH, 0, BuildOption::default_ & ~BuildOption::loadMultiDict, };
 	EXPECT_TRUE(kw.addWord(u"대학생 선교회", POSTag::nnp, 0.0).second);
 	Kiwi kiwi = kw.build();
 
@@ -643,6 +643,22 @@ TEST(KiwiCpp, WordsWithSpaces)
 	EXPECT_EQ(res5.first[1].lineNumber, 2);
 }
 
+TEST(KiwiCpp, MultiDict)
+{
+	Kiwi& kiwi = reuseKiwiInstance();
+	auto res = kiwi.analyze(u"프렌치카페 로스터리 크리스마스에디션 인증샷", Match::all).first;
+	for (auto& r : res)
+	{
+		EXPECT_NE(r.str, u"리 크리스마스");
+	}
+
+	res = kiwi.analyze(u"추첨이벤트 2018년 리빙디자인페어 행사기간", Match::all).first;
+	for (auto& r : res)
+	{
+		EXPECT_NE(r.str, u"리 빙");
+	}
+}
+
 TEST(KiwiCpp, Pattern)
 {
 	Kiwi& kiwi = reuseKiwiInstance();
@@ -716,6 +732,7 @@ TEST(KiwiCpp, Pattern)
 	tokens = kiwi.analyze(u"2001. 01. 02. 에", Match::all).first;
 	EXPECT_EQ(tokens.size(), 2);
 	EXPECT_EQ(tokens[0].tag, POSTag::w_serial);
+	EXPECT_EQ(tokens[0].str.back(), u'.');
 
 	tokens = kiwi.analyze(u"010-1234-5678에", Match::all).first;
 	EXPECT_EQ(tokens.size(), 2);