Merge pull request #198 from bab2min/dev/saisiot

bab2min · web-flow · commit 2b204f666d82 · 2024-10-27T17:49:42.000+09:00
사이시옷 분석 기능 추가
diff --git a/ModelGenerator/morphemes.txt b/ModelGenerator/morphemes.txt
diff --git a/include/kiwi/Form.h b/include/kiwi/Form.h
@@ -109,6 +109,7 @@ namespace kiwi
 		/**< 선행형태소의 모음조화 조건 */
 		CondPolarity polar() const { return static_cast<CondPolarity>((vpPack >> 4) & 0x7); }
 
+		/**< 추가 분석이 가능한 형태소인지(파생어나 사이시옷이 포함된 합성명사 등) */
 		bool complex() const { return !!(vpPack & 0x80); }
 
 		void setVowel(CondVowel v)
@@ -141,8 +142,9 @@ namespace kiwi
 		const KString* kform = nullptr;
 		POSTag tag = POSTag::unknown;
 		CondVowel vowel : 4;
-		CondPolarity polar : 3;
+		CondPolarity polar : 2;
 		bool complex : 1;
+		bool saisiot : 1;
 		uint8_t senseId = 0;
 		uint8_t combineSocket = 0;
 		int32_t combined = 0;
@@ -205,7 +207,8 @@ namespace kiwi
 		CondVowel vowel = CondVowel::none;
 		CondPolarity polar = CondPolarity::none;
 		uint8_t formHash = 0;
-		uint8_t zCodaAppendable = 0;
+		uint8_t zCodaAppendable : 1;
+		uint8_t zSiotAppendable : 1;
 
 		Form();
 		~Form();
@@ -251,7 +254,7 @@ namespace kiwi
 	 * @param morphBase 형태소 배열의 시작 위치
 	 * @return 최적화된 형태 정보
 	 */
-	Form bake(const FormRaw& o, const Morpheme* morphBase, bool zCodaAppendable, const Vector<uint32_t>& additionalCands = {});
+	Form bake(const FormRaw& o, const Morpheme* morphBase, bool zCodaAppendable, bool zSiotAppendable, const Vector<uint32_t>& additionalCands = {});
 
 	/**
 	 * @brief 변경 가능한 형태소 정보를 bake하여 최적화한다.
diff --git a/include/kiwi/PatternMatcher.h b/include/kiwi/PatternMatcher.h
@@ -25,6 +25,8 @@ namespace kiwi
 		splitComplex = 1 << 22, /**< 더 작은 단위로 분할될 수 있는 형태소는 더 분할하여 매칭한다 */
 		zCoda = 1 << 23, /**< 어미 및 조사에 덧붙은 받침이 있는 경우 이를 분리하여 z_coda 태그로 매칭한다 */
 		compatibleJamo = 1 << 24, /**< 출력시 한글 첫가끝 자모를 호환가능한 자모로 변환한다. */
+		splitSaisiot = 1 << 25, /**< 사이시옷이 포함된 합성명사를 분리하여 매칭한다. */
+		mergeSaisiot = 1 << 26, /**< 사이시옷이 포함된 것으로 추정되는 명사를 결합하여 매칭한다. */
 		joinVSuffix = joinVerbSuffix | joinAdjSuffix,
 		joinAffix = joinNounPrefix | joinNounSuffix | joinVerbSuffix | joinAdjSuffix | joinAdvSuffix,
 		all = url | email | hashtag | mention | serial | emoji | zCoda,
diff --git a/include/kiwi/TagUtils.h b/include/kiwi/TagUtils.h
@@ -31,7 +31,12 @@ namespace kiwi
 	{
 		return POSTag::jks <= tag && tag <= POSTag::jc;
 	}
-	
+
+	inline bool isNNClass(POSTag tag)
+	{
+		return POSTag::nng <= tag && tag <= POSTag::nnb;
+	}
+
 	inline bool isSuffix(POSTag tag)
 	{
 		tag = clearIrregular(tag);
diff --git a/include/kiwi/Types.h b/include/kiwi/Types.h
@@ -211,7 +211,7 @@ namespace kiwi
 		w_url, w_email, w_mention, w_hashtag, w_serial, w_emoji,
 		jks, jkc, jkg, jko, jkb, jkv, jkq, jx, jc,
 		ep, ef, ec, etn, etm,
-		z_coda,
+		z_coda, z_siot, 
 		user0, user1, user2, user3, user4,
 		p, /**< 분할된 동사/형용사를 나타내는데 사용됨 */
 		max, /**< POSTag의 총 개수를 나타내는 용도 */
@@ -275,7 +275,7 @@ namespace kiwi
 	 * @brief 선행 형태소의 양/음성 조건(모음 조화)과 관련된 열거형
 	 * 
 	 */
-	enum class CondPolarity : char
+	enum class CondPolarity : uint8_t
 	{
 		none, /**< 조건이 설정되지 않음 */
 		positive, /**< 선행 형태소가 양성(ㅏ,ㅑ,ㅗ)인 경우만 등장 가능 */
diff --git a/models/base/default.dict b/models/base/default.dict
@@ -17,8 +17,6 @@
 # * (점수)는 생략시 0으로 처리됩니다.
 # 예) 사겼다	사귀/VV + 었/EP + 다/EF	-1.0
 #
-# 현재는 공백을 포함하는 다어절 형태를 등록할 수 없습니다.
-#
 # <규칙 기반의 변형된 이형태를 추가하는 경우>
 # (형태 규칙)$ \t (변형된 형태/품사태그) \t (점수)
 # 예) 요$	용/EF	-5
diff --git a/models/base/sj.knlm b/models/base/sj.knlm
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52d50761ed4aeea82e3be9f6fbb4724b75f526e56368bdc76dd530049ef9a07e
-size 35828400
+oid sha256:00f93f6abdc6bc31b3995c063564ffd558c475c2e5f5ea1e2ac38b64b4e06842
+size 35836336
diff --git a/models/base/sj.morph b/models/base/sj.morph
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c8a9999841059a00ef2c76dc190f651e698c0516209da524930c9818f01279c
-size 3581294
+oid sha256:125fb05ad20c0d8d7ebb45591b8acaadcea0e740197aceff1ee2d14e8c8195e4
+size 3586754
diff --git a/models/base/skipbigram.mdl b/models/base/skipbigram.mdl
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae4f0ee268f516bf4dfe282337f8773ab900ce081fefc1167b5c29259b1c465b
-size 3186444
+oid sha256:3f5239d9542be89970c454336538375c8c59df2a678988ab79c512bed2301e78
+size 3186448
diff --git a/models/base/typo.dict b/models/base/typo.dict
@@ -15,3 +15,4 @@
 편찬	편찮/VA	-5
 귀찬	귀찮/VA	-5
 하찬	하찮/VA	-5
+시끄러	시끄럽/VA-I + 어/EF	-5
diff --git a/src/Form.cpp b/src/Form.cpp
@@ -1,4 +1,4 @@
-﻿#include <cassert>
+#include <cassert>
 #include <algorithm>
 #include <kiwi/Utils.h>
 #include <kiwi/Form.h>
@@ -70,7 +70,10 @@ namespace kiwi
 
 	DEFINE_SERIALIZER_OUTSIDE(FormRaw, form, candidate);
 
-	Form::Form() = default;
+	Form::Form()
+		: zCodaAppendable(0), zSiotAppendable(0)
+	{
+	}
 
 	Form::~Form() = default;
 
@@ -87,7 +90,7 @@ namespace kiwi
 		return ComparatorIgnoringSpace::less(form, o.form);
 	}
 
-	Form bake(const FormRaw& o, const Morpheme* morphBase, bool zCodaAppendable, const Vector<uint32_t>& additionalCands)
+	Form bake(const FormRaw& o, const Morpheme* morphBase, bool zCodaAppendable, bool zSiotAppendable, const Vector<uint32_t>& additionalCands)
 	{
 		Form ret;
 		ret.numSpaces = count(o.form.begin(), o.form.end(), u' ');
@@ -102,6 +105,7 @@ namespace kiwi
 			ret.candidate[i + o.candidate.size()] = morphBase + additionalCands[i];
 		}
 		ret.zCodaAppendable = zCodaAppendable ? 1 : 0;
+		ret.zSiotAppendable = zSiotAppendable ? 1 : 0;
 		return ret;
 	}
 
@@ -112,19 +116,25 @@ namespace kiwi
 		ret.tag = o.tag;
 		ret.vowel = o.vowel();
 		ret.polar = o.polar();
-		ret.complex = o.complex();
 		ret.combineSocket = o.combineSocket;
 		ret.combined = o.combined;
 		ret.userScore = o.userScore;
 		ret.lmMorphemeId = o.lmMorphemeId;
 		ret.origMorphemeId = o.origMorphemeId;
 		ret.senseId = o.senseId;
 		ret.chunks = FixedPairVector<const Morpheme*, std::pair<uint8_t, uint8_t>>{ o.chunks.size() };
+		
+		bool hasSaisiot = false;
 		for (size_t i = 0; i < o.chunks.size(); ++i)
 		{
 			ret.chunks[i] = morphBase + o.chunks[i];
 			ret.chunks.getSecond(i) = o.chunkPositions[i];
+			hasSaisiot = hasSaisiot || (morphBase[o.chunks[i]].tag == POSTag::z_siot);
 		}
+		// 사이시옷이 포함된 경우는 saisiot을 true로, 그 외에는 complex를 true로 설정
+		ret.complex = o.complex() && !hasSaisiot;
+		ret.saisiot = o.complex() && hasSaisiot;
+
 		return ret;
 	}
 
diff --git a/src/Joiner.cpp b/src/Joiner.cpp
@@ -27,6 +27,7 @@ namespace kiwi
 			if (l == POSTag::sso || l == POSTag::ssc) return false;
 			if (r == POSTag::sso) return true;
 			if ((isJClass(l) || isEClass(l)) && r == POSTag::ss) return true;
+			if (l == POSTag::z_siot && isNNClass(r)) return false;
 
 			if (r == POSTag::vx && rform.size() == 1 && (rform[0] == u'하' || rform[0] == u'지')) return false;
 
diff --git a/src/KTrie.cpp b/src/KTrie.cpp
@@ -158,19 +158,16 @@ namespace kiwi
 	};
 
 	template<bool typoTolerant>
-	bool getZCodaAppendable(
-		const Form* foundCand,
-		const Form* formBase
-	)
+	const Form& getForm(const Form* foundCand, const Form* formBase)
 	{
 		if (typoTolerant)
 		{
 			auto tCand = reinterpret_cast<const TypoForm*>(foundCand);
-			return tCand->form(formBase).zCodaAppendable;
+			return tCand->form(formBase);
 		}
 		else
 		{
-			return foundCand->zCodaAppendable;
+			return *foundCand;
 		}
 	}
 
@@ -229,23 +226,6 @@ namespace kiwi
 		return true;
 	}
 
-	template<bool typoTolerant>
-	size_t getFormLength(
-		const Form* form,
-		const Form* formBase
-	)
-	{
-		if (typoTolerant)
-		{
-			auto tCand = reinterpret_cast<const TypoForm*>(form);
-			return tCand->form(formBase).form.size();
-		}
-		else
-		{
-			return form->form.size();
-		}
-	}
-
 	inline void removeUnconnected(Vector<KGraphNode>& ret, const Vector<KGraphNode>& graph, const Vector<std::pair<uint32_t, uint32_t>>& endPosMap)
 	{
 		thread_local Vector<uint8_t> connectedList;
@@ -549,7 +529,7 @@ namespace kiwi
 			if (!cand) break;
 			else if (!trie.hasSubmatch(cand))
 			{
-				if (getFormLength<typoTolerant>(cand, formBase) <= 1) break;
+				if (getForm<typoTolerant>(cand, formBase).form.size() <= 1) break;
 				inserted = true;
 				if (!insertCandidates(candidates, cand, formBase, typoPtrs, str, nonSpaces, 0, boundary, continualTypoCost / 2)) break;
 			}
@@ -753,7 +733,7 @@ size_t kiwi::splitByTrie(
 		}
 	};
 
-	bool zCodaFollowable = false;
+	bool zCodaFollowable = false, zSiotFollowable = false;
 	const Form* const fallbackFormBegin = trie.value((size_t)POSTag::nng);
 	const Form* const fallbackFormEnd = trie.value((size_t)POSTag::max);
 	for (; n < str.size(); ++n)
@@ -1006,7 +986,12 @@ size_t kiwi::splitByTrie(
 				{
 					candidates.emplace_back(formBase + defaultTagSize + (c - 0x11A8) - 1, 0, nonSpaces.size() - 1);
 				}
+				else if (!!(matchOptions & (Match::splitSaisiot | Match::mergeSaisiot)) && zSiotFollowable && c == 0x11BA && n + 1 < str.size() && isHangulSyllable(str[n + 1]))
+				{
+					candidates.emplace_back(formBase + defaultTagSize + (0x11BA - 0x11A8) - 1, 0, nonSpaces.size() - 1);
+				}
 				zCodaFollowable = false;
+				zSiotFollowable = false;
 
 				// invalidate typo nodes
 				if (continualTypoTolerant)
@@ -1107,7 +1092,12 @@ size_t kiwi::splitByTrie(
 		{
 			candidates.emplace_back(formBase + defaultTagSize + (c - 0x11A8) - 1, 0, nonSpaces.size() - 1);
 		}
+		else if (!!(matchOptions & (Match::splitSaisiot | Match::mergeSaisiot)) && zSiotFollowable && c == 0x11BA && n + 1 < str.size() && isHangulSyllable(str[n + 1]))
+		{
+			candidates.emplace_back(formBase + defaultTagSize + (0x11BA - 0x11A8) - 1, 0, nonSpaces.size() - 1);
+		}
 		zCodaFollowable = false;
+		zSiotFollowable = false;
 
 		if (continualTypoTolerant && lastChrType == POSTag::max)
 		{
@@ -1128,7 +1118,8 @@ size_t kiwi::splitByTrie(
 			if (!cand) break;
 			else if (!trie.hasSubmatch(cand))
 			{
-				zCodaFollowable = zCodaFollowable || getZCodaAppendable<typoTolerant>(cand, formBase);
+				zCodaFollowable = zCodaFollowable || getForm<typoTolerant>(cand, formBase).zCodaAppendable;
+				zSiotFollowable = zSiotFollowable || getForm<typoTolerant>(cand, formBase).zSiotAppendable;
 				if (!insertCandidates(candidates, cand, formBase, typoPtrs, str, nonSpaces)) break;
 			}
 		}
diff --git a/src/Kiwi.cpp b/src/Kiwi.cpp
@@ -541,7 +541,12 @@ namespace kiwi
 	template<class TokenInfoIt>
 	TokenInfoIt joinAffixTokens(TokenInfoIt first, TokenInfoIt last, Match matchOptions)
 	{
-		if (!(matchOptions & (Match::joinNounPrefix | Match::joinNounSuffix | Match::joinVerbSuffix | Match::joinAdjSuffix | Match::joinAdvSuffix))) return last;
+		if (!(matchOptions & (Match::joinNounPrefix 
+							| Match::joinNounSuffix 
+							| Match::joinVerbSuffix 
+							| Match::joinAdjSuffix 
+							| Match::joinAdvSuffix 
+							| Match::mergeSaisiot))) return last;
 		if (std::distance(first, last) < 2) return last;
 
 		auto next = first;
@@ -554,7 +559,7 @@ namespace kiwi
 			// XPN + (NN. | SN) => (NN. | SN)
 			if (!!(matchOptions & Match::joinNounPrefix) 
 				&& current.tag == POSTag::xpn 
-				&& ((POSTag::nng <= nextToken.tag && nextToken.tag <= POSTag::nnb) || nextToken.tag == POSTag::sn)
+				&& (isNNClass(nextToken.tag) || nextToken.tag == POSTag::sn)
 			)
 			{
 				concatTokens(current, nextToken, nextToken.tag);
@@ -563,7 +568,7 @@ namespace kiwi
 			// (NN. | SN) + XSN => (NN. | SN)
 			else if (!!(matchOptions & Match::joinNounSuffix)
 				&& nextToken.tag == POSTag::xsn
-				&& ((POSTag::nng <= current.tag && current.tag <= POSTag::nnb) || current.tag == POSTag::sn)
+				&& (isNNClass(current.tag) || current.tag == POSTag::sn)
 			)
 			{
 				concatTokens(current, nextToken, current.tag);
@@ -572,7 +577,7 @@ namespace kiwi
 			// (NN. | XR) + XSV => VV
 			else if (!!(matchOptions & Match::joinVerbSuffix)
 				&& clearIrregular(nextToken.tag) == POSTag::xsv
-				&& ((POSTag::nng <= current.tag && current.tag <= POSTag::nnb) || current.tag == POSTag::xr)
+				&& (isNNClass(current.tag) || current.tag == POSTag::xr)
 			)
 			{
 				concatTokens(current, nextToken, setIrregular(POSTag::vv, isIrregular(nextToken.tag)));
@@ -581,7 +586,7 @@ namespace kiwi
 			// (NN. | XR) + XSA => VA
 			else if (!!(matchOptions & Match::joinAdjSuffix)
 				&& clearIrregular(nextToken.tag) == POSTag::xsa
-				&& ((POSTag::nng <= current.tag && current.tag <= POSTag::nnb) || current.tag == POSTag::xr)
+				&& (isNNClass(current.tag) || current.tag == POSTag::xr)
 			)
 			{
 				concatTokens(current, nextToken, setIrregular(POSTag::va, isIrregular(nextToken.tag)));
@@ -590,12 +595,24 @@ namespace kiwi
 			// (NN. | XR) + XSM => MAG
 			else if (!!(matchOptions & Match::joinAdvSuffix)
 				&& nextToken.tag == POSTag::xsm
-				&& ((POSTag::nng <= current.tag && current.tag <= POSTag::nnb) || current.tag == POSTag::xr)
+				&& (isNNClass(current.tag) || current.tag == POSTag::xr)
 				)
 			{
 				concatTokens(current, nextToken, POSTag::mag);
 				++next;
 			}
+			// NN. + Z_SIOT + NN. => NN
+			else if (!!(matchOptions & Match::mergeSaisiot)
+				&& nextToken.tag == POSTag::z_siot
+				&& isNNClass(current.tag)
+				&& next + 1 != last
+				&& isNNClass((next + 1)->tag))
+			{
+				current.str.back() += (0x11BA - 0x11A7);
+				concatTokens(current, *(next + 1), POSTag::nng);
+				++next;
+				++next;
+			}
 			else
 			{
 				++first;
@@ -1047,6 +1064,7 @@ namespace kiwi
 				topN,
 				false,
 				!!(matchOptions & Match::splitComplex),
+				!!(matchOptions & Match::splitSaisiot),
 				blocklist
 			);
 			insertPathIntoResults(ret, spStatesByRet, res, topN, matchOptions, integrateAllomorph, positionTable, wordPositions, pretokenizedGroup, nodeInWhichPretokenized);
diff --git a/src/KiwiBuilder.cpp b/src/KiwiBuilder.cpp
diff --git a/src/PathEvaluator.hpp b/src/PathEvaluator.hpp
diff --git a/src/StrUtils.h b/src/StrUtils.h
diff --git a/src/Utils.cpp b/src/Utils.cpp
diff --git a/test/test_cpp.cpp b/test/test_cpp.cpp

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,12 @@ namespace kiwi`
`31`	`31`	`{`
`32`	`32`	`return POSTag::jks <= tag && tag <= POSTag::jc;`
`33`	`33`	`}`
`34`		`-`
	`34`	`+`
	`35`	`+ inline bool isNNClass(POSTag tag)`
	`36`	`+ {`
	`37`	`+ return POSTag::nng <= tag && tag <= POSTag::nnb;`
	`38`	`+ }`
	`39`	`+`
`35`	`40`	`inline bool isSuffix(POSTag tag)`
`36`	`41`	`{`
`37`	`42`	`tag = clearIrregular(tag);`
Original file line number	Diff line number	Diff line change
`@@ -17,8 +17,6 @@`
`17`	`17`	`# * (점수)는 생략시 0으로 처리됩니다.`
`18`	`18`	`# 예) 사겼다 사귀/VV + 었/EP + 다/EF -1.0`
`19`	`19`	`#`
`20`		`-# 현재는 공백을 포함하는 다어절 형태를 등록할 수 없습니다.`
`21`		`-#`
`22`	`20`	`# <규칙 기반의 변형된 이형태를 추가하는 경우>`
`23`	`21`	`# (형태 규칙)$ \t (변형된 형태/품사태그) \t (점수)`
`24`	`22`	`# 예) 요$ 용/EF -5`