Merge pull request #167 from bab2min/dev_emoji

emoji 태그 추가

Author: bab2min

Diff for: ModelGenerator/sj.knlm

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe03f0a1fdd25186113e8ec192df4369975fdf174426d46411bc5ff5c4d6caf2
-size 35867382
+oid sha256:e36c3d16e1f305169f977840e7257b80de153e613ac5809fd7e25ee59cda4f6e
+size 35861136

Diff for: ModelGenerator/sj.morph

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31aea323cf870f614fb07681aa01c76d83d1fa27e325d626d4995b4e064d5b05
-size 3581056
+oid sha256:76f155f03b402b866af32ed5e759560fac72b6174982ac5835f9d56c3a997e34
+size 3581100

Diff for: ModelGenerator/skipbigram.mdl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:152de02fa696817564fad589460d8874ce37ef3e912442c0ed37fb2955471288
-size 3186824
+oid sha256:edce9fbe938b9f21eeb915c0695c4c160708ed5879a6ce2e4dd20117ff7f9ca5
+size 3186748

Diff for: bindings/java/kr/pe/bab2min/Kiwi.java

@@ -21,6 +21,7 @@ public static class Match {
 		hashtag = 1 << 2,
 		mention = 1 << 3,
 		serial = 1 << 4,
+		emoji = 1 << 5,
 		normalizeCoda = 1 << 16,
 		joinNounPrefix = 1 << 17,
 		joinNounSuffix = 1 << 18,
@@ -48,13 +49,13 @@ public static class POSTag {
 		vcp = 19, vcn = 20,
 		sf = 21, sp = 22, ss = 23, sso = 24, ssc = 25, se = 26, so = 27, sw = 28, sb = 29,
 		sl = 30, sh = 31, sn = 32,
-		w_url = 33, w_email = 34, w_mention = 35, w_hashtag = 36, w_serial = 37,
-		jks = 38, jkc = 39, jkg = 40, jko = 41, jkb = 42, jkv = 43, jkq = 44, jx = 45, jc = 46,
-		ep = 47, ef = 48, ec = 49, etn = 50, etm = 51,
-		z_coda = 52,
-		user0 = 53, user1 = 54, user2 = 55, user3 = 56, user4 = 57,
-		p = 58,
-		max = 59,
+		w_url = 33, w_email = 34, w_mention = 35, w_hashtag = 36, w_serial = 37, w_emoji = 38,
+		jks = 39, jkc = 40, jkg = 41, jko = 42, jkb = 43, jkv = 44, jkq = 45, jx = 46, jc = 47,
+		ep = 48, ef = 49, ec = 50, etn = 51, etm = 52,
+		z_coda = 53,
+		user0 = 54, user1 = 55, user2 = 56, user3 = 57, user4 = 58,
+		p = 59,
+		max = 60,
 		pv = p,
 		pa = (byte)(p + 1),
 		irregular = - 128,
@@ -106,6 +107,7 @@ static String toString(byte tag) {
 				case w_mention: return "W_MENTION";
 				case w_hashtag: return "W_HASHTAG";
 				case w_serial: return "W_SERIAL";
+				case w_emoji: return "W_EMOJI";
 				case jks: return "JKS";
 				case jkc: return "JKC";
 				case jkg: return "JKG";

Diff for: bindings/java/kr/pe/bab2min/KiwiBuilder.java

@@ -8,7 +8,8 @@ public static class BuildOption	{
 		integrateAllomorph = 1 << 0,
 		loadDefaultDict = 1 << 1,
 		loadTypoDict = 1 << 2,
-		default_ = integrateAllomorph | loadDefaultDict | loadTypoDict;
+		loadMultiDict = 1 << 3,
+		default_ = integrateAllomorph | loadDefaultDict | loadTypoDict | loadMultiDict;
 	}
 
 	public static class AnalyzedMorph {

Diff for: include/kiwi/PatternMatcher.h

@@ -15,6 +15,7 @@ namespace kiwi
 		hashtag = 1 << 2, /**< 해시태그 형태의 텍스트(#해시)를 w_hashtag 태그에 매칭한다 */
 		mention = 1 << 3, /**< 멘션 형태의 텍스트(@멘션)를 w_mention 태그에 매칭한다 */
 		serial = 1 << 4, /**< 일련 번호 형태의 텍스트를 w_serial 태그에 매칭한다 */
+		emoji = 1 << 5, /**< 이모지 문자를 w_emoji 태그에 매칭한다 */
 		normalizeCoda = 1 << 16, /**< 초성체가 앞 어절의 받침에 따라붙은 경우를 정규화하여 매칭한다 */
 		joinNounPrefix = 1 << 17, /**< 체언접두사(XPN)를 분리하지 않고 합쳐서 매칭한다 */
 		joinNounSuffix = 1 << 18, /**< 명사파생접미사(XSN)를 분리하지 않고 합쳐서 매칭한다 */
@@ -25,7 +26,7 @@ namespace kiwi
 		zCoda = 1 << 23, /**< 어미 및 조사에 덧붙은 받침이 있는 경우 이를 분리하여 z_coda 태그로 매칭한다 */
 		joinVSuffix = joinVerbSuffix | joinAdjSuffix,
 		joinAffix = joinNounPrefix | joinNounSuffix | joinVerbSuffix | joinAdjSuffix | joinAdvSuffix,
-		all = url | email | hashtag | mention | serial | zCoda,
+		all = url | email | hashtag | mention | serial | emoji | zCoda,
 		allWithNormalizing = all | normalizeCoda,
 	};
 

Diff for: include/kiwi/ScriptType.h

@@ -241,5 +241,10 @@ namespace kiwi
 
     const char* getScriptName(ScriptType type);
 
-    bool isEmoji(char32_t c0, char32_t c1 = 0);
+    /**
+     * @brief Check if the character is an emoji
+     * 
+     * @return 0 if the character is not an emoji, 1 if c0 is an emoji, 2 if c0 and c1 are combined to form an emoji.
+     */
+    int isEmoji(char32_t c0, char32_t c1 = 0);
 }

Diff for: include/kiwi/Types.h

@@ -202,7 +202,7 @@ namespace kiwi
 		vcp, vcn,
 		sf, sp, ss, sso, ssc, se, so, sw, sb,
 		sl, sh, sn,
-		w_url, w_email, w_mention, w_hashtag, w_serial,
+		w_url, w_email, w_mention, w_hashtag, w_serial, w_emoji,
 		jks, jkc, jkg, jko, jkb, jkv, jkq, jx, jc,
 		ep, ef, ec, etn, etm,
 		z_coda,

Diff for: include/kiwi/Utils.h

@@ -1,4 +1,4 @@
-#pragma once
+#pragma once
 #include <iostream>
 #include <string>
 #include <memory>
@@ -30,7 +30,7 @@ namespace kiwi
 
 	inline bool isWebTag(POSTag t)
 	{
-		return POSTag::w_url <= t && t <= POSTag::w_hashtag;
+		return POSTag::w_url <= t && t <= POSTag::w_emoji;
 	}
 
 	POSTag toPOSTag(const std::u16string& tagStr);

Diff for: src/Kiwi.cpp

@@ -600,7 +600,7 @@ namespace kiwi
 
 	inline void updateTokenInfoScript(TokenInfo& info)
 	{
-		if (!(info.tag == POSTag::sl || info.tag == POSTag::sh || info.tag == POSTag::sw)) return;
+		if (!(info.tag == POSTag::sl || info.tag == POSTag::sh || info.tag == POSTag::sw || info.tag == POSTag::w_emoji)) return;
 		if ((info.morph && info.morph->kform && !info.morph->kform->empty())) return;
 		if (info.str.empty()) return;
 		char32_t c = info.str[0];

Diff for: src/PatternMatcher.cpp

@@ -1,6 +1,8 @@
 #include <kiwi/PatternMatcher.h>
 #include <kiwi/Utils.h>
+#include <kiwi/ScriptType.h>
 #include "pattern.hpp"
+#include "StrUtils.h"
 
 using namespace std;
 using namespace kiwi;
@@ -26,6 +28,7 @@ namespace kiwi
 		size_t testNumeric(const char16_t left, const char16_t* first, const char16_t* last) const;
 		size_t testSerial(const char16_t* first, const char16_t* last) const;
 		size_t testAbbr(const char16_t* first, const char16_t* last) const;
+		size_t testEmoji(const char16_t* first, const char16_t* last) const;
 
 	public:
 		std::pair<size_t, POSTag> match(char16_t left, const char16_t* first, const char16_t* last, Match matchOptions) const;
@@ -290,6 +293,77 @@ size_t PatternMatcherImpl::testAbbr(const char16_t* first, const char16_t* last)
 	return b - first;
 }
 
+size_t PatternMatcherImpl::testEmoji(const char16_t* first, const char16_t* last) const
+{
+	const char16_t* b = first;
+	while (b + 1 < last)
+	{
+		char32_t c0 = 0, c1 = 0;
+		const char16_t* b1 = b;
+		if (isHighSurrogate(*b1))
+		{
+			c0 = mergeSurrogate(b1[0], b1[1]);
+			b1 += 2;
+		}
+		else
+		{
+			c0 = *b1++;
+		}
+
+		const char16_t* b2 = b1;
+		if (b2 < last)
+		{
+			if (isHighSurrogate(*b2) && b2 + 1 < last)
+			{
+				c1 = mergeSurrogate(b2[0], b2[1]);
+				b2 += 2;
+			}
+			else
+			{
+				c1 = *b2++;
+			}
+		}
+
+		auto r = isEmoji(c0, c1);
+		if (r == 1)
+		{
+			b = b1;
+		}
+		else if (r == 2)
+		{
+			b = b2;
+		}
+		else
+		{
+			break;
+		}
+
+		if (b == last) return b - first;
+		if (0xfe00 <= *b && *b <= 0xfe0f) // variation selectors
+		{
+			++b;
+			if (b == last) return b - first;
+		}
+		else if (b + 1 < last && isHighSurrogate(b[0]))
+		{
+			c1 = mergeSurrogate(b[0], b[1]);
+			if (0x1f3fb <= c1 && c1 <= 0x1f3ff) // skin color modifier
+			{
+				b += 2;
+				if (b == last) return b - first;
+			}
+		}
+
+		if (*b == 0x200d) // zero width joiner
+		{
+			++b;
+			continue;
+		}
+		break;
+	}
+	return b - first;
+}
+
 pair<size_t, POSTag> PatternMatcherImpl::match(char16_t left, const char16_t * first, const char16_t * last, Match matchOptions) const
 {
 	size_t size;
@@ -299,6 +373,7 @@ pair<size_t, POSTag> PatternMatcherImpl::match(char16_t left, const char16_t * f
 	if (!!(matchOptions & Match::email) && (size = testEmail(first, last))) return make_pair(size, POSTag::w_email);
 	if (!!(matchOptions & Match::mention) && (size = testMention(first, last))) return make_pair(size, POSTag::w_mention);
 	if (!!(matchOptions & Match::url) && (size = testUrl(first, last))) return make_pair(size, POSTag::w_url);
+	if (!!(matchOptions & Match::emoji) && (size = testEmoji(first, last))) return make_pair(size, POSTag::w_emoji);
 	if ((size = testAbbr(first, last))) return make_pair(size, POSTag::sl);
 	return make_pair(0, POSTag::unknown);
 }