Merge pull request #187 from bab2min/dev/fix_sb_errors

Fix SB errors & Optimize path search

Author: bab2min

Diff for: ModelGenerator/morphemes.txt

@@ -62,4 +62,49 @@ namespace kiwi
 		size_t cluster(size_t i) const;
 		float score(size_t i) const;
 	};
+
+	class Kiwi;
+
+	class NgramExtractor
+	{
+		const Kiwi* kiwi = nullptr;
+		bool gatherLmScore = true;
+		UnorderedMap<std::u16string, size_t> morph2id;
+		Vector<std::u16string> id2morph;
+		Vector<uint16_t> buf;
+		Vector<int16_t> scores;
+		Vector<size_t> docBoundaries;
+		Vector<uint32_t> positions;
+		Vector<std::u16string> rawDocs;
+
+		size_t addTokens(const std::vector<TokenInfo>& tokens);
+
+	public:
+		struct Candidate
+		{
+			std::u16string text;
+			std::vector<std::u16string> tokens;
+			std::vector<float> tokenScores;
+			size_t cnt = 0;
+			size_t df = 0;
+			float score = 0;
+			float npmi = 0;
+			float leftBranch = 0;
+			float rightBranch = 0;
+			float lmScore = 0;
+		};
+
+		NgramExtractor();
+		NgramExtractor(const Kiwi& kiwi, bool gatherLmScore = true);
+		NgramExtractor(const NgramExtractor&);
+		NgramExtractor(NgramExtractor&&) noexcept;
+		NgramExtractor& operator=(const NgramExtractor&);
+		NgramExtractor& operator=(NgramExtractor&&) noexcept;
+		~NgramExtractor();
+
+		size_t addText(const std::u16string& text);
+		size_t addTexts(const U16Reader& reader);
+
+		std::vector<Candidate> extract(size_t maxCandidates = 1000, size_t minCnt = 10, size_t maxLength = 5, float minScore = 1e-3, size_t numWorkers = 1) const;
+	};
 }

Diff for: include/kiwi/SubstringExtractor.h

@@ -972,16 +972,6 @@ size_t kiwi::splitByTrie(
 			if (curNode->fail())
 			{
 				curNode = curNode->fail();
-				for (auto submatcher = curNode; submatcher; submatcher = submatcher->fail())
-				{
-					const Form* cand = submatcher->val(trie);
-					if (!cand) break;
-					else if (!trie.hasSubmatch(cand))
-					{
-						zCodaFollowable = zCodaFollowable || getZCodaAppendable<typoTolerant>(cand, formBase);
-						if (!insertCandidates(candidates, cand, formBase, typoPtrs, str, nonSpaces)) break;
-					}
-				}
 				nextNode = curNode->template nextOpt<arch>(trie, c);
 			}
 			else
@@ -1161,7 +1151,7 @@ size_t kiwi::splitByTrie(
 				const Form* cand = node.second->val(trie);
 				if (cand && !trie.hasSubmatch(cand))
 				{
-					insertCandidates(candidates, cand, formBase, typoPtrs, str, nonSpaces, 0, 0, lengtheningTypoCost * node.first, node.first);
+					insertCandidates(candidates, cand, formBase, typoPtrs, str, nonSpaces, 0, 0, lengtheningTypoCost * (3 + node.first), node.first);
 				}
 			}
 		}

Diff for: src/KTrie.cpp

@@ -775,8 +775,25 @@ namespace kiwi
 			spStateCnt[r.curState]++;
 			validTarget++;
 		}
-		ret.erase(ret.begin() + validTarget, ret.end());
-		spStatesByRet.erase(spStatesByRet.begin() + validTarget, spStatesByRet.end());
+		Vector<size_t> idx(validTarget);
+		iota(idx.begin(), idx.end(), 0);
+		sort(idx.begin(), idx.end(), [&](size_t a, size_t b) { return ret[a].second > ret[b].second; });
+		
+		Vector<TokenResult> sortedRet;
+		Vector<SpecialState> sortedSpStatesByRet;
+		const size_t maxCands = min(topN * 2, validTarget);
+		for (size_t i = 0; i < maxCands; ++i)
+		{
+			sortedRet.emplace_back(move(ret[idx[i]]));
+			sortedSpStatesByRet.emplace_back(spStatesByRet[idx[i]]);
+		}
+		ret.clear();
+		spStatesByRet.clear();
+		for (size_t i = 0; i < maxCands; ++i)
+		{
+			ret.emplace_back(move(sortedRet[i]));
+			spStatesByRet.emplace_back(sortedSpStatesByRet[i]);
+		}
 	}
 
 	inline void makePretokenizedSpanGroup(