feat(script_translator): allow multiple sentence candidates

Author: ksqsf

src/rime/gear/poet.cc

@@ -258,7 +258,8 @@ an<Sentence> Poet::MakeSentence(const WordGraph& graph,
 deque<an<Sentence>> Poet::MakeSentences(const WordGraph& graph,
                                         size_t total_length,
                                         const string& preceding_text,
-                                        size_t max_sentences) {
+                                        size_t max_sentences,
+                                        double cutoff_threshold) {
   size_t beam_width =
       max_sentences * 3;  // allow more possibilities during search
   using State = std::list<Line>;
@@ -320,17 +321,24 @@ deque<an<Sentence>> Poet::MakeSentences(const WordGraph& graph,
     return {};
 
   deque<an<Sentence>> results;
-  size_t i = 0;
-  double last_weight = found->second.front().weight;
-  for (const auto& candidate : found->second) {
-    i++;
-    if (i > max_sentences)
-      break;
+  double last_weight;
+  double acceleration = 1.0 - 1.0 / (double)max_sentences;
+  auto iter = found->second.begin();
+  for (size_t i = 0; iter != found->second.end() && i < max_sentences;
+       ++i, ++iter) {
+    const auto& candidate = *iter;
     double cur_weight = candidate.weight;
-    // idea: if the current sentence is, on average, not too rare when
-    // compared to last sentence, we should consider it too
-    if (fabs(cur_weight - last_weight) / fabs(last_weight) > 0.05) {
-      break;
+    if (i > 0) {
+      // idea: if the current sentence is, on average, not too rare when
+      // compared to last sentence, we should consider it too
+      if (fabs(cur_weight - last_weight) / fabs(last_weight) >
+          cutoff_threshold) {
+        break;
+      }
+      // but don't deviate too far from the first weight by accelerating
+      // the cutoff threshold. cutoff_threshold becomes
+      // ~0.36*cutoff_threshold after N candidates are added.
+      cutoff_threshold *= acceleration;
     }
     last_weight = cur_weight;
     auto sentence = New<Sentence>(language_);

src/rime/gear/poet.h

@@ -42,7 +42,8 @@ class Poet {
   deque<an<Sentence>> MakeSentences(const WordGraph& graph,
                                     size_t total_length,
                                     const string& preceding_text,
-                                    size_t count);
+                                    size_t count,
+                                    double cutoff_threshold);
 
   template <class TranslatorT>
   an<Translation> ContextualWeighted(an<Translation> translation,

src/rime/gear/script_translator.cc

@@ -115,14 +115,18 @@ class ScriptTranslation : public Translation {
                     Poet* poet,
                     const string& input,
                     size_t start,
-                    size_t end_of_input)
+                    size_t end_of_input,
+                    int max_sentences,
+                    double sentence_cutoff_threshold)
       : translator_(translator),
         poet_(poet),
         start_(start),
         end_of_input_(end_of_input),
         syllabifier_(
             New<ScriptSyllabifier>(translator, corrector, input, start)),
-        enable_correction_(corrector) {
+        enable_correction_(corrector),
+        max_sentences_(max_sentences),
+        sentence_cutoff_threshold_(sentence_cutoff_threshold) {
     set_exhausted(true);
   }
   bool Evaluate(Dictionary* dict, UserDictionary* user_dict);
@@ -136,7 +140,11 @@ class ScriptTranslation : public Translation {
   template <class QueryResult>
   void EnrollEntries(map<int, DictEntryList>& entries_by_end_pos,
                      const an<QueryResult>& query_result);
+  WordGraph PrepareForMakingSentence(Dictionary* dict,
+                                     UserDictionary* user_dict);
   an<Sentence> MakeSentence(Dictionary* dict, UserDictionary* user_dict);
+  deque<an<Sentence>> MakeSentences(Dictionary* dict,
+                                    UserDictionary* user_dict);
 
   ScriptTranslator* translator_;
   Poet* poet_;
@@ -146,7 +154,7 @@ class ScriptTranslation : public Translation {
 
   an<DictEntryCollector> phrase_;
   an<UserDictEntryCollector> user_phrase_;
-  an<Sentence> sentence_;
+  deque<an<Sentence>> sentences_;
 
   an<Phrase> candidate_ = nullptr;
   size_t candidate_index_ = 0;
@@ -163,6 +171,8 @@ class ScriptTranslation : public Translation {
 
   const size_t max_corrections_ = 4;
   size_t correction_count_ = 0;
+  int max_sentences_ = 1;
+  double sentence_cutoff_threshold_ = 0.1;
 
   bool enable_correction_;
 };
@@ -210,8 +220,9 @@ an<Translation> ScriptTranslator::Query(const string& input,
 
   size_t end_of_input = engine_->context()->input().length();
   // the translator should survive translations it creates
-  auto result = New<ScriptTranslation>(this, corrector_.get(), poet_.get(),
-                                       input, segment.start, end_of_input);
+  auto result = New<ScriptTranslation>(
+      this, corrector_.get(), poet_.get(), input, segment.start, end_of_input,
+      max_sentences_, sentence_cutoff_threshold_);
   if (!result || !result->Evaluate(
                      dict_.get(), enable_user_dict ? user_dict_.get() : NULL)) {
     return nullptr;
@@ -484,7 +495,15 @@ bool ScriptTranslation::Evaluate(Dictionary* dict, UserDictionary* user_dict) {
   // make sentences when there is no exact-matching phrase candidate
   if (has_at_least_two_syllables && !has_reliable_phrase &&
       !has_reliable_user_phrase) {
-    sentence_ = MakeSentence(dict, user_dict);
+    if (max_sentences_ > 1)
+      sentences_ = MakeSentences(dict, user_dict);
+    else if (max_sentences_) {
+      auto sentence = MakeSentence(dict, user_dict);
+      if (sentence)
+        sentences_ = {sentence};
+      else
+        sentences_.clear();
+    }
   }
 
   return !CheckEmpty();
@@ -501,7 +520,8 @@ bool ScriptTranslation::Next() {
       case kUninitialized:
         break;
       case kSentence:
-        sentence_.reset();
+        if (!sentences_.empty())
+          sentences_.pop_front();
         break;
       case kUserPhrase: {
         UserDictEntryIterator& uter(user_phrase_iter_->second);
@@ -575,9 +595,9 @@ bool ScriptTranslation::PrepareCandidate() {
     candidate_ = nullptr;
     return false;
   }
-  if (sentence_) {
+  if (!sentences_.empty()) {
     candidate_source_ = kSentence;
-    candidate_ = sentence_;
+    candidate_ = sentences_[0];
     return true;
   }
   const size_t full_code_length = end_of_input_ - start_;
@@ -675,8 +695,9 @@ void ScriptTranslation::EnrollEntries(
   }
 }
 
-an<Sentence> ScriptTranslation::MakeSentence(Dictionary* dict,
-                                             UserDictionary* user_dict) {
+WordGraph ScriptTranslation::PrepareForMakingSentence(
+    Dictionary* dict,
+    UserDictionary* user_dict) {
   const int kMaxSyllablesForUserPhraseQuery = 5;
   const auto& syllable_graph = syllabifier_->syllable_graph();
   WordGraph graph;
@@ -691,6 +712,29 @@ an<Sentence> ScriptTranslation::MakeSentence(Dictionary* dict,
     EnrollEntries(same_start_pos, dict->Lookup(syllable_graph, x.first,
                                                &translator_->blacklist()));
   }
+  return graph;
+}
+
+deque<an<Sentence>> ScriptTranslation::MakeSentences(
+    Dictionary* dict,
+    UserDictionary* user_dict) {
+  const auto& syllable_graph = syllabifier_->syllable_graph();
+  WordGraph graph = PrepareForMakingSentence(dict, user_dict);
+  auto sentences =
+      poet_->MakeSentences(graph, syllable_graph.interpreted_length,
+                           translator_->GetPrecedingText(start_),
+                           max_sentences_, sentence_cutoff_threshold_);
+  for (auto& sentence : sentences) {
+    sentence->Offset(start_);
+    sentence->set_syllabifier(syllabifier_);
+  }
+  return sentences;
+}
+
+an<Sentence> ScriptTranslation::MakeSentence(Dictionary* dict,
+                                             UserDictionary* user_dict) {
+  const auto& syllable_graph = syllabifier_->syllable_graph();
+  WordGraph graph = PrepareForMakingSentence(dict, user_dict);
   if (auto sentence =
           poet_->MakeSentence(graph, syllable_graph.interpreted_length,
                               translator_->GetPrecedingText(start_))) {

src/rime/gear/translator_commons.cc

@@ -125,12 +125,18 @@ TranslatorOptions::TranslatorOptions(const Ticket& ticket) {
     config->GetBool(ticket.name_space + "/strict_spelling", &strict_spelling_);
     config->GetDouble(ticket.name_space + "/initial_quality",
                       &initial_quality_);
+    config->GetInt(ticket.name_space + "/max_sentences", &max_sentences_);
+    max_sentences_ = std::min(std::max(1, max_sentences_), 100);
+    config->GetDouble(ticket.name_space + "/sentence_cutoff_threshold",
+                      &sentence_cutoff_threshold_);
+
     preedit_formatter_.Load(
         config->GetList(ticket.name_space + "/preedit_format"));
     comment_formatter_.Load(
         config->GetList(ticket.name_space + "/comment_format"));
     user_dict_disabling_patterns_.Load(
         config->GetList(ticket.name_space + "/disable_user_dict_for_patterns"));
+
     string tag;
     if (config->GetString(ticket.name_space + "/tag", &tag)) {
       // replace the first tag, and understand /tags as extra tags

src/rime/gear/translator_commons.h

@@ -176,6 +176,8 @@ class TranslatorOptions {
   bool enable_completion_ = true;
   bool strict_spelling_ = false;
   double initial_quality_ = 0.;
+  int max_sentences_ = 1;
+  double sentence_cutoff_threshold_ = 0.1;
   Projection preedit_formatter_;
   Projection comment_formatter_;
   Patterns user_dict_disabling_patterns_;