FEATURE: Detect locale and translate posts from core table

nattsw · nattsw · commit 5b008e518ccc · 2025-04-30T16:20:40.000+08:00
diff --git a/app/jobs/regular/detect_posts_locale.rb b/app/jobs/regular/detect_posts_locale.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Jobs
+  class DetectPostsLocale < ::Jobs::Base
+    cluster_concurrency 1
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      posts =
+        Post
+          .where(locale: nil)
+          .where(deleted_at: nil)
+          .where("posts.user_id > 0")
+          .where.not(raw: [nil, ""])
+          .order(id: :desc)
+          .limit(BATCH_SIZE)
+      return if posts.empty?
+
+      posts.each do |post|
+        begin
+          DiscourseTranslator::PostLocaleDetector.detect_locale(post)
+        rescue => e
+          Rails.logger.error(
+            "Discourse Translator: Failed to detect post #{post.id}'s locale: #{e.message}",
+          )
+        end
+      end
+
+      DiscourseTranslator::VerboseLogger.log("Detected #{posts.size} post locales")
+    end
+  end
+end
diff --git a/app/jobs/regular/translate_posts.rb b/app/jobs/regular/translate_posts.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Jobs
+  class TranslatePosts < ::Jobs::Base
+    cluster_concurrency 1
+    BATCH_SIZE = 50
+
+    def execute(args)
+      return unless SiteSetting.translator_enabled
+      return unless SiteSetting.experimental_content_translation
+
+      locales = SiteSetting.automatic_translation_target_languages.split("|")
+      return if locales.blank?
+
+      # keeping this query simple by just getting any post with a missing localization
+      posts =
+        Post
+          .left_joins(:post_localizations)
+          .where(deleted_at: nil)
+          .where("posts.user_id > 0")
+          .where.not(raw: [nil, ""])
+          .group("posts.id")
+          .having(
+            "COUNT(DISTINCT CASE WHEN post_localizations.locale IN (?) THEN post_localizations.locale END) < ?",
+            locales,
+            locales.size,
+          )
+          .order(id: :desc)
+          .limit(BATCH_SIZE)
+
+      return if posts.empty?
+
+      posts.each do |post|
+        locales.each do |locale|
+          next if post.locale == locale
+          next if post.has_localization?(locale)
+
+          begin
+            DiscourseTranslator::PostTranslator.translate(post, locale)
+          rescue => e
+            Rails.logger.error(
+              "Discourse Translator: Failed to translate post #{post.id} to #{locale}: #{e.message}",
+            )
+          end
+        end
+      end
+
+      DiscourseTranslator::VerboseLogger.log(
+        "Translated #{posts.size} posts to #{locales.join(", ")}",
+      )
+    end
+  end
+end
diff --git a/app/services/discourse_translator/post_locale_detector.rb b/app/services/discourse_translator/post_locale_detector.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class PostLocaleDetector
+    def self.detect_locale(post)
+      return if post.blank?
+
+      translator = DiscourseTranslator::Provider::TranslatorProvider.get
+      detected_locale = translator.detect!(post)
+      post.update!(locale: detected_locale)
+      detected_locale
+    end
+  end
+end
diff --git a/app/services/discourse_translator/post_translator.rb b/app/services/discourse_translator/post_translator.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module DiscourseTranslator
+  class PostTranslator
+    def self.translate(post, target_locale = I18n.locale)
+      return if post.blank? || target_locale.blank?
+
+      target_locale_sym = target_locale.to_s.sub("-", "_").to_sym
+
+      translator = DiscourseTranslator::Provider::TranslatorProvider.get
+      translated_raw = translator.translate_post!(post, target_locale_sym)
+
+      localization =
+        PostLocalization.find_or_initialize_by(post_id: post.id, locale: target_locale_sym.to_s)
+
+      localization.raw = translated_raw
+      localization.cooked = PrettyText.cook(translated_raw)
+      localization.post_version = post.version
+      localization.localizer_user_id = Discourse.system_user.id
+      localization.save!
+      localization
+    end
+  end
+end
diff --git a/app/services/discourse_translator/provider/base_provider.rb b/app/services/discourse_translator/provider/base_provider.rb
@@ -55,12 +55,7 @@ def self.translate(translatable, target_locale_sym = I18n.locale)
         [detected_lang, translated]
       end
 
-      # Subclasses must implement this method to translate the text of a
-      # post or topic and return only the translated text.
-      # Subclasses should use text_for_translation
-      # @param translatable [Post|Topic]
-      # @param target_locale_sym [Symbol]
-      # @return [String]
+      # TODO: Deprecate this in favour of translate_<model>
       def self.translate_translatable!(translatable, target_locale_sym = I18n.locale)
         raise "Not Implemented"
       end
@@ -69,6 +64,14 @@ def self.translate_text!(text, target_locale_sym = I18n.locale)
         raise "Not Implemented"
       end
 
+      def self.translate_post!(post, target_locale_sym = I18n.locale)
+        translate_translatable!(post, target_locale_sym)
+      end
+
+      def self.translate_topic!(topic, target_locale_sym = I18n.locale)
+        translate_translatable!(topic, target_locale_sym)
+      end
+
       # Returns the stored detected locale of a post or topic.
       # If the locale does not exist yet, it will be detected first via the API then stored.
       # @param translatable [Post|Topic]
diff --git a/app/services/discourse_translator/provider/discourse_ai.rb b/app/services/discourse_translator/provider/discourse_ai.rb
@@ -16,25 +16,32 @@ def self.detect!(topic_or_post)
       end
 
       def self.translate_translatable!(translatable, target_locale_sym = I18n.locale)
+        if (translatable.class.name == "Post")
+          translate_post!(translatable, target_locale_sym)
+        elsif (translatable.class.name == "Topic")
+          translate_topic!(translatable, target_locale_sym)
+        end
+      end
+
+      def self.translate_post!(post, target_locale_sym = I18n.locale)
         validate_required_settings!
 
-        language = get_language_name(target_locale_sym)
+        text = text_for_translation(post, raw: true)
+        chunks = DiscourseTranslator::ContentSplitter.split(text)
         translated =
-          case translatable.class.name
-          when "Post"
-            text = text_for_translation(translatable, raw: true)
-            chunks = DiscourseTranslator::ContentSplitter.split(text)
-            chunks
-              .map { |chunk| ::DiscourseAi::PostTranslator.new(chunk, target_locale_sym).translate }
-              .join("")
-          when "Topic"
-            ::DiscourseAi::TopicTranslator.new(
-              text_for_translation(translatable),
-              language,
-            ).translate
-          end
+          chunks
+            .map { |chunk| ::DiscourseAi::PostTranslator.new(chunk, target_locale_sym).translate }
+            .join("")
+        DiscourseTranslator::TranslatedContentNormalizer.normalize(post, translated)
+      end
 
-        DiscourseTranslator::TranslatedContentNormalizer.normalize(translatable, translated)
+      def self.translate_topic!(topic, target_locale_sym = I18n.locale)
+        validate_required_settings!
+
+        language = get_language_name(target_locale_sym)
+        translated =
+          ::DiscourseAi::TopicTranslator.new(text_for_translation(topic), language).translate
+        DiscourseTranslator::TranslatedContentNormalizer.normalize(topic, translated)
       end
 
       def self.translate_text!(text, target_locale_sym = I18n.locale)
diff --git a/spec/jobs/detect_posts_locale_spec.rb b/spec/jobs/detect_posts_locale_spec.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+describe Jobs::DetectPostsLocale do
+  fab!(:post) { Fabricate(:post, locale: nil) }
+  subject(:job) { described_class.new }
+
+  before do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+  end
+
+  it "does nothing when translator is disabled" do
+    SiteSetting.translator_enabled = false
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "does nothing when content translation is disabled" do
+    SiteSetting.experimental_content_translation = false
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "does nothing when there are no posts to detect" do
+    Post.update_all(locale: "en")
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).never
+
+    job.execute({})
+  end
+
+  it "detects locale for posts with nil locale" do
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).once
+    job.execute({})
+  end
+
+  it "skips bot posts" do
+    post.update!(user: Discourse.system_user)
+    DiscourseTranslator::PostLocaleDetector.expects(:detect_locale).with(post).never
+
+    job.execute({})
+  end
+
+  it "handles detection errors gracefully" do
+    DiscourseTranslator::PostLocaleDetector
+      .expects(:detect_locale)
+      .with(post)
+      .raises(StandardError.new("jiboomz"))
+      .once
+
+    expect { job.execute({}) }.not_to raise_error
+  end
+
+  it "logs a summary after running" do
+    DiscourseTranslator::PostLocaleDetector.stubs(:detect_locale)
+    DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Detected 1 post locales"))
+
+    job.execute({})
+  end
+end
diff --git a/spec/jobs/translate_posts_spec.rb b/spec/jobs/translate_posts_spec.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+describe Jobs::TranslatePosts do
+  fab!(:post)
+  subject(:job) { described_class.new }
+
+  let(:locales) { %w[en ja] }
+
+  before do
+    SiteSetting.translator_enabled = true
+    SiteSetting.experimental_content_translation = true
+    SiteSetting.automatic_translation_backfill_rate = 1
+    SiteSetting.automatic_translation_target_languages = locales.join("|")
+  end
+
+  it "does nothing when translator is disabled" do
+    SiteSetting.translator_enabled = false
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({})
+  end
+
+  it "does nothing when content translation is disabled" do
+    SiteSetting.experimental_content_translation = false
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({})
+  end
+
+  it "does nothing when no target languages are configured" do
+    SiteSetting.automatic_translation_target_languages = ""
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({})
+  end
+
+  it "does nothing when there are no posts to translate" do
+    Post.destroy_all
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({})
+  end
+
+  it "translates posts to the configured locales" do
+    DiscourseTranslator::PostTranslator.expects(:translate).with(post, "en").at_least_once
+    DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").at_least_once
+
+    job.execute({})
+  end
+
+  it "skips posts that already have localizations" do
+    Post.all.each do |post|
+      Fabricate(:post_localization, post:, locale: "en")
+      Fabricate(:post_localization, post:, locale: "ja")
+    end
+    DiscourseTranslator::PostTranslator.expects(:translate).never
+
+    job.execute({})
+  end
+
+  it "skips bot posts" do
+    post.update!(user: Discourse.system_user)
+    DiscourseTranslator::PostTranslator.expects(:translate).with(post, "en").never
+    DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").never
+
+    job.execute({})
+  end
+
+  it "handles translation errors gracefully" do
+    DiscourseTranslator::PostTranslator
+      .expects(:translate)
+      .with(post, "en")
+      .raises(StandardError.new("API error"))
+    DiscourseTranslator::PostTranslator.expects(:translate).with(post, "ja").once
+
+    expect { job.execute({}) }.not_to raise_error
+  end
+
+  it "logs a summary after translation" do
+    DiscourseTranslator::PostTranslator.stubs(:translate)
+    DiscourseTranslator::VerboseLogger.expects(:log).with(includes("Translated 1 posts to en, ja"))
+
+    job.execute({})
+  end
+end
diff --git a/spec/services/post_locale_detector_spec.rb b/spec/services/post_locale_detector_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+describe DiscourseTranslator::PostLocaleDetector do
+  describe ".detect_locale" do
+    fab!(:post) { Fabricate(:post, raw: "Hello world", locale: nil) }
+
+    let(:translator) { mock }
+
+    before { DiscourseTranslator::Provider::TranslatorProvider.stubs(:get).returns(translator) }
+
+    it "returns nil if post is blank" do
+      expect(described_class.detect_locale(nil)).to eq(nil)
+    end
+
+    it "calls detect! on the provider with the post" do
+      translator.expects(:detect!).with(post).returns("ja")
+      expect(described_class.detect_locale(post)).to eq("ja")
+    end
+
+    it "updates the post locale with the detected locale" do
+      translator.stubs(:detect!).with(post).returns("ja")
+      expect { described_class.detect_locale(post) }.to change { post.reload.locale }.from(nil).to(
+        "ja",
+      )
+    end
+  end
+end
diff --git a/spec/services/post_translator_spec.rb b/spec/services/post_translator_spec.rb
