diff --git a/app/jobs/regular/translate_categories.rb b/app/jobs/regular/translate_categories.rb new file mode 100644 index 0000000..ca008ae --- /dev/null +++ b/app/jobs/regular/translate_categories.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Jobs + class TranslateCategories < ::Jobs::Base + cluster_concurrency 1 + BATCH_SIZE = 50 + + def execute(args) + return unless SiteSetting.translator_enabled + return unless SiteSetting.experimental_category_translation + + locales = SiteSetting.automatic_translation_target_languages.split("|") + return if locales.blank? + + cat_id = args[:from_category_id] || Category.order(:id).first&.id + last_id = nil + + # we're just gonna take all categories and keep it simple + # instead of checking in the db which ones are absent + categories = Category.where("id >= ?", cat_id).order(:id).limit(BATCH_SIZE) + return if categories.empty? + + categories.each do |category| + CategoryLocalization.transaction do + locales.each do |locale| + next if CategoryLocalization.exists?(category_id: category.id, locale: locale) + begin + DiscourseTranslator::CategoryTranslator.translate(category, locale) + rescue => e + Rails.logger.error( + "Discourse Translator: Failed to translate category #{category.id} to #{locale}: #{e.message}", + ) + end + end + end + last_id = category.id + end + + # from batch if needed + if categories.size == BATCH_SIZE + Jobs.enqueue_in(10.seconds, :translate_categories, from_category_id: last_id + 1) + end + end + end +end diff --git a/app/jobs/scheduled/automatic_category_translation.rb b/app/jobs/scheduled/automatic_category_translation.rb new file mode 100644 index 0000000..ef28e47 --- /dev/null +++ b/app/jobs/scheduled/automatic_category_translation.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Jobs + class AutomaticCategoryTranslation < ::Jobs::Scheduled + every 12.hours + cluster_concurrency 1 + + def execute(args) + return unless SiteSetting.translator_enabled + return unless SiteSetting.experimental_category_translation + + locales = SiteSetting.automatic_translation_target_languages.split("|") + return if locales.blank? + + Jobs.enqueue(:translate_categories) + end + end +end diff --git a/app/services/discourse_translator/category_translator.rb b/app/services/discourse_translator/category_translator.rb new file mode 100644 index 0000000..89fb0cf --- /dev/null +++ b/app/services/discourse_translator/category_translator.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module DiscourseTranslator + class CategoryTranslator + # unlike post and topics, categories do not have a detected locale + # and will translate two fields, name and description + + def self.translate(category, target_locale = I18n.locale) + return if category.blank? || target_locale.blank? + + # locale can come in various forms + # standardize it to a _ symbol + target_locale_sym = target_locale.to_s.sub("-", "_").to_sym + + translator = DiscourseTranslator::Provider::TranslatorProvider.get + translated_name = translator.translate_text!(category.name, target_locale_sym) + translated_description = translator.translate_text!(category.description, target_locale_sym) + + category.update!(name: translated_name, description: translated_description) + end + end +end diff --git a/app/services/discourse_translator/translator.rb b/app/services/discourse_translator/translator.rb new file mode 100644 index 0000000..0171693 --- /dev/null +++ b/app/services/discourse_translator/translator.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module DiscourseTranslator + # The canonical class for all your translation needs + class Translator + # this invokes the specific methods + def translate(translatable, target_locale = I18n.locale) + target_locale_sym = target_locale.to_s.sub("-", "_").to_sym + + case translatable.class.name + when "Post", "Topic" + DiscourseTranslator::Provider.TranslatorProvider.get.translate( + translatable, + target_locale_sym, + ) + when "Category" + CategoryTranslator.translate(translatable, target_locale) + end + end + end +end diff --git a/config/settings.yml b/config/settings.yml index bb54394..e293e9e 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -129,6 +129,9 @@ discourse_translator: experimental_inline_translation: default: false client: true + experimental_category_translation: + default: false + hidden: true discourse_translator_verbose_logs: default: false client: false diff --git a/spec/jobs/translate_categories_spec.rb b/spec/jobs/translate_categories_spec.rb new file mode 100644 index 0000000..0392ad7 --- /dev/null +++ b/spec/jobs/translate_categories_spec.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe Jobs::TranslateCategories do + subject(:job) { described_class.new } + + let(:translator) { mock } + + def localize_all_categories(*locales) + Category.all.each do |category| + locales.each { |locale| Fabricate(:category_localization, category:, locale:, name: "x") } + end + end + + before do + SiteSetting.translator_enabled = true + SiteSetting.experimental_category_translation = true + SiteSetting.automatic_translation_backfill_rate = 100 + SiteSetting.automatic_translation_target_languages = "pt|zh_CN" + + DiscourseTranslator::Provider.stubs(:get).returns(translator) + Jobs.run_immediately! + end + + it "does nothing when translator is disabled" do + SiteSetting.translator_enabled = false + + translator.expects(:translate_text!).never + + job.execute({}) + end + + it "does nothing when experimental_category_translation is disabled" do + SiteSetting.experimental_category_translation = false + + translator.expects(:translate_text!).never + + job.execute({}) + end + + it "does nothing when no target languages are configured" do + SiteSetting.automatic_translation_target_languages = "" + + translator.expects(:translate_text!).never + + job.execute({}) + end + + it "does nothing when no categories exist" do + Category.destroy_all + + translator.expects(:translate_text!).never + + job.execute({}) + end + + it "translates categories to the configured locales" do + number_of_categories = Category.count + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(is_a(Category), "pt") + .times(number_of_categories) + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(is_a(Category), "zh_CN") + .times(number_of_categories) + + job.execute({}) + end + + it "skips categories that already have localizations" do + localize_all_categories("pt", "zh_CN") + + category1 = + Fabricate(:category, name: "First Category", description: "First category description") + Fabricate(:category_localization, category: category1, locale: "pt", name: "Primeira Categoria") + + # It should only translate to Chinese, not Portuguese + DiscourseTranslator::CategoryTranslator.expects(:translate).with(category1, "pt").never + DiscourseTranslator::CategoryTranslator.expects(:translate).with(category1, "zh_CN").once + + job.execute({}) + end + + it "continues from a specified category ID" do + category1 = Fabricate(:category, name: "First", description: "First description") + category2 = Fabricate(:category, name: "Second", description: "Second description") + + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(category1, any_parameters) + .never + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(category2, any_parameters) + .twice + + job.execute(from_category_id: category2.id) + end + + it "handles translation errors gracefully" do + localize_all_categories("pt", "zh_CN") + + category1 = Fabricate(:category, name: "First", description: "First description") + DiscourseTranslator::CategoryTranslator + .expects(:translate) + .with(category1, "pt") + .raises(StandardError.new("API error")) + DiscourseTranslator::CategoryTranslator.expects(:translate).with(category1, "zh_CN").once + + expect { job.execute({}) }.not_to raise_error + end + + it "enqueues the next batch when there are more categories" do + Jobs.run_later! + freeze_time + Jobs::TranslateCategories.const_set(:BATCH_SIZE, 1) + + job.execute({}) + + Category.all.each do |category| + puts category.id + expect_job_enqueued( + job: :translate_categories, + args: { + from_category_id: category.id + 1, + }, + at: 10.seconds.from_now, + ) + end + + Jobs::TranslateCategories.send(:remove_const, :BATCH_SIZE) + Jobs::TranslateCategories.const_set(:BATCH_SIZE, 50) + end +end diff --git a/spec/services/category_translator_spec.rb b/spec/services/category_translator_spec.rb new file mode 100644 index 0000000..8e2d1b5 --- /dev/null +++ b/spec/services/category_translator_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +describe DiscourseTranslator::CategoryTranslator do + fab!(:category) do + Fabricate(:category, name: "Test Category", description: "This is a test category") + end + + describe ".translate" do + let(:target_locale) { :fr } + let(:translator) { mock } + + before { DiscourseTranslator::Provider::TranslatorProvider.stubs(:get).returns(translator) } + + it "translates the category name and description" do + translator + .expects(:translate_text!) + .with(category.name, target_locale) + .returns("Catégorie de Test") + translator + .expects(:translate_text!) + .with(category.description, target_locale) + .returns("C'est une catégorie de test") + + DiscourseTranslator::CategoryTranslator.translate(category, target_locale) + + expect(category.name).to eq("Catégorie de Test") + expect(category.description).to eq("C'est une catégorie de test") + end + + it "handles locale format standardization" do + translator.expects(:translate_text!).with(category.name, :fr_CA).returns("Catégorie de Test") + translator + .expects(:translate_text!) + .with(category.description, :fr_CA) + .returns("C'est une catégorie de test") + + DiscourseTranslator::CategoryTranslator.translate(category, "fr-CA") + + expect(category.name).to eq("Catégorie de Test") + expect(category.description).to eq("C'est une catégorie de test") + end + + it "returns nil if category is blank" do + expect(DiscourseTranslator::CategoryTranslator.translate(nil)).to be_nil + end + + it "returns nil if target locale is blank" do + expect(DiscourseTranslator::CategoryTranslator.translate(category, nil)).to be_nil + end + + it "uses I18n.locale as default when no target locale is provided" do + I18n.locale = :es + translator.expects(:translate_text!).with(category.name, :es).returns("Categoría de Prueba") + translator + .expects(:translate_text!) + .with(category.description, :es) + .returns("Esta es una categoría de prueba") + + DiscourseTranslator::CategoryTranslator.translate(category) + + expect(category.name).to eq("Categoría de Prueba") + expect(category.description).to eq("Esta es una categoría de prueba") + end + end +end