Add CSV plural support

Author: SkyLucilfer

core/compressed_translation.cpp

@@ -31,6 +31,7 @@
 #include "compressed_translation.h"
 
 #include "core/pair.h"
+#include "core/translation_plural_rules.h"
 
 extern "C" {
 #include "thirdparty/misc/smaz.h"
@@ -272,8 +273,10 @@ StringName PHashTranslation::get_message(const StringName &p_src_text, const Str
 }
 
 StringName PHashTranslation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const {
-	// The use of plurals translation is not yet supported in PHashTranslation.
-	return get_message(p_src_text, p_context);
+	// It is assumed for now that PHashTranslation is only used by CSV translation. So get_plural_message() method follows the CSV plural format to fetch the translation.
+	int index = TranslationPluralRules::get_plural_index(get_locale(), p_n);
+	String search_src = String(p_src_text) + "[" + String::num_int64(index) + "]";
+	return get_message(search_src, p_context);
 }
 
 void PHashTranslation::_get_property_list(List<PropertyInfo> *p_list) const {

core/io/translation_loader_po.cpp

@@ -139,12 +139,19 @@ RES TranslationLoaderPO::load_translation(FileAccess *f, Error *r_error) {
 				}
 			} else if (config == "") {
 				config = msg_str;
-				// Record plural rule.
-				int p_start = config.find("Plural-Forms");
-				if (p_start != -1) {
-					int p_end = config.find("\n", p_start);
-					translation->set_plural_rule(config.substr(p_start, p_end - p_start));
-					plural_forms = translation->get_plural_forms();
+				// Set plural_forms and locale.
+				int nplural_start = config.find("nplurals=");
+				if (nplural_start != -1) {
+					nplural_start += String("nplurals=").length();
+					int nplural_end = config.find(";", nplural_start);
+					plural_forms = config.substr(nplural_start, nplural_end - nplural_start).to_int();
+
+					int lang_start = config.find("Language:") + String("Language:").length();
+					if (lang_start - String("Language:").length() == -1) {
+						lang_start = config.find("X-Language:") + String("X-Language:").length();
+					}
+					int lang_end = config.find("\n", lang_start);
+					translation->set_locale(config.substr(lang_start, lang_end - lang_start).strip_edges());
 				}
 			}
 

core/object.cpp

@@ -1439,10 +1439,10 @@ String Object::tr(const StringName &p_message, const StringName &p_context) cons
 	return TranslationServer::get_singleton()->translate(p_message, p_context);
 }
 
-String Object::tr_n(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const {
+String Object::tr_n(int p_n, const StringName &p_message, const StringName &p_message_plural, const StringName &p_context) const {
 	if (!_can_translate || !TranslationServer::get_singleton()) {
-		// Return message based on English plural rule if translation is not possible.
-		if (p_n == 1) {
+		// If no plural message, just return the key (for CSV). Else, return message based on English plural rule.
+		if (p_message_plural == StringName() || p_n == 1) {
 			return p_message;
 		}
 		return p_message_plural;
@@ -1589,7 +1589,7 @@ void Object::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_message_translation", "enable"), &Object::set_message_translation);
 	ClassDB::bind_method(D_METHOD("can_translate_messages"), &Object::can_translate_messages);
 	ClassDB::bind_method(D_METHOD("tr", "message", "context"), &Object::tr, DEFVAL(""));
-	ClassDB::bind_method(D_METHOD("tr_n", "message", "plural_message", "n", "context"), &Object::tr_n, DEFVAL(""));
+	ClassDB::bind_method(D_METHOD("tr_n", "n", "message", "plural_message", "context"), &Object::tr_n, DEFVAL(""), DEFVAL(""));
 
 	ClassDB::bind_method(D_METHOD("is_queued_for_deletion"), &Object::is_queued_for_deletion);
 

core/object.h

@@ -720,7 +720,7 @@ class Object {
 	virtual void get_argument_options(const StringName &p_function, int p_idx, List<String> *r_options) const;
 
 	String tr(const StringName &p_message, const StringName &p_context = "") const; // translate message (internationalization)
-	String tr_n(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context = "") const;
+	String tr_n(int p_n, const StringName &p_message, const StringName &p_message_plural = "", const StringName &p_context = "") const;
 
 	bool _is_queued_for_deletion = false; // set to true by SceneTree::queue_delete()
 	bool is_queued_for_deletion() const;

core/plural_rules_builder.py

@@ -0,0 +1,78 @@
+#### TO MENTORS:
+#### I'm not sure what is the right way to write a Python script in Godot? I saw other Python files all have def func_name(target, source, env).
+#### Also I'm not sure of the location.
+#### Right now I call this script using the "python" command.
+
+# This script is used to generate core/translation_plural_rules.h
+# The plural data used in this Python script is from POEdit Github repository - https://github.com/vslavik/poedit/raw/master/src/language_impl_plurals.h
+# POEdit in turn parsed and extracted the plural data from Unicode Consortium (see http://www.unicode.org/cldr/charts/supplemental/language_plural_rules.html).
+
+#!/bin/python
+
+import sys
+import urllib.request
+
+target_file = "translation_plural_rules.h"
+
+plural_data_url = "https://github.com/vslavik/poedit/raw/master/src/language_impl_plurals.h"
+generate_text = "// The rest of the file is generated via plural_rules_builder.py (DO NOT MODIFY THIS LINE).\n"
+
+# Get plural data from POEdit github repository. See above for the link.
+text = ""
+with urllib.request.urlopen(plural_data_url) as response:
+    text = response.read().decode("utf-8")
+
+# Build dictionary mapping plural rule to a list of locales from plural data. Also store the number of plural forms each local has.
+rules = text.split("\n")
+rule_to_locales = {}
+locale_nplural = {}
+for rule in rules:
+    if rule.find("plural=") == -1:
+        continue
+
+    first_quote = rule.find('"')
+    second_quote = rule.find('"', first_quote + 1)
+    locale = rule[first_quote + 1 : second_quote]
+
+    plural_start = rule.find("plural=") + len("plural=")
+    plural_end = rule.find(";", plural_start)
+    plural_rule = rule[plural_start:plural_end]
+
+    if plural_rule in rule_to_locales:
+        rule_to_locales[plural_rule].append(locale)
+    else:
+        rule_to_locales[plural_rule] = [locale]
+
+    nplural_start = rule.find("nplurals=") + len("nplurals=")
+    locale_nplural[locale] = rule[nplural_start : nplural_start + 1]
+
+# Find the position in translation_plural_rules.h for code generation.
+with open(target_file, "r") as f:
+    text = f.read()
+gen_start_pos = text.find(generate_text) + len(generate_text)
+pretext = text[:gen_start_pos]
+
+# Generate mapping in build_mapping().
+gen_content = "\n"
+rule_number = 1
+for rule in rule_to_locales:
+    for locale in rule_to_locales[rule]:
+        gen_content += (
+            '\t\ttemp.set("' + locale + '", { ' + locale_nplural[locale] + ", &rule" + str(rule_number) + " });\n"
+        )
+    rule_number += 1
+gen_content += "\t\treturn temp;\n\t}\n"
+
+# Generate rule functions.
+rule_number = 1
+for rule in rule_to_locales:
+    gen_content += "\n\tstatic int rule" + str(rule_number) + "(const int p_n) {\n\t\t// "
+    gen_content += ", ".join(rule_to_locales[rule]) + ".\n\t\t"
+    gen_content += "return " + rule.replace("n", "p_n") + ";\n\t}\n"
+    rule_number += 1
+
+gen_content += "};\n\n#endif // TRANSLATION_PLURAL_RULES_H\n"
+
+# Write to file.
+with open(target_file, "w") as f:
+    f.write(pretext + gen_content)

core/translation.cpp

@@ -33,6 +33,7 @@
 #include "core/io/resource_loader.h"
 #include "core/os/os.h"
 #include "core/project_settings.h"
+#include "core/translation_plural_rules.h"
 
 // ISO 639-1 language codes, with the addition of glibc locales with their
 // regional identifiers. This list must match the language names (in English)
@@ -845,9 +846,7 @@ void Translation::add_message(const StringName &p_src_text, const StringName &p_
 }
 
 void Translation::add_plural_message(const StringName &p_src_text, const Vector<String> &p_plural_xlated_texts, const StringName &p_context) {
-	WARN_PRINT("Translation class doesn't handle plural messages. Calling add_plural_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles plurals, such as TranslationPO class");
-	ERR_FAIL_COND_MSG(p_plural_xlated_texts.empty(), "Parameter vector p_plural_xlated_texts passed in is empty.");
-	translation_map[p_src_text] = p_plural_xlated_texts[0];
+	WARN_PRINT("Translation class doesn't use add_plural_message(), as plural keys are appended with subscript by users. Calling add_plural_message() on a Translation instance is probably a mistake.");
 }
 
 StringName Translation::get_message(const StringName &p_src_text, const StringName &p_context) const {
@@ -864,8 +863,24 @@ StringName Translation::get_message(const StringName &p_src_text, const StringNa
 }
 
 StringName Translation::get_plural_message(const StringName &p_src_text, const StringName &p_plural_text, int p_n, const StringName &p_context) const {
-	WARN_PRINT("Translation class doesn't handle plural messages. Calling get_plural_message() on a Translation instance is probably a mistake. \nUse a derived Translation class that handles plurals, such as TranslationPO class");
-	return get_message(p_src_text);
+	int index = TranslationPluralRules::get_plural_index(locale, p_n);
+	String search_src = String(p_src_text) + "[" + String::num_int64(index) + "]";
+
+	const Map<StringName, StringName>::Element *E = translation_map.find(search_src);
+	if (E) {
+		return E->get();
+	}
+
+	// Fallback to singular translation.
+	WARN_PRINT("CSV plural translation for \"" + String(search_src) + "\" with locale \"" + locale + "\" is not found. Returning singular translation...");
+	search_src.set(search_src.length() - 2, '0');
+	E = translation_map.find(search_src);
+	if (!E) {
+		WARN_PRINT("Singular translation \"" + String(search_src) + "\" with locale \"" + locale + "\" is not found.");
+		return StringName();
+	}
+
+	return E->get();
 }
 
 void Translation::erase_message(const StringName &p_src_text, const StringName &p_context) {
@@ -979,6 +994,14 @@ String TranslationServer::get_locale() const {
 	return locale;
 }
 
+int TranslationServer::get_plural_index(const String &p_locale, int p_n) {
+	return TranslationPluralRules::get_plural_index(p_locale, p_n);
+}
+
+int TranslationServer::get_plural_forms(const String &p_locale) {
+	return TranslationPluralRules::get_plural_forms(p_locale);
+}
+
 String TranslationServer::get_locale_name(const String &p_locale) const {
 	if (!locale_name_map.has(p_locale)) {
 		return String();
@@ -1085,7 +1108,7 @@ StringName TranslationServer::translate(const StringName &p_message, const Strin
 
 StringName TranslationServer::translate_plural(const StringName &p_message, const StringName &p_message_plural, int p_n, const StringName &p_context) const {
 	if (!enabled) {
-		if (p_n == 1) {
+		if (p_message_plural == StringName() || p_n == 1) {
 			return p_message;
 		}
 		return p_message_plural;
@@ -1100,7 +1123,7 @@ StringName TranslationServer::translate_plural(const StringName &p_message, cons
 	}
 
 	if (!res) {
-		if (p_n == 1) {
+		if (p_message_plural == StringName() || p_n == 1) {
 			return p_message;
 		}
 		return p_message_plural;
@@ -1269,6 +1292,8 @@ StringName TranslationServer::doc_translate_plural(const StringName &p_message,
 void TranslationServer::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_locale", "locale"), &TranslationServer::set_locale);
 	ClassDB::bind_method(D_METHOD("get_locale"), &TranslationServer::get_locale);
+	ClassDB::bind_method(D_METHOD("get_plural_index", "locale", "n"), &TranslationServer::get_plural_index);
+	ClassDB::bind_method(D_METHOD("get_plural_forms", "locale"), &TranslationServer::get_plural_forms);
 
 	ClassDB::bind_method(D_METHOD("get_locale_name", "locale"), &TranslationServer::get_locale_name);
 

core/translation.h

@@ -92,6 +92,8 @@ class TranslationServer : public Object {
 
 	void set_locale(const String &p_locale);
 	String get_locale() const;
+	int get_plural_index(const String &p_locale, int p_n);
+	int get_plural_forms(const String &p_locale);
 	Ref<Translation> get_translation_object(const String &p_locale);
 
 	String get_locale_name(const String &p_locale) const;

core/translation_plural_rules.cpp

@@ -0,0 +1,34 @@
+/*************************************************************************/
+/*  translation_plural_rules.cpp                                         */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2020 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2020 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "core/translation_plural_rules.h"
+
+// Initialize static member plural_rule_mapping.
+HashMap<String, TranslationPluralRules::PluralData> TranslationPluralRules::plural_rule_mapping = TranslationPluralRules::build_mapping();