Integration of tatoeba for simple sentences translation

app/build.gradle

@@ -125,6 +125,10 @@ dependencies {
     implementation "ai.djl.android:core:0.33.0"
     implementation "ai.djl.huggingface:tokenizers:0.33.0"
     implementation "ai.djl.android:tokenizer-native:0.33.0"
+    //protobuf
+    implementation "com.google.protobuf:protobuf-javalite:4.33.5"
+    // bouncycastle crypto lib
+    implementation "org.bouncycastle:bcprov-jdk15to18:1.83"
 
     //implementation 'androidx.core:core-ktx:1.10.0'
     implementation 'androidx.work:work-runtime:2.7.1'
@@ -140,6 +144,7 @@ dependencies {
     implementation 'com.google.mlkit:language-id:17.0.5'
     //JWS parser
     implementation group: 'com.nimbusds', name: 'nimbus-jose-jwt', version: '5.1'
+    implementation files('libs/sqlite4java-android-release.aar')
     //Room library (database)
     def room_version = '2.1.0'
     implementation "androidx.room:room-runtime:$room_version"

app/src/main/cpp/src/BergamotTranslator.cpp

@@ -73,11 +73,11 @@ std::vector<std::string> translateMultiple(std::vector<std::string> &&inputs, co
 
     // Assume models are already loaded in cache
     if (srcLang != "en") {
-        if(model_cache.find(srcLang) == model_cache.end()) throw std::runtime_error("Missing loaded src model");
+        if(model_cache.find(srcLang) == model_cache.end()) throw std::runtime_error("Missing loaded src model: "+srcLang);
         firstModel = model_cache[srcLang]->toEnglishModel;
     }
     if (trgLang != "en") {
-        if(model_cache.find(trgLang) == model_cache.end()) throw std::runtime_error("Missing loaded trg model");
+        if(model_cache.find(trgLang) == model_cache.end()) throw std::runtime_error("Missing loaded trg model: "+trgLang);
         secondModel = model_cache[trgLang]->fromEnglishModel;
     }
 

app/src/main/java/nie/translator/rtranslator/Global.java

@@ -43,6 +43,7 @@
 import nie.translator.rtranslator.access.AccessActivity;
 import nie.translator.rtranslator.tools.CustomLocale;
 import nie.translator.rtranslator.tools.TTS;
+import nie.translator.rtranslator.voice_translation._conversation_mode._conversation.ConversationService;
 import nie.translator.rtranslator.voice_translation._conversation_mode.communication.ConversationBluetoothCommunicator;
 import nie.translator.rtranslator.bluetooth.BluetoothCommunicator;
 import nie.translator.rtranslator.bluetooth.Peer;
@@ -85,6 +86,7 @@ public enum RTranslatorMode {
     private Handler mainHandler;
     private static Handler mHandler = new Handler();
     private final Object lock = new Object();
+    private boolean useTatoeba;
 
     @Override
     public void onCreate() {
@@ -98,6 +100,8 @@ public void onCreate() {
         //initializeBluetoothCommunicator();
         getMicSensitivity();
         createNotificationChannel();
+        SharedPreferences sharedPreferences = getSharedPreferences("default", Context.MODE_PRIVATE);
+        useTatoeba = sharedPreferences.getBoolean("useTatoeba", false);
     }
 
     public void initializeTranslator(Translator.GeneralListener initListener){
@@ -430,6 +434,12 @@ public void setLanguage(CustomLocale language) {
         SharedPreferences.Editor editor = sharedPreferences.edit();
         editor.putString("language", language.getCode());
         editor.apply();
+        if(ConversationService.isRunning){
+            translator.loadTgtLangResourcesForConversation(language, null);
+            if(getBluetoothCommunicator() != null){
+                getBluetoothCommunicator().sendLanguage(language);
+            }
+        }
     }
 
     public void setFirstLanguage(CustomLocale language, @Nullable Translator.GeneralListener listener) {
@@ -438,11 +448,7 @@ public void setFirstLanguage(CustomLocale language, @Nullable Translator.General
         SharedPreferences.Editor editor = sharedPreferences.edit();
         editor.putString("firstLanguage", language.getCode());
         editor.apply();
-        if(translator.getMode() == Translator.MOZILLA) {
-            translator.loadMozillaModels(language, getSecondLanguage(true), RTranslatorMode.WALKIE_TALKIE_MODE, listener);
-        }else{
-            if(listener != null) listener.onSuccess();
-        }
+        loadLanguagesResources(language, getSecondLanguage(true), RTranslatorMode.WALKIE_TALKIE_MODE, listener);
     }
 
     public void setSecondLanguage(CustomLocale language, @Nullable Translator.GeneralListener listener) {
@@ -451,11 +457,7 @@ public void setSecondLanguage(CustomLocale language, @Nullable Translator.Genera
         SharedPreferences.Editor editor = sharedPreferences.edit();
         editor.putString("secondLanguage", language.getCode());
         editor.apply();
-        if(translator.getMode() == Translator.MOZILLA) {
-            translator.loadMozillaModels(getFirstLanguage(true), language, RTranslatorMode.WALKIE_TALKIE_MODE, listener);
-        }else{
-            if(listener != null) listener.onSuccess();
-        }
+        loadLanguagesResources(getFirstLanguage(true), language, RTranslatorMode.WALKIE_TALKIE_MODE, listener);
     }
 
     public void setFirstTextLanguage(CustomLocale language, @Nullable Translator.GeneralListener listener) {
@@ -464,11 +466,7 @@ public void setFirstTextLanguage(CustomLocale language, @Nullable Translator.Gen
         SharedPreferences.Editor editor = sharedPreferences.edit();
         editor.putString("firstTextLanguage", language.getCode());
         editor.apply();
-        if(translator.getMode() == Translator.MOZILLA) {
-            translator.loadMozillaModels(language, getSecondTextLanguage(true), RTranslatorMode.TEXT_TRANSLATION_MODE, listener);
-        }else{
-            if(listener != null) listener.onSuccess();
-        }
+        loadLanguagesResources(language, getSecondTextLanguage(true), RTranslatorMode.TEXT_TRANSLATION_MODE, listener);
     }
 
     public void setSecondTextLanguage(CustomLocale language, @Nullable Translator.GeneralListener listener) {
@@ -478,24 +476,20 @@ public void setSecondTextLanguage(CustomLocale language, @Nullable Translator.Ge
         SharedPreferences.Editor editor = sharedPreferences.edit();
         editor.putString("secondTextLanguage", language.getCode());
         editor.apply();
-        if(translator.getMode() == Translator.MOZILLA) {
-            translator.loadMozillaModels(getFirstTextLanguage(true), language, RTranslatorMode.TEXT_TRANSLATION_MODE, listener);
-        }else{
-            if(listener != null) listener.onSuccess();
-        }
+        loadLanguagesResources(getFirstTextLanguage(true), language, RTranslatorMode.TEXT_TRANSLATION_MODE, listener);
     }
 
-    public void switchTextLanguages(){
+    public void switchTextLanguages(@Nullable Translator.GeneralListener listener) {
         CustomLocale firstLanguage = getFirstTextLanguage(true);
         CustomLocale secondLanguage = getSecondTextLanguage(true);
         this.firstTextLanguage = secondLanguage;
         this.secondTextLanguage = firstLanguage;
-        if(translator.getMode() == Translator.MOZILLA) translator.loadMozillaModels(secondLanguage, firstLanguage, RTranslatorMode.TEXT_TRANSLATION_MODE, null);
         SharedPreferences sharedPreferences = this.getSharedPreferences("default", Context.MODE_PRIVATE);
         SharedPreferences.Editor editor = sharedPreferences.edit();
         editor.putString("firstTextLanguage", this.firstTextLanguage.getCode());
         editor.putString("secondTextLanguage", this.secondTextLanguage.getCode());
         editor.apply();
+        loadLanguagesResources(secondLanguage, firstLanguage, RTranslatorMode.TEXT_TRANSLATION_MODE, listener);
     }
 
 
@@ -579,6 +573,24 @@ public void setBeamSize(int value) {
         editor.apply();
     }
 
+    public boolean isUseTatoeba(){
+        return useTatoeba;
+    }
+
+    public void setUseTatoeba(boolean useTatoeba) {
+        this.useTatoeba = useTatoeba;
+        final SharedPreferences sharedPreferences = this.getSharedPreferences("default", Context.MODE_PRIVATE);
+        SharedPreferences.Editor editor = sharedPreferences.edit();
+        editor.putBoolean("useTatoeba", useTatoeba);
+        editor.apply();
+        // loading of tatoeba
+        if(useTatoeba) {
+            translator.loadAllTatoebaResources(null);
+        }else{
+            translator.unloadAllTatoebaResources();
+        }
+    }
+
     public String getName() {
         if (name.length() == 0) {
             final SharedPreferences sharedPreferences = this.getSharedPreferences("default", Context.MODE_PRIVATE);
@@ -594,7 +606,7 @@ public void setName(String savedName) {
         editor.putString("name", savedName);
         editor.apply();
         if(getBluetoothCommunicator() != null) {
-            getBluetoothCommunicator().setName(savedName);  //si aggiorna il nome anche per il comunicator
+            getBluetoothCommunicator().setName(savedName);  //we update the name also for communicator
         }
     }
 
@@ -718,5 +730,9 @@ public boolean isNetworkOnWifi() {
             return false; // user turned off wifi
         }
     }
+
+    private void loadLanguagesResources(CustomLocale firstLanguage, CustomLocale secondLanguage, RTranslatorMode mode, Translator.GeneralListener listener){
+        translator.loadLanguageResources(firstLanguage, secondLanguage, mode, listener);
+    }
 }
 

app/src/main/java/nie/translator/rtranslator/database/AppDatabase.java → app/src/main/java/nie/translator/rtranslator/databases/general/AppDatabase.java

@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-package nie.translator.rtranslator.database;
+package nie.translator.rtranslator.databases.general;
 
 import androidx.room.RoomDatabase;
-import nie.translator.rtranslator.database.dao.MyDao;
-import nie.translator.rtranslator.database.entities.RecentPeerEntity;
+import nie.translator.rtranslator.databases.general.dao.MyDao;
+import nie.translator.rtranslator.databases.general.entities.RecentPeerEntity;
 
 
 @androidx.room.Database(version = 1, entities = {RecentPeerEntity.class})

app/src/main/java/nie/translator/rtranslator/database/dao/MyDao.java → app/src/main/java/nie/translator/rtranslator/databases/general/dao/MyDao.java

@@ -14,15 +14,15 @@
  * limitations under the License.
  */
 
-package nie.translator.rtranslator.database.dao;
+package nie.translator.rtranslator.databases.general.dao;
 
 import androidx.room.Dao;
 import androidx.room.Delete;
 import androidx.room.Insert;
 import androidx.room.OnConflictStrategy;
 import androidx.room.Query;
 import androidx.room.Update;
-import nie.translator.rtranslator.database.entities.RecentPeerEntity;
+import nie.translator.rtranslator.databases.general.entities.RecentPeerEntity;
 
 @Dao
 public interface MyDao{

app/src/main/java/nie/translator/rtranslator/database/entities/RecentPeerEntity.java → app/src/main/java/nie/translator/rtranslator/databases/general/entities/RecentPeerEntity.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package nie.translator.rtranslator.database.entities;
+package nie.translator.rtranslator.databases.general.entities;
 
 import androidx.room.ColumnInfo;
 import androidx.room.Entity;