Skip to content

Commit 6118b05

Browse files
niedevniedev
committed
Implemented language resource management for Conversation with bluetooth communication protocol changes
1 parent 301ed2d commit 6118b05

6 files changed

Lines changed: 66 additions & 19 deletions

File tree

‎app/src/main/cpp/src/BergamotTranslator.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,11 +73,11 @@ std::vector<std::string> translateMultiple(std::vector<std::string> &&inputs, co
7373

7474
// Assume models are already loaded in cache
7575
if (srcLang != "en") {
76-
if(model_cache.find(srcLang) == model_cache.end()) throw std::runtime_error("Missing loaded src model");
76+
if(model_cache.find(srcLang) == model_cache.end()) throw std::runtime_error("Missing loaded src model: "+srcLang);
7777
firstModel = model_cache[srcLang]->toEnglishModel;
7878
}
7979
if (trgLang != "en") {
80-
if(model_cache.find(trgLang) == model_cache.end()) throw std::runtime_error("Missing loaded trg model");
80+
if(model_cache.find(trgLang) == model_cache.end()) throw std::runtime_error("Missing loaded trg model: "+trgLang);
8181
secondModel = model_cache[trgLang]->fromEnglishModel;
8282
}
8383

‎app/src/main/java/nie/translator/rtranslator/Global.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,9 @@ public void setLanguage(CustomLocale language) {
436436
editor.apply();
437437
if(ConversationService.isRunning){
438438
translator.loadTgtLangResourcesForConversation(language, null);
439+
if(getBluetoothCommunicator() != null){
440+
getBluetoothCommunicator().sendLanguage(language);
441+
}
439442
}
440443
}
441444

‎app/src/main/java/nie/translator/rtranslator/voice_translation/_conversation_mode/_conversation/ConversationService.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,6 @@ public void onMessageReceived(final Message message) {
154154
int languageCodeSize = Integer.valueOf(completeText.substring(completeText.length() - 1));
155155
String text = completeText.substring(0, completeText.length() - (languageCodeSize + 1));
156156
String languageCode = completeText.substring(completeText.length() - (languageCodeSize + 1), completeText.length() - 1);
157-
158157
ConversationMessage conversationMessage = new ConversationMessage(message.getSender(), new NeuralNetworkApiText(text, CustomLocale.getInstance(languageCode)));
159158
translator.translateMessage(conversationMessage, language, TRANSLATOR_BEAM_SIZE, new Translator.TranslateMessageListener() {
160159
@Override

‎app/src/main/java/nie/translator/rtranslator/voice_translation/_conversation_mode/communication/ConversationBluetoothCommunicator.java‎

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,12 @@
2020
import android.graphics.Bitmap;
2121
import android.os.Handler;
2222
import android.os.Looper;
23+
import android.util.Log;
2324

2425
import nie.translator.rtranslator.bluetooth.BluetoothCommunicator;
2526
import nie.translator.rtranslator.bluetooth.Message;
2627
import nie.translator.rtranslator.bluetooth.Peer;
28+
import nie.translator.rtranslator.tools.CustomLocale;
2729
import nie.translator.rtranslator.tools.GalleryImageSelector;
2830

2931
import java.util.ArrayList;
@@ -179,6 +181,21 @@ public void onRecentPeerObtained(RecentPeer recentPeer) {
179181
});
180182
break;
181183
}
184+
case "l": {
185+
//we have received an empty message that contains only the other peer language, we load that language resources
186+
String completeText = message.getText();
187+
int srcLanguageCodeSize = Integer.valueOf(completeText.substring(completeText.length() - 1));
188+
String srcLanguageCode = completeText.substring(completeText.length() - (srcLanguageCodeSize + 1), completeText.length() - 1);
189+
CustomLocale srcLanguage = new CustomLocale(srcLanguageCode);
190+
Log.d("ConvBluetoothCommunicator", "language received: "+srcLanguageCode);
191+
boolean firstLoad = global.getTranslator().getLanguageResourcesManager().isPeerLoaded(message.getSender());
192+
global.getTranslator().loadSrcLangResourcesForPeer(srcLanguage, message.getSender(), null);
193+
// if we are the server and the language indication received is the one sent at the beginning of connection (and not after, when the peer updates the language), we respond with our language
194+
if (source == BluetoothCommunicator.SERVER && firstLoad) {
195+
sendLanguage(global.getLanguage(true), message.getSender());
196+
}
197+
break;
198+
}
182199
}
183200
}
184201

@@ -190,24 +207,28 @@ public void onDataReceived(final Message data, int source) {
190207
// adding the peer image to recent devices
191208
if (source == BluetoothCommunicator.SERVER) {
192209
sendImage(data.getSender());
210+
}else{
211+
sendLanguage(global.getLanguage(true), data.getSender());
193212
}
194213
global.getRecentPeersDataManager().getRecentPeers(new RecentPeersDataManager.RecentPeersListener() {
195214
@Override
196215
public void onRecentPeersObtained(ArrayList<RecentPeer> recentPeers) {
197-
Bitmap image = null;
198-
if (!data.getText().equals("null")) {
199-
image = Tools.convertBytesToBitmap(data.getData());
200-
}
201-
for (RecentPeer recentPeer : recentPeers) {
202-
if (data.getSender().getUniqueName().equals(recentPeer.getUniqueName())) {
203-
global.getRecentPeersDataManager().insertRecentPeer(recentPeer.getDeviceID(), recentPeer.getUniqueName(), image);
216+
if(data.getSender() != null) {
217+
Bitmap image = null;
218+
if (!data.getText().equals("null")) {
219+
image = Tools.convertBytesToBitmap(data.getData());
220+
}
221+
for (RecentPeer recentPeer : recentPeers) {
222+
if (data.getSender().getUniqueName().equals(recentPeer.getUniqueName())) {
223+
global.getRecentPeersDataManager().insertRecentPeer(recentPeer.getDeviceID(), recentPeer.getUniqueName(), image);
224+
}
225+
}
226+
int index = connectedPeers.indexOf(data.getSender());
227+
if (index != -1) {
228+
GuiPeer clonePeer = (GuiPeer) connectedPeers.get(index).clone();
229+
connectedPeers.get(index).setUserImage(image);
230+
notifyPeerUpdated(clonePeer, connectedPeers.get(index));
204231
}
205-
}
206-
int index = connectedPeers.indexOf(data.getSender());
207-
if (index != -1) {
208-
GuiPeer clonePeer = (GuiPeer) connectedPeers.get(index).clone();
209-
connectedPeers.get(index).setUserImage(image);
210-
notifyPeerUpdated(clonePeer, connectedPeers.get(index));
211232
}
212233
}
213234
});
@@ -300,6 +321,16 @@ public void sendMessage(Message message) {
300321
bluetoothCommunicator.sendMessage(message);
301322
}
302323

324+
public void sendLanguage(CustomLocale lang, Peer receiver) {
325+
String languageCode = lang.getCode();
326+
bluetoothCommunicator.sendMessage(new Message(global, "l", languageCode + languageCode.length(), receiver)); //this message will be used by the other Peer to load the language resources for our language
327+
}
328+
329+
public void sendLanguage(CustomLocale lang) {
330+
String languageCode = lang.getCode();
331+
bluetoothCommunicator.sendMessage(new Message(global, "l", languageCode + languageCode.length(), null)); //this message will be used by the other Peer to load the language resources for our language
332+
}
333+
303334
private void sendID(final Peer receiver) {
304335
global.getMyID(new Global.MyIDListener() {
305336
@Override

‎app/src/main/java/nie/translator/rtranslator/voice_translation/neural_networks/translation/LanguageResourcesManager.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
import android.os.Environment;
66
import android.os.Looper;
7+
import android.util.Log;
8+
import android.widget.Toast;
79

810
import androidx.annotation.NonNull;
911
import androidx.annotation.Nullable;
@@ -87,6 +89,13 @@ public void loadLanguageResources(@NonNull CustomLocale srcLang, @NonNull Custom
8789
}
8890
}
8991

92+
public boolean isPeerLoaded(@Nullable Peer peer){
93+
if(peer != null) {
94+
return languageResourcesIndicator.conversationSrcResources.containsKey(peer.getUniqueName());
95+
}
96+
return false;
97+
}
98+
9099
public void loadSrcLangResourcesForPeer(CustomLocale srcLang, Peer peer) throws Exception {
91100
CustomLocale tgtLang = languageResourcesIndicator.conversationTgtResource != null ? languageResourcesIndicator.conversationTgtResource : global.getLanguage(true);
92101
HashMap<String, CustomLocale> conversationSrcModels = languageResourcesIndicator.conversationSrcResources;
@@ -235,13 +244,14 @@ public void updatePeer(Peer oldPeer, Peer newPeer){
235244
}
236245

237246
private void performLoadLanguageResources(@NonNull CustomLocale srcLang, @NonNull CustomLocale tgtLang, Global.RTranslatorMode rtranslatorMode) throws Exception {
247+
Log.d("language_resources", "Language loaded: "+srcLang.getLanguage());
238248
if(modelMode == MOZILLA){
239249
ArrayList<CustomLocale> allUniqueResources = languageResourcesIndicator.getAllUniqueResources();
240250
boolean initialLoad = languageResourcesIndicator.isResourceTypeLoaded(rtranslatorMode, LanguageResourcesIndicator.ResourceType.MOZILLA);
241-
if (initialLoad || (!allUniqueResources.contains(srcLang) && !srcLang.getLanguage().equals("en"))) {
251+
if (!srcLang.getLanguage().equals("en") && (initialLoad || !allUniqueResources.contains(srcLang))) {
242252
BergamotTranslator.loadModelIntoCache(global, srcLang);
243253
}
244-
if (initialLoad || (!allUniqueResources.contains(tgtLang) && !tgtLang.getLanguage().equals("en"))) {
254+
if (!tgtLang.getLanguage().equals("en") && (initialLoad || !allUniqueResources.contains(tgtLang))) {
245255
BergamotTranslator.loadModelIntoCache(global, tgtLang);
246256
}
247257
}

‎app/src/main/java/nie/translator/rtranslator/voice_translation/neural_networks/translation/Translator.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,7 @@ private void translateMessage(){
349349
final CustomLocale languageInput = data.conversationMessageToTranslate.getPayload().getLanguage();
350350
if (!languageInput.equals(data.languageOutput)) {
351351
Peer sender = data.conversationMessageToTranslate.getSender();
352-
loadSrcLangResourcesForPeer(data.languageOutput, sender, new GeneralListener() {
352+
loadSrcLangResourcesForPeer(languageInput, sender, new GeneralListener() {
353353
@Override
354354
public void onSuccess() {
355355
performTextTranslation(text, languageInput, data.languageOutput, data.beamSize, false, new TranslateListener() {
@@ -688,6 +688,10 @@ public void unloadAllLangResourcesForConversation(@Nullable GeneralListener list
688688
}).start();
689689
}
690690

691+
public LanguageResourcesManager getLanguageResourcesManager() {
692+
return languageResourcesManager;
693+
}
694+
691695
public int getMode(){
692696
return mode;
693697
}

0 commit comments

Comments
 (0)