Add word span info to results (#9)

* build refactoring
* extand api to get phrase letter offset arrays
* export symbol fix

Author: aleksas

Build.ps1

@@ -37,19 +37,12 @@ if ($build) {
 	} else {
 
 		# Build native linux libraries
-
-		sudo apt-get install -y libc6-dev-i386 gcc-multilib g++-multilib
-
 		cd native/source
 
-		mkdir -p ../../phonology_engine/Linux_x86
-		g++ -fPIC -m32 -g -I ../include/ Transkr.cpp transcrLUSS.cpp Skiemen.cpp Kircdb.cpp fv2id.cpp ArKirciuoti.cpp Engine.cpp strtokf.cpp stringWithLetterPosition.cpp TextNormalization.cpp -shared -o ../../phonology_engine/Linux_x86/libPhonologyEngine.so -Wno-write-strings
-		  
-		mkdir -p ../../phonology_engine/Linux_x86_64
-		g++ -fPIC -g -I ../include/ Transkr.cpp transcrLUSS.cpp Skiemen.cpp Kircdb.cpp fv2id.cpp ArKirciuoti.cpp Engine.cpp strtokf.cpp stringWithLetterPosition.cpp TextNormalization.cpp -shared -o ../../phonology_engine/Linux_x86_64/libPhonologyEngine.so -Wno-write-strings
+		make
 
 		cd ../..
-		
+				
 	}
 
 	python setup.py build

native/include/Engine.h

@@ -25,6 +25,7 @@ EXPORT Result PhonologyEngineNormalizeText(char * szText, NormalizedTextHandle *
 EXPORT Result PhonologyEngineNormalizedTextFree(NormalizedTextHandle * pHandle);
 EXPORT Result PhonologyEngineNormalizedTextGetPhraseCount(NormalizedTextHandle handle, int * pValue);
 EXPORT Result PhonologyEngineNormalizedTextGetPhrase(NormalizedTextHandle handle, int index, char ** pSzValue);
+EXPORT Result PhonologyEngineNormalizedTextGetPhraseLetterMap(NormalizedTextHandle handle, int index, int ** pArValue, int * count);
 
 EXPORT Result PhonologyEngineProcessPhrase(char * szNormalizedText, PhonologyEngineOutputHandle * pHandle);
 EXPORT Result PhonologyEngineOutputFree(PhonologyEngineOutputHandle * pHandle);

native/source/.VS2017/PhonologyEngine.def

@@ -3,8 +3,13 @@ EXPORTS
 	PhonologyEngineInit
 	PhonologyEngineProcessPhrase
 
-	PhonologyEngineOutputFree
+	PhonologyEngineNormalizeText
+	PhonologyEngineNormalizedTextFree
+	PhonologyEngineNormalizedTextGetPhraseCount
+	PhonologyEngineNormalizedTextGetPhrase
+	PhonologyEngineNormalizedTextGetPhraseLetterMap
 
+	PhonologyEngineOutputFree
 	PhonologyEngineOutputGetWordCount
 	PhonologyEngineOutputGetWord
 	PhonologyEngineOutputGetWordSyllables

native/source/Engine.cpp

@@ -30,14 +30,14 @@ EXPORT Result PhonologyEngineNormalizeText(char * szText, NormalizedTextHandle *
 	char *pos = szNormalizedText;
 	char sakinys[200];
 	char * pPhrases[1024];
+	int * pPhraseLetterMap[1024];
 	int n, phraseCount = 0;
 
 	while ((pos != (void*)1))
 	{
 		int hr2 = 0;
 		int lp = (int)(pos - szNormalizedText);
 		n = sscanf(pos, "%[^\n]", sakinys);
-		pos = strchr(pos, '\n') + 1;
 		if (n < 0)
 			break;
 		if (n == 0)
@@ -48,17 +48,23 @@ EXPORT Result PhonologyEngineNormalizeText(char * szText, NormalizedTextHandle *
 			sakinys[0] = 0; //jei eilute tuscia arba tik neskaitytini simboliai
 
 		pPhrases[phraseCount] = (char*) malloc((strlen(sakinys) + 1) * sizeof(char));
-		if (!pPhrases[phraseCount]) return -3;
+		pPhraseLetterMap[phraseCount] = (int*) malloc((strlen(sakinys)) * sizeof(int));
+		if (!pPhrases[phraseCount] || !pPhraseLetterMap[phraseCount]) return -3;
 		strcpy(pPhrases[phraseCount], sakinys);
+		memcpy(pPhraseLetterMap[phraseCount], pArLetterPosistion + lp, strlen(sakinys) * sizeof(int));
+
+		pos = strchr(pos, '\n') + 1;
 		phraseCount++;
 	}
 
 	pNormalizedText->phraseCount = phraseCount;
 	pNormalizedText->pSzPhrases = (char**) malloc(sizeof(char*) * phraseCount);
+	pNormalizedText->pArLetterMap = (int**) malloc(sizeof(int*) * phraseCount);
 
-	if (!pNormalizedText->pSzPhrases) return -4;
+	if (!pNormalizedText->pSzPhrases || !pNormalizedText->pArLetterMap) return -4;
 
 	memcpy(pNormalizedText->pSzPhrases, pPhrases, sizeof(char*) * phraseCount);
+	memcpy(pNormalizedText->pArLetterMap, pPhraseLetterMap, sizeof(int*) * phraseCount);
 
 	*pHandle = pNormalizedText;	
 
@@ -79,10 +85,12 @@ EXPORT Result PhonologyEngineNormalizedTextFree(NormalizedTextHandle * pHandle)
 		for (int i = 0; i < pNormalizedText->phraseCount; i++)
 		{
 			free(pNormalizedText->pSzPhrases[i]);
+			free(pNormalizedText->pArLetterMap[i]);
 		}
 	}
 
 	free(pNormalizedText->pSzPhrases);
+	free(pNormalizedText->pArLetterMap);
 	free(pNormalizedText);
 	*pHandle = NULL;
 
@@ -117,6 +125,24 @@ EXPORT Result PhonologyEngineNormalizedTextGetPhrase(NormalizedTextHandle handle
 	return 0;
 }
 
+EXPORT Result PhonologyEngineNormalizedTextGetPhraseLetterMap(NormalizedTextHandle handle, int index, int ** pArValue, int * pCount)
+{
+	NormalizedText * pOutput = GetObjectPtr(NormalizedText, handle);
+
+	if (!handle) return -1;
+	if (!pArValue) return -2;
+	if (!pCount) return -3;
+
+	int count = 0;
+
+	if (index < 0 || index >= pOutput->phraseCount) return -4;
+
+	*pArValue = pOutput->pArLetterMap[index];
+	*pCount = strlen(pOutput->pSzPhrases[index]);
+
+	return 0;
+}
+
 ////////////Pagrindine sintezavimo funkcija////////////////////////////////////////////////////////////////////////////////
 EXPORT Result PhonologyEngineProcessPhrase(char * szNormalizedText, PhonologyEngineOutputHandle * pHandle)
 {

native/source/Makefile

@@ -0,0 +1,16 @@
+all: install_dependencies compile_x86 compile_x86_64
+
+install_dependencies:
+	sudo apt-get install -y libc6-dev-i386 gcc-multilib g++-multilib
+
+compile_x86:
+	mkdir -p ../../phonology_engine/Linux_x86
+	g++ -fPIC -m32 -g -I ../include/ Transkr.cpp transcrLUSS.cpp Skiemen.cpp Kircdb.cpp fv2id.cpp ArKirciuoti.cpp Engine.cpp strtokf.cpp stringWithLetterPosition.cpp TextNormalization.cpp -shared -o ../../phonology_engine/Linux_x86/libPhonologyEngine.so -Wno-write-strings
+	
+compile_x86_64:	
+	mkdir -p ../../phonology_engine/Linux_x86_64
+	g++ -fPIC -g -I ../include/ Transkr.cpp transcrLUSS.cpp Skiemen.cpp Kircdb.cpp fv2id.cpp ArKirciuoti.cpp Engine.cpp strtokf.cpp stringWithLetterPosition.cpp TextNormalization.cpp -shared -o ../../phonology_engine/Linux_x86_64/libPhonologyEngine.so -Wno-write-strings
+
+clean:
+	rm ../../phonology_engine/Linux_x86/libPhonologyEngine.so
+	rm ../../phonology_engine/Linux_x86_64/libPhonologyEngine.so

native/source/transcrLUSSInternal.h

@@ -16,6 +16,7 @@ extern "C" {
 typedef struct _NormalizedText {
 	int phraseCount;
 	char ** pSzPhrases;
+	int ** pArLetterMap;
 } NormalizedText;
 
 typedef struct _WordStressOptions {

native/test/LithUSStest.cpp

@@ -45,7 +45,7 @@ int main(int argc, char *argv[])
 	int pastrSk;
 
 	int hr = 0;
-	if ((df = fopen("duom.txt", "rb")) == NULL)
+	if ((df = fopen("../tests/duom.txt", "rb")) == NULL)
 	{
 		spausdinti_loga("Nepavyko atidaryti duom.txt failo");
 		hr = -1;

phonology_engine/pe_native.py

@@ -54,6 +54,9 @@
 _PhonologyEngineNormalizedTextGetPhrase = _PhonologyEngineLibrary.PhonologyEngineNormalizedTextGetPhrase#(NormalizedTextHandle handle, int index, char ** pSzValue);
 _PhonologyEngineNormalizedTextGetPhrase.argtypes = [c_void_p, c_int, POINTER(c_char_p)]
 
+_PhonologyEngineNormalizedTextGetPhraseLetterMap = _PhonologyEngineLibrary.PhonologyEngineNormalizedTextGetPhraseLetterMap#(NormalizedTextHandle handle, int index, int ** pArValue, int * pCount)
+_PhonologyEngineNormalizedTextGetPhraseLetterMap.argtypes = [c_void_p, c_int, POINTER(POINTER(c_int)), POINTER(c_int)]
+
 _PhonologyEngineProcessPhrase = _PhonologyEngineLibrary.PhonologyEngineProcessPhrase#(char * szNormalizedText, PhonologyEngineOutputHandle * pHandle);
 _PhonologyEngineProcessPhrase.argtypes = [c_char_p, POINTER(c_void_p)]
 
@@ -111,6 +114,14 @@ def phonology_engine_normalized_text_get_phrase(handle, index):
         return ''
     return cs.value.decode(_native_encoding)
 
+def phonology_engine_normalized_text_get_phrase_letter_map(handle, index):
+    ci = POINTER(c_int)()
+    c = c_int(0)
+
+    _check( _PhonologyEngineNormalizedTextGetPhraseLetterMap( handle, c_int(index), byref(ci), byref(c) ) )
+
+    return [ci[i] for i in range(c.value)]
+
 def phonology_engine_process_phrase(text):
 
     handle = c_void_p()

phonology_engine/pe_output.py

@@ -34,6 +34,7 @@ def next(self):  # Python 2
             raise StopIteration
         else:
             value = self.get_phrase(self.current)
+            letter_map = self.get_phrase_letter_map(self.current)
 
             if self.remove_stress_chars:
                 for c in _stress_ascii_chars:
@@ -44,13 +45,16 @@ def next(self):  # Python 2
                     value = value.replace(c, '')
 
             self.current += 1
-            return value
+            return value, letter_map
 
     def get_phrase_count(self):
         return pe_native.phonology_engine_normalized_text_get_phrase_count(self.handle)
 
     def get_phrase(self, index):
         return pe_native.phonology_engine_normalized_text_get_phrase(self.handle, index)
+    
+    def get_phrase_letter_map(self, index):
+        return pe_native.phonology_engine_normalized_text_get_phrase_letter_map(self.handle, index)
 
 class PhonologyEngineOutput:
     def __init__(self, handle):

phonology_engine/phonology_engine.py

@@ -30,11 +30,14 @@ def _process(self, text, separators, normalize=True, include_syllables=True, nor
                     processed_phrases = []
 
                     if normalize_only:
-                        return ('', [' '.join(normalized_phrases)])
+                        return ('', [' '.join([p for p,_ in normalized_phrases])])
                     else:
-                        for phrase in normalized_phrases:
+                        r = []
+                        for phrase, letter_map in normalized_phrases:
                             processed_phrases.append(self._process(text=phrase, separators=separators, normalize=False, include_syllables=include_syllables, normalize_only=normalize_only))
-                        return ('', processed_phrases, list(normalized_phrases))
+                            r.append((phrase, letter_map))
+
+                        return ('', processed_phrases, r)
 
             else:
                 if len(text) > _max_prase_length: