Isolate a French tokenizer logic with no lucene dependancies

Author: glorieux-f

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/CleanupFilter.java

@@ -15,6 +15,7 @@
 import com.github.oeuvres.alix.common.Upos;
 import com.github.oeuvres.alix.util.CSVReader;
 import com.github.oeuvres.alix.util.Char;
+import com.github.oeuvres.alix.util.MweLexicon;
 import com.github.oeuvres.alix.util.Report;
 
 import opennlp.tools.postag.POSModel;

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/LexiconHelper.java

@@ -43,6 +43,8 @@
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.apache.lucene.util.AttributeSource;
 
+import com.github.oeuvres.alix.util.MweLexicon;
+
 
 /**
  * A {@link TokenFilter} that merges multi-word expressions (MWEs) into single tokens,
@@ -179,7 +181,10 @@ private void emitMerged(final int matchPos, final int matchOrd)
         // Restore all attributes from first component (posIncr, startOffset, flags, ...).
         queue.restoreTo(this, 0);
 
-        lexicon.formToAttribute(matchOrd, termAtt);
+        final int len = lexicon.formLength(matchOrd);
+        final char[] buf = termAtt.resizeBuffer(len);
+        lexicon.formToChars(matchOrd, buf, 0);
+        termAtt.setLength(len);
 
         // Fix endOffset and type.
         offsetAtt.setOffset(offsetAtt.startOffset(), endOffset);

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/MweFilter.java

@@ -53,10 +53,10 @@
 import com.github.oeuvres.alix.lucene.analysis.MarkupTokenizer;
 import com.github.oeuvres.alix.lucene.analysis.MarkupZoneFilter;
 import com.github.oeuvres.alix.lucene.analysis.MweFilter;
-import com.github.oeuvres.alix.lucene.analysis.MweLexicon;
 import com.github.oeuvres.alix.lucene.analysis.PosTaggingFilter;
 import com.github.oeuvres.alix.lucene.analysis.SentenceStartLowerCaseFilter;
 import com.github.oeuvres.alix.lucene.analysis.TermReplaceFilter;
+import com.github.oeuvres.alix.util.MweLexicon;
 
 import opennlp.tools.postag.POSModel;
 

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/MweLexicon.java

@@ -44,8 +44,8 @@
 
 import com.github.oeuvres.alix.lucene.analysis.LemmaLexicon;
 import com.github.oeuvres.alix.lucene.analysis.LexiconHelper;
-import com.github.oeuvres.alix.lucene.analysis.MweLexicon;
 import com.github.oeuvres.alix.lucene.analysis.LexiconHelper.PosResolver;
+import com.github.oeuvres.alix.util.MweLexicon;
 
 public class FrenchLexicons
 {

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/fr/FrenchAnalyzer.java

@@ -4,7 +4,7 @@
 import com.github.oeuvres.alix.lucene.analysis.LexiconHelper;
 import com.github.oeuvres.alix.lucene.analysis.MarkupTokenizer;
 import com.github.oeuvres.alix.lucene.analysis.MweFilter;
-import com.github.oeuvres.alix.lucene.analysis.MweLexicon;
+import com.github.oeuvres.alix.util.MweLexicon;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.TokenStream;