Select Names is working a bi better

Author: glorieux-f

alix-cli/lib/alix-analysis-1.0.0.jar

@@ -82,6 +82,7 @@ public class FilterHTML extends TokenFilter
         PARA.add("h5");
         PARA.add("h6");
         PARA.add("item"); 
+        PARA.add("label");
         PARA.add("li"); 
         PARA.add("p"); 
         PARA.add("tr"); 

alix-cli/lib/alix-common-1.0.0.jar

@@ -480,7 +480,7 @@ else if (other instanceof Chain) {
             if (chain.length() != len)
                 return false;
             char[] test = chain.array();
-            int start = chain.start();
+            int start = chain.offset();
             for (int i = zero; i < zero + len; i++) {
                 if (test[start] != chars[i])
                     return false;

alix-cli/lib/alix-fr-1.0.0.jar

@@ -47,6 +47,10 @@ public abstract class Cli
         }
     }
 
+    /** Current path processed */
+    Path path;
+
+    
     public class AnaCli extends Analyzer
     {
         /**

alix-cli/lib/alix-util-1.0.0.jar

@@ -43,7 +43,32 @@
 public class Names  extends Cli implements Callable<Integer>
 {
     HashMap<Chain, IntMutable> forms = new HashMap<>(16384);
-    Chain form = new Chain();
+    final static CharArraySet STOP = new CharArraySet(200, false);
+    static {
+        String[] words = new String[]{
+            "A' B",
+            "BC",
+            "Einstellung",
+            "IB",
+            "Ib",
+            "IIA",
+            "IIa",
+            "IIB",
+            "IIb",
+            "IIIA",
+            "IIIa",
+            "IIIB",
+            "IIIb",
+            "I-IV",
+            "Melle",
+            "Pr",
+            "The",
+        };
+        for (final String word: words) {
+            STOP.add(word);
+        }
+    }
+
 
 
     @Override
@@ -56,6 +81,7 @@ public Integer call() throws Exception
         paths.sort(null);
         for (final Path path: paths) {
             System.err.println(path);
+            this.path = path;
             BufferedReader reader = new BufferedReader(
                 new InputStreamReader(
                     new FileInputStream(path.toFile())
@@ -81,36 +107,60 @@ public Integer call() throws Exception
     private void analyze(final TokenStream tokenStream) throws IOException
     {
 
-
+        final Chain form = new Chain();
         final CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);
         final FlagsAttribute flagsAtt = tokenStream.addAttribute(FlagsAttribute.class);
         final LemAtt lemAtt = tokenStream.addAttribute(LemAtt.class);
         final OrthAtt orthAtt = tokenStream.addAttribute(OrthAtt.class);
-        
         final CharsAttImpl testAtt = new CharsAttImpl();
         tokenStream.reset();
         int words = 0;
+        Chain glob = new Chain("? ?");
         while(tokenStream.incrementToken()) {
             final int flags = flagsAtt.getFlags();
             final int group = (flags & 0xF0);
             if (termAtt.isEmpty()) {
                 continue; // skip empty position
             }
             // candidate name, append
-            if (group == NAME.code) {
+            if (
+                group == NAME.code 
+                && flags != NAMEspec.code
+                && flags != NAMEplace.code
+                && flags != NAMEorg.code
+                && !Char.isDigit(termAtt.charAt(termAtt.length() - 1)) // A1
+            ) {
                 if (!form.isEmpty()) form.append(" ");
                 if (!orthAtt.isEmpty()) form.append(orthAtt);
                 else form.append(termAtt);
-                words++;
+                
+                if (STOP.contains(form.array(), form.offset(), form.length())) {
+                    form.setLength(0);
+                    words = 0;
+                }
+                else {
+                    words++;
+                }
                 continue;
             }
             // breaks
             if (
                     PUN.isPun(flags)
+                 || Char.isMath(termAtt.charAt(0)) // < >
                  || Char.isDigit(termAtt.charAt(0))
                  || !lemAtt.isEmpty() // token known from dictionary as a word
             ) {
-                if (form.isEmpty()) continue;
+                if (
+                    form.isEmpty()
+                 || form.length() == 1 // variable
+                 || form.last() == '\'' // A'
+                 || form.last() == '.' // A.
+                 || glob.glob(form) // W q
+                ) {
+                    form.setLength(0);
+                    words = 0;
+                    continue;
+                }
                 IntMutable count = forms.get(form);
                 if (count == null) {
                     count = new IntMutable(0);
@@ -119,17 +169,10 @@ private void analyze(final TokenStream tokenStream) throws IOException
                 count.inc();
                 form.setLength(0);
                 words = 0;
-            }
-            // ?
-            if (form.isEmpty()) {
-                System.out.println(termAtt);
                 continue;
             }
-            // Arion subfuscus ? (maybe foreign words like Piaget said)
-            form.append(" ");
-            if (!orthAtt.isEmpty()) form.append(orthAtt);
-            else form.append(termAtt);
-            words++;
+            // not yet a name, append nothing
+            if (form.isEmpty()) continue;
             continue;
         }
         tokenStream.close();

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterHTML.java

@@ -94,7 +94,9 @@ public enum TagFr implements Tag {
     /** Fiction character name */
     NAMEfict(0x49, "Personnage", "Rodogune, Chicot… (nom de personnage fictif, dictionnaire)."),
     /** title name */
-    NAMEtitle(0x4A, "Titre", " Titre d’œuvre (dictionnaire)") { },
+    NAMEtitle(0x4A, "Titre", " Titre d’œuvre (dictionnaire)"),
+    /** title name */
+    NAMEspec(0x4B, "Espèce", " Xerophila candidula"),
     /** People name */
     NAMEpeople(0x4E, "Peuple", " (nom de peuple, dictionnaire)."),
     /** God name */

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/tokenattributes/CharsAttImpl.java

@@ -18,8 +18,8 @@ Cf.,,,cf.
 ch.,,,chapitre
 chap.,,,chapitre
 Chap.,,,chapitre
-cit.,SUB,,
-coll.,SUB,,
+cit.,REF,,
+coll.,REF,,
 D.,NAME
 Dec.,,,décembre
 Déc.,,,décembre
@@ -32,21 +32,27 @@ ed.,,,édition
 eq.,,,équation
 éq.,,,équation
 etc.,ADV
-exp.,SUB,,
+exp.,REF,,
 Exp.,,,exp.
+Expér.,,,exp.
+EXP.,,,exp.
+EXP,,,exp.
 F.,NAME
-Fam.,,,Fam.
-fam.,SUB,,
-fasc.,,,fascicule
+fam.,,,Fam.
+Fam.,REF,,
+fasc.,REF,,
+Fasc.,,,fasc.
 fev.,,,février
 Fev.,,,février
 fevr.,,,février
 Fevr.,,,février
-fig.,NAME,,
-fol.,NAME,,
+fig.,REF,,
+fol.,REF,,
 fr.,,,francs
 Fr.,,,francs
 G.,NAME
+gen.,,,Gen.
+Gen.,REF,,
 H.,NAME
 h.,,,heure
 Ib.,,,ibid
@@ -69,8 +75,8 @@ J.-C.,NAME
 K.,NAME
 L.,NAME
 liv.,,,livre
-loc. cit.,SUB
-loc.,SUB
+loc. cit.,REF
+loc.,REF
 M.,,,monsieur
 Me.,,,maître
 Melle.,,,mademoiselle
@@ -94,55 +100,56 @@ obs.,,,observation
 op.,,,opus
 oct.,,,octobre
 P.,NAME
+PUF,NAME
 P.U.F.,,,PUF
 P.u.F.,,,PUF
 p.u.f.,,,PUF
-p.,,,page
+p.,REF,,
 pag.,,,page
 phr.,,,phrase
-pp.,,,pages
-P.P.,,,pages
+pp.,REF,,
+P.P.,,,pp.
 Pr.,,,Professeur
-probl.,,,problème
-Probl.,,,problème
+probl.,REF,,
+Probl.,,,probl.
 prop.,,,proposition
 Prs.,,,Professeur
 P.S.,,,post-scriptum
 PS.,,,post-scriptum
 Psychol.,,,Psychologie
 pt.,,,point
 Q.,NAME
-quest.,,,question
-Quest.,,,question
+quest.,REF,,
+Quest.,,,quest.
 R.,NAME
 S.,NAME
-sc.,,,scène
-Sect.,,,section
-sect.,,,section
+sc.,REF,,
+Sect.,,,sect.
+sect.,REF,,
 sept.,,,septembre
-séq.,,,suivante
-sq.,,,suivante
-sqq.,,,suivantes
-suppl.,,,supplément
+séq.,,,sq.
+sq.,REF,,
+sqq.,REF,,
+suppl.,REF,,
 T.,NAME
-tabl.,,,table
+tabl.,REF,,
 techn.,,,technique
 tél.,,,tél
 tel.,,,tél
 télép.,,,tél
 téléph.,,,tél
 téleph.,,,tél
 teleph.,,,tél
-trad.,,,traduction
-Trad.,,,traduction
+trad.,SUB,,
+Trad.,,,trad.
 U.,NAME
 U.S.A.,,,USA
 V.,NAME
 v.,,,voir
 Var.,,,var.
-var.,SUB,,
+var.,REF,,
 vol.,,,volume
-voy.,VERB,,
+voy.,REF,,
 Voy.,,,voy.
 W.,NAME
 X.,NAME