Universal pos

glorieux-f · glorieux-f · commit 49c6f40e7ea2 · 2026-02-07T23:56:41.000+01:00
diff --git a/.gitignore b/.gitignore
@@ -16,5 +16,5 @@ buildNumber.properties
 .project
 .~*
 test/src/java/com/github/oeuvres/alix/fr/affix_miner_java_prefix_suffix_stem_extraction_from_lemma_udpos_count.java
-test/stemming/
 *.bak
+/fr/dic/lexique*
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterAposHyphenFr.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterAposHyphenFr.java
@@ -9,7 +9,7 @@
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.CharsAttImpl;
 /**
  * A filter that decomposes words on a list of suffixes and prefixes, mainly to handle 
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterCloud.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterCloud.java
@@ -40,7 +40,7 @@
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import static com.github.oeuvres.alix.fr.TagFr.*;
 
 import com.github.oeuvres.alix.fr.TagFr;
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterFind.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterFind.java
@@ -40,7 +40,7 @@
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.LemAtt;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.OrthAtt;
 
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterHTML.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterHTML.java
@@ -41,7 +41,7 @@
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.CharsAttImpl;
 
 /**
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterLemmatize.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterLemmatize.java
@@ -39,7 +39,7 @@
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import static com.github.oeuvres.alix.fr.TagFr.*;
 
 import com.github.oeuvres.alix.fr.TagFr;
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterLocution.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterLocution.java
@@ -41,7 +41,7 @@
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 
 import com.github.oeuvres.alix.fr.FrDics;
 import com.github.oeuvres.alix.fr.TagFr;
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterOrth.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterOrth.java
@@ -40,7 +40,7 @@
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.LemAtt;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.OrthAtt;
 
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterPosFin.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/FilterPosFin.java
@@ -40,7 +40,7 @@
 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import com.github.oeuvres.alix.fr.TagFr;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.LemAtt;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.OrthAtt;
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/Lexicon.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/Lexicon.java
@@ -13,7 +13,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.github.oeuvres.alix.common.Flags;
+import com.github.oeuvres.alix.common.Upos;
 import com.github.oeuvres.alix.common.Tag;
 import com.github.oeuvres.alix.fr.TagFr;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.CharsAttImpl;
@@ -79,7 +79,7 @@ public LexEntry(Logger logger, final Chain graph, final Chain tag, final Chain l
             }
             catch (Exception e) {
                 try {
-                    tagEnum = Flags.valueOf(tagKey);
+                    tagEnum = Upos.valueOf(tagKey);
                 }
                 catch (Exception ee) {
                     logger.debug(res + " graph=\"" + graph + "\" tag=\"" + tag + "\"? tag not found.");
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/PosNlpFilter.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/PosNlpFilter.java
@@ -45,7 +45,7 @@
 
 import com.github.oeuvres.alix.common.Tag;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 
 import opennlp.tools.postag.POSModel;
 import opennlp.tools.postag.POSTaggerME;
diff --git a/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/TokenizerML.java b/analysis/src/java/com/github/oeuvres/alix/lucene/analysis/TokenizerML.java
@@ -14,7 +14,7 @@
 import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
 import org.apache.lucene.util.AttributeFactory;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 
 import com.github.oeuvres.alix.fr.FrDics;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.AttributeFactoryAlix;
@@ -23,7 +23,7 @@
 
 /**
  * A tokenizer for latin script languages and possible XML like tags. Tags are kept in token stream for further analysis.
- * A {@link FlagsAttribute} is set with an int define in {@link Flags}.
+ * A {@link FlagsAttribute} is set with an int define in {@link Upos}.
  *
  * <pre>
  * &lt;p&gt;            XML
diff --git a/common/src/java/com/github/oeuvres/alix/common/Upos.java b/common/src/java/com/github/oeuvres/alix/common/Upos.java
@@ -4,7 +4,7 @@
  * List of flags inferred by an analyzer without dictionaries or tagger,
  * on alphabetic scripts.
  */
-public enum Flags implements Tag
+public enum Upos implements Tag
 {
     // 0x, possible flags 
     
@@ -20,10 +20,8 @@ public enum Flags implements Tag
     STOP(0x04),
     /** Non stop word */
     NOSTOP(0x05),
-    /** Locution  (maybe substantive, conjunction…) */
+    /** Locution (maybe substantive, conjunction…) */
     LOC(0x06),
-    /** [0-9\-] numbers */
-    DIGIT(0x07),
     
     /** UDPOS, Punctuation */
     PUNCT(0x08),
@@ -37,15 +35,15 @@ public enum Flags implements Tag
     PUNCTclause(0x0C),
     
     // 1x, verbs
-    /** UDPOS, verb */
+    /** https://universaldependencies.org/u/pos/VERB.html */
     VERB(0x10),
-    /** UDPOS, auxiliary verb */
+    /** https://universaldependencies.org/u/pos/AUX_.html */
     AUX(0x11),
-    /** Infinitive */
+    /** https://universaldependencies.org/u/feat/VerbForm.html#inf-infinitive */
     VERBinf(0x12),
-    /** Participle past */
+    /** https://universaldependencies.org/u/feat/VerbForm.html#part-participle-verbal-adjective */
     VERBpartpast(0x13),
-    /** Participle present */
+    /** https://universaldependencies.org/u/feat/VerbForm.html#part-participle-verbal-adjective */
     VERBpartpres(0x14),
 
     // 2x, substantifs
@@ -88,22 +86,22 @@ public enum Flags implements Tag
     PROPNgod(0x4F),
 
     // 5x, Adverbes
-    /** UDPOS Adverb */
+    /** https://universaldependencies.org/u/pos/ADV.html */
     ADV(0x50),
     /** Interrogative adverb */
     ADVint(0x51),
     /** Negation adverb */
     ADVneg(0x52),
-    // non UD feats
+    /** https://universaldependencies.org/u/pos/PART.html fr: n’, ne */
+    PART(0x53),
     /** Situation (space-time) adverb */
     ADVsit(0x58),
     /** Aspect adverb */
     ADVasp(0x59),
     /** Gradation adverb */
     ADVdeg(0x5A),
 
-    // 6x, déterminants
-    /** UDPOS, determiner*/
+    /** https://universaldependencies.org/u/pos/DET.html */
     DET(0x60),
     /** Article */
     DETart(0x61),
@@ -120,8 +118,8 @@ public enum Flags implements Tag
     /** Prepositional determiner, UD ADP+DET, fr: du, aux */
     ADP_DET(0x67),
 
-    // 7x, pronoms
-    /** UDPOS Pronoun */
+    // 7x, pronouns
+    /** https://universaldependencies.org/u/pos/PRON.html */
     PRON(0x70),
     /** Demonstrative pronoun */
     PRONdem(0x71),
@@ -138,60 +136,63 @@ public enum Flags implements Tag
     /** UD ADP+PRON, ex fr: duquel, auquel… */
     ADP_PRON(0x89),
 
-    // 8x, connecteurs
-    /** UDPOS “ad”position (pre or post in some languages) */
+    // 8x, connectors
+    /** https://universaldependencies.org/u/pos/ADP.html “ad”position (pre or post in some languages) */
     ADP(0x80),
-    /** UDPOS Coordinating conjunction */
+    /** https://universaldependencies.org/u/pos/CCONJ.html */
     CCONJ(0x81),
-    /** UDPOS Subordinating conjunction */
+    /** https://universaldependencies.org/u/pos/SCONJ.html */
     SCONJ(0x82),
     
     /** Adverbial conjunction */
-    ADVconj(0x88, "Adv. conj.", "Cependant, désormais… (adverbe de connexion)."),
+    // ADVconj(0x88, "Adv. conj.", "Cependant, désormais… (adverbe de connexion)."),
     
-    // Ax, Numéraux divers
-    /** UDPOSNumeral */
+    // Ax, numerals
+    /** https://universaldependencies.org/u/pos/NUM.html */
     NUM(0xA0),
     /** Ordinal */
     NUMord(0xA1),
-    /** UDPOS Symbol  */
+    /** https://universaldependencies.org/u/pos/SYM.html  */
     SYM(0xA8),
-    /** Reference */
-    REF(0xA2, "Référence", "p. 50, f. 2… (un numéro utilisé comme référence, comme une page, une note…"),
+    /** [0-9\-] numbers */
+    DIGIT(0x07),
     /** Math operator */
     MATH(0xA3),
     /** Physical unit (cm, kg…) */
     UNIT(0xA4),
+    /** A number reference, for example p. 50 */
+    REF(0xA2),
 
-    // Fx, divers
-    /** Misc */
-    MISC(0xF0, "Divers", ""),
-    /** Abbreviation (maybe name, substantive…) */
-    ABBR(0xF1, "Abréviation", ""),
-    /** Exclamation */
-    EXCL(0xF2, "Exclamation", "Ho, Ô, haha… (interjections)"),
-    /** Exclamation */
-    MG(0xF3, "Mot grammatical", "Déterminants, pronoms, connecteurs…"),
+    // misc
+    /** https://universaldependencies.org/u/pos/X.html */
+    X(0xF0),
+    /** https://universaldependencies.org/u/pos/INTJ.html */
+    INTJ(0xF1),
+    /** Abbreviation before resolution to a pos */
+    ABBR(0xF8),
+    /** Grammatical word, before resolution */
+    MG(0xF9)
+    ;
     
     
     public final int code;
     static final Index index = new Index(0, 15);
     static
     {
-        for (Flags tag : Flags.values()) index.add(tag.code, tag);
+        for (Upos tag : Upos.values()) index.add(tag.code, tag);
     }
 
-    private Flags(final int code)
+    private Upos(final int code)
     {
         this.code = code;
     }
     public boolean isPun(final int code)
     {
-        return (code == PUN.code 
-          || code == PUNsection.code 
-          || code == PUNpara.code 
-          || code == PUNsent.code 
-          || code == PUNclause.code
+        return (code == PUNCT.code 
+          || code == PUNCTsection.code 
+          || code == PUNCTpara.code 
+          || code == PUNCTsent.code 
+          || code == PUNCTclause.code
         );
     }
     
diff --git a/fr/dic/fr-google1gram.csv b/fr/dic/fr-google1gram.csv
diff --git a/fr/dic/fr-grammalecte.csv b/fr/dic/fr-grammalecte.csv
diff --git a/fr/dic/fr-lemma-ud.csv b/fr/dic/fr-lemma-ud.csv
diff --git a/fr/dic/prefixes.tsv b/fr/dic/prefixes.tsv
diff --git a/fr/src/resources/com/github/oeuvres/alix/fr/word.csv b/fr/src/resources/com/github/oeuvres/alix/fr/word.csv
@@ -1,4 +1,4 @@
-GRAPH,POS,LEMMA
+INFLECTION,POS,LEMMA
 1er,ADJ,1er
 1ᵉʳ,ADJ,1ᵉʳ
 1ers,ADJ,1ers
diff --git a/search/src/java/com/github/oeuvres/alix/lucene/search/Doc.java b/search/src/java/com/github/oeuvres/alix/lucene/search/Doc.java
@@ -53,7 +53,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import static com.github.oeuvres.alix.common.Names.*;
 import com.github.oeuvres.alix.common.TagFilter;
 import com.github.oeuvres.alix.lucene.index.BytesDic;
diff --git a/search/src/java/com/github/oeuvres/alix/lucene/search/FieldRail.java b/search/src/java/com/github/oeuvres/alix/lucene/search/FieldRail.java
@@ -65,7 +65,7 @@
 import org.apache.lucene.util.BytesRefHash;
 import org.apache.lucene.util.FixedBitSet;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import com.github.oeuvres.alix.common.TagFilter;
 import com.github.oeuvres.alix.util.Chain;
 import com.github.oeuvres.alix.util.Edge;
diff --git a/search/src/java/com/github/oeuvres/alix/lucene/search/FieldText.java b/search/src/java/com/github/oeuvres/alix/lucene/search/FieldText.java
@@ -53,7 +53,7 @@
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.SparseFixedBitSet;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 
 import com.github.oeuvres.alix.common.Tag;
 import com.github.oeuvres.alix.common.TagFilter;
diff --git a/search/src/java/com/github/oeuvres/alix/lucene/search/FormEnum.java b/search/src/java/com/github/oeuvres/alix/lucene/search/FormEnum.java
@@ -46,7 +46,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import com.github.oeuvres.alix.common.TagFilter;
 import com.github.oeuvres.alix.util.Chain;
 import com.github.oeuvres.alix.util.MI;
diff --git a/search/src/java/com/github/oeuvres/alix/lucene/search/SuggestForm.java b/search/src/java/com/github/oeuvres/alix/lucene/search/SuggestForm.java
@@ -49,7 +49,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefHash;
 
-import static com.github.oeuvres.alix.common.Flags.*;
+import static com.github.oeuvres.alix.common.Upos.*;
 import com.github.oeuvres.alix.common.TagFilter;
 import com.github.oeuvres.alix.util.Char;
 import com.github.oeuvres.alix.util.IntList;
diff --git a/test/src/java/com/github/oeuvres/alix/lucene/analysis/TokenStreamsTest.java b/test/src/java/com/github/oeuvres/alix/lucene/analysis/TokenStreamsTest.java
@@ -18,7 +18,7 @@
 // import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
 import org.junit.Test;
 
-import com.github.oeuvres.alix.common.Flags;
+import com.github.oeuvres.alix.common.Upos;
 import com.github.oeuvres.alix.fr.FrDics;
 import com.github.oeuvres.alix.fr.TagFr;
 import com.github.oeuvres.alix.lucene.analysis.tokenattributes.CharsAttImpl;
@@ -106,7 +106,7 @@ private static void analyze(TokenStream tokenStream, String text) throws IOExcep
             final int startOffset = offsetAttribute.startOffset();
             final int flags = flagsAttribute.getFlags();
             String tag = TagFr.name(flags);
-            if (tag == null) tag = Flags.name(flags);
+            if (tag == null) tag = Upos.name(flags);
             if (tag == null) tag = "" + flags;
             System.out.print(""
               + "term=" + termAttribute.toString() + "\t" 
