First draft of alix xml ingester

Author: glorieux-f

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/PosTaggingFilter.java

@@ -250,7 +250,10 @@ private void tagBufferedQueue()
             final int pos = p.getPos();
 
             // Skip structural XML tags entirely (1 -> 0)
-            if (pos == XML.code) continue;
+            if (pos == XML.code) {
+                probAtt.setProb(1);
+                continue;
+            }
 
             // Sentence boundaries are submitted as punctuation token (1 -> 1)
             if (isSentenceBoundary(pos)) {

analysis/src/java/com/github/oeuvres/alix/lucene/analysis/tokenattributes/ProbAttributeImpl.java

@@ -4,7 +4,7 @@
 import org.apache.lucene.util.AttributeReflector;
 
 public final class ProbAttributeImpl extends AttributeImpl implements ProbAttribute {
-    private static double UNKNOWN = -1;
+    private static double UNKNOWN = 0;
     private double prob = UNKNOWN;
 
 

test/src/main/java/com/github/oeuvres/alix/lucene/analysis/AnalysisDemoHelper.java

@@ -87,10 +87,10 @@ public static void dump(final TokenStream ts, final String input) throws IOExcep
                 final int start = offAtt.startOffset();
                 final int end = offAtt.endOffset();
                 final int pos = (posAtt != null)?posAtt.getPos():0;
-                final double prob = (probAtt != null)?probAtt.getProb():-1;
+                final String prob = (probAtt != null)?String.format(java.util.Locale.ROOT, "%.5f",probAtt.getProb()):"";
 
                 System.out.printf(
-                    "%5d\t[%d,%d)\t|%s|\t%s\t%s\t%s\t%.5f%n",
+                    "%5d\t[%d,%d)\t|%s|\t%s\t%s\t%s\t%s%n",
                     i++,
                     start, end,
                     safeSlice(input, start, end),

test/src/main/java/com/github/oeuvres/alix/lucene/analysis/MLTokenizerBenchmark.java

@@ -0,0 +1,42 @@
+package com.github.oeuvres.alix.lucene.analysis;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharArrayMap;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
+
+public class MLTokenizerDemo
+{
+    /** Minimal Analyzer for StandardTokenizer ->TermReplaceFilter. */
+    private static Analyzer buildAnalyzer() {
+        return new Analyzer() {
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                Tokenizer tokenizer = new MLTokenizer();
+                TokenStream stream = tokenizer;
+                return new TokenStreamComponents(tokenizer, stream);
+            }
+        };
+    }
+
+    static public void main(String[] args) throws IOException
+    {
+        Path path = Paths.get("src/test/test-data/ingest.html");
+        String input = Files.readString(path, StandardCharsets.UTF_8);
+        try (
+            Analyzer analyzer = buildAnalyzer();
+            BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)
+        ){
+            TokenStream ts = analyzer.tokenStream("contents", reader);
+            AnalysisDemoHelper.dump(ts, input);
+        }
+    }
+}

test/src/main/java/com/github/oeuvres/alix/lucene/analysis/MLTokenizerDemo.java

@@ -0,0 +1,59 @@
+package com.github.oeuvres.alix.lucene.index;
+
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public final class  AlixSaxParserSinkDebug implements AlixSaxParser.AlixSink {
+
+    @Override
+    public void startUnit(AlixSaxParser.Unit unit) {
+        System.out.printf("START %-7s id=%s%n", unit.kind(), unit.xmlId());
+    }
+
+    @Override
+    public void field(AlixSaxParser.Unit unit, AlixSaxParser.FieldSpec f) {
+        System.out.printf("  FIELD %-12s type=%-8s analyzer=%-12s",
+            f.name(), f.type().name().toLowerCase(), f.analyzerHint());
+
+        if (f.value() != null) {
+            System.out.printf(" value=%s", shortStr(f.value()));
+        }
+        if (f.contentXml() != null) {
+            System.out.printf(" contentXml=%s", shortStr(f.contentXml()));
+        }
+        if (f.source() != null) {
+            System.out.printf(" source=%s selectors=%d", f.source(), f.selectors().size());
+        }
+        System.out.println();
+
+        for (var s : f.selectors()) {
+            System.out.printf("    %s element=%s attribute=%s value=%s%n",
+                s.mode().name().toLowerCase(), s.element(), s.attribute(), s.value());
+        }
+    }
+
+    @Override
+    public void endUnit(AlixSaxParser.Unit unit) {
+        System.out.printf("END   %-7s id=%s%n", unit.kind(), unit.xmlId());
+    }
+
+    private static String shortStr(String s) {
+        if (s == null) return "null";
+        s = s.replace('\n', ' ').replace('\r', ' ');
+        return (s.length() > 90) ? s.substring(0, 87) + "..." : s;
+    }
+    
+    public static void main(String[] args) throws Exception {
+        // final String res = "/ingest/test-alix.xml";
+        Path path = Paths.get("src/main/resources/ingest/test-alix.xml");
+        System.out.println(path.toAbsolutePath());
+        // try (InputStream in = Thread.currentThread().getContextClassLoader() .getResourceAsStream(res)) {
+        try (InputStream in = Files.newInputStream(path)) {
+            if (in == null) throw new IllegalStateException(path + ": resource not found");
+            AlixSaxParser.parse(in, new AlixSaxParserSinkDebug());
+        }
+    }
+}