PartScorer works, but doesn’t do yet interesting things

Author: glorieux-f

common/src/java/com/github/oeuvres/alix/lucene/FlucNum.java

@@ -384,21 +384,9 @@ public int numBytes()
      * </pre>
      *
      * <p>
-     * The returned {@link Partition} is query-specific. Documents are assigned
-     * only if all of the following are true:
-     * </p>
-     *
-     * <ul>
-     *   <li>the document has a value for this numeric field;</li>
-     *   <li>the document is present in {@code acceptedDocs}, unless
-     *       {@code acceptedDocs == null};</li>
-     *   <li>the document's value belongs to a retained part.</li>
-     * </ul>
-     *
-     * <p>
      * Documents outside the partition are assigned {@link Partition#NO_PART}.
-     * Valid part ids are chronological/value order. The focus part is the part
-     * corresponding to {@code [start, end]} and is available through
+     * Valid part ids are in chronological/value order. The focus part is the
+     * part corresponding to {@code [start, end]} and is available through
      * {@link Partition#focusPart()}.
      * </p>
      *
@@ -424,11 +412,11 @@ public int numBytes()
     public Partition partition(
         final int start,
         final int end,
-        final PartialExtremityPolicy policy,
+        PartialExtremityPolicy policy,
         final FixedBitSet acceptedDocs
     ) throws IOException {
         if (policy == null) {
-            throw new NullPointerException("policy");
+            policy = PartialExtremityPolicy.ABSORB;
         }
         if (start > end) {
             throw new IllegalArgumentException(
@@ -445,69 +433,72 @@ public Partition partition(
                 + " < reader.maxDoc()=" + maxDoc);
         }
 
-        if (end < min || start > max) {
+        // After cacheDense() we know min/max round-trip exactly through int.
+        final int intMin = (int) min;
+        final int intMax = (int) max;
+
+        if (end < intMin || start > intMax) {
             throw new IllegalArgumentException(
                 "Focus interval [" + start + ',' + end
                 + "] does not overlap field range ["
-                + min + ',' + max + ']');
+                + intMin + ',' + intMax + ']');
         }
 
-        final int width = end - start + 1;
-        if (width <= 0L) {
+        // Compute width in long to detect overflow when start/end span the int
+        // range. start <= end has already been validated, so widthLong >= 1.
+        final long widthLong = (long) end - (long) start + 1L;
+        if (widthLong > Integer.MAX_VALUE) {
             throw new IllegalArgumentException(
-                "Invalid focus width for interval [" + start + ',' + end + ']');
+                "Focus width too large: " + widthLong);
         }
+        final int width = (int) widthLong;
 
-        final long kMin = Math.floorDiv((long) min - (long) start, width);
-        final long kMax = Math.floorDiv((long) max - (long) start, width);
-
-        if (kMin > 0L || kMax < 0L) {
-            throw new IllegalArgumentException(
-                "Focus interval [" + start + ',' + end
-                + "] is not represented in field range ["
-                + min + ',' + max + ']');
-        }
+        // Anchored part index k: the focus is at k = 0, parts before are k < 0,
+        // parts after are k > 0. Each k stands for the inclusive value range
+        // [start + k*width, start + (k+1)*width - 1].
+        // The overlap test above guarantees kMin <= 0 <= kMax.
+        final long kMin = Math.floorDiv((long) intMin - (long) start, width);
+        final long kMax = Math.floorDiv((long) intMax - (long) start, width);
 
         final long rawCountLong = kMax - kMin + 1L;
-        if (rawCountLong < 1L || rawCountLong > Integer.MAX_VALUE) {
+        if (rawCountLong > Integer.MAX_VALUE) {
             throw new IllegalArgumentException(
-                "Invalid raw part count: " + rawCountLong);
+                "Too many raw parts: " + rawCountLong);
         }
-
         final int rawCount = (int) rawCountLong;
-        final long[] groupStart = new long[rawCount];
-        final long[] groupEnd = new long[rawCount];
 
-        for (int i = 0; i < rawCount; i++) {
-            final long k = kMin + i;
-            groupStart[i] = k;
-            groupEnd[i] = k;
-        }
-
-        int first = 0;
-        int last = rawCount - 1;
-
-        final boolean leftPartial = rawPartStart(start, width, kMin) < min;
-        final boolean rightPartial = rawPartEnd(start, width, kMax) > max;
+        // A leftmost / rightmost raw part is "partial" when the corpus does not
+        // cover its full anchored width.
+        final boolean leftPartial =
+            ((long) start + kMin * width) < intMin;
+        final boolean rightPartial =
+            ((long) start + (kMax + 1L) * width - 1L) > intMax;
+
+        // Active raw range. Indices are into [0, rawCount); raw index i
+        // corresponds to k = kMin + i. firstRaw/lastRaw narrow under DROP and
+        // ABSORB; the absorbed raw is then redirected back into its neighbour
+        // through mergeLeft/mergeRight.
+        int firstRaw = 0;
+        int lastRaw = rawCount - 1;
+        boolean mergeLeft = false;
+        boolean mergeRight = false;
 
         if (leftPartial && kMin < 0L) {
             switch (policy) {
                 case KEEP:
                     break;
                 case DROP:
-                    first++;
+                    firstRaw++;
                     break;
                 case ABSORB:
-                    if (kMin + 1L == 0L) {
-                        // Never absorb a partial extremity into the focus part.
-                        first++;
-                    } else {
-                        groupStart[1] = groupStart[0];
-                        first++;
+                    // Refuse to grow the focus part: when the only neighbour of
+                    // the partial extremity is the focus (kMin == -1), fall back
+                    // to DROP rather than absorb.
+                    firstRaw++;
+                    if (kMin != -1L) {
+                        mergeLeft = true;
                     }
                     break;
-                default:
-                    throw new IllegalStateException("Unhandled policy: " + policy);
             }
         }
 
@@ -516,23 +507,18 @@ public Partition partition(
                 case KEEP:
                     break;
                 case DROP:
-                    last--;
+                    lastRaw--;
                     break;
                 case ABSORB:
-                    if (kMax - 1L == 0L) {
-                        // Never absorb a partial extremity into the focus part.
-                        last--;
-                    } else {
-                        groupEnd[last - 1] = groupEnd[last];
-                        last--;
+                    lastRaw--;
+                    if (kMax != 1L) {
+                        mergeRight = true;
                     }
                     break;
-                default:
-                    throw new IllegalStateException("Unhandled policy: " + policy);
             }
         }
 
-        final int partCount = last - first + 1;
+        final int partCount = lastRaw - firstRaw + 1;
         if (partCount < 1) {
             throw new IllegalArgumentException("No part left after applying " + policy);
         }
@@ -541,46 +527,53 @@ public Partition partition(
                 "Too many parts for byte partition: " + partCount);
         }
 
-        int focusPart = Partition.NO_FOCUS;
+        // Map each raw index to its part id. Indices outside [firstRaw, lastRaw]
+        // are NO_PART unless absorbed into a neighbour. The focus part is the
+        // one that contains k = 0.
         final int[] rawToPart = new int[rawCount];
         Arrays.fill(rawToPart, Partition.NO_PART);
 
-        for (int part = 0; part < partCount; part++) {
-            final int groupIndex = first + part;
-            final long from = groupStart[groupIndex];
-            final long to = groupEnd[groupIndex];
-
-            if (from <= 0L && 0L <= to) {
+        int focusPart = Partition.NO_FOCUS;
+        for (int i = firstRaw; i <= lastRaw; i++) {
+            final int part = i - firstRaw;
+            rawToPart[i] = part;
+            if ((kMin + i) == 0L) {
                 focusPart = part;
             }
-
-            for (long k = from; k <= to; k++) {
-                rawToPart[(int) (k - kMin)] = part;
-            }
+        }
+        if (mergeLeft) {
+            rawToPart[firstRaw - 1] = 0;
+        }
+        if (mergeRight) {
+            rawToPart[lastRaw + 1] = partCount - 1;
         }
 
         if (focusPart == Partition.NO_FOCUS) {
+            // Unreachable: kMin <= 0 <= kMax and the focus part is never dropped
+            // or absorbed. Defensive only.
             throw new IllegalStateException(
-                "Focus part disappeared while building partition.");
+                "Focus part missing while building partition.");
         }
 
+        // Assign documents. When acceptedDocs is null we walk only docs that
+        // have a value; otherwise we walk the filter and skip docs without a
+        // value.
         final Partition partition = new Partition(maxDoc, partCount, focusPart);
-
         final FixedBitSet docsToScan =
             acceptedDocs == null ? densint.docHasValue : acceptedDocs;
+        final int scanLimit = Math.min(docsToScan.length(), maxDoc);
 
         for (int docId = docsToScan.nextSetBit(0);
-             docId != DocIdSetIterator.NO_MORE_DOCS;
+             docId != DocIdSetIterator.NO_MORE_DOCS && docId < scanLimit;
              docId = docsToScan.nextSetBit(docId + 1)) {
 
             if (!densint.docHasValue.get(docId)) {
                 continue;
             }
 
             final int value = densint.docValues[docId];
-            final long k = Math.floorDiv((long) value - (long) start, width);
-            final long rawIndexLong = k - kMin;
-
+            final long rawIndexLong =
+                Math.floorDiv((long) value - (long) start, (long) width) - kMin;
             if (rawIndexLong < 0L || rawIndexLong >= rawCount) {
                 continue;
             }
@@ -595,7 +588,7 @@ public Partition partition(
 
         return partition;
     }
-
+    
     /**
      * Returns a human-readable point type label.
      *
@@ -741,50 +734,6 @@ private static boolean probeStored(
         return false;
     }
 
-    /**
-     * Returns the inclusive end value of one raw anchored part.
-     *
-     * <p>
-     * Raw part {@code k == 0} is the focus interval
-     * {@code [start, start + width - 1]}. Negative parts precede it; positive
-     * parts follow it.
-     * </p>
-     *
-     * @param start focus start value
-     * @param width part width
-     * @param k     raw anchored part index
-     * @return inclusive raw part end
-     */
-    private static long rawPartEnd(
-        final int start,
-        final long width,
-        final long k
-    ) {
-        return (long) start + (k + 1L) * width - 1L;
-    }
-
-    /**
-     * Returns the inclusive start value of one raw anchored part.
-     *
-     * <p>
-     * Raw part {@code k == 0} is the focus interval
-     * {@code [start, start + width - 1]}. Negative parts precede it; positive
-     * parts follow it.
-     * </p>
-     *
-     * @param start focus start value
-     * @param width part width
-     * @param k     raw anchored part index
-     * @return inclusive raw part start
-     */
-    private static long rawPartStart(
-        final int start,
-        final long width,
-        final long k
-    ) {
-        return (long) start + k * width;
-    }
-    
     /**
      * Policy for incomplete extremity parts when a fixed-width value partition is
      * anchored on a focus interval.

common/src/java/com/github/oeuvres/alix/lucene/terms/KeynessScorer.java

@@ -85,6 +85,87 @@ public double score(
             return ppmFocus / ppmOther;
         }
     }
+    
+    /**
+     * Signed Pearson chi-square X² (Pearson 1900), 2×2 contingency.
+     *
+     * <p>
+     * Same null model as {@link LogLikelihood} — independence of "term" and
+     * "focus", expected counts from row/column marginals — but using the
+     * Pearson statistic
+     * </p>
+     *
+     * <pre>{@code
+     * X² = Σ (O − E)² / E
+     * }</pre>
+     *
+     * <p>
+     * instead of the LL G². Sign convention matches {@code LogLikelihood}:
+     * positive when the term's focus rate meets or exceeds its other rate.
+     * </p>
+     *
+     * <p>
+     * Pearson and LL agree closely for moderate counts but diverge on the
+     * rare-term tail. Pearson's variance estimate {@code E} (a single
+     * cell's expectation) reacts more harshly than LL's log-ratio when an
+     * observed count is very far from a small expected count, which tends
+     * to push very rare terms to the top of a Pearson ranking. LL's
+     * {@code O · log(O/E)} is gentler in the same regime. This is the
+     * usual reason corpus-linguistics work prefers LL over X² for keyness;
+     * Dunning (1993) makes the argument explicit.
+     * </p>
+     *
+     * <p>
+     * Returns {@link Double#NaN} on invalid inputs (negative counts, count
+     * exceeding tokens). Returns {@code 0} on degenerate marginals where
+     * no expectation can be formed.
+     * </p>
+     */
+    class Chi2 implements KeynessScorer {
+        @Override
+        public double score(
+            final long focusTermCount,
+            final long focusTokens,
+            final long otherTermCount,
+            final long otherTokens
+        ) {
+            if (focusTokens <= 0L || otherTokens <= 0L) return 0d;
+            if (focusTermCount < 0L || otherTermCount < 0L) return Double.NaN;
+            if (focusTermCount > focusTokens || otherTermCount > otherTokens) return Double.NaN;
+
+            final long focusNonTermCount = focusTokens - focusTermCount;
+            final long otherNonTermCount = otherTokens - otherTermCount;
+
+            final long allTokens = focusTokens + otherTokens;
+            final long allTermCount = focusTermCount + otherTermCount;
+            final long allNonTermCount = focusNonTermCount + otherNonTermCount;
+
+            final double expectedFocusTerm = (double) focusTokens * allTermCount / allTokens;
+            final double expectedOtherTerm = (double) otherTokens * allTermCount / allTokens;
+            final double expectedFocusNonTerm = (double) focusTokens * allNonTermCount / allTokens;
+            final double expectedOtherNonTerm = (double) otherTokens * allNonTermCount / allTokens;
+
+            double x2 = 0d;
+            x2 += cell(focusTermCount, expectedFocusTerm);
+            x2 += cell(otherTermCount, expectedOtherTerm);
+            x2 += cell(focusNonTermCount, expectedFocusNonTerm);
+            x2 += cell(otherNonTermCount, expectedOtherNonTerm);
+
+            return ((double) focusTermCount / focusTokens
+                    >= (double) otherTermCount / otherTokens) ? x2 : -x2;
+        }
+
+        /**
+         * One cell of the Pearson sum. Returns 0 when expected is non-positive
+         * (degenerate marginal — corresponding row or column is empty).
+         */
+        private static double cell(final long observed, final double expected)
+        {
+            if (expected <= 0d) return 0d;
+            final double d = observed - expected;
+            return (d * d) / expected;
+        }
+    }
 
     /**
      * Log-Likelihood G² (Dunning 1993), for use as a significance pre-filter,