Fix KWIC capping

The properties krill.match.max.token and krill.context.max.token and,
correspondingly variables, and parameters like maxTokenMatchSize, were
introduced to configure the maximum visible token length of search hits
with context ("KWICs") and exports, to adhere with copyright and license
restrictions, which are very important.
However, the implementation was flawed and apparently based on a
misunderstanding between linguists, lawyers and programmers.
The only point that matters legally is the total number of tokens shown
in a KWIC snippet (left context + match + right context). If an actual
match is larger than krill.kwic.max.token, it must be cut down to
krill.kwic.max.token, if not the remaining token budget should be
distributed  between left and right context, either equally or in such a
way that the total number of capped words in minimized.

Change-Id: Ib0afd476fcd84144d4d9db18839ed8b9952f92e3

Author: kupietz

src/main/java/de/ids_mannheim/korap/KrillIndex.java

@@ -997,8 +997,9 @@ public Match getMatchInfo (String idString, String field, boolean info,
         if (DEBUG)
             log.trace("Get info on {}", idString);
 
-        int maxTokenMatchSize = KrillProperties.maxTokenMatchSize;
-        Match match = new Match(maxTokenMatchSize, idString, includeHighlights);
+        // Use total KWIC cap to limit match size at most to the total snippet cap
+        int kwicMax = de.ids_mannheim.korap.util.KrillProperties.getMaxTokenKwicSize();
+        Match match = new Match(kwicMax, idString, includeHighlights);
 
         if (this.getVersion() != null)
             match.setVersion(this.getVersion());
@@ -1569,11 +1570,8 @@ public Result search (Krill ks) {
                             ? lreader.document(localDocID, fieldsSet)
                             : lreader.document(localDocID);
 
-                    int maxMatchSize = ks.getMaxTokenMatchSize();
-                    if (maxMatchSize <= 0
-                            || maxMatchSize > KrillProperties.maxTokenMatchSize) {
-                        maxMatchSize = KrillProperties.maxTokenMatchSize;
-                    };
+                    // Use total KWIC cap for match capping, ignore per-query match limits
+                    int maxMatchSize = KrillProperties.maxTokenKwicSize;
 
                     // Create new Match
                     final Match match = new Match(maxMatchSize, pto, localDocID,

src/main/java/de/ids_mannheim/korap/response/Match.java

@@ -1672,6 +1672,60 @@ public ObjectNode getSnippetTokens () {
                 log.debug("Set endContext {}", endContext);
         };
 
+        // Enforce total KWIC token cap (left + match + right)
+        int kwicMax = KrillProperties.getMaxTokenKwicSize();
+        if (kwicMax > 0) {
+            // Convert endContext to exclusive bound for iteration ease
+            int leftLen = (startContext < this.startPos) ? (this.startPos - startContext) : 0;
+            int matchLen = (this.endPos > this.startPos) ? (this.endPos - this.startPos) : 0;
+            int rightLen = (endContext > this.endPos) ? (endContext - this.endPos) : 0;
+            int total = leftLen + matchLen + rightLen;
+
+            if (matchLen >= kwicMax) {
+                // Cut match to kwicMax, drop all context
+                this.endPos = this.startPos + kwicMax;
+                this.endCutted = true;
+                startContext = this.startPos;
+                endContext = this.endPos; // exclusive bound
+            }
+            else if (total > kwicMax) {
+                int toReduce = total - kwicMax;
+                int reduceLeft = Math.min((toReduce + 1) / 2, leftLen);
+                int reduceRight = Math.min(toReduce - reduceLeft, rightLen);
+
+                int rest = toReduce - (reduceLeft + reduceRight);
+                if (rest > 0) {
+                    int extraRight = Math.min(rest, rightLen - reduceRight);
+                    reduceRight += extraRight;
+                    rest -= extraRight;
+                }
+                if (rest > 0) {
+                    int extraLeft = Math.min(rest, leftLen - reduceLeft);
+                    reduceLeft += extraLeft;
+                    rest -= extraLeft;
+                }
+
+                startContext += reduceLeft;
+                endContext -= reduceRight;
+
+                if (rest > 0) {
+                    // Trim remaining from match end
+                    int newMatchLen = matchLen - rest;
+                    if (newMatchLen < 0) newMatchLen = 0;
+                    this.endPos = this.startPos + newMatchLen;
+                    this.endCutted = true;
+                    if (endContext < this.endPos)
+                        endContext = this.endPos;
+                }
+            }
+            else {
+                // No trimming necessary
+                if (DEBUG)
+                    log.debug("KWIC cap not reached: total={} ≤ cap={}",
+                              leftLen + matchLen + rightLen, kwicMax);
+            }
+        }
+        
         // Retrieve the character offsets for all tokens
         for (int i = startContext; i < endContext; i++) {
             pto.add(ldid, i);
@@ -2347,6 +2401,7 @@ private int[] _processOffsetChars (int ldid, int startPosChar,
 
         int startOffsetChar = -1, endOffsetChar = -1;
         int startOffset = -1, endOffset = -1;
+        PositionsToOffset pto = this.positionsToOffset;
 
         // The offset is defined by a span
         if (this.getContext().isSpanDefined()) {
@@ -2368,20 +2423,23 @@ private int[] _processOffsetChars (int ldid, int startPosChar,
             if (DEBUG)
                 log.trace("Got context based on span {}-{}/{}-{}",
                         startOffset, endOffset, startOffsetChar, endOffsetChar);
+            // Make sure we can (re)compute character offsets after adjustments
+            this.positionsToOffset.add(ldid, startOffset);
+            this.positionsToOffset.add(ldid, endOffset);
         };
 
         // The offset is defined by tokens or characters
         if (endOffset == -1) {
 
-            PositionsToOffset pto = this.positionsToOffset;
+            PositionsToOffset ptoTok = pto;
 
             // The left offset is defined by tokens
             if (this.context.left.isToken()) {
                 startOffset = this.startPos - this.context.left.getLength();
                 if (DEBUG)
                     log.trace("PTO will retrieve {} (Left context)",
                             startOffset);
-                pto.add(ldid, startOffset);
+                ptoTok.add(ldid, startOffset);
             }
 
             // The left offset is defined by characters
@@ -2395,7 +2453,7 @@ private int[] _processOffsetChars (int ldid, int startPosChar,
                 if (DEBUG)
                     log.trace("PTO will retrieve {} (Right context)",
                             endOffset);
-                pto.add(ldid, endOffset);
+                ptoTok.add(ldid, endOffset);
             }
 
             // The right context is defined by characters
@@ -2404,24 +2462,138 @@ private int[] _processOffsetChars (int ldid, int startPosChar,
                         : endPosChar + this.context.right.getLength();
             };
 
-            if (startOffset != -1)
-                startOffsetChar = pto.start(ldid, startOffset);
+        // Enforce total KWIC token cap (left + match + right) on token offsets
+            int kwicMax = KrillProperties.getMaxTokenKwicSize();
+        if (kwicMax > 0) {
+                int leftLen = (startOffset != -1) ? (this.startPos - startOffset) : 0;
+                if (leftLen < 0) leftLen = 0;
+                int matchLen = (this.endPos > this.startPos) ? (this.endPos - this.startPos) : 0;
+                int rightLen = (endOffset != -1) ? (endOffset - (this.endPos - 1)) : 0;
+                if (rightLen < 0) rightLen = 0;
+                int total = leftLen + matchLen + rightLen;
+
+                if (matchLen >= kwicMax) {
+                    // Cut match to kwicMax and drop context
+                    this.endPos = this.startPos + kwicMax;
+                    this.endCutted = true;
+                    startOffset = this.startPos;
+                    endOffset = this.endPos - 1;
+                }
+                else if (total > kwicMax) {
+                    int toReduce = total - kwicMax;
+                    int reduceLeft = Math.min((toReduce + 1) / 2, leftLen);
+                    int reduceRight = Math.min(toReduce - reduceLeft, rightLen);
+
+                    int rest = toReduce - (reduceLeft + reduceRight);
+                    if (rest > 0) {
+                        int extraRight = Math.min(rest, rightLen - reduceRight);
+                        reduceRight += extraRight;
+                        rest -= extraRight;
+                    }
+                    if (rest > 0) {
+                        int extraLeft = Math.min(rest, leftLen - reduceLeft);
+                        reduceLeft += extraLeft;
+                        rest -= extraLeft;
+                    }
+
+                    if (startOffset != -1)
+                        startOffset += reduceLeft;
+                    if (endOffset != -1)
+                        endOffset -= reduceRight;
+
+                    if (rest > 0) {
+                        int newMatchLen = matchLen - rest;
+                        if (newMatchLen < 0) newMatchLen = 0;
+                        this.endPos = this.startPos + newMatchLen;
+                        this.endCutted = true;
+                        if (endOffset != -1)
+                            endOffset = Math.max(endOffset, this.endPos - 1);
+                }
+                else {
+                    if (DEBUG)
+                        log.debug("KWIC cap not reached (offset path): total={} ≤ cap={}",
+                                  leftLen + matchLen + rightLen, kwicMax);
+                }
+        }
+            }
 
-            if (endOffset != -1)
-                endOffsetChar = pto.end(ldid, endOffset);
         };
 
+        // Enforce total KWIC token cap (left + match + right), regardless of span or token context
+        int kwicMax = KrillProperties.getMaxTokenKwicSize();
+        if (kwicMax > 0) {
+            int leftLen = (startOffset != -1) ? (this.startPos - startOffset) : 0;
+            if (leftLen < 0) leftLen = 0;
+            int matchLen = (this.endPos > this.startPos) ? (this.endPos - this.startPos) : 0;
+            int rightLen = (endOffset != -1) ? (endOffset - (this.endPos - 1)) : 0;
+            if (rightLen < 0) rightLen = 0;
+            int total = leftLen + matchLen + rightLen;
+
+            if (matchLen >= kwicMax) {
+                this.endPos = this.startPos + kwicMax;
+                this.endCutted = true;
+                startOffset = this.startPos;
+                endOffset = this.endPos - 1;
+            }
+            else if (total > kwicMax) {
+                int toReduce = total - kwicMax;
+                int reduceLeft = Math.min((toReduce + 1) / 2, leftLen);
+                int reduceRight = Math.min(toReduce - reduceLeft, rightLen);
+                int rest = toReduce - (reduceLeft + reduceRight);
+                if (rest > 0) {
+                    int extraRight = Math.min(rest, rightLen - reduceRight);
+                    reduceRight += extraRight;
+                    rest -= extraRight;
+                }
+                if (rest > 0) {
+                    int extraLeft = Math.min(rest, leftLen - reduceLeft);
+                    reduceLeft += extraLeft;
+                    rest -= extraLeft;
+                }
+
+                if (startOffset != -1)
+                    startOffset += reduceLeft;
+                if (endOffset != -1)
+                    endOffset -= reduceRight;
+
+                if (rest > 0) {
+                    int newMatchLen = matchLen - rest;
+                    if (newMatchLen < 0) newMatchLen = 0;
+                    this.endPos = this.startPos + newMatchLen;
+                    this.endCutted = true;
+                    if (endOffset != -1)
+                        endOffset = Math.max(endOffset, this.endPos - 1);
+                }
+            }
+            else {
+                if (DEBUG)
+                    log.debug("KWIC cap not reached (unified path): total={} ≤ cap={}", total, kwicMax);
+            }
+        }
+
+        // Compute character offsets according to potentially adjusted token offsets
+        if (startOffset != -1)
+            startOffsetChar = pto.start(ldid, startOffset);
+        if (endOffset != -1)
+            endOffsetChar = pto.end(ldid, endOffset);
+
         if (DEBUG)
             log.trace("Premature found offsets at {}-{}", startOffsetChar,
                     endOffsetChar);
 
+        // Ensure zero-context means match-only and not full document
+        if (startOffset == -1 && (startOffsetChar < 0 || this.context.left.getLength() == 0))
+            startOffsetChar = startPosChar;
+        if (endOffset == -1 && (endOffsetChar < 0 || this.context.right.getLength() == 0))
+            endOffsetChar = endPosChar;
+
 
         // This can happen in case of non-token characters
         // in the match and null offsets
         if (startOffsetChar > startPosChar)
             startOffsetChar = startPosChar;
         else if (startOffsetChar < 0)
-            startOffsetChar = 0;
+            startOffsetChar = startPosChar;
 
         // No "..." at the beginning
         if (startOffsetChar == 0)

src/main/java/de/ids_mannheim/korap/util/KrillProperties.java

@@ -23,6 +23,9 @@ public class KrillProperties {
 
     public static int maxTokenMatchSize = 50;
     public static int maxTokenContextSize = 60;
+    // New: Total KWIC size cap (match + left + right)
+    // Default to derived value even if properties are never loaded
+    public static int maxTokenKwicSize = (2 * maxTokenContextSize) + maxTokenMatchSize;
     public static int maxCharContextSize = 500;
     public static int defaultSearchContextLength = 6;
     public static int maxTextSize = DEFAULT_MAX_STRING_LEN; // Default max text size
@@ -89,19 +92,21 @@ public static Properties loadProperties (String propFile) {
     public static void updateConfigurations (Properties  prop) {
         String maxTokenMatchSize = prop.getProperty("krill.match.max.token");
         String maxTokenContextSize = prop.getProperty("krill.context.max.token");
+        String maxTokenKwicSize = prop.getProperty("krill.kwic.max.token");
         // EM: not implemented yet
 //        String maxCharContextSize = prop.getProperty("krill.context.max.char");
         String defaultSearchContextLength = prop.getProperty("krill.search.context.default");
         String maxTextSizeValue = prop.getProperty("krill.index.textSize.max");
 
         try {
             if (maxTokenMatchSize != null) {
-                KrillProperties.maxTokenMatchSize = Integer
-                        .parseInt(maxTokenMatchSize);
+                KrillProperties.maxTokenMatchSize = Integer.parseInt(maxTokenMatchSize);
             }
             if (maxTokenContextSize != null) {
-                KrillProperties.maxTokenContextSize = Integer
-                        .parseInt(maxTokenContextSize);
+                KrillProperties.maxTokenContextSize = Integer.parseInt(maxTokenContextSize);
+            }
+            if (maxTokenKwicSize != null) {
+                KrillProperties.maxTokenKwicSize = Integer.parseInt(maxTokenKwicSize);
             }
 //            if (maxCharContextSize != null) {
 //                KrillProperties.maxCharContextSize = Integer
@@ -128,6 +133,34 @@ public static void updateConfigurations (Properties  prop) {
             log.error("A Krill property expects numerical values: "
                     + e.getMessage());
         };
+
+        // Always ensure kwic cap has a sensible value, even if not configured
+        if (KrillProperties.maxTokenKwicSize <= 0) {
+            KrillProperties.maxTokenKwicSize = (2 * KrillProperties.maxTokenContextSize)
+                    + KrillProperties.maxTokenMatchSize;
+        }
+
+        // Handle deprecation and fallback for KWIC size
+        if (KrillProperties.maxTokenKwicSize <= 0) {
+            boolean legacyMatchSet = (maxTokenMatchSize != null);
+            boolean legacyContextSet = (maxTokenContextSize != null);
+            if (legacyMatchSet || legacyContextSet) {
+                if (legacyMatchSet)
+                    log.warn("Property 'krill.match.max.token' is deprecated. Use 'krill.kwic.max.token'.");
+                if (legacyContextSet)
+                    log.warn("Property 'krill.context.max.token' is deprecated. Use 'krill.kwic.max.token'.");
+                // Compute sensible default from deprecated settings
+                KrillProperties.maxTokenKwicSize = (2 * KrillProperties.maxTokenContextSize)
+                        + KrillProperties.maxTokenMatchSize;
+                log.warn("Computed 'krill.kwic.max.token' as {} from deprecated settings.",
+                        KrillProperties.maxTokenKwicSize);
+            }
+            else {
+                // Neither new nor legacy; derive from current defaults
+                KrillProperties.maxTokenKwicSize = (2 * KrillProperties.maxTokenContextSize)
+                        + KrillProperties.maxTokenMatchSize;
+            }
+        }
 
         String p = prop.getProperty("krill.test", "false");
         isTest = Boolean.parseBoolean(p);
@@ -139,6 +172,15 @@ public static void updateConfigurations (Properties  prop) {
         matchExpansionIncludeContextSize = Boolean.parseBoolean(matchExpansion);
 
         secret = prop.getProperty("krill.secretB64", "");
+
+        log.info("Effective krill.kwic.max.token = {}", KrillProperties.maxTokenKwicSize);
+    }
+
+    public static int getMaxTokenKwicSize() {
+        // In case properties were never loaded, return a derived sensible default
+        if (maxTokenKwicSize <= 0)
+            maxTokenKwicSize = (2 * maxTokenContextSize) + maxTokenMatchSize;
+        return maxTokenKwicSize;
     }
 
 

src/main/resources/krill.properties.info

@@ -15,3 +15,15 @@ krill.index.commit.log = log/krill.commit.log
 krill.index.commit.auto = 500
 krill.index.relations.max = 100
 krill.index.textSize.max = 20000000
+
+# Snippet (KWIC) settings
+# New: Maximum total number of tokens per KWIC snippet (left + match + right)
+# If unset, and deprecated properties below are set, Krill will compute this as
+# 2*krill.context.max.token + krill.match.max.token and log a deprecation warning.
+# krill.kwic.max.token = 100
+
+# Deprecated: These are ignored when 'krill.kwic.max.token' is set and will be removed.
+# They were previously used to cap match length and per-side context lengths, but
+# licensing limits apply to the total snippet size, not to the match alone.
+#krill.match.max.token = [DEPRECATED]
+#krill.context.max.token = [DEPRECATED]