Port Lucene 4.8.1 fixes for AnalyzingSuggester, OfflineSorter, and SpecialOperations, #1381

src/Lucene.Net.Suggest/Suggest/Analyzing/AnalyzingSuggester.cs

@@ -979,7 +979,8 @@ internal ISet<Int32sRef> ToFiniteStrings(BytesRef surfaceForm, TokenStreamToAuto
             ReplaceSep(automaton);
             automaton = ConvertAutomaton(automaton);
 
-            if (Debugging.AssertsEnabled) Debugging.Assert(SpecialOperations.IsFinite(automaton));
+            // TODO: LUCENE-5660 re-enable this once we disallow massive suggestion strings
+            // if (Debugging.AssertsEnabled) Debugging.Assert(SpecialOperations.IsFinite(automaton));
 
             // Get all paths from the automaton (there can be
             // more than one path, eg if the analyzer created a

src/Lucene.Net.TestFramework/Support/ApiScanTestBase.cs

@@ -1109,6 +1109,63 @@ private static IEnumerable<string> GetMembersAcceptingOrReturningType(Type lookF
         private static IEnumerable<Type> GetTypesToTest(Assembly assembly) =>
             assembly.GetTypes()
                 .Where(t => !t.HasAttribute<GeneratedCodeAttribute>(inherit: false)
-                            && !t.HasAttribute<CompilerGeneratedAttribute>(inherit: false));
+                            && !t.HasAttribute<CompilerGeneratedAttribute>(inherit: false)
+                            && IsPartOfEffectiveApi(t));
+
+        /// <summary>
+        /// Returns whether the type is part of the effective API surface.
+        /// <para />
+        /// Types that are part of the effective API surface include <c>public</c> types, as well as
+        /// nested types that are either <c>public</c>, <c>protected</c>, or <c>protected internal</c> and where the
+        /// declaring type hierarchy is public.
+        /// </summary>
+        /// <param name="type"></param>
+        /// <returns></returns>
+        private static bool IsPartOfEffectiveApi(Type type)
+        {
+            while (type != null)
+            {
+                if (!type.IsNested)
+                {
+                    return type.IsPublic;
+                }
+
+                if (type.IsNestedPublic)
+                {
+                    type = type.DeclaringType!;
+                    continue;
+                }
+
+                if (type.IsNestedFamily || type.IsNestedFamORAssem) // protected or protected internal
+                {
+                    type = type.DeclaringType!;
+
+                    // Every containing type must itself be externally inheritable to see protected members
+                    while (type != null)
+                    {
+                        if (!IsExternallyInheritable(type))
+                            return false;
+
+                        type = type.DeclaringType;
+                    }
+
+                    return true;
+                }
+
+                return false;
+            }
+
+            return false;
+        }
+
+        private static bool IsExternallyInheritable(Type type)
+        {
+            if (!type.IsNested)
+                return type.IsPublic;
+
+            return type.IsNestedPublic ||
+                   type.IsNestedFamily ||
+                   type.IsNestedFamORAssem;
+        }
     }
 }

src/Lucene.Net.TestFramework/Util/Automaton/AutomatonTestUtil.cs

@@ -1,12 +1,15 @@
 using J2N;
 using J2N.Runtime.CompilerServices;
-using Lucene.Net.Support;
 using Lucene.Net.Diagnostics;
 using RandomizedTesting.Generators;
 using System;
 using System.Collections.Generic;
 using JCG = J2N.Collections.Generic;
 
+#if !FEATURE_QUEUE_TRYDEQUEUE_TRYPEEK
+using Lucene.Net.Support;
+#endif
+
 namespace Lucene.Net.Util.Automaton
 {
     /*
@@ -49,7 +52,7 @@ public static string RandomRegexp(Random r)
                 }
                 try
                 {
-                    new RegExp(regexp, RegExpSyntax.NONE);
+                    _ = new RegExp(regexp, RegExpSyntax.NONE);
                     return regexp;
                 }
                 catch (Exception e) when (e.IsException())
@@ -355,6 +358,84 @@ public static void DeterminizeSimple(Automaton a, ISet<State> initialset)
             a.RemoveDeadTransitions();
         }
 
+        /// <summary>
+        /// Simple, original implementation of getFiniteStrings.
+        ///
+        /// <para/>
+        /// Returns the set of accepted strings, assuming that at most
+        /// <paramref name="limit"/> strings are accepted. If more than <paramref name="limit"/>
+        /// strings are accepted, the first <paramref name="limit"/> strings found are returned. If <paramref name="limit"/>&lt;0, then
+        /// the limit is infinite.
+        ///
+        /// <para/>
+        /// This implementation is recursive: it uses one stack
+        /// frame for each digit in the returned strings (i.e., max
+        /// is the max length returned string).
+        /// </summary>
+        /// <param name="a">The automaton.</param>
+        /// <param name="limit">The maximum number of strings to return.</param>
+        /// <returns>A set of accepted strings.</returns>
+        public static ISet<Int32sRef> GetFiniteStringsRecursive(Automaton a, int limit)
+        {
+            JCG.HashSet<Int32sRef> strings = new JCG.HashSet<Int32sRef>();
+            if (a.IsSingleton)
+            {
+                if (limit > 0)
+                {
+                    strings.Add(Fst.Util.ToUTF32(a.Singleton, new Int32sRef()));
+                }
+            }
+            else if (!GetFiniteStrings(a.initial, new JCG.HashSet<State>(), strings, new Int32sRef(), limit))
+            {
+                return strings;
+            }
+
+            return strings;
+        }
+
+        /// <summary>
+        /// Returns the strings that can be produced from the given state, or
+        /// false if more than <paramref name="limit"/> strings are found.
+        /// <paramref name="limit"/>&lt;0 means "infinite".
+        /// </summary>
+        private static bool GetFiniteStrings(State s, JCG.HashSet<State> pathstates,
+            JCG.HashSet<Int32sRef> strings, Int32sRef path, int limit)
+        {
+            pathstates.Add(s);
+            foreach (Transition t in s.GetTransitions())
+            {
+                if (pathstates.Contains(t.to))
+                {
+                    return false;
+                }
+
+                for (int n = t.min; n <= t.max; n++)
+                {
+                    path.Grow(path.Length + 1);
+                    path.Int32s[path.Length] = n;
+                    path.Length++;
+                    if (t.to.accept)
+                    {
+                        strings.Add(Int32sRef.DeepCopyOf(path));
+                        if (limit >= 0 && strings.Count > limit)
+                        {
+                            return false;
+                        }
+                    }
+
+                    if (!GetFiniteStrings(t.to, pathstates, strings, path, limit))
+                    {
+                        return false;
+                    }
+
+                    path.Length--;
+                }
+            }
+
+            pathstates.Remove(s);
+            return true;
+        }
+
         /// <summary>
         /// Returns true if the language of this automaton is finite.
         /// <para/>

src/Lucene.Net.Tests.Suggest/Suggest/Analyzing/AnalyzingSuggesterTest.cs

@@ -1499,24 +1499,24 @@ internal static IEnumerable<Input> Shuffle(params Input[] values)
             return asList;
         }
 
-        // LUCENENET TODO: This is a test from Lucene 4.8.1 that currently produces a stack overflow
-        //// TODO: we need BaseSuggesterTestCase?
-        //[Test]
-        //public void TestTooLongSuggestion()
-        //{
-        //    Analyzer a = new MockAnalyzer(Random);
-        //    AnalyzingSuggester suggester = new AnalyzingSuggester(a);
-        //    String bigString = TestUtil.RandomSimpleString(Random, 60000, 60000);
-        //    try
-        //    {
-        //        suggester.Build(new InputArrayEnumerator(new Input[] {
-        //            new Input(bigString, 7)}));
-        //        fail("did not hit expected exception");
-        //    }
-        //    catch (Exception iae) when (iae.IsIllegalArgumentException())
-        //    {
-        //        // expected
-        //    }
-        //}
+        // TODO: we need BaseSuggesterTestCase?
+        [Test]
+        public void TestTooLongSuggestion()
+        {
+            Analyzer a = new MockAnalyzer(Random);
+            AnalyzingSuggester suggester = new AnalyzingSuggester(a);
+            string bigString = TestUtil.RandomSimpleString(Random, 60000, 60000);
+            try
+            {
+                suggester.Build(new InputArrayEnumerator([
+                    new Input(bigString, 7)
+                ]));
+                Assert.Fail("did not hit expected exception");
+            }
+            catch (ArgumentOutOfRangeException /*iae*/) // LUCENENET-specific AOORE for .NET semantics
+            {
+                // expected
+            }
+        }
     }
 }