(FabianNitsche) Tests and fix for issue pieterderycke#66 pieterderycke#67 pieterderycke#68 - Problems with ambiguity of e - Invalid scientific notation leads to IndexOutOfBoundsException instead of ParseException

Author: mrxrsd

Jace.Tests/TokenReaderTests.cs

@@ -652,5 +652,94 @@ public void TestTokenReader31()
                 List<Token> tokens = reader.Read("0..1");
             });
         }
+
+        [TestMethod]
+        public void TestTokenReader32()
+        {
+            var reader = new TokenReader<double>(CultureInfo.InvariantCulture, DoubleNumericalOperations.Instance);
+            List<Token> tokens = reader.Read("-e");
+
+            Assert.AreEqual(2, tokens.Count);
+
+            Assert.AreEqual('_', tokens[0].Value);
+            Assert.AreEqual(0, tokens[0].StartPosition);
+            Assert.AreEqual(1, tokens[0].Length);
+
+            Assert.AreEqual("e", tokens[1].Value);
+            Assert.AreEqual(1, tokens[1].StartPosition);
+            Assert.AreEqual(1, tokens[1].Length);
+        }
+
+        [TestMethod]
+        public void TestTokenReader33()
+        {
+            var reader = new TokenReader<double>(CultureInfo.InvariantCulture, DoubleNumericalOperations.Instance);
+            List<Token> tokens = reader.Read("1-e");
+
+            Assert.AreEqual(3, tokens.Count);
+
+            Assert.AreEqual(1, tokens[0].Value);
+            Assert.AreEqual(0, tokens[0].StartPosition);
+            Assert.AreEqual(1, tokens[0].Length);
+
+            Assert.AreEqual('-', tokens[1].Value);
+            Assert.AreEqual(1, tokens[1].StartPosition);
+            Assert.AreEqual(1, tokens[1].Length);
+
+            Assert.AreEqual("e", tokens[2].Value);
+            Assert.AreEqual(2, tokens[2].StartPosition);
+            Assert.AreEqual(1, tokens[2].Length);
+        }
+
+        [TestMethod]
+        public void TestTokenReader34()
+        {
+            var reader = new TokenReader<double>(CultureInfo.InvariantCulture, DoubleNumericalOperations.Instance);
+            List<Token> tokens = reader.Read("1+e");
+
+            Assert.AreEqual(3, tokens.Count);
+
+            Assert.AreEqual(1, tokens[0].Value);
+            Assert.AreEqual(0, tokens[0].StartPosition);
+            Assert.AreEqual(1, tokens[0].Length);
+
+            Assert.AreEqual('+', tokens[1].Value);
+            Assert.AreEqual(1, tokens[1].StartPosition);
+            Assert.AreEqual(1, tokens[1].Length);
+
+            Assert.AreEqual("e", tokens[2].Value);
+            Assert.AreEqual(2, tokens[2].StartPosition);
+            Assert.AreEqual(1, tokens[2].Length);
+        }
+
+        [TestMethod]
+        public void TestTokenReader35()
+        {
+            AssertExtensions.ThrowsException<ParseException>(() =>
+            {
+                var reader = new TokenReader<double>(CultureInfo.InvariantCulture, DoubleNumericalOperations.Instance);
+                List<Token> tokens = reader.Read("2.11E-e3");
+            });
+        }
+
+        [TestMethod]
+        public void TestTokenReader36()
+        {
+            AssertExtensions.ThrowsException<ParseException>(() =>
+            {
+                var reader = new TokenReader<double>(CultureInfo.InvariantCulture, DoubleNumericalOperations.Instance);
+                List<Token> tokens = reader.Read("2.11E-e");
+            });
+        }
+
+        [TestMethod]
+        public void TestTokenReader37()
+        {
+            AssertExtensions.ThrowsException<ParseException>(() =>
+            {
+                var reader = new TokenReader<double>(CultureInfo.InvariantCulture, DoubleNumericalOperations.Instance);
+                List<Token> tokens = reader.Read("3e");
+            });
+        }
     }
 }

Jace/Tokenizer/TokenReader.cs

@@ -56,15 +56,15 @@ public List<Token> Read(string formula)
 
             for(int i = 0; i < characters.Length; i++)
             {
-                if (IsPartOfNumeric(characters[i], true, isFormulaSubPart))
+                if (IsPartOfNumeric(characters[i], true, false, isFormulaSubPart))
                 {
                     StringBuilder buffer = new StringBuilder();
                     buffer.Append(characters[i]);
                     //string buffer = "" + characters[i];
                     int startPosition = i;
 
 
-                    while (++i < characters.Length && IsPartOfNumeric(characters[i], false, isFormulaSubPart))
+                    while (++i < characters.Length && IsPartOfNumeric(characters[i], false, characters[i - 1] == '-', isFormulaSubPart))
                     {
                         if (isScientific && IsScientificNotation(characters[i]))
                             throw new ParseException(string.Format("Invalid token \"{0}\" detected at position {1}.", characters[i], i));
@@ -73,7 +73,7 @@ public List<Token> Read(string formula)
                         {
                             isScientific = IsScientificNotation(characters[i]);
 
-                            if (characters[i + 1] == '-')
+                            if (characters.Length > i + 1 && characters[i + 1] == '-')
                             {
                                 buffer.Append(characters[i++]);
                             }
@@ -235,9 +235,9 @@ public List<Token> Read(string formula)
             return tokens;
         }
 
-        private bool IsPartOfNumeric(char character, bool isFirstCharacter, bool isFormulaSubPart)
+        private bool IsPartOfNumeric(char character, bool isFirstCharacter, bool afterMinus, bool isFormulaSubPart)
         {
-            return character == decimalSeparator || (character >= '0' && character <= '9') || (isFormulaSubPart && isFirstCharacter && character == '-') || (!isFirstCharacter && character == 'e') || (!isFirstCharacter && character == 'E');
+            return character == decimalSeparator || (character >= '0' && character <= '9') || (isFormulaSubPart && isFirstCharacter && character == '-') || (!isFirstCharacter && !afterMinus && character == 'e') || (!isFirstCharacter && character == 'E');
         }
 
         private bool IsPartOfVariable(char character, bool isFirstCharacter)