Switched tokenizer implementation's empty to consider also the unterminated token in the buffer

Author: andsel

Diff for: logstash-core/src/main/java/org/logstash/common/BufferedTokenizer.java

@@ -120,6 +120,11 @@ public String flush() {
             return flushed;
         }
 
+        // considered empty if caught up to the accumulator
+        public synchronized boolean isBufferEmpty() {
+            return currentIdx >= accumulator.length();
+        }
+
         @Override
         public synchronized String toString() {
             return "accumulator=" + accumulator + ", currentIdx=" + currentIdx;
@@ -171,6 +176,7 @@ public String toString() {
     }
 
     public boolean isEmpty() {
-        return !dataSplitter.hasNext();
+//        return !dataSplitter.hasNext();
+        return dataSplitter.isBufferEmpty();
     }
 }