Skip to content

Commit 247d52d

Browse files
committed
Fix
1 parent dcdb0db commit 247d52d

File tree

1 file changed

+4
-0
lines changed

1 file changed

+4
-0
lines changed

operators/tokenizer/case_encoder.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ class CaseEncoder {
9898
buffer_.clear();
9999
buffer_queue_.clear();
100100
offset_ = 0;
101+
// Reset all state for new tokenization
102+
signature_.clear();
103+
spans_ = 0;
104+
seen_three_spans_ = false;
101105
}
102106

103107
if (isUpper(sp)) {

0 commit comments

Comments
 (0)