Skip to content

Commit 194e1c2

Browse files
committed
Update sample code
1 parent f005417 commit 194e1c2

File tree

1 file changed

+7
-4
lines changed

1 file changed

+7
-4
lines changed

Sample/Program.cs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,12 +79,15 @@ private static void Main(string[] args)
7979

8080
outputs.Dispose();
8181

82-
var output = tokenizer.Tokenize("Hi");
82+
const bool TEST_SINGLE_TOKENIZE = false;
8383

84-
// Console.WriteLine(output.IDs.AsReadOnlySpan().GetSpanPrintString());
85-
Console.WriteLine($"Overflowing Tokens Length: {output.OverflowingTokens.Length}");
84+
if (TEST_SINGLE_TOKENIZE)
85+
{
86+
using var output = tokenizer.Tokenize("Hi");
8687

87-
output.Dispose();
88+
// Console.WriteLine(output.IDs.AsReadOnlySpan().GetSpanPrintString());
89+
Console.WriteLine($"Overflowing Tokens Length: {output.OverflowingTokens.Length}");
90+
}
8891

8992
Console.WriteLine("Done!");
9093

0 commit comments

Comments
 (0)