We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent f005417 commit 194e1c2Copy full SHA for 194e1c2
Sample/Program.cs
@@ -79,12 +79,15 @@ private static void Main(string[] args)
79
80
outputs.Dispose();
81
82
- var output = tokenizer.Tokenize("Hi");
+ const bool TEST_SINGLE_TOKENIZE = false;
83
84
- // Console.WriteLine(output.IDs.AsReadOnlySpan().GetSpanPrintString());
85
- Console.WriteLine($"Overflowing Tokens Length: {output.OverflowingTokens.Length}");
+ if (TEST_SINGLE_TOKENIZE)
+ {
86
+ using var output = tokenizer.Tokenize("Hi");
87
- output.Dispose();
88
+ // Console.WriteLine(output.IDs.AsReadOnlySpan().GetSpanPrintString());
89
+ Console.WriteLine($"Overflowing Tokens Length: {output.OverflowingTokens.Length}");
90
+ }
91
92
Console.WriteLine("Done!");
93
0 commit comments