Skip to content

Commit d021488

Browse files
authored
Fix bug in PACKAGE.md examples (#7561)
* Fix bug in PACKAGE.md examples * Apply suggestions from code review
1 parent 753abcc commit d021488

File tree

1 file changed

+4
-4
lines changed

1 file changed

+4
-4
lines changed

src/Microsoft.ML.Tokenizers/PACKAGE.md

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,12 @@ string source = "Text tokenization is the process of splitting a string into a l
3030
Console.WriteLine($"Tokens: {tokenizer.CountTokens(source)}");
3131
// prints: Tokens: 16
3232
33-
var trimIndex = tokenizer.GetIndexByTokenCountFromEnd(source, 5, out string processedText, out _);
34-
Console.WriteLine($"5 tokens from end: {processedText.Substring(trimIndex)}");
33+
var trimIndex = tokenizer.GetIndexByTokenCountFromEnd(source, 5, out string normalizedText, out _);
34+
Console.WriteLine($"5 tokens from end: {(normalizedText ?? source).Substring(trimIndex)}");
3535
// prints: 5 tokens from end: a list of tokens.
3636
37-
trimIndex = tokenizer.GetIndexByTokenCount(source, 5, out processedText, out _);
38-
Console.WriteLine($"5 tokens from start: {processedText.Substring(0, trimIndex)}");
37+
trimIndex = tokenizer.GetIndexByTokenCount(source, 5, out normalizedText, out _);
38+
Console.WriteLine($"5 tokens from start: {(normalizedText ?? source).Substring(0, trimIndex)}");
3939
// prints: 5 tokens from start: Text tokenization is the
4040
4141
IReadOnlyList<int> ids = tokenizer.EncodeToIds(source);

0 commit comments

Comments
 (0)