File tree Expand file tree Collapse file tree 1 file changed +4
-4
lines changed
src/Microsoft.ML.Tokenizers Expand file tree Collapse file tree 1 file changed +4
-4
lines changed Original file line number Diff line number Diff line change @@ -30,12 +30,12 @@ string source = "Text tokenization is the process of splitting a string into a l
3030Console .WriteLine ($" Tokens: {tokenizer .CountTokens (source )}" );
3131// prints: Tokens: 16
3232
33- var trimIndex = tokenizer .GetIndexByTokenCountFromEnd (source , 5 , out string processedText , out _ );
34- Console .WriteLine ($" 5 tokens from end: {processedText .Substring (trimIndex )}" );
33+ var trimIndex = tokenizer .GetIndexByTokenCountFromEnd (source , 5 , out string normalizedText , out _ );
34+ Console .WriteLine ($" 5 tokens from end: {( normalizedText ?? source ) .Substring (trimIndex )}" );
3535// prints: 5 tokens from end: a list of tokens.
3636
37- trimIndex = tokenizer .GetIndexByTokenCount (source , 5 , out processedText , out _ );
38- Console .WriteLine ($" 5 tokens from start: {processedText .Substring (0 , trimIndex )}" );
37+ trimIndex = tokenizer .GetIndexByTokenCount (source , 5 , out normalizedText , out _ );
38+ Console .WriteLine ($" 5 tokens from start: {( normalizedText ?? source ) .Substring (0 , trimIndex )}" );
3939// prints: 5 tokens from start: Text tokenization is the
4040
4141IReadOnlyList < int > ids = tokenizer .EncodeToIds (source );
You can’t perform that action at this time.
0 commit comments