File tree Expand file tree Collapse file tree 1 file changed +2
-2
lines changed
Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Original file line number Diff line number Diff line change @@ -76,13 +76,13 @@ TEST(OrtxTokenizerTest, ClipTokenizer) {
7676 // validate tokenizer is not null
7777 ASSERT_NE (tokenizer.get (), nullptr ) << " Tokenizer is null, stopping the test." ;
7878
79- std::vector<std::string_view> input = {" <|startoftext|>| this is a test<|endoftext|> " , " the second one" };
79+ std::vector<std::string_view> input = {" this is a test . " , " the second one" };
8080
8181 std::vector<std::vector<extTokenId_t>> token_ids;
8282 status = tokenizer->Tokenize (input, token_ids);
8383 EXPECT_TRUE (status.IsOk ());
8484 EXPECT_EQ (token_ids.size (), 2 );
85- EXPECT_EQ (token_ids[0 ].size (), 6 );
85+ EXPECT_EQ (token_ids[0 ].size (), 7 );
8686 EXPECT_EQ (token_ids[1 ].size (), 5 );
8787
8888 std::vector<std::string> out_text;
You can’t perform that action at this time.
0 commit comments