Skip to content

Commit f03d6a1

Browse files
committed
revert the test case
1 parent 7b7913a commit f03d6a1

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

test/pp_api_test/test_tokenizer.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,13 +76,13 @@ TEST(OrtxTokenizerTest, ClipTokenizer) {
7676
// validate tokenizer is not null
7777
ASSERT_NE(tokenizer.get(), nullptr) << "Tokenizer is null, stopping the test.";
7878

79-
std::vector<std::string_view> input = {"<|startoftext|>|this is a test<|endoftext|>", "the second one"};
79+
std::vector<std::string_view> input = {"this is a test .", "the second one"};
8080

8181
std::vector<std::vector<extTokenId_t>> token_ids;
8282
status = tokenizer->Tokenize(input, token_ids);
8383
EXPECT_TRUE(status.IsOk());
8484
EXPECT_EQ(token_ids.size(), 2);
85-
EXPECT_EQ(token_ids[0].size(), 6);
85+
EXPECT_EQ(token_ids[0].size(), 7);
8686
EXPECT_EQ(token_ids[1].size(), 5);
8787

8888
std::vector<std::string> out_text;

0 commit comments

Comments
 (0)