File tree Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Original file line number Diff line number Diff line change @@ -76,13 +76,13 @@ TEST(OrtxTokenizerTest, ClipTokenizer) {
76
76
// validate tokenizer is not null
77
77
ASSERT_NE (tokenizer.get (), nullptr ) << " Tokenizer is null, stopping the test." ;
78
78
79
- std::vector<std::string_view> input = {" <|startoftext|>| this is a test<|endoftext|> " , " the second one" };
79
+ std::vector<std::string_view> input = {" this is a test . " , " the second one" };
80
80
81
81
std::vector<std::vector<extTokenId_t>> token_ids;
82
82
status = tokenizer->Tokenize (input, token_ids);
83
83
EXPECT_TRUE (status.IsOk ());
84
84
EXPECT_EQ (token_ids.size (), 2 );
85
- EXPECT_EQ (token_ids[0 ].size (), 6 );
85
+ EXPECT_EQ (token_ids[0 ].size (), 7 );
86
86
EXPECT_EQ (token_ids[1 ].size (), 5 );
87
87
88
88
std::vector<std::string> out_text;
You can’t perform that action at this time.
0 commit comments