[ggma] Fix hardcoded bos_id in TokenizerSentencePiece (#16275)

glistening · web-flow · commit af14f55419c6 · 2025-11-10T10:16:56.000+09:00
TokenizerSentencePiece should use bos_id from vocabulary.
Hardcoded 1 was wrong.

ONE-DCO-1.0-Signed-off-by: Sanggyu Lee &lt;sg5.lee@samsung.com&gt;
diff --git a/runtime/ggma/src/tokenize/TokenizerSentencePiece.cc b/runtime/ggma/src/tokenize/TokenizerSentencePiece.cc
@@ -76,10 +76,11 @@ size_t SentencePieceTokenizer::tokenize(const char *text, size_t text_len, int32
   int bos_id = _processor->bos_id();
   size_t bos_offset = 0;
 
+  // TODO: Make BOS token prepending configurable
   if (bos_id >= 0 && max_tokens > 0)
   {
-    tokens[0] = 1;  // Add BOS token
-    bos_offset = 1; // Start actual tokens from index 1
+    tokens[0] = bos_id; // Add BOS token
+    bos_offset = 1;     // Start actual tokens from index 1
   }
 
   size_t available_space = max_tokens - bos_offset;

Original file line number	Diff line number	Diff line change
`@@ -76,10 +76,11 @@ size_t SentencePieceTokenizer::tokenize(const char *text, size_t text_len, int32`
`76`	`76`	`int bos_id = _processor->bos_id();`
`77`	`77`	`size_t bos_offset = 0;`
`78`	`78`
	`79`	`+ // TODO: Make BOS token prepending configurable`
`79`	`80`	`if (bos_id >= 0 && max_tokens > 0)`
`80`	`81`	`{`
`81`		`- tokens[0] = 1; // Add BOS token`
`82`		`- bos_offset = 1; // Start actual tokens from index 1`
	`82`	`+ tokens[0] = bos_id; // Add BOS token`
	`83`	`+ bos_offset = 1; // Start actual tokens from index 1`
`83`	`84`	`}`
`84`	`85`
`85`	`86`	`size_t available_space = max_tokens - bos_offset;`