docs: add comments clarifying google/sentencepiece proto spec compliance for default token IDs (#21)

Closes #20

https://claude.ai/code/session_01SeqvRT7UqoHg9gCx2j3YV4

Co-authored-by: Claude <noreply@anthropic.com>

Author: brody-0125

README.md

@@ -177,6 +177,11 @@ final seqIdx = encoding.tokenToSequence(1);
 
 ### Vocabulary Access
 
+> **Note:** Default special token IDs (`unkId=0, bosId=1, eosId=2, padId=-1`) follow the
+> [google/sentencepiece proto spec](https://github.com/google/sentencepiece/blob/master/src/sentencepiece_model.proto).
+> Some models use different values (e.g. Gemma: `pad=0, eos=1, bos=2`), which are
+> automatically parsed from the model file at load time.
+
 ```dart
 print(tokenizer.vocabSize);     // 32000
 print(tokenizer.vocab.unkId);   // 0

lib/src/sentencepiece/model/model_proto.dart

@@ -62,6 +62,11 @@ class SentencePiece {
 }
 
 /// Training specification from the model.
+///
+/// Default token IDs (unkId=0, bosId=1, eosId=2, padId=-1) comply with the
+/// google/sentencepiece proto spec (sentencepiece_model.proto). Some models
+/// (e.g. Gemma: pad=0, eos=1, bos=2) use different values, which are
+/// correctly parsed from the model file at runtime.
 class TrainerSpec {
   final ModelType modelType;
   final int vocabSize;
@@ -78,6 +83,7 @@ class TrainerSpec {
   const TrainerSpec({
     this.modelType = ModelType.unigram,
     this.vocabSize = 8000,
+    // Default token IDs per google/sentencepiece proto spec.
     this.unkId = 0,
     this.bosId = 1,
     this.eosId = 2,

lib/src/sentencepiece/model/sentencepiece_model.dart

@@ -120,6 +120,8 @@ class SentencePieceModelLoader {
   static TrainerSpec _parseTrainerSpec(ProtobufReader reader) {
     ModelType modelType = ModelType.unigram;
     int vocabSize = 8000;
+    // Defaults per google/sentencepiece proto spec (sentencepiece_model.proto).
+    // These are overridden by actual values parsed from the model file below.
     int unkId = 0;
     int bosId = 1;
     int eosId = 2;

test/test_utils.dart

@@ -96,13 +96,14 @@ Uint8List _createMinimalTestModel() {
   }
 
   // Add trainer_spec (field 2)
+  // Default token IDs follow the google/sentencepiece proto spec.
   final trainerSpec = _ProtobufBuilder();
   trainerSpec.writeVarint(1, 1); // model_type = unigram
   trainerSpec.writeVarint(3, testPieces.length); // vocab_size
-  trainerSpec.writeVarint(40, 0); // unk_id
-  trainerSpec.writeVarint(41, 1); // bos_id
-  trainerSpec.writeVarint(42, 2); // eos_id
-  trainerSpec.writeVarint(43, -1); // pad_id (not set)
+  trainerSpec.writeVarint(40, 0); // unk_id (proto default)
+  trainerSpec.writeVarint(41, 1); // bos_id (proto default)
+  trainerSpec.writeVarint(42, 2); // eos_id (proto default)
+  trainerSpec.writeVarint(43, -1); // pad_id (proto default, -1 = unused)
   builder.writeBytes(2, trainerSpec.toBytes());
 
   // Add normalizer_spec (field 3)