Add comment noting that CurRank assumes rank == token Id (Tiktoken-specific)

Co-authored-by: tarekgh <10833894+tarekgh@users.noreply.github.com>

Author: Copilot

src/Microsoft.ML.Tokenizers/Utils/BytePairEncoder.cs

@@ -131,6 +131,9 @@ private struct State
             public int End;
             public int NextEnd;
             public int NextRank;
+            // Note: In the Tiktoken tokenizer, the rank is also the token Id.
+            // This field is used to cache the rank/Id after a merge so we don't need to re-look it up.
+            // Using this code with a different tokenizer where rank != token Id would produce wrong results.
             public int CurRank;
         }