Support gpt-5.1 model in Tiktoken tokenizer (#7556)

* Support gpt-5.1 model in Tiktoken tokenizer

* Fix spaces

Author: tarekgh

src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs

@@ -1044,13 +1044,14 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
                                                                 ( "o4-mini-", ModelEncoding.O200kBase ),  // e.g. o4-mini
 
                                                                 // chat
-                                                                ( "gpt-5-", ModelEncoding.O200kBase),
-                                                                ( "gpt-4.1-", ModelEncoding.O200kBase),   // e.g., gpt-4.1-mini
-                                                                ( "gpt-4.5-", ModelEncoding.O200kBase),   // e.g., gpt-4.5
-                                                                ( "gpt-4o-", ModelEncoding.O200kBase),    // e.g., gpt-4o-2024-05-13
-                                                                ( "chatgpt-4o-", ModelEncoding.O200kBase),
-                                                                ( "gpt-4-", ModelEncoding.Cl100kBase),    // e.g., gpt-4-0314, etc., plus gpt-4-32k
-                                                                ( "gpt-3.5-", ModelEncoding.Cl100kBase),  // e.g, gpt-3.5-turbo-0301, -0401, etc.
+                                                                ( "gpt-5.1-", ModelEncoding.O200kBase ),
+                                                                ( "gpt-5-", ModelEncoding.O200kBase ),
+                                                                ( "gpt-4.1-", ModelEncoding.O200kBase ),   // e.g., gpt-4.1-mini
+                                                                ( "gpt-4.5-", ModelEncoding.O200kBase ),   // e.g., gpt-4.5
+                                                                ( "gpt-4o-", ModelEncoding.O200kBase ),    // e.g., gpt-4o-2024-05-13
+                                                                ( "chatgpt-4o-", ModelEncoding.O200kBase ),
+                                                                ( "gpt-4-", ModelEncoding.Cl100kBase ),    // e.g., gpt-4-0314, etc., plus gpt-4-32k
+                                                                ( "gpt-3.5-", ModelEncoding.Cl100kBase ),  // e.g, gpt-3.5-turbo-0301, -0401, etc.
                                                                 ( "gpt-35-", ModelEncoding.Cl100kBase ),  // Azure deployment name
                                                                 ( "gpt-oss-", ModelEncoding.O200kHarmony ),
 
@@ -1071,6 +1072,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
                                                                 { "o4-mini", ModelEncoding.O200kBase },
 
                                                                 // chat
+                                                                { "gpt-5.1", ModelEncoding.O200kBase },
                                                                 { "gpt-5", ModelEncoding.O200kBase },
                                                                 { "gpt-4.1", ModelEncoding.O200kBase },
                                                                 { "gpt-4o", ModelEncoding.O200kBase },

test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs

@@ -36,6 +36,7 @@ public class TiktokenTests
         public static Tokenizer P50kEdit { get; } = TiktokenTokenizer.CreateForModel("text-davinci-edit-001");
         public static Tokenizer GPT4o { get; } = TiktokenTokenizer.CreateForModel("gpt-4o");
         public static Tokenizer GPT5 { get; } = TiktokenTokenizer.CreateForModel("gpt-5");
+        public static Tokenizer GPT5_1 { get; } = TiktokenTokenizer.CreateForModel("gpt-5.1");
         public static Tokenizer Phi4 { get; } = TiktokenTokenizer.CreateForModel("phi-4");
         public static TiktokenTokenizer GptOss { get; } = TiktokenTokenizer.CreateForModel("gpt-oss-20b");
 
@@ -286,7 +287,7 @@ public void TestEncode5()
         [Fact]
         public void TestEncodeO200kBaseEncoding()
         {
-            foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5 })
+            foreach (TiktokenTokenizer tokenizer in new[] { GPT4o, GptOss, GPT5, GPT5_1 })
             {
                 string text = ReadAndSanitizeFile("./Data/lib.rs.txt");
                 IReadOnlyList<int> encoded = tokenizer.EncodeToIds(text);
@@ -415,6 +416,8 @@ public void TestEncodeR50kBase()
         [InlineData("gpt-4o-")]
         [InlineData("gpt-5")]
         [InlineData("gpt-5-chat")]
+        [InlineData("gpt-5.1")]
+        [InlineData("gpt-5.1-mini")]
         [InlineData("chatgpt-4o-")]
         [InlineData("gpt-4")]
         [InlineData("gpt-4-")]
@@ -533,6 +536,7 @@ public void TestEncodingNamesNegativeCases()
         [InlineData("gpt-4.1")]
         [InlineData("gpt-4o")]
         [InlineData("gpt-5")]
+        [InlineData("gpt-5.1")]
         [InlineData("o1")]
         [InlineData("o3")]
         [InlineData("o4-mini")]