Check for end-of-turn tokens in tokenizer config top-level

swift/Sources/CoreAILanguageModels/Bundle/LanguageConfig.swift

@@ -134,7 +134,7 @@ public struct LanguageConfig: Codable, Sendable, Equatable {
         // 3. Check added_tokens_decoder for turn-ending special tokens
         //    (e.g. Gemma's <end_of_turn> ID 106, Qwen's <|im_end|>)
         //    Only include tokens whose content matches known turn-ending patterns.
-        let turnEndPatterns = ["end_of_turn", "im_end", "eot_id", "endoftext"]
+        let turnEndPatterns = ["end_of_turn", "im_end", "eot_id", "endoftext", "eot_token"]
         if let addedTokens = json["added_tokens_decoder"] as? [String: Any] {
             for (idString, value) in addedTokens {
                 guard let dict = value as? [String: Any],
@@ -149,6 +149,18 @@ public struct LanguageConfig: Codable, Sendable, Equatable {
             }
         }
 
+        // 4. Check for turn-ending tokens in in top level of config
+        for turnEndPattern in turnEndPatterns {
+            if let eotToken = json[turnEndPattern] as? String {
+                if let id = tokenizer.convertTokenToId(eotToken) {
+                    let id32 = Int32(id)
+                    if id32 != mainEos {
+                        result.insert(id32)
+                    }
+                }
+            }
+        }
+
         return Array(result)
     }
 }

swift/Tests/LanguageModelsTests/AdditionalStopTokensTests.swift

@@ -0,0 +1,110 @@
+// Copyright 2026 Apple Inc.
+//
+// Use of this source code is governed by a BSD-3-clause license that can
+// be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause
+
+import Foundation
+import TestUtilities
+import Testing
+import Tokenizers
+
+@testable import CoreAILanguageModels
+
+@Suite("LanguageConfig.additionalStopTokenIds")
+struct AdditionalStopTokensTests {
+    /// Vocabulary shared by the tests. `<eos>` must be ID 2 to match
+    /// `MockTokenizer.eosTokenId`, so it is expected to be filtered out.
+    private static let vocab: [String: Int] = [
+        "<eot>": 1,
+        "<eos>": 2,
+        "<end_of_turn>": 3,
+        "<|im_end|>": 4,
+        "<|endoftext|>": 5,
+    ]
+
+    private static func tokenizer() -> any Tokenizer {
+        MockTokenizer(vocab: vocab)
+    }
+
+    /// Write `tokenizer_config.json` into a fresh temp directory and return it.
+    private static func tokenizerDir(config: String) throws -> URL {
+        let dir = FileManager.default.temporaryDirectory.appending(
+            path: "AdditionalStopTokensTests-\(UUID().uuidString)"
+        )
+        try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
+        try config.write(
+            to: dir.appending(path: "tokenizer_config.json"),
+            atomically: true, encoding: .utf8
+        )
+        return dir
+    }
+
+    private static func stopIds(config: String) throws -> Set<Int32> {
+        let dir = try tokenizerDir(config: config)
+        defer { try? FileManager.default.removeItem(at: dir) }
+        return Set(
+            LanguageConfig.additionalStopTokenIds(from: dir, tokenizer: tokenizer())
+        )
+    }
+
+    // MARK: - Top-level turn-ending tokens
+
+    @Test("top-level eot_token string is picked up")
+    func topLevelEotToken() throws {
+        let ids = try Self.stopIds(
+            config: """
+                {
+                  "eos_token": "<eos>",
+                  "eot_token": "<eot>"
+                }
+                """)
+        #expect(ids == [1])
+    }
+
+    @Test("top-level end_of_turn / im_end / endoftext keys are picked up")
+    func topLevelOtherPatterns() throws {
+        let ids = try Self.stopIds(
+            config: """
+                {
+                  "end_of_turn": "<end_of_turn>",
+                  "im_end": "<|im_end|>",
+                  "endoftext": "<|endoftext|>"
+                }
+                """)
+        #expect(ids == [3, 4, 5])
+    }
+
+    @Test("top-level token equal to the main EOS is not duplicated")
+    func topLevelSkipsMainEos() throws {
+        let ids = try Self.stopIds(
+            config: """
+                {
+                  "eos_token": "<eos>",
+                  "eot_token": "<eos>"
+                }
+                """)
+        #expect(ids.isEmpty)
+    }
+
+    @Test("top-level token missing from the vocab is ignored")
+    func topLevelUnknownToken() throws {
+        let ids = try Self.stopIds(
+            config: """
+                {
+                  "eot_token": "<not_in_vocab>"
+                }
+                """)
+        #expect(ids.isEmpty)
+    }
+
+    @Test("non-string top-level value is ignored")
+    func topLevelNonStringValue() throws {
+        let ids = try Self.stopIds(
+            config: """
+                {
+                  "eot_token": { "content": "<eot>" }
+                }
+                """)
+        #expect(ids.isEmpty)
+    }
+}

swift/Tests/TestUtilities/Utilities.swift

@@ -27,7 +27,16 @@ public struct CIEnvironment {
 /// - encode("hello") → [104, 101, 108, 108, 111] (UTF-8 bytes)
 /// - decode([104, 101, 108, 108, 111]) → "hello"
 public struct MockTokenizer: Tokenizer, Sendable {
-    public init() {}
+    /// Optional explicit vocabulary for multi-character (special) tokens.
+    /// When non-empty it becomes authoritative for `convertTokenToId`: known
+    /// tokens map to their listed ID and unknown tokens return `nil`, instead of
+    /// the default first-UTF-8-byte behaviour (which collides for tokens sharing
+    /// a leading character, e.g. `<eos>` and `<end_of_turn>`).
+    public let vocab: [String: Int]
+
+    public init(vocab: [String: Int] = [:]) {
+        self.vocab = vocab
+    }
 
     public var bosToken: String? { nil }
     public var bosTokenId: Int? { nil }
@@ -62,7 +71,8 @@ public struct MockTokenizer: Tokenizer, Sendable {
     }
 
     public func convertTokenToId(_ token: String) -> Int? {
-        token.utf8.first.map { Int($0) }
+        if !vocab.isEmpty { return vocab[token] }
+        return token.utf8.first.map { Int($0) }
     }
 
     public func convertTokensToIds(_ tokens: [String]) -> [Int?] {