From 89de43c6c8c36f037da3db22230fa5356463b594 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Fri, 3 Apr 2026 12:05:24 +0200 Subject: [PATCH 1/4] Add Mistral3, Nemotron, and Qwen3.5 tool call integration test helpers --- .../IntegrationTestHelpers.swift | 348 +++++++++++++++++- Libraries/MLXVLM/MediaProcessing.swift | 2 +- 2 files changed, 336 insertions(+), 14 deletions(-) diff --git a/Libraries/IntegrationTestHelpers/IntegrationTestHelpers.swift b/Libraries/IntegrationTestHelpers/IntegrationTestHelpers.swift index ed4089134..50e50db05 100644 --- a/Libraries/IntegrationTestHelpers/IntegrationTestHelpers.swift +++ b/Libraries/IntegrationTestHelpers/IntegrationTestHelpers.swift @@ -35,6 +35,9 @@ public enum IntegrationTestModelIDs { public static let vlm = "mlx-community/Qwen3-VL-4B-Instruct-4bit" public static let lfm2 = "mlx-community/LFM2-2.6B-Exp-4bit" public static let glm4 = "mlx-community/GLM-4-9B-0414-4bit" + public static let mistral3 = "mlx-community/Ministral-3-3B-Instruct-2512-4bit" + public static let nemotron = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-4bit" + public static let qwen35 = "mlx-community/Qwen3.5-2B-4bit" } // MARK: - Model Loading @@ -48,6 +51,9 @@ public actor IntegrationTestModels { private var vlmTask: Task? private var lfm2Task: Task? private var glm4Task: Task? + private var mistral3Task: Task? + private var nemotronTask: Task? + private var qwen35Task: Task? public init(downloader: any Downloader, tokenizerLoader: any TokenizerLoader) { self.downloader = downloader @@ -138,6 +144,69 @@ public actor IntegrationTestModels { return try await task.value } + public func mistral3Container() async throws -> LMModelContainer { + if let task = mistral3Task { + return try await task.value + } + let downloader = self.downloader + let tokenizerLoader = self.tokenizerLoader + let id = IntegrationTestModelIDs.mistral3 + let task = Task { + print("Loading Mistral3: \(id)") + let container = try await LLMModelFactory.shared.loadContainer( + from: downloader, using: tokenizerLoader, + configuration: .init(id: id), + progressHandler: logProgress(id) + ) + print("Loaded Mistral3: \(id)") + return container + } + mistral3Task = task + return try await task.value + } + + public func nemotronContainer() async throws -> LMModelContainer { + if let task = nemotronTask { + return try await task.value + } + let downloader = self.downloader + let tokenizerLoader = self.tokenizerLoader + let id = IntegrationTestModelIDs.nemotron + let task = Task { + print("Loading Nemotron: \(id)") + let container = try await LLMModelFactory.shared.loadContainer( + from: downloader, using: tokenizerLoader, + configuration: .init(id: id), + progressHandler: logProgress(id) + ) + print("Loaded Nemotron: \(id)") + return container + } + nemotronTask = task + return try await task.value + } + + public func qwen35Container() async throws -> LMModelContainer { + if let task = qwen35Task { + return try await task.value + } + let downloader = self.downloader + let tokenizerLoader = self.tokenizerLoader + let id = IntegrationTestModelIDs.qwen35 + let task = Task { + print("Loading Qwen3.5: \(id)") + let container = try await LLMModelFactory.shared.loadContainer( + from: downloader, using: tokenizerLoader, + configuration: .init(id: id), + progressHandler: logProgress(id) + ) + print("Loaded Qwen3.5: \(id)") + return container + } + qwen35Task = task + return try await task.value + } + public func embeddingContainer() async throws -> EmbeddingModelContainer { let downloader = self.downloader let tokenizerLoader = self.tokenizerLoader @@ -470,6 +539,8 @@ public enum EmbedderTests { public enum ToolCallTests { + // MARK: LFM2 + public static func lfm2FormatAutoDetection(container: LMModelContainer) async throws { let config = await container.configuration try check( @@ -501,6 +572,8 @@ public enum ToolCallTests { } } + // MARK: GLM4 + public static func glm4FormatAutoDetection(container: LMModelContainer) async throws { let config = await container.configuration try check( @@ -532,27 +605,239 @@ public enum ToolCallTests { } } + // MARK: Mistral3 + + public static func mistral3FormatAutoDetection(container: LMModelContainer) async throws { + let config = await container.configuration + try check( + config.toolCallFormat == ToolCallFormat.mistral, + "Expected .mistral tool call format, got: \(String(describing: config.toolCallFormat))" + ) + } + + public static func mistral3EndToEndGeneration(container: LMModelContainer) async throws { + let input = UserInput( + chat: [ + .system( + "You are a helpful assistant with access to tools. When asked about weather, use the get_weather function." + ), + .user("What's the weather in Tokyo?"), + ], + tools: [weatherToolSchema] + ) + + let (result, toolCalls) = try await generateWithTools( + container: container, input: input, maxTokens: 150) + + print("Mistral3 Output:", result) + print("Mistral3 Tool Calls:", toolCalls) + + if !toolCalls.isEmpty { + let toolCall = toolCalls[0] + try check( + toolCall.function.name == "get_weather", + "Expected tool name 'get_weather', got: \(toolCall.function.name)" + ) + if case .string(let location) = toolCall.function.arguments["location"] { + try check( + location.lowercased().contains("tokyo"), + "Expected location containing 'Tokyo', got: \(location)" + ) + } + } + } + + public static func mistral3MultiToolGeneration(container: LMModelContainer) async throws { + let input = UserInput( + chat: [ + .system( + "You are a helpful assistant with access to tools. Always use the available tools to answer questions. Call multiple tools in parallel when needed." + ), + .user("What's the weather in Tokyo and what time is it there?"), + ], + tools: multiToolSchemas + ) + + let (result, toolCalls) = try await generateWithTools( + container: container, input: input, maxTokens: 150) + + print("Mistral3 Output:", result) + print("Mistral3 Calls:", toolCalls) + + let validNames: Set = ["get_weather", "get_time"] + for toolCall in toolCalls { + try check( + validNames.contains(toolCall.function.name), + "Unexpected tool call: \(toolCall.function.name)" + ) + } + + if toolCalls.count > 1 { + print("Successfully parsed \(toolCalls.count) tool calls from Mistral3") + } + } + + // MARK: Nemotron + + public static func nemotronFormatAutoDetection(container: LMModelContainer) async throws { + let config = await container.configuration + try check( + config.toolCallFormat == ToolCallFormat.xmlFunction, + "Expected .xmlFunction tool call format, got: \(String(describing: config.toolCallFormat))" + ) + } + + public static func nemotronEndToEndGeneration(container: LMModelContainer) async throws { + let input = UserInput( + chat: [ + .system( + "You are a helpful assistant with access to tools. When asked about weather, use the get_weather function." + ), + .user("What's the weather in Tokyo?"), + ], + tools: [weatherToolSchema], + additionalContext: ["enable_thinking": false] + ) + + let (result, toolCalls) = try await generateWithTools( + container: container, input: input, maxTokens: 150) + + print("Nemotron Output:", result) + print("Nemotron Tool Calls:", toolCalls) + + if !toolCalls.isEmpty { + let toolCall = toolCalls[0] + try check( + toolCall.function.name == "get_weather", + "Expected tool name 'get_weather', got: \(toolCall.function.name)" + ) + if case .string(let location) = toolCall.function.arguments["location"] { + try check( + location.lowercased().contains("tokyo"), + "Expected location containing 'Tokyo', got: \(location)" + ) + } + } + } + + public static func nemotronMultiToolGeneration(container: LMModelContainer) async throws { + let input = UserInput( + chat: [ + .system( + "You are a helpful assistant with access to tools. Always use the available tools to answer questions. Call multiple tools in parallel when needed." + ), + .user("What's the weather in Tokyo and what time is it there?"), + ], + tools: multiToolSchemas, + additionalContext: ["enable_thinking": false] + ) + + let (result, toolCalls) = try await generateWithTools( + container: container, input: input, maxTokens: 600) + + print("Nemotron Output:", result) + print("Nemotron Calls:", toolCalls) + + let validNames: Set = ["get_weather", "get_time"] + for toolCall in toolCalls { + try check( + validNames.contains(toolCall.function.name), + "Unexpected tool call: \(toolCall.function.name)" + ) + } + + if toolCalls.count > 1 { + print("Successfully parsed \(toolCalls.count) tool calls from Nemotron") + } + } + + // MARK: Qwen3.5 + + public static func qwen35FormatAutoDetection(container: LMModelContainer) async throws { + let config = await container.configuration + try check( + config.toolCallFormat == ToolCallFormat.xmlFunction, + "Expected .xmlFunction tool call format, got: \(String(describing: config.toolCallFormat))" + ) + } + + public static func qwen35EndToEndGeneration(container: LMModelContainer) async throws { + let input = UserInput( + chat: [ + .system( + "You are a helpful assistant with access to tools. When asked about weather, use the get_weather function." + ), + .user("What's the weather in Tokyo?"), + ], + tools: [weatherToolSchema] + ) + + let (result, toolCalls) = try await generateWithTools( + container: container, input: input, maxTokens: 150) + + print("Qwen3.5 Output:", result) + print("Qwen3.5 Tool Calls:", toolCalls) + + if !toolCalls.isEmpty { + let toolCall = toolCalls[0] + try check( + toolCall.function.name == "get_weather", + "Expected tool name 'get_weather', got: \(toolCall.function.name)" + ) + if case .string(let location) = toolCall.function.arguments["location"] { + try check( + location.lowercased().contains("tokyo"), + "Expected location containing 'Tokyo', got: \(location)" + ) + } + } + } + + public static func qwen35MultiToolGeneration(container: LMModelContainer) async throws { + let input = UserInput( + chat: [ + .system( + "You are a helpful assistant with access to tools. Always use the available tools to answer questions. Call multiple tools in parallel when needed." + ), + .user("What's the weather in Tokyo and what time is it there?"), + ], + tools: multiToolSchemas, + additionalContext: ["enable_thinking": true] + ) + + let (result, toolCalls) = try await generateWithTools( + container: container, input: input, maxTokens: 300) + + print("Qwen3.5 Output:", result) + print("Qwen3.5 Calls:", toolCalls) + + let validNames: Set = ["get_weather", "get_time"] + for toolCall in toolCalls { + try check( + validNames.contains(toolCall.function.name), + "Unexpected tool call: \(toolCall.function.name)" + ) + } + + if toolCalls.count > 1 { + print("Successfully parsed \(toolCalls.count) tool calls from Qwen3.5") + } + } + + // MARK: Helpers + private static func generateWithTools( container: LMModelContainer, - userMessage: String + input: UserInput, + maxTokens: Int = 100 ) async throws -> (text: String, toolCalls: [ToolCall]) { - try await container.perform { context in - let input = UserInput( - chat: [ - .system( - "You are a helpful assistant with access to tools. When asked about weather, use the get_weather function." - ), - .user(userMessage), - ], - tools: [weatherToolSchema] - ) + try await container.perform(nonSendable: input) { context, input in let lmInput = try await context.processor.prepare(input: input) let stream = try generate( input: lmInput, - parameters: GenerateParameters(maxTokens: 100), + parameters: GenerateParameters(maxTokens: maxTokens), context: context ) - var text = "" var toolCalls: [ToolCall] = [] for try await generation in stream { @@ -568,6 +853,23 @@ public enum ToolCallTests { return (text, toolCalls) } } + + private static func generateWithTools( + container: LMModelContainer, + userMessage: String + ) async throws -> (text: String, toolCalls: [ToolCall]) { + let input = UserInput( + chat: [ + .system( + "You are a helpful assistant with access to tools. When asked about weather, use the get_weather function." + ), + .user(userMessage), + ], + tools: [weatherToolSchema] + ) + return try await generateWithTools( + container: container, input: input) + } } // MARK: - Progress Logging @@ -612,3 +914,23 @@ private let weatherToolSchema: ToolSpec = [ ] as [String: any Sendable], ] as [String: any Sendable], ] + +private let timeToolSchema: ToolSpec = [ + "type": "function", + "function": [ + "name": "get_time", + "description": "Get the current time in a given timezone", + "parameters": [ + "type": "object", + "properties": [ + "timezone": [ + "type": "string", + "description": "The timezone, e.g. America/New_York, Asia/Tokyo", + ] as [String: any Sendable] + ] as [String: any Sendable], + "required": ["timezone"], + ] as [String: any Sendable], + ] as [String: any Sendable], +] + +private let multiToolSchemas: [ToolSpec] = [weatherToolSchema, timeToolSchema] diff --git a/Libraries/MLXVLM/MediaProcessing.swift b/Libraries/MLXVLM/MediaProcessing.swift index 0cd9a5b6d..4d69939a9 100644 --- a/Libraries/MLXVLM/MediaProcessing.swift +++ b/Libraries/MLXVLM/MediaProcessing.swift @@ -341,7 +341,7 @@ public enum MediaProcessing { static public func asProcessedSequence( _ video: UserInput.Video, samplesPerSecond: Int, - frameProcessing: (VideoFrame) throws -> VideoFrame = { $0 }, + frameProcessing: (VideoFrame) throws -> VideoFrame = { $0 } ) async throws -> ProcessedFrames { return try await asProcessedSequence( video, From 6d3fafae391e057db4a7c92ad8db9ed87d0f3b0d Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Sat, 4 Apr 2026 14:58:38 +0200 Subject: [PATCH 2/4] Add MLXLMIntegrationTests --- Package.swift | 16 ++++ .../ToolCallIntegrationTests.swift | 91 +++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 Tests/MLXLMIntegrationTests/ToolCallIntegrationTests.swift diff --git a/Package.swift b/Package.swift index 8dcf54c51..960267421 100644 --- a/Package.swift +++ b/Package.swift @@ -38,6 +38,11 @@ let package = Package( dependencies: [ .package(url: "https://github.com/ml-explore/mlx-swift", .upToNextMinor(from: "0.31.3")), .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "600.0.0-latest"), + .package(url: "https://github.com/huggingface/swift-huggingface", from: "0.9.0"), + .package( + url: "https://github.com/huggingface/swift-transformers", + .upToNextMinor(from: "1.3.0") + ), ], targets: [ .target( @@ -146,6 +151,17 @@ let package = Package( ], path: "Libraries/MLXHuggingFace" ), + .testTarget( + name: "MLXLMIntegrationTests", + dependencies: [ + "IntegrationTestHelpers", + "MLXHuggingFace", + "MLXLMCommon", + .product(name: "HuggingFace", package: "swift-huggingface"), + .product(name: "Tokenizers", package: "swift-transformers"), + ], + path: "Tests/MLXLMIntegrationTests" + ), ] ) diff --git a/Tests/MLXLMIntegrationTests/ToolCallIntegrationTests.swift b/Tests/MLXLMIntegrationTests/ToolCallIntegrationTests.swift new file mode 100644 index 000000000..1cafe9d85 --- /dev/null +++ b/Tests/MLXLMIntegrationTests/ToolCallIntegrationTests.swift @@ -0,0 +1,91 @@ +import Foundation +import HuggingFace +import IntegrationTestHelpers +import MLXHuggingFace +import MLXLMCommon +import Testing +import Tokenizers + +private let models = IntegrationTestModels( + downloader: #hubDownloader(), + tokenizerLoader: #huggingFaceTokenizerLoader() +) + +@Suite(.serialized) +struct ToolCallIntegrationTests { + + // MARK: - LFM2 + + @Test func lfm2FormatAutoDetection() async throws { + let container = try await models.lfm2Container() + try await ToolCallTests.lfm2FormatAutoDetection(container: container) + } + + @Test func lfm2EndToEnd() async throws { + let container = try await models.lfm2Container() + try await ToolCallTests.lfm2EndToEndGeneration(container: container) + } + + // MARK: - GLM4 + + @Test func glm4FormatAutoDetection() async throws { + let container = try await models.glm4Container() + try await ToolCallTests.glm4FormatAutoDetection(container: container) + } + + @Test func glm4EndToEnd() async throws { + let container = try await models.glm4Container() + try await ToolCallTests.glm4EndToEndGeneration(container: container) + } + + // MARK: - Mistral3 + + @Test func mistral3FormatAutoDetection() async throws { + let container = try await models.mistral3Container() + try await ToolCallTests.mistral3FormatAutoDetection(container: container) + } + + @Test func mistral3EndToEnd() async throws { + let container = try await models.mistral3Container() + try await ToolCallTests.mistral3EndToEndGeneration(container: container) + } + + @Test func mistral3MultiTool() async throws { + let container = try await models.mistral3Container() + try await ToolCallTests.mistral3MultiToolGeneration(container: container) + } + + // MARK: - Nemotron + + @Test func nemotronFormatAutoDetection() async throws { + let container = try await models.nemotronContainer() + try await ToolCallTests.nemotronFormatAutoDetection(container: container) + } + + @Test func nemotronEndToEnd() async throws { + let container = try await models.nemotronContainer() + try await ToolCallTests.nemotronEndToEndGeneration(container: container) + } + + @Test func nemotronMultiTool() async throws { + let container = try await models.nemotronContainer() + try await ToolCallTests.nemotronMultiToolGeneration(container: container) + } + + // MARK: - Qwen3.5 + + @Test func qwen35FormatAutoDetection() async throws { + let container = try await models.qwen35Container() + try await ToolCallTests.qwen35FormatAutoDetection(container: container) + } + + @Test func qwen35EndToEnd() async throws { + let container = try await models.qwen35Container() + try await ToolCallTests.qwen35EndToEndGeneration(container: container) + } + + @Test func qwen35MultiTool() async throws { + let container = try await models.qwen35Container() + try await ToolCallTests.qwen35MultiToolGeneration(container: container) + } +} From 39cd9fc5371389a19808e2a7ca979463debd6f54 Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Sat, 4 Apr 2026 15:07:28 +0200 Subject: [PATCH 3/4] Update documentation --- CONTRIBUTING.md | 19 +++++++++++++++++ Libraries/IntegrationTestHelpers/README.md | 24 +++++++++++++++++++--- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1fbe678e..f8de13ce4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,25 @@ possible. or run `pre-commit run --all-files` to check all files in the repo. +## Running Tests + +Unit tests run without any special hardware and do not download models: + +``` +swift test +``` + +Integration tests verify end-to-end model loading and generation. They require macOS with Metal and download models from Hugging Face Hub on first run: + +```bash +xcodebuild test \ + -scheme mlx-swift-lm-Package \ + -destination 'platform=macOS' \ + -only-testing:MLXLMIntegrationTests +``` + +See [Libraries/IntegrationTestHelpers/README.md](Libraries/IntegrationTestHelpers/README.md) for more details. + ## Issues We use GitHub issues to track public bugs. Please ensure your description is diff --git a/Libraries/IntegrationTestHelpers/README.md b/Libraries/IntegrationTestHelpers/README.md index 39ec6ff14..826d69587 100644 --- a/Libraries/IntegrationTestHelpers/README.md +++ b/Libraries/IntegrationTestHelpers/README.md @@ -2,9 +2,27 @@ `IntegrationTestHelpers` and `BenchmarkHelpers` provide shared test logic for verifying end-to-end model loading, inference, tokenizer performance, and download performance. They are designed to be used by integration packages that supply their own `Downloader` and `TokenizerLoader` implementations. -## Integration packages +## Running integration tests locally + +The `MLXLMIntegrationTests` test target in this repo uses [Swift Hugging Face](https://github.com/huggingface/swift-huggingface) and [Swift Transformers](https://github.com/huggingface/swift-transformers) via the `MLXHuggingFace` macros to provide `Downloader` and `TokenizerLoader` implementations. Models are downloaded from Hugging Face Hub on first run and cached in `~/.cache/huggingface/`. + +```bash +# Run all integration tests (requires macOS with Metal) +xcodebuild test \ + -scheme mlx-swift-lm-Package \ + -destination 'platform=macOS' \ + -only-testing:MLXLMIntegrationTests + +# Run a single model's tests +xcodebuild test \ + -scheme mlx-swift-lm-Package \ + -destination 'platform=macOS' \ + -only-testing:MLXLMIntegrationTests/ToolCallIntegrationTests/qwen35FormatAutoDetection +``` + +## External integration packages + +Integration tests and benchmarks can also be run from external packages: - [Swift Tokenizers MLX](https://github.com/DePasqualeOrg/swift-tokenizers-mlx): Uses [Swift Tokenizers](https://github.com/DePasqualeOrg/swift-tokenizers) and [Swift HF API](https://github.com/DePasqualeOrg/swift-hf-api) - [Swift Transformers MLX](https://github.com/DePasqualeOrg/swift-transformers-mlx): Uses [Swift Transformers](https://github.com/huggingface/swift-transformers) and [Swift Hugging Face](https://github.com/huggingface/swift-huggingface) - -Integration tests and benchmarks are run from those packages. From 21a0ab0b78dac4f6ef68f1ceb956bc58c5d6d1cf Mon Sep 17 00:00:00 2001 From: Anthony Drendel Date: Sun, 5 Apr 2026 18:12:02 +0200 Subject: [PATCH 4/4] Add IntegrationTesting Xcode project and remove MLXLMIntegrationTests --- CONTRIBUTING.md | 21 +- .../project.pbxproj | 587 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + .../xcschemes/IntegrationTesting.xcscheme | 68 ++ .../IntegrationTesting.swift | 5 + .../ToolCallIntegrationTests.swift | 2 + Libraries/IntegrationTestHelpers/README.md | 19 +- Package.swift | 16 - 8 files changed, 698 insertions(+), 27 deletions(-) create mode 100644 IntegrationTesting/IntegrationTesting.xcodeproj/project.pbxproj create mode 100644 IntegrationTesting/IntegrationTesting.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 IntegrationTesting/IntegrationTesting.xcodeproj/xcshareddata/xcschemes/IntegrationTesting.xcscheme create mode 100644 IntegrationTesting/IntegrationTesting/IntegrationTesting.swift rename {Tests/MLXLMIntegrationTests => IntegrationTesting/IntegrationTestingTests}/ToolCallIntegrationTests.swift (98%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f8de13ce4..ed2a80d9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,7 +14,7 @@ possible. You can also run the formatters manually as follows: ``` - swift-format format --in-place --recursive Libraries Tools Applications + swift-format format --in-place --recursive Libraries Tools Applications IntegrationTesting ``` or run `pre-commit run --all-files` to check all files in the repo. @@ -27,13 +27,26 @@ Unit tests run without any special hardware and do not download models: swift test ``` -Integration tests verify end-to-end model loading and generation. They require macOS with Metal and download models from Hugging Face Hub on first run: +Integration tests verify end-to-end model loading and generation. They require +macOS with Metal and download models from Hugging Face Hub on first run. These +tests do not run in CI. + +Open `IntegrationTesting/IntegrationTesting.xcodeproj` in Xcode and run the +test target (`Cmd+U` or via the Test Navigator), or use `xcodebuild`: ```bash +# Run all integration tests +xcodebuild test \ + -project IntegrationTesting/IntegrationTesting.xcodeproj \ + -scheme IntegrationTesting \ + -destination 'platform=macOS' + +# Run a single test xcodebuild test \ - -scheme mlx-swift-lm-Package \ + -project IntegrationTesting/IntegrationTesting.xcodeproj \ + -scheme IntegrationTesting \ -destination 'platform=macOS' \ - -only-testing:MLXLMIntegrationTests + -only-testing:IntegrationTestingTests/ToolCallIntegrationTests/qwen35FormatAutoDetection\(\) ``` See [Libraries/IntegrationTestHelpers/README.md](Libraries/IntegrationTestHelpers/README.md) for more details. diff --git a/IntegrationTesting/IntegrationTesting.xcodeproj/project.pbxproj b/IntegrationTesting/IntegrationTesting.xcodeproj/project.pbxproj new file mode 100644 index 000000000..04174f7d3 --- /dev/null +++ b/IntegrationTesting/IntegrationTesting.xcodeproj/project.pbxproj @@ -0,0 +1,587 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXBuildFile section */ + 57408EB82F82A947001E2121 /* HuggingFace in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EB72F82A947001E2121 /* HuggingFace */; }; + 57408EBC2F82A947001E2121 /* Tokenizers in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EBB2F82A947001E2121 /* Tokenizers */; }; + 57408EDB2F82B8FB001E2121 /* BenchmarkHelpers in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EDA2F82B8FB001E2121 /* BenchmarkHelpers */; }; + 57408EDD2F82B8FB001E2121 /* IntegrationTestHelpers in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EDC2F82B8FB001E2121 /* IntegrationTestHelpers */; }; + 57408EDF2F82B8FB001E2121 /* MLXEmbedders in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EDE2F82B8FB001E2121 /* MLXEmbedders */; }; + 57408EE12F82B8FB001E2121 /* MLXLLM in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EE02F82B8FB001E2121 /* MLXLLM */; }; + 57408EE32F82B8FB001E2121 /* MLXLMCommon in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EE22F82B8FB001E2121 /* MLXLMCommon */; }; + 57408EE52F82B8FB001E2121 /* MLXVLM in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EE42F82B8FB001E2121 /* MLXVLM */; }; + 57408EE72F82B922001E2121 /* MLXHuggingFace in Frameworks */ = {isa = PBXBuildFile; productRef = 57408EE62F82B922001E2121 /* MLXHuggingFace */; }; + 578E559D2F82A3B9001FEF6B /* IntegrationTesting.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 578E55932F82A3B9001FEF6B /* IntegrationTesting.framework */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 578E559E2F82A3B9001FEF6B /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 578E558A2F82A3B9001FEF6B /* Project object */; + proxyType = 1; + remoteGlobalIDString = 578E55922F82A3B9001FEF6B; + remoteInfo = IntegrationTesting; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXFileReference section */ + 578E55932F82A3B9001FEF6B /* IntegrationTesting.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = IntegrationTesting.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + 578E559C2F82A3B9001FEF6B /* IntegrationTestingTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = IntegrationTestingTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; +/* End PBXFileReference section */ + +/* Begin PBXFileSystemSynchronizedRootGroup section */ + 578E55952F82A3B9001FEF6B /* IntegrationTesting */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = IntegrationTesting; + sourceTree = ""; + }; + 578E55A02F82A3B9001FEF6B /* IntegrationTestingTests */ = { + isa = PBXFileSystemSynchronizedRootGroup; + path = IntegrationTestingTests; + sourceTree = ""; + }; +/* End PBXFileSystemSynchronizedRootGroup section */ + +/* Begin PBXFrameworksBuildPhase section */ + 578E55902F82A3B9001FEF6B /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 578E55992F82A3B9001FEF6B /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 57408EE12F82B8FB001E2121 /* MLXLLM in Frameworks */, + 57408EBC2F82A947001E2121 /* Tokenizers in Frameworks */, + 57408EDD2F82B8FB001E2121 /* IntegrationTestHelpers in Frameworks */, + 57408EDF2F82B8FB001E2121 /* MLXEmbedders in Frameworks */, + 578E559D2F82A3B9001FEF6B /* IntegrationTesting.framework in Frameworks */, + 57408EE72F82B922001E2121 /* MLXHuggingFace in Frameworks */, + 57408EE52F82B8FB001E2121 /* MLXVLM in Frameworks */, + 57408EDB2F82B8FB001E2121 /* BenchmarkHelpers in Frameworks */, + 57408EE32F82B8FB001E2121 /* MLXLMCommon in Frameworks */, + 57408EB82F82A947001E2121 /* HuggingFace in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 57408EB62F82A947001E2121 /* Frameworks */ = { + isa = PBXGroup; + children = ( + ); + name = Frameworks; + sourceTree = ""; + }; + 578E55892F82A3B9001FEF6B = { + isa = PBXGroup; + children = ( + 578E55952F82A3B9001FEF6B /* IntegrationTesting */, + 578E55A02F82A3B9001FEF6B /* IntegrationTestingTests */, + 57408EB62F82A947001E2121 /* Frameworks */, + 578E55942F82A3B9001FEF6B /* Products */, + ); + sourceTree = ""; + }; + 578E55942F82A3B9001FEF6B /* Products */ = { + isa = PBXGroup; + children = ( + 578E55932F82A3B9001FEF6B /* IntegrationTesting.framework */, + 578E559C2F82A3B9001FEF6B /* IntegrationTestingTests.xctest */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXHeadersBuildPhase section */ + 578E558E2F82A3B9001FEF6B /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXHeadersBuildPhase section */ + +/* Begin PBXNativeTarget section */ + 578E55922F82A3B9001FEF6B /* IntegrationTesting */ = { + isa = PBXNativeTarget; + buildConfigurationList = 578E55A52F82A3B9001FEF6B /* Build configuration list for PBXNativeTarget "IntegrationTesting" */; + buildPhases = ( + 578E558E2F82A3B9001FEF6B /* Headers */, + 578E558F2F82A3B9001FEF6B /* Sources */, + 578E55902F82A3B9001FEF6B /* Frameworks */, + 578E55912F82A3B9001FEF6B /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + 578E55952F82A3B9001FEF6B /* IntegrationTesting */, + ); + name = IntegrationTesting; + packageProductDependencies = ( + ); + productName = IntegrationTesting; + productReference = 578E55932F82A3B9001FEF6B /* IntegrationTesting.framework */; + productType = "com.apple.product-type.framework"; + }; + 578E559B2F82A3B9001FEF6B /* IntegrationTestingTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = 578E55A82F82A3B9001FEF6B /* Build configuration list for PBXNativeTarget "IntegrationTestingTests" */; + buildPhases = ( + 578E55982F82A3B9001FEF6B /* Sources */, + 578E55992F82A3B9001FEF6B /* Frameworks */, + 578E559A2F82A3B9001FEF6B /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 578E559F2F82A3B9001FEF6B /* PBXTargetDependency */, + ); + fileSystemSynchronizedGroups = ( + 578E55A02F82A3B9001FEF6B /* IntegrationTestingTests */, + ); + name = IntegrationTestingTests; + packageProductDependencies = ( + 57408EB72F82A947001E2121 /* HuggingFace */, + 57408EBB2F82A947001E2121 /* Tokenizers */, + 57408EDA2F82B8FB001E2121 /* BenchmarkHelpers */, + 57408EDC2F82B8FB001E2121 /* IntegrationTestHelpers */, + 57408EDE2F82B8FB001E2121 /* MLXEmbedders */, + 57408EE02F82B8FB001E2121 /* MLXLLM */, + 57408EE22F82B8FB001E2121 /* MLXLMCommon */, + 57408EE42F82B8FB001E2121 /* MLXVLM */, + 57408EE62F82B922001E2121 /* MLXHuggingFace */, + ); + productName = IntegrationTestingTests; + productReference = 578E559C2F82A3B9001FEF6B /* IntegrationTestingTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 578E558A2F82A3B9001FEF6B /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastSwiftUpdateCheck = 2630; + LastUpgradeCheck = 2630; + TargetAttributes = { + 578E55922F82A3B9001FEF6B = { + CreatedOnToolsVersion = 26.3; + }; + 578E559B2F82A3B9001FEF6B = { + CreatedOnToolsVersion = 26.3; + }; + }; + }; + buildConfigurationList = 578E558D2F82A3B9001FEF6B /* Build configuration list for PBXProject "IntegrationTesting" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 578E55892F82A3B9001FEF6B; + minimizedProjectReferenceProxies = 1; + packageReferences = ( + 57408EAC2F82A8F0001E2121 /* XCRemoteSwiftPackageReference "swift-huggingface" */, + 57408EAF2F82A90E001E2121 /* XCRemoteSwiftPackageReference "swift-transformers" */, + 57408ED92F82B8FB001E2121 /* XCLocalSwiftPackageReference "../../mlx-swift-lm" */, + ); + preferredProjectObjectVersion = 77; + productRefGroup = 578E55942F82A3B9001FEF6B /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 578E55922F82A3B9001FEF6B /* IntegrationTesting */, + 578E559B2F82A3B9001FEF6B /* IntegrationTestingTests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 578E55912F82A3B9001FEF6B /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 578E559A2F82A3B9001FEF6B /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 578E558F2F82A3B9001FEF6B /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 578E55982F82A3B9001FEF6B /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 578E559F2F82A3B9001FEF6B /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 578E55922F82A3B9001FEF6B /* IntegrationTesting */; + targetProxy = 578E559E2F82A3B9001FEF6B /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 578E55A32F82A3B9001FEF6B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 1; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 15.7; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = macosx; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Debug; + }; + 578E55A42F82A3B9001FEF6B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 1; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_USER_SCRIPT_SANDBOXING = YES; + GCC_C_LANGUAGE_STANDARD = gnu17; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + LOCALIZATION_PREFERS_STRING_CATALOGS = YES; + MACOSX_DEPLOYMENT_TARGET = 15.7; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = macosx; + SWIFT_COMPILATION_MODE = wholemodule; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Release; + }; + 578E55A62F82A3B9001FEF6B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUILD_LIBRARY_FOR_DISTRIBUTION = YES; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + ENABLE_MODULE_VERIFIER = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + "@loader_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++"; + MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu17 gnu++20"; + PRODUCT_BUNDLE_IDENTIFIER = mlx.IntegrationTesting; + PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; + SKIP_INSTALL = YES; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_INSTALL_MODULE = YES; + SWIFT_INSTALL_OBJC_HEADER = NO; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + 578E55A72F82A3B9001FEF6B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUILD_LIBRARY_FOR_DISTRIBUTION = YES; + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + ENABLE_MODULE_VERIFIER = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/../Frameworks", + "@loader_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++"; + MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu17 gnu++20"; + PRODUCT_BUNDLE_IDENTIFIER = mlx.IntegrationTesting; + PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; + SKIP_INSTALL = YES; + STRING_CATALOG_GENERATE_SYMBOLS = YES; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_INSTALL_MODULE = YES; + SWIFT_INSTALL_OBJC_HEADER = NO; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; + 578E55A92F82A3B9001FEF6B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = mlx.IntegrationTestingTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = NO; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + }; + name = Debug; + }; + 578E55AA2F82A3B9001FEF6B /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = mlx.IntegrationTestingTests; + PRODUCT_NAME = "$(TARGET_NAME)"; + STRING_CATALOG_GENERATE_SYMBOLS = NO; + SWIFT_APPROACHABLE_CONCURRENCY = YES; + SWIFT_EMIT_LOC_STRINGS = NO; + SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; + SWIFT_VERSION = 5.0; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 578E558D2F82A3B9001FEF6B /* Build configuration list for PBXProject "IntegrationTesting" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 578E55A32F82A3B9001FEF6B /* Debug */, + 578E55A42F82A3B9001FEF6B /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 578E55A52F82A3B9001FEF6B /* Build configuration list for PBXNativeTarget "IntegrationTesting" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 578E55A62F82A3B9001FEF6B /* Debug */, + 578E55A72F82A3B9001FEF6B /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 578E55A82F82A3B9001FEF6B /* Build configuration list for PBXNativeTarget "IntegrationTestingTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 578E55A92F82A3B9001FEF6B /* Debug */, + 578E55AA2F82A3B9001FEF6B /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + +/* Begin XCLocalSwiftPackageReference section */ + 57408ED92F82B8FB001E2121 /* XCLocalSwiftPackageReference "../../mlx-swift-lm" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = "../../mlx-swift-lm"; + }; +/* End XCLocalSwiftPackageReference section */ + +/* Begin XCRemoteSwiftPackageReference section */ + 57408EAC2F82A8F0001E2121 /* XCRemoteSwiftPackageReference "swift-huggingface" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/huggingface/swift-huggingface"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 0.9.0; + }; + }; + 57408EAF2F82A90E001E2121 /* XCRemoteSwiftPackageReference "swift-transformers" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/huggingface/swift-transformers"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 1.3.0; + }; + }; +/* End XCRemoteSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + 57408EB72F82A947001E2121 /* HuggingFace */ = { + isa = XCSwiftPackageProductDependency; + package = 57408EAC2F82A8F0001E2121 /* XCRemoteSwiftPackageReference "swift-huggingface" */; + productName = HuggingFace; + }; + 57408EBB2F82A947001E2121 /* Tokenizers */ = { + isa = XCSwiftPackageProductDependency; + package = 57408EAF2F82A90E001E2121 /* XCRemoteSwiftPackageReference "swift-transformers" */; + productName = Tokenizers; + }; + 57408EDA2F82B8FB001E2121 /* BenchmarkHelpers */ = { + isa = XCSwiftPackageProductDependency; + productName = BenchmarkHelpers; + }; + 57408EDC2F82B8FB001E2121 /* IntegrationTestHelpers */ = { + isa = XCSwiftPackageProductDependency; + productName = IntegrationTestHelpers; + }; + 57408EDE2F82B8FB001E2121 /* MLXEmbedders */ = { + isa = XCSwiftPackageProductDependency; + productName = MLXEmbedders; + }; + 57408EE02F82B8FB001E2121 /* MLXLLM */ = { + isa = XCSwiftPackageProductDependency; + productName = MLXLLM; + }; + 57408EE22F82B8FB001E2121 /* MLXLMCommon */ = { + isa = XCSwiftPackageProductDependency; + productName = MLXLMCommon; + }; + 57408EE42F82B8FB001E2121 /* MLXVLM */ = { + isa = XCSwiftPackageProductDependency; + productName = MLXVLM; + }; + 57408EE62F82B922001E2121 /* MLXHuggingFace */ = { + isa = XCSwiftPackageProductDependency; + package = 57408ED92F82B8FB001E2121 /* XCLocalSwiftPackageReference "../../mlx-swift-lm" */; + productName = MLXHuggingFace; + }; +/* End XCSwiftPackageProductDependency section */ + }; + rootObject = 578E558A2F82A3B9001FEF6B /* Project object */; +} diff --git a/IntegrationTesting/IntegrationTesting.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/IntegrationTesting/IntegrationTesting.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 000000000..919434a62 --- /dev/null +++ b/IntegrationTesting/IntegrationTesting.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/IntegrationTesting/IntegrationTesting.xcodeproj/xcshareddata/xcschemes/IntegrationTesting.xcscheme b/IntegrationTesting/IntegrationTesting.xcodeproj/xcshareddata/xcschemes/IntegrationTesting.xcscheme new file mode 100644 index 000000000..a1bea13f4 --- /dev/null +++ b/IntegrationTesting/IntegrationTesting.xcodeproj/xcshareddata/xcschemes/IntegrationTesting.xcscheme @@ -0,0 +1,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/IntegrationTesting/IntegrationTesting/IntegrationTesting.swift b/IntegrationTesting/IntegrationTesting/IntegrationTesting.swift new file mode 100644 index 000000000..6d108fac7 --- /dev/null +++ b/IntegrationTesting/IntegrationTesting/IntegrationTesting.swift @@ -0,0 +1,5 @@ +// Copyright © 2026 Apple Inc. + +import Foundation + +public struct IntegrationTesting {} diff --git a/Tests/MLXLMIntegrationTests/ToolCallIntegrationTests.swift b/IntegrationTesting/IntegrationTestingTests/ToolCallIntegrationTests.swift similarity index 98% rename from Tests/MLXLMIntegrationTests/ToolCallIntegrationTests.swift rename to IntegrationTesting/IntegrationTestingTests/ToolCallIntegrationTests.swift index 1cafe9d85..42aae454b 100644 --- a/Tests/MLXLMIntegrationTests/ToolCallIntegrationTests.swift +++ b/IntegrationTesting/IntegrationTestingTests/ToolCallIntegrationTests.swift @@ -1,3 +1,5 @@ +// Copyright © 2026 Apple Inc. + import Foundation import HuggingFace import IntegrationTestHelpers diff --git a/Libraries/IntegrationTestHelpers/README.md b/Libraries/IntegrationTestHelpers/README.md index 826d69587..d050c6675 100644 --- a/Libraries/IntegrationTestHelpers/README.md +++ b/Libraries/IntegrationTestHelpers/README.md @@ -4,22 +4,27 @@ ## Running integration tests locally -The `MLXLMIntegrationTests` test target in this repo uses [Swift Hugging Face](https://github.com/huggingface/swift-huggingface) and [Swift Transformers](https://github.com/huggingface/swift-transformers) via the `MLXHuggingFace` macros to provide `Downloader` and `TokenizerLoader` implementations. Models are downloaded from Hugging Face Hub on first run and cached in `~/.cache/huggingface/`. +The `IntegrationTesting/IntegrationTesting.xcodeproj` Xcode project in this repo uses [Swift Hugging Face](https://github.com/huggingface/swift-huggingface) and [Swift Transformers](https://github.com/huggingface/swift-transformers) via the `MLXHuggingFace` macros to provide `Downloader` and `TokenizerLoader` implementations. Models are downloaded from Hugging Face Hub on first run and cached in `~/.cache/huggingface/`. + +To run integration tests, open `IntegrationTesting/IntegrationTesting.xcodeproj` in Xcode and run the test target (`Cmd+U` or via the Test Navigator), or use `xcodebuild`: ```bash # Run all integration tests (requires macOS with Metal) xcodebuild test \ - -scheme mlx-swift-lm-Package \ - -destination 'platform=macOS' \ - -only-testing:MLXLMIntegrationTests + -project IntegrationTesting/IntegrationTesting.xcodeproj \ + -scheme IntegrationTesting \ + -destination 'platform=macOS' -# Run a single model's tests +# Run a single test xcodebuild test \ - -scheme mlx-swift-lm-Package \ + -project IntegrationTesting/IntegrationTesting.xcodeproj \ + -scheme IntegrationTesting \ -destination 'platform=macOS' \ - -only-testing:MLXLMIntegrationTests/ToolCallIntegrationTests/qwen35FormatAutoDetection + -only-testing:IntegrationTestingTests/ToolCallIntegrationTests/qwen35FormatAutoDetection\(\) ``` +These tests do not run in CI. + ## External integration packages Integration tests and benchmarks can also be run from external packages: diff --git a/Package.swift b/Package.swift index 960267421..8dcf54c51 100644 --- a/Package.swift +++ b/Package.swift @@ -38,11 +38,6 @@ let package = Package( dependencies: [ .package(url: "https://github.com/ml-explore/mlx-swift", .upToNextMinor(from: "0.31.3")), .package(url: "https://github.com/swiftlang/swift-syntax.git", from: "600.0.0-latest"), - .package(url: "https://github.com/huggingface/swift-huggingface", from: "0.9.0"), - .package( - url: "https://github.com/huggingface/swift-transformers", - .upToNextMinor(from: "1.3.0") - ), ], targets: [ .target( @@ -151,17 +146,6 @@ let package = Package( ], path: "Libraries/MLXHuggingFace" ), - .testTarget( - name: "MLXLMIntegrationTests", - dependencies: [ - "IntegrationTestHelpers", - "MLXHuggingFace", - "MLXLMCommon", - .product(name: "HuggingFace", package: "swift-huggingface"), - .product(name: "Tokenizers", package: "swift-transformers"), - ], - path: "Tests/MLXLMIntegrationTests" - ), ] )