From 16957fd67513c1a5c9f2f715a67f60bf5291c861 Mon Sep 17 00:00:00 2001 From: FL33TW00D Date: Mon, 11 Aug 2025 16:21:14 +0100 Subject: [PATCH 1/6] chore: start Xetification --- Sources/Hub/HubApi.swift | 56 +++++++++++++++++++++++++++++--- Tests/HubTests/HubApiTests.swift | 12 +++++++ 2 files changed, 63 insertions(+), 5 deletions(-) diff --git a/Sources/Hub/HubApi.swift b/Sources/Hub/HubApi.swift index fb54ce1e..e64d7d8c 100644 --- a/Sources/Hub/HubApi.swift +++ b/Sources/Hub/HubApi.swift @@ -10,6 +10,23 @@ import Foundation import Network import os +enum HFHttpHeaders { + static let location = "Location" + static let etag = "Etag" + static let contentLength = "Content-Length" + static let repoCommit = "X-Repo-Commit" + static let linkedEtag = "X-Linked-Etag" + static let linkedSize = "X-Linked-Size" + static let xetHash = "X-Xet-Hash" + static let xetRefreshRoute = "X-Xet-Refresh-Route" + static let linkXetAuthKey = "xet-auth" +} + +public struct XetFileData { + let fileHash: String + let refreshRoute: String +} + public struct HubApi: Sendable { var downloadBase: URL var hfToken: String? @@ -24,7 +41,7 @@ public struct HubApi: Sendable { public init( downloadBase: URL? = nil, hfToken: String? = nil, - endpoint: String = "https://huggingface.co", + endpoint: String? = nil, useBackgroundSession: Bool = false, useOfflineMode: Bool? = nil ) { @@ -35,7 +52,7 @@ public struct HubApi: Sendable { let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first! self.downloadBase = documents.appending(component: "huggingface") } - self.endpoint = endpoint + self.endpoint = endpoint ?? Self.hfEndpointfromEnv() self.useBackgroundSession = useBackgroundSession self.useOfflineMode = useOfflineMode NetworkMonitor.shared.startMonitoring() @@ -50,6 +67,10 @@ public struct HubApi: Sendable { } private extension HubApi { + static func hfEndpointfromEnv() -> String { + ProcessInfo.processInfo.environment["HF_ENDPOINT"] ?? "https://huggingface.co" + } + static func hfTokenFromEnv() -> String? { let possibleTokens = [ { ProcessInfo.processInfo.environment["HF_TOKEN"] }, @@ -573,6 +594,9 @@ public extension HubApi { /// Size of the file. In case of an LFS file, contains the size of the actual LFS file, not the pointer. public let size: Int? + + /// Xet file data, if available. Contains the file hash and the refresh route. + public let xetFileData: XetFileData? } /// Metadata about a file in the local directory related to a download process @@ -600,13 +624,35 @@ public extension HubApi { let (_, response) = try await httpHead(for: url) let location = response.statusCode == 302 ? response.value(forHTTPHeaderField: "Location") : response.url?.absoluteString + print("getFileMetadata: \(url) -> \(response)") + return FileMetadata( - commitHash: response.value(forHTTPHeaderField: "X-Repo-Commit"), + commitHash: response.value(forHTTPHeaderField: HFHttpHeaders.repoCommit), etag: normalizeEtag( - (response.value(forHTTPHeaderField: "X-Linked-Etag")) ?? (response.value(forHTTPHeaderField: "Etag")) + (response.value(forHTTPHeaderField: HFHttpHeaders.linkedEtag)) ?? (response.value(forHTTPHeaderField: HFHttpHeaders.etag)) ), location: location ?? url.absoluteString, - size: Int(response.value(forHTTPHeaderField: "X-Linked-Size") ?? response.value(forHTTPHeaderField: "Content-Length") ?? "") + size: Int(response.value(forHTTPHeaderField: HFHttpHeaders.linkedSize) ?? response.value(forHTTPHeaderField: HFHttpHeaders.contentLength) ?? ""), + xetFileData: parseXetFileDataFromResponse(response: response, endpoint: endpoint) + ) + } + + //https://github.com/huggingface/huggingface_hub/blob/b698915d6b582c72806ac3e91c43bfd8dde35856/src/huggingface_hub/utils/_xet.py#L29 + private func parseXetFileDataFromResponse( + response: HTTPURLResponse?, + endpoint: String? = nil + ) -> XetFileData? { + guard let response = response else { + return nil + } + + guard let fileHash = response.allHeaderFields[HFHttpHeaders.xetHash] as? String else { + return nil + } + + return XetFileData( + fileHash: fileHash, + refreshRoute: "" ) } diff --git a/Tests/HubTests/HubApiTests.swift b/Tests/HubTests/HubApiTests.swift index 40e75453..2140883b 100644 --- a/Tests/HubTests/HubApiTests.swift +++ b/Tests/HubTests/HubApiTests.swift @@ -103,6 +103,18 @@ class HubApiTests: XCTestCase { } } + func testGetXetMetadata() async throws { + do { + let url = URL(string: "https://huggingface.co/Qwen/Qwen3-0.6B/resolve/main/tokenizer.json") + let metadata = try await Hub.getFileMetadata(fileURL: url!) + + print("Metadata: \(metadata)") + + } catch { + XCTFail("\(error)") + } + } + func testGetFileMetadataBlobPath() async throws { do { let url = URL(string: "https://huggingface.co/enterprise-explorers/Llama-2-7b-chat-coreml/resolve/main/config.json") From 646da2def79ec4f3fba202c6a85605142cc6d0ba Mon Sep 17 00:00:00 2001 From: FL33TW00D Date: Mon, 11 Aug 2025 16:50:11 +0100 Subject: [PATCH 2/6] chore: need to fuzz Xet headers --- Sources/Hub/HubApi.swift | 47 +++++++++++++++++++++++++++++--- Tests/HubTests/HubApiTests.swift | 3 -- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/Sources/Hub/HubApi.swift b/Sources/Hub/HubApi.swift index e64d7d8c..073712b3 100644 --- a/Sources/Hub/HubApi.swift +++ b/Sources/Hub/HubApi.swift @@ -10,6 +10,10 @@ import Foundation import Network import os +// https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2 +// `requests` in Python leaves headers as their original casing, +// where as Swift strictly adheres to RFC 7540 and can force lower case. +// This is relevant for Xet enum HFHttpHeaders { static let location = "Location" static let etag = "Etag" @@ -17,7 +21,7 @@ enum HFHttpHeaders { static let repoCommit = "X-Repo-Commit" static let linkedEtag = "X-Linked-Etag" static let linkedSize = "X-Linked-Size" - static let xetHash = "X-Xet-Hash" + static let xetHash = "x-xet-hash" static let xetRefreshRoute = "X-Xet-Refresh-Route" static let linkXetAuthKey = "xet-auth" } @@ -27,6 +31,30 @@ public struct XetFileData { let refreshRoute: String } +extension HTTPURLResponse { + func getLinkURL(for rel: String) -> String? { + guard let linkHeader = allHeaderFields["Link"] as? String else { + return nil + } + + + for link in linkHeader.split(separator: ",") { + let trimmed = link.trimmingCharacters(in: .whitespaces) + + if trimmed.contains("rel=\"\(rel)\"") || trimmed.contains("rel=\(rel)") { + if let start = trimmed.firstIndex(of: "<"), + let end = trimmed.firstIndex(of: ">"), + start < end { + let startIndex = trimmed.index(after: start) + return String(trimmed[startIndex.. \(response)") - return FileMetadata( commitHash: response.value(forHTTPHeaderField: HFHttpHeaders.repoCommit), etag: normalizeEtag( @@ -649,10 +675,23 @@ public extension HubApi { guard let fileHash = response.allHeaderFields[HFHttpHeaders.xetHash] as? String else { return nil } + + guard let refreshRoute = response.getLinkURL(for: HFHttpHeaders.linkXetAuthKey) + ?? response.allHeaderFields[HFHttpHeaders.xetRefreshRoute] as? String else { + return nil + } + + let finalEndpoint = endpoint ?? "https://huggingface.co" + let finalRefreshRoute = refreshRoute.hasPrefix("https://huggingface.co") + ? refreshRoute.replacingOccurrences( + of: "https://huggingface.co".trimmingCharacters(in: CharacterSet(charactersIn: "/")), + with: finalEndpoint.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + ) + : refreshRoute return XetFileData( fileHash: fileHash, - refreshRoute: "" + refreshRoute: finalRefreshRoute ) } diff --git a/Tests/HubTests/HubApiTests.swift b/Tests/HubTests/HubApiTests.swift index 2140883b..3846255a 100644 --- a/Tests/HubTests/HubApiTests.swift +++ b/Tests/HubTests/HubApiTests.swift @@ -107,9 +107,6 @@ class HubApiTests: XCTestCase { do { let url = URL(string: "https://huggingface.co/Qwen/Qwen3-0.6B/resolve/main/tokenizer.json") let metadata = try await Hub.getFileMetadata(fileURL: url!) - - print("Metadata: \(metadata)") - } catch { XCTFail("\(error)") } From ef876722b862a3bf28398efc6e814bbf8cd77324 Mon Sep 17 00:00:00 2001 From: FL33TW00D Date: Mon, 11 Aug 2025 16:51:01 +0100 Subject: [PATCH 3/6] chore: comment --- Sources/Hub/HubApi.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/Hub/HubApi.swift b/Sources/Hub/HubApi.swift index 073712b3..6eb9add8 100644 --- a/Sources/Hub/HubApi.swift +++ b/Sources/Hub/HubApi.swift @@ -31,6 +31,7 @@ public struct XetFileData { let refreshRoute: String } +// `requests` automatically parses Link headers into `response.links`. Swift does not. extension HTTPURLResponse { func getLinkURL(for rel: String) -> String? { guard let linkHeader = allHeaderFields["Link"] as? String else { From 51789e0e045601b960abf700c12a904079710ab2 Mon Sep 17 00:00:00 2001 From: FL33TW00D Date: Mon, 11 Aug 2025 16:53:21 +0100 Subject: [PATCH 4/6] chore: fmt --- Sources/Hub/HubApi.swift | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/Sources/Hub/HubApi.swift b/Sources/Hub/HubApi.swift index 6eb9add8..ac1e1ebc 100644 --- a/Sources/Hub/HubApi.swift +++ b/Sources/Hub/HubApi.swift @@ -10,10 +10,10 @@ import Foundation import Network import os -// https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2 -// `requests` in Python leaves headers as their original casing, -// where as Swift strictly adheres to RFC 7540 and can force lower case. -// This is relevant for Xet +/// https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2 +/// `requests` in Python leaves headers as their original casing, +/// where as Swift strictly adheres to RFC 7540 and can force lower case. +/// This is relevant for Xet enum HFHttpHeaders { static let location = "Location" static let etag = "Etag" @@ -31,27 +31,27 @@ public struct XetFileData { let refreshRoute: String } -// `requests` automatically parses Link headers into `response.links`. Swift does not. +/// `requests` automatically parses Link headers into `response.links`. Swift does not. extension HTTPURLResponse { func getLinkURL(for rel: String) -> String? { guard let linkHeader = allHeaderFields["Link"] as? String else { return nil } - for link in linkHeader.split(separator: ",") { let trimmed = link.trimmingCharacters(in: .whitespaces) - + if trimmed.contains("rel=\"\(rel)\"") || trimmed.contains("rel=\(rel)") { if let start = trimmed.firstIndex(of: "<"), let end = trimmed.firstIndex(of: ">"), - start < end { + start < end + { let startIndex = trimmed.index(after: start) return String(trimmed[startIndex.. XetFileData? { - guard let response = response else { + guard let response else { return nil } - + guard let fileHash = response.allHeaderFields[HFHttpHeaders.xetHash] as? String else { return nil } - - guard let refreshRoute = response.getLinkURL(for: HFHttpHeaders.linkXetAuthKey) - ?? response.allHeaderFields[HFHttpHeaders.xetRefreshRoute] as? String else { + + guard let refreshRoute = response.getLinkURL(for: HFHttpHeaders.linkXetAuthKey) + ?? response.allHeaderFields[HFHttpHeaders.xetRefreshRoute] as? String + else { return nil } - - let finalEndpoint = endpoint ?? "https://huggingface.co" - let finalRefreshRoute = refreshRoute.hasPrefix("https://huggingface.co") + + let finalEndpoint = endpoint ?? "https://huggingface.co" + let finalRefreshRoute = refreshRoute.hasPrefix("https://huggingface.co") ? refreshRoute.replacingOccurrences( of: "https://huggingface.co".trimmingCharacters(in: CharacterSet(charactersIn: "/")), with: finalEndpoint.trimmingCharacters(in: CharacterSet(charactersIn: "/")) From ff9bb295b0686c2daaf6efb5c5b04646aafff868 Mon Sep 17 00:00:00 2001 From: FL33TW00D Date: Wed, 20 Aug 2025 08:50:15 +0100 Subject: [PATCH 5/6] chore: move to repo I control --- Tests/HubTests/HubApiTests.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Tests/HubTests/HubApiTests.swift b/Tests/HubTests/HubApiTests.swift index 3846255a..cbb70e3a 100644 --- a/Tests/HubTests/HubApiTests.swift +++ b/Tests/HubTests/HubApiTests.swift @@ -105,8 +105,11 @@ class HubApiTests: XCTestCase { func testGetXetMetadata() async throws { do { - let url = URL(string: "https://huggingface.co/Qwen/Qwen3-0.6B/resolve/main/tokenizer.json") + let url = URL(string: "https://huggingface.co/FL33TW00D-HF/xet-test/resolve/main/tokenizer.json") let metadata = try await Hub.getFileMetadata(fileURL: url!) + + XCTAssertNotNil(metadata.xetFileData) + XCTAssertEqual(metadata.xetFileData?.fileHash, "6aec39639a0a2d1ca966356b8c2b8426a484f80ff80731f44fa8482040713bdf") } catch { XCTFail("\(error)") } From 355f381b809e99329a295744d2cd267b6fb646ae Mon Sep 17 00:00:00 2001 From: FL33TW00D Date: Wed, 20 Aug 2025 10:03:26 +0100 Subject: [PATCH 6/6] chore: simplify --- Sources/Hub/HubApi.swift | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Sources/Hub/HubApi.swift b/Sources/Hub/HubApi.swift index ac1e1ebc..86679b04 100644 --- a/Sources/Hub/HubApi.swift +++ b/Sources/Hub/HubApi.swift @@ -31,7 +31,8 @@ public struct XetFileData { let refreshRoute: String } -/// `requests` automatically parses Link headers into `response.links`. Swift does not. +/// `requests` automatically parses Link headers into `response.links`, +/// we implement similar functionality here. extension HTTPURLResponse { func getLinkURL(for rel: String) -> String? { guard let linkHeader = allHeaderFields["Link"] as? String else { @@ -677,23 +678,26 @@ public extension HubApi { return nil } - guard let refreshRoute = response.getLinkURL(for: HFHttpHeaders.linkXetAuthKey) + guard var refreshRoute = response.getLinkURL(for: HFHttpHeaders.linkXetAuthKey) ?? response.allHeaderFields[HFHttpHeaders.xetRefreshRoute] as? String else { return nil } - let finalEndpoint = endpoint ?? "https://huggingface.co" - let finalRefreshRoute = refreshRoute.hasPrefix("https://huggingface.co") - ? refreshRoute.replacingOccurrences( - of: "https://huggingface.co".trimmingCharacters(in: CharacterSet(charactersIn: "/")), - with: finalEndpoint.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + let endpoint = endpoint ?? "https://huggingface.co" + + let defaultEndpoint = "https://huggingface.co" + + if refreshRoute.hasPrefix(defaultEndpoint) { + refreshRoute = refreshRoute.replacingOccurrences( + of: defaultEndpoint.trimmingCharacters(in: CharacterSet(charactersIn: "/")), + with: endpoint.trimmingCharacters(in: CharacterSet(charactersIn: "/")) ) - : refreshRoute + } return XetFileData( fileHash: fileHash, - refreshRoute: finalRefreshRoute + refreshRoute: refreshRoute ) }