Skip to content

Commit 65418e4

Browse files
FL33TW00DFL33TW00D
authored
fix: fix 210 (#211)
* fix: fix 210 * chore: add back comments * chore: format * chore: format * chore: pin swiftformat --------- Co-authored-by: FL33TW00D <FL33TW00D@users.noreply.github.com>
1 parent f68d109 commit 65418e4

12 files changed

Lines changed: 75 additions & 56 deletions

File tree

‎Sources/Hub/HubApi.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ public struct HubApi: Sendable {
1515
var hfToken: String?
1616
var endpoint: String
1717
var useBackgroundSession: Bool
18-
var useOfflineMode: Bool? = nil
18+
var useOfflineMode: Bool?
1919

2020
private let networkMonitor = NetworkMonitor()
2121
public typealias RepoType = Hub.RepoType

‎Sources/TensorUtils/Weights.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ public struct Weights {
55
case notSupported(message: String)
66
case invalidFile
77

8-
public var errorDescription: String? {
8+
var errorDescription: String? {
99
switch self {
1010
case let .notSupported(message):
1111
String(localized: "The weight format '\(message)' is not supported by this application.", comment: "Error when weight format is not supported")

‎Sources/Tokenizers/BPETokenizer.swift‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,14 @@ class BPETokenizer: PreTrainedTokenizerModel {
3939

4040
var vocabCount: Int { tokensToIds.count }
4141

42-
public let bosToken: String?
43-
public let bosTokenId: Int?
44-
public let eosToken: String?
45-
public let eosTokenId: Int?
46-
public let unknownToken: String?
47-
public let unknownTokenId: Int?
48-
49-
public let fuseUnknownTokens: Bool
42+
let bosToken: String?
43+
let bosTokenId: Int?
44+
let eosToken: String?
45+
let eosTokenId: Int?
46+
let unknownToken: String?
47+
let unknownTokenId: Int?
48+
49+
let fuseUnknownTokens: Bool
5050

5151
static func mergesFromConfig(_ config: Config?) -> [[String]]? {
5252
guard let config else { return nil }

‎Sources/Tokenizers/Decoder.swift‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ class WordPieceDecoder: Decoder {
6060
/// https://github.com/huggingface/tokenizers/blob/main/tokenizers/src/decoders/wordpiece.rs#L31
6161
private let re = try! NSRegularExpression(pattern: "\\s(\\.|\\?|\\!|\\,|'\\s|n't|'m|'s|'ve|'re)", options: [])
6262

63-
public required init(config: Config) {
63+
required init(config: Config) {
6464
guard let prefix = config.prefix.string() else { fatalError("Missing `prefix` configuration for WordPieceDecoder.") }
6565
self.prefix = prefix
6666
cleanup = config.cleanup.boolean(or: false)
@@ -85,7 +85,7 @@ class WordPieceDecoder: Decoder {
8585
class DecoderSequence: Decoder {
8686
let decoders: [Decoder]
8787

88-
public required init(config: Config) {
88+
required init(config: Config) {
8989
guard let configs = config.decoders.array() else { fatalError("No decoders in Sequence") }
9090
decoders = configs.compactMap { DecoderFactory.fromConfig(config: $0) }
9191
}
@@ -100,7 +100,7 @@ class DecoderSequence: Decoder {
100100
class ByteLevelDecoder: Decoder {
101101
let addedTokens: Set<String>
102102

103-
public required init(config: Config) {
103+
required init(config: Config) {
104104
addedTokens = []
105105
}
106106

@@ -142,7 +142,7 @@ class ByteLevelDecoder: Decoder {
142142
class ReplaceDecoder: Decoder {
143143
let pattern: StringReplacePattern?
144144

145-
public required init(config: Config) {
145+
required init(config: Config) {
146146
pattern = StringReplacePattern.from(config: config)
147147
}
148148

@@ -153,7 +153,7 @@ class ReplaceDecoder: Decoder {
153153
}
154154

155155
class ByteFallbackDecoder: Decoder {
156-
public required init(config: Config) { }
156+
required init(config: Config) { }
157157

158158
func decode(tokens: [String]) -> [String] {
159159
var newTokens: [String] = []
@@ -186,7 +186,7 @@ class ByteFallbackDecoder: Decoder {
186186
}
187187

188188
class FuseDecoder: Decoder {
189-
public required init(config: Config) { }
189+
required init(config: Config) { }
190190

191191
func decode(tokens: [String]) -> [String] {
192192
[tokens.joined(separator: "")]
@@ -198,7 +198,7 @@ class StripDecoder: Decoder {
198198
let start: Int
199199
let stop: Int
200200

201-
public required init(config: Config) {
201+
required init(config: Config) {
202202
guard let content = config.content.string() else { fatalError("Incorrect StripDecoder configuration: can't parse `content`.") }
203203
guard let start = config.start.integer() else { fatalError("Incorrect StripDecoder configuration: can't parse `start`.") }
204204
guard let stop = config.stop.integer() else { fatalError("Incorrect StripDecoder configuration: can't parse `stop`.") }
@@ -218,7 +218,7 @@ class MetaspaceDecoder: Decoder {
218218
let addPrefixSpace: Bool
219219
let replacement: String
220220

221-
public required init(config: Config) {
221+
required init(config: Config) {
222222
addPrefixSpace = config.addPrefixSpace.boolean(or: false)
223223
replacement = config.replacement.string(or: "_")
224224
}

‎Sources/Tokenizers/Normalizer.swift‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -64,14 +64,14 @@ struct NormalizerFactory {
6464
class NormalizerSequence: Normalizer {
6565
let normalizers: [Normalizer]
6666

67-
public required init(config: Config) {
67+
required init(config: Config) {
6868
guard let configs = config.normalizers.array() else {
6969
fatalError("No normalizers in Sequence")
7070
}
7171
normalizers = configs.compactMap { NormalizerFactory.fromConfig(config: $0) }
7272
}
7373

74-
public func normalize(text: String) -> String {
74+
func normalize(text: String) -> String {
7575
normalizers.reduce(text) { current, normalizer in
7676
normalizer(text: current)
7777
}
@@ -81,48 +81,48 @@ class NormalizerSequence: Normalizer {
8181
class PrependNormalizer: Normalizer {
8282
let prepend: String
8383

84-
public required init(config: Config) {
84+
required init(config: Config) {
8585
prepend = config.prepend.string(or: "")
8686
}
8787

88-
public func normalize(text: String) -> String {
88+
func normalize(text: String) -> String {
8989
prepend + text
9090
}
9191
}
9292

9393
class ReplaceNormalizer: Normalizer {
9494
let pattern: StringReplacePattern?
9595

96-
public required init(config: Config) {
96+
required init(config: Config) {
9797
pattern = StringReplacePattern.from(config: config)
9898
}
9999

100-
public func normalize(text: String) -> String {
100+
func normalize(text: String) -> String {
101101
guard let pattern else { return text }
102102
return pattern.replace(text)
103103
}
104104
}
105105

106106
class LowercaseNormalizer: Normalizer {
107-
public required init(config: Config) { }
107+
required init(config: Config) { }
108108

109-
public func normalize(text: String) -> String {
109+
func normalize(text: String) -> String {
110110
text.lowercased()
111111
}
112112
}
113113

114114
class NFDNormalizer: Normalizer {
115-
public required init(config: Config) { }
115+
required init(config: Config) { }
116116

117-
public func normalize(text: String) -> String {
117+
func normalize(text: String) -> String {
118118
text.decomposedStringWithCanonicalMapping
119119
}
120120
}
121121

122122
class NFCNormalizer: Normalizer {
123-
public required init(config: Config) { }
123+
required init(config: Config) { }
124124

125-
public func normalize(text: String) -> String {
125+
func normalize(text: String) -> String {
126126
text.precomposedStringWithCanonicalMapping
127127
}
128128
}

‎Sources/Tokenizers/PostProcessor.swift‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ class TemplateProcessing: PostProcessor {
4949
let single: [Config]
5050
let pair: [Config]
5151

52-
public required init(config: Config) {
52+
required init(config: Config) {
5353
guard let single = config.single.array() else { fatalError("Missing `single` processor configuration") }
5454
guard let pair = config.pair.array() else { fatalError("Missing `pair` processor configuration") }
5555

@@ -77,7 +77,7 @@ class TemplateProcessing: PostProcessor {
7777
}
7878

7979
class ByteLevelPostProcessor: PostProcessor {
80-
public required init(config: Config) { }
80+
required init(config: Config) { }
8181
func postProcess(tokens: [String], tokensPair: [String]? = nil, addSpecialTokens: Bool = true) -> [String] { tokens }
8282
}
8383

@@ -89,7 +89,7 @@ class RobertaProcessing: PostProcessor {
8989
/// Keep one space character on each side. Depends on `trimOffsets` being `true`.
9090
private let addPrefixSpace: Bool
9191

92-
public required init(config: Config) {
92+
required init(config: Config) {
9393
guard let sep = config.sep.token() else { fatalError("Missing `sep` processor configuration") }
9494
guard let cls = config.cls.token() else { fatalError("Missing `cls` processor configuration") }
9595
self.sep = sep
@@ -146,7 +146,7 @@ class BertProcessing: PostProcessor {
146146
private let sep: (UInt, String)
147147
private let cls: (UInt, String)
148148

149-
public required init(config: Config) {
149+
required init(config: Config) {
150150
guard let sep = config.sep.token() else { fatalError("Missing `sep` processor configuration") }
151151
guard let cls = config.cls.token() else { fatalError("Missing `cls` processor configuration") }
152152
self.sep = sep
@@ -168,7 +168,7 @@ class BertProcessing: PostProcessor {
168168
class SequenceProcessing: PostProcessor {
169169
private let processors: [PostProcessor]
170170

171-
public required init(config: Config) {
171+
required init(config: Config) {
172172
guard let processorConfigs = config.processors.array() else {
173173
fatalError("Missing `processors` configuration")
174174
}

‎Sources/Tokenizers/PreTokenizer.swift‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -308,21 +308,24 @@ public extension String {
308308

309309
var result: [String] = []
310310
var start = startIndex
311+
311312
for match in matches {
312-
// Safely move the prefix end to the start of the current match
313-
let safePrefixEnd = index(startIndex, offsetBy: match.range.lowerBound, limitedBy: endIndex) ?? endIndex
314-
if start < safePrefixEnd {
315-
result.append(String(self[start..<safePrefixEnd]))
313+
// IMPORTANT: convert from NSRange to Range<String.Index>
314+
// https://stackoverflow.com/questions/75543272/convert-a-given-utf8-nsrange-in-a-string-to-a-utf16-nsrange
315+
guard let matchRange = Range(match.range, in: self) else { continue }
316+
317+
// Add text before the match
318+
if start < matchRange.lowerBound {
319+
result.append(String(self[start..<matchRange.lowerBound]))
316320
}
317321

318-
// Safely move the start index to the end of the current match
319-
let matchEndIndex = index(startIndex, offsetBy: match.range.upperBound, limitedBy: endIndex) ?? endIndex
320-
start = matchEndIndex
322+
// Move start to after the match
323+
start = matchRange.upperBound
321324

322325
// Append separator, supporting capture groups
323326
for r in (0..<match.numberOfRanges).reversed() {
324-
let matchRange = match.range(at: r)
325-
if let sepRange = Range(matchRange, in: self) {
327+
let nsRange = match.range(at: r)
328+
if let sepRange = Range(nsRange, in: self) {
326329
result.append(String(self[sepRange]))
327330
break
328331
}

‎Sources/Tokenizers/Tokenizer.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ struct TokenizerModel {
122122
tokenizerConfig.unkToken.content.string() ?? tokenizerConfig.unkToken.string()
123123
}
124124

125-
public static func from(tokenizerConfig: Config, tokenizerData: Config, addedTokens: [String: Int]) throws -> TokenizingModel {
125+
static func from(tokenizerConfig: Config, tokenizerData: Config, addedTokens: [String: Int]) throws -> TokenizingModel {
126126
guard let tokenizerClassName = tokenizerConfig.tokenizerClass.string() else {
127127
throw TokenizerError.missingTokenizerClassInConfig
128128
}

‎Sources/Tokenizers/UnigramTokenizer.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ class UnigramTokenizer: PreTrainedTokenizerModel {
2020
let unknownPiece: SentencePieceToken
2121
var unknownTokenScore: Float { unknownPiece.score }
2222

23-
public let unknownTokenId: Int?
24-
public var unknownToken: String? { unknownPiece.token }
23+
let unknownTokenId: Int?
24+
var unknownToken: String? { unknownPiece.token }
2525

2626
let minScore: Float
2727
let tokensToIds: [NSString: Int]

‎Tests/HubTests/HubApiTests.swift‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,7 @@ class SnapshotDownloadTests: XCTestCase {
375375
])
376376
)
377377

378-
let _ = try await hubApi.snapshot(from: repo, matching: "*.json") { progress in
378+
_ = try await hubApi.snapshot(from: repo, matching: "*.json") { progress in
379379
print("Total Progress: \(progress.fractionCompleted)")
380380
print("Files Completed: \(progress.completedUnitCount) of \(progress.totalUnitCount)")
381381
lastProgress = progress
@@ -419,7 +419,7 @@ class SnapshotDownloadTests: XCTestCase {
419419

420420
let originalMetadata = try String(contentsOf: metadataPath, encoding: .utf8)
421421

422-
let _ = try await hubApi.snapshot(from: repo, matching: "tokenizer.json") { progress in
422+
_ = try await hubApi.snapshot(from: repo, matching: "tokenizer.json") { progress in
423423
print("Total Progress: \(progress.fractionCompleted)")
424424
print("Files Completed: \(progress.completedUnitCount) of \(progress.totalUnitCount)")
425425
lastProgress = progress
@@ -483,7 +483,7 @@ class SnapshotDownloadTests: XCTestCase {
483483
print("Testing corrupted file.")
484484
try "a".write(to: metadataDestination.appendingPathComponent("config.json.metadata"), atomically: true, encoding: .utf8)
485485

486-
let _ = try await hubApi.snapshot(from: repo, matching: "*.json") { progress in
486+
_ = try await hubApi.snapshot(from: repo, matching: "*.json") { progress in
487487
print("Total Progress: \(progress.fractionCompleted)")
488488
print("Files Completed: \(progress.completedUnitCount) of \(progress.totalUnitCount)")
489489
lastProgress = progress
@@ -499,7 +499,7 @@ class SnapshotDownloadTests: XCTestCase {
499499
print("Testing corrupted timestamp.")
500500
try "a\nb\nc\n".write(to: metadataDestination.appendingPathComponent("config.json.metadata"), atomically: true, encoding: .utf8)
501501

502-
let _ = try await hubApi.snapshot(from: repo, matching: "*.json") { progress in
502+
_ = try await hubApi.snapshot(from: repo, matching: "*.json") { progress in
503503
print("Total Progress: \(progress.fractionCompleted)")
504504
print("Files Completed: \(progress.completedUnitCount) of \(progress.totalUnitCount)")
505505
lastProgress = progress
@@ -552,7 +552,7 @@ class SnapshotDownloadTests: XCTestCase {
552552
let metadataFile = metadataDestination.appendingPathComponent("llama-2-7b-chat.mlpackage/Data/com.apple.CoreML/model.mlmodel.metadata")
553553
try corruptedMetadataString.write(to: metadataFile, atomically: true, encoding: .utf8)
554554

555-
let _ = try await hubApi.snapshot(from: repo, matching: "*.mlmodel") { progress in
555+
_ = try await hubApi.snapshot(from: repo, matching: "*.mlmodel") { progress in
556556
print("Total Progress: \(progress.fractionCompleted)")
557557
print("Files Completed: \(progress.completedUnitCount) of \(progress.totalUnitCount)")
558558
lastProgress = progress
@@ -702,7 +702,7 @@ class SnapshotDownloadTests: XCTestCase {
702702
let metadataFile = metadataDestination.appendingPathComponent("x.bin.metadata")
703703
try FileManager.default.removeItem(atPath: metadataFile.path)
704704

705-
let _ = try await hubApi.snapshot(from: lfsRepo, matching: "x.bin") { progress in
705+
_ = try await hubApi.snapshot(from: lfsRepo, matching: "x.bin") { progress in
706706
print("Total Progress: \(progress.fractionCompleted)")
707707
print("Files Completed: \(progress.completedUnitCount) of \(progress.totalUnitCount)")
708708
lastProgress = progress
@@ -753,7 +753,7 @@ class SnapshotDownloadTests: XCTestCase {
753753
let metadataFile = metadataDestination.appendingPathComponent("x.bin.metadata")
754754
try "a".write(to: metadataFile, atomically: true, encoding: .utf8)
755755

756-
let _ = try await hubApi.snapshot(from: lfsRepo, matching: "x.bin") { progress in
756+
_ = try await hubApi.snapshot(from: lfsRepo, matching: "x.bin") { progress in
757757
print("Total Progress: \(progress.fractionCompleted)")
758758
print("Files Completed: \(progress.completedUnitCount) of \(progress.totalUnitCount)")
759759
lastProgress = progress
@@ -804,7 +804,7 @@ class SnapshotDownloadTests: XCTestCase {
804804
let metadataFile = metadataDestination.appendingPathComponent("config.json.metadata")
805805
try FileManager.default.removeItem(atPath: metadataFile.path)
806806

807-
let _ = try await hubApi.snapshot(from: repo, matching: "config.json") { progress in
807+
_ = try await hubApi.snapshot(from: repo, matching: "config.json") { progress in
808808
print("Total Progress: \(progress.fractionCompleted)")
809809
print("Files Completed: \(progress.completedUnitCount) of \(progress.totalUnitCount)")
810810
lastProgress = progress

0 commit comments

Comments
 (0)