Skip to content

Commit e13adf5

Browse files
committed
Fix F-Droid catalog format issues
1 parent 465d163 commit e13adf5

9 files changed

Lines changed: 1665 additions & 250 deletions

File tree

Sources/FairExpo/FDroidCatalog.swift

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public struct FDroidIndex : Codable, Equatable {
2727
public var repo: Repo
2828

2929
/// The list of packages (i.e., apps) that make up the catalog
30-
public var packages: Dictionary<String, Package>
30+
public var packages: Dictionary<String, Package>?
3131

3232
/// A map of language code to the translated text. E.g.: `["en-US": "Name", "fr-FR": "Nom"]`
3333
public typealias LocalizedText = Dictionary<String, String>
@@ -51,18 +51,18 @@ public struct FDroidIndex : Codable, Equatable {
5151
///
5252
public typealias LocalizedFileList = Dictionary<String, Array<File>>
5353

54-
public init(repo: Repo, packages: Dictionary<String, Package>) {
54+
public init(repo: Repo, packages: Dictionary<String, Package>?) {
5555
self.repo = repo
5656
self.packages = packages
5757
}
5858

5959
/// A reference to a resource path
6060
public struct File : Codable, Equatable {
61-
public var name: String?
61+
public var name: String
6262
public var sha256: String?
6363
public var size: Int64?
6464

65-
public init(name: String? = nil, sha256: String? = nil, size: Int64? = nil) {
65+
public init(name: String, sha256: String? = nil, size: Int64? = nil) {
6666
self.name = name
6767
self.sha256 = sha256
6868
self.size = size
@@ -106,14 +106,14 @@ public struct FDroidIndex : Codable, Equatable {
106106
public var address: String
107107
public var webBaseUrl: String?
108108
public var description: LocalizedText?
109-
public var mirrors: Array<Mirror>
109+
public var mirrors: Array<Mirror>?
110110
public var timestamp: Int64
111111
public var antiFeatures: Dictionary<String, AntiFeature>?
112112
/// A mapping of the category name of metadata about the category
113113
public var categories: Dictionary<String, Category>?
114114
public var releaseChannels: Dictionary<String, ReleaseChannel>?
115115

116-
public init(name: LocalizedText, icon: LocalizedFile, address: String, webBaseUrl: String? = nil, description: LocalizedText? = nil, mirrors: Array<Mirror>, timestamp: Int64, antiFeatures: Dictionary<String, AntiFeature>? = nil, categories: Dictionary<String, Category>? = nil, releaseChannels: Dictionary<String, ReleaseChannel>? = nil) {
116+
public init(name: LocalizedText, icon: LocalizedFile, address: String, webBaseUrl: String? = nil, description: LocalizedText? = nil, mirrors: Array<Mirror>? = nil, timestamp: Int64, antiFeatures: Dictionary<String, AntiFeature>? = nil, categories: Dictionary<String, Category>? = nil, releaseChannels: Dictionary<String, ReleaseChannel>? = nil) {
117117
self.name = name
118118
self.icon = icon
119119
self.address = address
@@ -129,12 +129,12 @@ public struct FDroidIndex : Codable, Equatable {
129129

130130
public struct Mirror : Codable, Equatable {
131131
public var url: String
132-
public var location: String?
132+
public var countryCode: String?
133133
public var isPrimary: Bool? // undocumented
134134

135-
public init(url: String, location: String? = nil, isPrimary: Bool? = nil) {
135+
public init(url: String, countryCode: String? = nil, isPrimary: Bool? = nil) {
136136
self.url = url
137-
self.location = location
137+
self.countryCode = countryCode
138138
self.isPrimary = isPrimary
139139
}
140140
}

Sources/FairExpo/HubAPI.swift

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,6 @@ public let appfairName = "appfair"
1010

1111
public let appfairRoot = URL(string: "https://appfair.net")!
1212

13-
/// The canonical location of the catalog for the Fair Ground
14-
public let appfairCatalogURLMacOS = URL(string: "fairapps-macos.json", relativeTo: appfairRoot)!
15-
16-
/// The canonical location of the iOS catalog for the Fair Ground
17-
public let appfairCatalogURLIOS = URL(string: "fairapps-ios.json", relativeTo: appfairRoot)!
18-
19-
/// The canonical location of the enhanced cask app metadata
20-
public let appfairCaskAppsURL = URL(string: "appcasks.json", relativeTo: appfairRoot)!
21-
2213
/// A Fair Ground based on an online git service such as GitHub or GitLab.
2314
public struct FairHub : GraphQLEndpointService {
2415
/// The root of the FairGround-compatible service

Sources/fairtool/SourceCommand.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public struct SourceCommand : AsyncParsableCommand {
8787
var packages: Dictionary<String, FDroidIndex.Package> = [:]
8888
packages[token] = package
8989

90-
let repo = FDroidIndex.Repo(name: FDroidIndex.LocalizedText(), icon: FDroidIndex.LocalizedFile(), address: "", mirrors: [], timestamp: 0)
90+
let repo = FDroidIndex.Repo(name: ["en-US": "name"], icon: ["en-US": .init(name: "images/icon/english.svg")], address: "", timestamp: 0)
9191
let catalog = FDroidIndex(repo: repo, packages: packages)
9292

9393
let json = try outputOptions.writeCatalog(catalog)

Tests/FairExpoTests/FairHubTests.swift

Lines changed: 45 additions & 199 deletions
Original file line numberDiff line numberDiff line change
@@ -139,35 +139,6 @@ final class FairHubTests: XCTestCase {
139139
XCTAssertEqual(false, sig.wasSignedByGitHub)
140140
}
141141

142-
// func testFetchSponsorshipListings() async throws {
143-
// if runningFromCI || true { // not permitted with default action token: GraphQLError(message: "Resource not accessible by integration", type: Optional("FORBIDDEN"), path: Optional(["repository", "owner", "sponsorsListing"])
144-
// throw XCTSkip("disabled to reduce API load")
145-
// }
146-
//
147-
// let hub = try Self.hub(skipNoAuth: true)
148-
//
149-
// do {
150-
// let fundingSources = try await Self.hub(skipNoAuth: true).buildFundingSources(owner: appfairName, baseRepository: baseFairgroundRepoName)
151-
// let response = try await hub.request(FairHub.GetSponsorsQuery(owner: appfairName, name: baseFairgroundRepoName)).get().data
152-
// XCTAssertLessThan(20, response.repository.forks.totalCount ?? 0)
153-
//
154-
// XCTAssertEqual("The App Fair!", response.repository.owner.name)
155-
//
156-
//
157-
// do {
158-
// let goal = try XCTUnwrap(fundingSources.first?.goals.first, "missing goal")
159-
// XCTAssertEqual("TOTAL_SPONSORS_COUNT", goal.kind)
160-
// XCTAssertEqual("100 sponsors", goal.title)
161-
// XCTAssertEqual(100, goal.targetValue)
162-
// //XCTAssertEqual(0, goal.percentComplete)
163-
// //XCTAssertEqual("Attaining our sponsorship goal will enable us to set out a firm roadmap for version 1.0 of the project, as well as break ground on implementing support for additional platforms and integrations.", goal.description)
164-
// }
165-
// } catch {
166-
// //print("error: \(error)")
167-
// XCTFail("error: \(error)")
168-
// }
169-
// }
170-
171142
func testCatalogQuery() async throws {
172143
if runningFromCI {
173144
throw XCTSkip("disabled to reduce API load")
@@ -204,187 +175,62 @@ final class FairHubTests: XCTestCase {
204175
}
205176
}
206177

207-
/// Debugging slow connections to GH API
208-
// func XXXtestGHAPISpeed() async throws {
209-
// let token = wip("XXX")
210-
// var req = URLRequest(url: URL(string: "https://api.github.com/graphql")!)
211-
// req.addValue("token \(token)", forHTTPHeaderField: "Authorization")
212-
// req.addValue("application/vnd.github.v3+json", forHTTPHeaderField: "Accept")
213-
// req.httpMethod = "POST"
214-
// req.httpBody = #"{"query":"query { viewer { login } }"} "#.data(using: .utf8)
215-
//
216-
// // dbg("requesting:", req.cURL(pretty: false))
217-
// let t1 = DispatchTime.now().uptimeNanoseconds
218-
// var response: URLResponse?
219-
// let data = try NSURLConnection.sendSynchronousRequest(req, returning: &response)
220-
// //let (data, response) = try await URLSession.shared.data(for: req)
221-
// let t2 = DispatchTime.now().uptimeNanoseconds
222-
// print("response in:", Double(t2 - t1) / 1_000_000_000, data.count, response)
223-
// }
224-
225-
// func testIngestCatalogData() throws {
226-
// var app = AltCatalogAppItem(name: "X", bundleIdentifier: "X", downloadURL: URL(string: "about:blank")!)
227-
// XCTAssertTrue(try app.ingest(json: #"```{ "localizedDescription": "XYZ" }```"#))
228-
// XCTAssertEqual("XYZ", app.localizedDescription)
229-
// XCTAssertTrue(try app.ingest(json: #"```json { "localizedDescription": "ABC" }```"#))
230-
// XCTAssertEqual("ABC", app.localizedDescription)
231-
// XCTAssertTrue(try app.ingest(json: #"```{ "localizedDescription": "XYZ", "tintColor": "AABBCC" }```"#))
232-
// XCTAssertEqual("AABBCC", app.tintColor)
233-
// }
234-
//
235-
// func testBuildAppCasks() async throws {
236-
// if runningFromCI {
237-
// // this quickly exhausts the API limit for the default actions token
238-
// throw XCTSkip("disabled to reduce API load")
239-
// }
240-
//
241-
// let _ = try Self.hub(skipNoAuth: true) // just to throw a skipwhen there is no token
242-
//
243-
// let api = HomebrewAPI(caskAPIEndpoint: HomebrewAPI.defaultEndpoint)
244-
// let maxApps: Int? = 123 // wip(3808) // 123 // _000_000
245-
// let catalog = try await Self.hub(skipNoAuth: true).buildAppCasks(owner: appfairName, catalogName: "Catalog", catalogIdentifier: "net.catalog.id", baseRepository: "appcasks", topicName: "appfair-cask", starrerName: "appfairbot", maxApps: maxApps, mergeCasksURL: api.caskList, caskStatsURL: api.caskStats30, boostFactor: 1000, caskQueryCount: 10, releaseQueryCount: 10, assetQueryCount: 10)
246-
// let names = Set(catalog.apps.map({ $0.name })) // + " " + ($0.version ?? "") }))
247-
// let ids = Set(catalog.apps.map({ $0.bundleIdentifier }))
248-
// dbg("catalog", names.sorted())
249-
//
250-
// XCTAssertEqual(ids.count, catalog.apps.count, "expected to have unique identifiers")
251-
//
252-
// if let maxApps = maxApps {
253-
// XCTAssertEqual(ids.count, maxApps)
254-
// XCTAssertEqual(catalog.apps.count, maxApps)
255-
// }
256-
//
257-
//// XCTAssertTrue(names.contains("CotEditor"))
258-
//// XCTAssertTrue(ids.contains("coteditor"))
259-
//
260-
// XCTAssertGreaterThanOrEqual(names.count, 1)
261-
//
262-
// //dbg(catalog.prettyJSON)
263-
// dbg("created app casks catalog count:", ids.count, "size:", try? catalog.prettyJSON.count.localizedByteCount())
264-
// }
265-
//
266-
// @discardableResult private func checkApp(_ id: String, catalog: AppCatalog, fundingPlatform: AppFundingPlatform? = nil) -> AltCatalogAppItem? {
267-
// guard let app = catalog.apps.first(where: { $0.bundleIdentifier == id }) else {
268-
// XCTFail("no app \(id) found in app list: \(catalog.apps.map(\.bundleIdentifier))")
269-
// return nil
270-
// }
271-
//
272-
// XCTAssertNotNil(app.subtitle, "missing subtitle in app: \(app.bundleIdentifier ?? "")")
273-
// XCTAssertNotNil(app.version, "missing version in app: \(app.bundleIdentifier ?? "")")
274-
// XCTAssertNotNil(app.versionDate, "missing versionDate in app: \(app.bundleIdentifier ?? "")")
275-
// XCTAssertNotNil(app.sha256, "missing sha256 in app: \(app.bundleIdentifier ?? "")")
276-
// XCTAssertNotNil(app.stats?.downloadCount, "missing downloadCount in app: \(app.bundleIdentifier ?? "")")
277-
//
278-
// if let fundingPlatform = fundingPlatform {
279-
// if let link = app.fundingLinks?.first {
280-
// XCTAssertEqual(fundingPlatform, link.platform, "unexpected funding platform")
281-
// } else {
282-
// //XCTAssertNotNil(app.fundingLinks)
283-
// //XCTFail("no funding links")
284-
// }
285-
// }
286-
//
287-
// return app
288-
// }
289-
//
290-
// func testBuildMacOSCatalog() async throws {
291-
// if runningFromCI {
292-
// throw XCTSkip("disabled to reduce API load")
293-
// }
294-
//
295-
// let target = ArtifactTarget(artifactType: "macOS.zip", devices: ["mac"])
296-
// let configuration = try FairHub.ProjectConfiguration()
297-
// let catalog = try await Self.hub(skipNoAuth: true).buildAppCatalog(title: "The App Fair macOS Catalog", identifier: "net.appfair.catalog", owner: appfairName, baseRepository: baseFairgroundRepoName, fairsealCheck: true, artifactTarget: target, configuration: configuration, requestLimit: nil)
298-
// let names = Set(catalog.apps.map({ $0.name })) // + " " + ($0.version ?? "") }))
299-
// dbg("catalog", names.sorted())
300-
// //dbg("### catalog", wip(catalog.prettyJSON))
301-
//
302-
// XCTAssertFalse(names.contains(baseFairgroundRepoName))
303-
// XCTAssertEqual("net.appfair.catalog", catalog.identifier)
304-
//
305-
// checkApp("app.App-Fair", catalog: catalog)
306-
// checkApp("app.Cloud-Cuckoo", catalog: catalog, fundingPlatform: .GITHUB)
307-
// checkApp("app.Tune-Out", catalog: catalog, fundingPlatform: .GITHUB)
308-
//
309-
// dbg("created macOS catalog count:", names.count, "size:", try? catalog.prettyJSON.count.localizedByteCount())
310-
// }
311-
//
312-
// func testBuildIOSAppSourceCatalog() async throws {
313-
// if runningFromCI {
314-
// throw XCTSkip("disabled to reduce API load")
315-
// }
316-
//
317-
// let target = ArtifactTarget(artifactType: "iOS.ipa", devices: ["iphone", "ipad"])
318-
// let configuration = try FairHub.ProjectConfiguration()
319-
// let catalog = try await Self.hub(skipNoAuth: true).buildAppCatalog(title: "The App Fair iOS Catalog", identifier: "net.appfair.catalog", owner: appfairName, baseRepository: baseFairgroundRepoName, fairsealCheck: true, artifactTarget: target, configuration: configuration, requestLimit: nil)
320-
// let names = Set(catalog.apps.map({ $0.name })) // + " " + ($0.version ?? "") }))
321-
// dbg("catalog", names.sorted())
322-
//
323-
// XCTAssertFalse(names.contains(baseFairgroundRepoName))
324-
// XCTAssertEqual("net.appfair.catalog", catalog.identifier)
325-
//
326-
// checkApp("app.Cloud-Cuckoo", catalog: catalog, fundingPlatform: .GITHUB)
327-
// checkApp("app.Tune-Out", catalog: catalog, fundingPlatform: .GITHUB)
328-
//
329-
// dbg("created iOS catalog count:", names.count, "size:", try? catalog.prettyJSON.count.localizedByteCount())
330-
// try print(catalog.prettyJSON)
331-
// }
332-
//
333178
func testParseDroidCatalog() async throws {
334179
// let catalogData = try Data(contentsOf: URL(fileURLWithPath: "f-droid-index-v2.json", relativeTo: baseDir))
335180
let catalogData = try await URLSession.shared.fetch(request: URLRequest(url: FDroidEndpoint.defaultEndpoint)).data
336181
let catalog = try FDroidIndex(fromJSON: catalogData)
337-
XCTAssertLessThan(3_900, catalog.packages.count, "F-Droid catalog should have contained packages")
182+
XCTAssertLessThan(3_900, catalog.packages?.count ?? -1, "F-Droid catalog should have contained packages")
338183

339184
let complete = try FDroidIndex.codableComplete(data: catalogData)
340185
//XCTAssertTrue(complete.difference == nil, "catalog serialized differently")
341186
let _ = complete // FIXME: catalog fidelity
342187
}
343188

344-
// func testFetchCatalog() async throws {
345-
// let url = appfairCatalogURLMacOS
346-
//
347-
// let (data, response) = try await URLSession.shared.fetch(request: URLRequest(url: url, cachePolicy: .reloadIgnoringLocalAndRemoteCacheData, timeoutInterval: 10.0))
348-
// XCTAssertEqual(200, (response as? HTTPURLResponse)?.statusCode)
349-
//
350-
// let catalog = try AppCatalog.parse(jsonData: data)
351-
// XCTAssertEqual("Fair Apps", catalog.name)
352-
// dbg("loaded catalog apps:", catalog.apps.count)
353-
// }
354-
355-
// func testFairHubAllowDenyPatterns() throws {
356-
// func check(success successEmail: String? = nil, failure failureEmail: String? = nil, verification reason: String = "valid", allow: [String] = [], deny: [String] = []) throws {
357-
// var hub = try Self.hub()
358-
// hub.allowFrom = allow
359-
// hub.denyFrom = deny
360-
//
361-
// let mkcommit = { (email: String) in
362-
// FairHub.CommitInfo(sha: "", node_id: "", url: nil, html_url: nil, comments_url: nil, commit: FairHub.CommitInfo.Commit(author: FairHub.User(name: "Some Name", email: email, date: nil), committer: FairHub.User(name: "Some Name", email: email, date: nil), message: "message", url: .tmpdir, comment_count: nil, verification: FairHub.CommitInfo.Verification(verified: true, reason: reason, signature: "", payload: "")))
363-
// }
364-
//
365-
// if let successEmail = successEmail {
366-
// XCTAssertNoThrow(try hub.authorize(commit: mkcommit(successEmail)))
367-
// }
368-
//
369-
// if let failureEmail = failureEmail {
370-
// XCTAssertThrowsError(try hub.authorize(commit: mkcommit(failureEmail)))
371-
// }
372-
// }
373-
//
374-
// try check(success: "abc@QiZ.edu", allow: [".*@QIZ.EDU"])
375-
// try check(failure: "abc@AQiZ.edu", allow: [".*@QIZ.EDU"])
376-
//
377-
// try check(success: "abc@qiz.edu", allow: [".*@QIZ.EDU", ".*@ZIQ.EDU"])
378-
// try check(success: "abc@ziq.edu", allow: [".*@QIZ.EDU", ".*@ZIQ.EDU"])
379-
// try check(failure: "abc@ziz.edu", allow: [".*@QIZ.EDU", ".*@ZIQ.EDU"])
380-
// try check(failure: "abc@qiq.edu", allow: [".*@QIZ.EDU", ".*@ZIQ.EDU"])
381-
//
382-
// try check(failure: "abc@badbadbad.edu", deny: [".*@badbadbad.edu"])
383-
// try check(failure: "abc@badbadbad.edu", allow: ["abc@badbadbad.edu"], deny: [".*@badbadbad.edu"]) // deny trumps allow
384-
// try check(success: "abc@badbad.edu", deny: [".*@badbadbad.edu"])
385-
//
386-
// }
189+
func testParseDroidCatalogs() async throws {
190+
/// Parses the FDroidIndex at the given test resource path, ensuring that it is codable complete
191+
func parseResource(_ name: String, roundtrip: Bool) throws -> FDroidIndex {
192+
let data = try Data(contentsOf: XCTUnwrap(Bundle.module.url(forResource: name, withExtension: nil)))
193+
let decoder = JSONDecoder()
194+
195+
// bug in round-trip codableComplete checking where large numbers are serialized differently depending whether they are an Int or Double
196+
// "size":9223372036854775807
197+
// "size":9.223372036854776e+18
198+
/*
199+
let (index, diff) = try FDroidIndex.codableComplete(data: data)
200+
if roundtrip {
201+
//let json = decoder.decode(JSON.self, from: data)
202+
//print("raw JSON: \(try json.prettyJSON)")
203+
//print("cat JSON: \(try index.prettyJSON)")
204+
XCTAssertNil(diff, "index at \(name) has serialization differences")
205+
}
206+
*/
207+
208+
let index = try decoder.decode(FDroidIndex.self, from: data)
209+
if roundtrip {
210+
var rawJSON = try decoder.decode(JSON.self, from: data)
211+
rawJSON["unknownKey"] = nil // trick with fdroid-index-max-v2.json ("should get ignored")
212+
let rawPretty = try rawJSON.prettyJSON
213+
214+
let rtJSON = try decoder.decode(JSON.self, from: try JSONEncoder().encode(index))
215+
let rtPretty = try rtJSON.prettyJSON
216+
XCTAssertTrue(rawJSON == rtJSON, "mismatch between raw parse: \(rawPretty) and round-tripped: \(rtPretty)")
217+
}
218+
219+
return index
220+
}
221+
222+
let empty = try parseResource("fdroid-index-empty-v2.json", roundtrip: true)
223+
XCTAssertEqual(nil, empty.packages?.count)
224+
225+
let min = try parseResource("fdroid-index-min-v2.json", roundtrip: true)
226+
XCTAssertEqual(1, min.packages?.count)
387227

228+
let mid = try parseResource("fdroid-index-mid-v2.json", roundtrip: true)
229+
XCTAssertEqual(2, mid.packages?.count)
230+
231+
let max = try parseResource("fdroid-index-max-v2.json", roundtrip: true)
232+
XCTAssertEqual(3, max.packages?.count)
233+
}
388234

389235
/// Verifies the default name validation strategy
390236
func testNameValidation() throws {
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"repo": {
3+
"name": {
4+
"en-US": "EmptyV1"
5+
},
6+
"icon": {
7+
"en-US": {
8+
"name": "/icons/empty-v1.png",
9+
"sha256": "824a109b2352138c3699760e1683385d0ed50ce526fc7982f8d65757743374bf",
10+
"size": 32492
11+
}
12+
},
13+
"address": "https://empty-v1.org",
14+
"description": {
15+
"en-US": "This is a repo with empty data."
16+
},
17+
"timestamp": 23
18+
}
19+
}

0 commit comments

Comments
 (0)