Skip to content

Commit 042569a

Browse files
jaclyncjaclync
authored
[Local Catalog] Incremental sync: remote sync (#16102)
2 parents 538663a + 1dd2103 commit 042569a

File tree

7 files changed

+455
-162
lines changed

7 files changed

+455
-162
lines changed

‎Modules/Sources/Networking/Remote/POSCatalogSyncRemote.swift‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,14 @@ public class POSCatalogSyncRemote: Remote, POSCatalogSyncRemoteProtocol {
6767
ParameterKey.fields: POSProduct.requestFields.joined(separator: ",")
6868
]
6969

70-
let request = JetpackRequest(wooApiVersion: .mark3, method: .get, siteID: siteID, path: path, parameters: parameters)
70+
let request = JetpackRequest(
71+
wooApiVersion: .mark3,
72+
method: .get,
73+
siteID: siteID,
74+
path: path,
75+
parameters: parameters,
76+
availableAsRESTRequest: true
77+
)
7178
let mapper = ListMapper<POSProduct>(siteID: siteID)
7279
let (products, responseHeaders) = try await enqueueWithResponseHeaders(request, mapper: mapper)
7380

@@ -92,7 +99,14 @@ public class POSCatalogSyncRemote: Remote, POSCatalogSyncRemoteProtocol {
9299
ParameterKey.fields: POSProductVariation.requestFields.joined(separator: ",")
93100
]
94101

95-
let request = JetpackRequest(wooApiVersion: .wcAnalytics, method: .get, siteID: siteID, path: path, parameters: parameters)
102+
let request = JetpackRequest(
103+
wooApiVersion: .wcAnalytics,
104+
method: .get,
105+
siteID: siteID,
106+
path: path,
107+
parameters: parameters,
108+
availableAsRESTRequest: true
109+
)
96110
let mapper = ListMapper<POSProductVariation>(siteID: siteID)
97111
let (variations, responseHeaders) = try await enqueueWithResponseHeaders(request, mapper: mapper)
98112

@@ -117,7 +131,14 @@ public class POSCatalogSyncRemote: Remote, POSCatalogSyncRemoteProtocol {
117131
ParameterKey.fields: POSProduct.requestFields.joined(separator: ",")
118132
]
119133

120-
let request = JetpackRequest(wooApiVersion: .mark3, method: .get, siteID: siteID, path: path, parameters: parameters)
134+
let request = JetpackRequest(
135+
wooApiVersion: .mark3,
136+
method: .get,
137+
siteID: siteID,
138+
path: path,
139+
parameters: parameters,
140+
availableAsRESTRequest: true
141+
)
121142
let mapper = ListMapper<POSProduct>(siteID: siteID)
122143
let (products, responseHeaders) = try await enqueueWithResponseHeaders(request, mapper: mapper)
123144

@@ -140,7 +161,14 @@ public class POSCatalogSyncRemote: Remote, POSCatalogSyncRemoteProtocol {
140161
ParameterKey.fields: POSProductVariation.requestFields.joined(separator: ",")
141162
]
142163

143-
let request = JetpackRequest(wooApiVersion: .wcAnalytics, method: .get, siteID: siteID, path: path, parameters: parameters)
164+
let request = JetpackRequest(
165+
wooApiVersion: .wcAnalytics,
166+
method: .get,
167+
siteID: siteID,
168+
path: path,
169+
parameters: parameters,
170+
availableAsRESTRequest: true
171+
)
144172
let mapper = ListMapper<POSProductVariation>(siteID: siteID)
145173
let (variations, responseHeaders) = try await enqueueWithResponseHeaders(request, mapper: mapper)
146174

‎Modules/Sources/Yosemite/Tools/POS/BatchedRequestLoader.swift‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import Foundation
2+
3+
/// Generic utility for loading paginated data with batch processing.
4+
final class BatchedRequestLoader {
5+
private let batchSize: Int
6+
7+
init(batchSize: Int) {
8+
self.batchSize = batchSize
9+
}
10+
11+
/// Loads all items using a paginated request function.
12+
/// - Parameters:
13+
/// - makeRequest: Function that takes a page number and returns PagedItems<T>.
14+
/// - Returns: Array of all loaded items.
15+
func loadAll<T>(makeRequest: @escaping (Int) async throws -> PagedItems<T>) async throws -> [T] {
16+
var allItems: [T] = []
17+
var currentPage = 1
18+
var hasMorePages = true
19+
20+
while hasMorePages {
21+
let pagesToFetch = Array(currentPage..<(currentPage + batchSize))
22+
23+
let batchResults = try await withThrowingTaskGroup(of: PageResult<T>.self) { group in
24+
for pageNumber in pagesToFetch {
25+
group.addTask {
26+
let result = try await makeRequest(pageNumber)
27+
return PageResult(pageNumber: pageNumber, items: result)
28+
}
29+
}
30+
31+
var results: [PageResult<T>] = []
32+
for try await result in group {
33+
results.append(result)
34+
}
35+
return results.sorted(by: { $0.pageNumber < $1.pageNumber })
36+
}
37+
38+
// Processes results in order and checks if there are more pages.
39+
let newItems = batchResults.flatMap { $0.items.items }
40+
allItems.append(contentsOf: newItems)
41+
42+
let highestPageResult = batchResults.last?.items
43+
hasMorePages = (highestPageResult?.hasMorePages ?? false) && !newItems.isEmpty
44+
currentPage += batchSize
45+
}
46+
47+
return allItems
48+
}
49+
}
50+
51+
private struct PageResult<T> {
52+
let pageNumber: Int
53+
let items: PagedItems<T>
54+
}

‎Modules/Sources/Yosemite/Tools/POS/POSCatalogFullSyncService.swift‎

Lines changed: 12 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ public final class POSCatalogFullSyncService: POSCatalogFullSyncServiceProtocol
2828
private let syncRemote: POSCatalogSyncRemoteProtocol
2929
private let batchSize: Int
3030
private let persistenceService: POSCatalogPersistenceServiceProtocol
31+
private let batchedLoader: BatchedRequestLoader
3132

3233
public convenience init?(credentials: Credentials?, batchSize: Int = 2, grdbManager: GRDBManagerProtocol) {
3334
guard let credentials else {
@@ -44,6 +45,7 @@ public final class POSCatalogFullSyncService: POSCatalogFullSyncServiceProtocol
4445
self.syncRemote = syncRemote
4546
self.batchSize = batchSize
4647
self.persistenceService = persistenceService
48+
self.batchedLoader = BatchedRequestLoader(batchSize: batchSize)
4749
}
4850

4951
// MARK: - Protocol Conformance
@@ -73,95 +75,19 @@ public final class POSCatalogFullSyncService: POSCatalogFullSyncServiceProtocol
7375
private extension POSCatalogFullSyncService {
7476
func loadCatalog(for siteID: Int64, syncRemote: POSCatalogSyncRemoteProtocol) async throws -> POSCatalog {
7577
// Loads products and variations in batches in parallel.
76-
async let productsTask = loadAllProducts(for: siteID, syncRemote: syncRemote)
77-
async let variationsTask = loadAllProductVariations(for: siteID, syncRemote: syncRemote)
78-
79-
let (products, variations) = try await (productsTask, variationsTask)
80-
return POSCatalog(products: products, variations: variations)
81-
}
82-
83-
func loadAllProducts(for siteID: Int64, syncRemote: POSCatalogSyncRemoteProtocol) async throws -> [POSProduct] {
84-
DDLogInfo("🔄 Starting products sync for site ID: \(siteID)")
85-
86-
var allProducts: [POSProduct] = []
87-
var currentPage = 1
88-
var hasMorePages = true
89-
90-
while hasMorePages {
91-
let pagesToFetch = Array(currentPage..<(currentPage + batchSize))
92-
93-
let batchResults = try await withThrowingTaskGroup(of: PageResult<POSProduct>.self) { group in
94-
for pageNumber in pagesToFetch {
95-
group.addTask {
96-
let result = try await syncRemote.loadProducts(siteID: siteID, pageNumber: pageNumber)
97-
return PageResult(pageNumber: pageNumber, items: result)
98-
}
99-
}
100-
101-
var results: [PageResult<POSProduct>] = []
102-
for try await result in group {
103-
results.append(result)
104-
}
105-
return results.sorted(by: { $0.pageNumber < $1.pageNumber })
78+
async let productsTask = batchedLoader.loadAll(
79+
makeRequest: { pageNumber in
80+
try await syncRemote.loadProducts(siteID: siteID, pageNumber: pageNumber)
10681
}
107-
108-
// Processes results in order and checks if there are more pages.
109-
let newProducts = batchResults.flatMap { $0.items.items }
110-
allProducts.append(contentsOf: newProducts)
111-
112-
let highestPageResult = batchResults.last?.items
113-
hasMorePages = (highestPageResult?.hasMorePages ?? false) && !newProducts.isEmpty
114-
currentPage += batchSize
115-
116-
DDLogInfo("📥 Loaded batch: \(batchResults.count) pages, total products: \(allProducts.count), hasMorePages: \(hasMorePages)")
117-
}
118-
119-
DDLogInfo("✅ Products sync complete: \(allProducts.count) products loaded")
120-
return allProducts
121-
}
122-
123-
func loadAllProductVariations(for siteID: Int64, syncRemote: POSCatalogSyncRemoteProtocol) async throws -> [POSProductVariation] {
124-
DDLogInfo("🔄 Starting variations sync for site ID: \(siteID)")
125-
126-
var allVariations: [POSProductVariation] = []
127-
var currentPage = 1
128-
var hasMorePages = true
129-
130-
while hasMorePages {
131-
let pagesToFetch = Array(currentPage..<(currentPage + batchSize))
132-
133-
let batchResults = try await withThrowingTaskGroup(of: PageResult<POSProductVariation>.self) { group in
134-
for pageNumber in pagesToFetch {
135-
group.addTask {
136-
let result = try await syncRemote.loadProductVariations(siteID: siteID, pageNumber: pageNumber)
137-
return PageResult(pageNumber: pageNumber, items: result)
138-
}
139-
}
140-
141-
var results: [PageResult<POSProductVariation>] = []
142-
for try await result in group {
143-
results.append(result)
144-
}
145-
return results.sorted(by: { $0.pageNumber < $1.pageNumber })
82+
)
83+
async let variationsTask = batchedLoader.loadAll(
84+
makeRequest: { pageNumber in
85+
try await syncRemote.loadProductVariations(siteID: siteID, pageNumber: pageNumber)
14686
}
87+
)
14788

148-
// Processes results in order and checks if there are more pages.
149-
let newVariations = batchResults.flatMap { $0.items.items }
150-
allVariations.append(contentsOf: newVariations)
151-
152-
let highestPageResult = batchResults.last?.items
153-
hasMorePages = (highestPageResult?.hasMorePages ?? false) && !newVariations.isEmpty
154-
currentPage += batchSize
155-
156-
DDLogInfo("📥 Loaded batch: \(batchResults.count) pages, total variations: \(allVariations.count), hasMorePages: \(hasMorePages)")
157-
}
158-
159-
DDLogInfo("✅ Variations sync complete: \(allVariations.count) variations loaded")
160-
return allVariations
89+
let (products, variations) = try await (productsTask, variationsTask)
90+
return POSCatalog(products: products, variations: variations)
16191
}
162-
}
16392

164-
private struct PageResult<T> {
165-
let pageNumber: Int
166-
let items: PagedItems<T>
16793
}

‎Modules/Sources/Yosemite/Tools/POS/POSCatalogIncrementalSyncService.swift‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import Foundation
2+
import protocol Networking.POSCatalogSyncRemoteProtocol
3+
import class Networking.AlamofireNetwork
4+
import class Networking.POSCatalogSyncRemote
5+
import CocoaLumberjackSwift
6+
import protocol Storage.GRDBManagerProtocol
7+
8+
// TODO - remove the periphery ignore comment when the service is integrated with POS.
9+
// periphery:ignore
10+
public protocol POSCatalogIncrementalSyncServiceProtocol {
11+
/// Starts an incremental catalog sync process.
12+
/// - Parameters:
13+
/// - siteID: The site ID to sync catalog for.
14+
/// - lastFullSyncDate: The date of the last full sync to use if no incremental sync date exists.
15+
func startIncrementalSync(for siteID: Int64, lastFullSyncDate: Date) async throws
16+
}
17+
18+
// TODO - remove the periphery ignore comment when the service is integrated with POS.
19+
// periphery:ignore
20+
public final class POSCatalogIncrementalSyncService: POSCatalogIncrementalSyncServiceProtocol {
21+
private let syncRemote: POSCatalogSyncRemoteProtocol
22+
private let batchSize: Int
23+
private var lastIncrementalSyncDates: [Int64: Date] = [:]
24+
private let batchedLoader: BatchedRequestLoader
25+
26+
public convenience init?(credentials: Credentials?, batchSize: Int = 1) {
27+
guard let credentials else {
28+
DDLogError("⛔️ Could not create POSCatalogIncrementalSyncService due missing credentials")
29+
return nil
30+
}
31+
let network = AlamofireNetwork(credentials: credentials, ensuresSessionManagerIsInitialized: true)
32+
let syncRemote = POSCatalogSyncRemote(network: network)
33+
self.init(syncRemote: syncRemote, batchSize: batchSize)
34+
}
35+
36+
init(syncRemote: POSCatalogSyncRemoteProtocol, batchSize: Int) {
37+
self.syncRemote = syncRemote
38+
self.batchSize = batchSize
39+
self.batchedLoader = BatchedRequestLoader(batchSize: batchSize)
40+
}
41+
42+
// MARK: - Protocol Conformance
43+
44+
public func startIncrementalSync(for siteID: Int64, lastFullSyncDate: Date) async throws {
45+
let modifiedAfter = lastIncrementalSyncDates[siteID] ?? lastFullSyncDate
46+
47+
DDLogInfo("🔄 Starting incremental catalog sync for site ID: \(siteID), modifiedAfter: \(modifiedAfter)")
48+
49+
do {
50+
let syncStartDate = Date()
51+
let catalog = try await loadCatalog(for: siteID, modifiedAfter: modifiedAfter, syncRemote: syncRemote)
52+
DDLogInfo("✅ Loaded \(catalog.products.count) products and \(catalog.variations.count) variations for siteID \(siteID)")
53+
54+
// TODO: WOOMOB-1298 - persist to database
55+
56+
// TODO: WOOMOB-1289 - replace with store settings persistence
57+
lastIncrementalSyncDates[siteID] = syncStartDate
58+
DDLogInfo("✅ Updated last incremental sync date to \(syncStartDate) for siteID \(siteID)")
59+
} catch {
60+
DDLogError("❌ Failed to sync and persist catalog incrementally: \(error)")
61+
throw error
62+
}
63+
}
64+
}
65+
66+
// MARK: - Remote Loading
67+
68+
private extension POSCatalogIncrementalSyncService {
69+
func loadCatalog(for siteID: Int64, modifiedAfter: Date, syncRemote: POSCatalogSyncRemoteProtocol) async throws -> POSCatalog {
70+
async let productsTask = batchedLoader.loadAll(
71+
makeRequest: { pageNumber in
72+
try await syncRemote.loadProducts(modifiedAfter: modifiedAfter, siteID: siteID, pageNumber: pageNumber)
73+
}
74+
)
75+
async let variationsTask = batchedLoader.loadAll(
76+
makeRequest: { pageNumber in
77+
try await syncRemote.loadProductVariations(modifiedAfter: modifiedAfter, siteID: siteID, pageNumber: pageNumber)
78+
}
79+
)
80+
81+
let (products, variations) = try await (productsTask, variationsTask)
82+
return POSCatalog(products: products, variations: variations)
83+
}
84+
}

0 commit comments

Comments
 (0)