Improve sync folder performance

kmg0308 · kmg0308 · commit e74d659694e4 · 2026-05-23T19:26:31.000+09:00
diff --git a/Sources/TokenMeter/DashboardModel.swift b/Sources/TokenMeter/DashboardModel.swift
@@ -106,7 +106,9 @@ final class DashboardModel: ObservableObject {
         let device = Self.loadLocalDevice(defaults: defaults)
         self.localDevice = device
         self.scanner = TokenLogScanner(localDevice: device)
-        self.syncFolderPath = defaults.string(forKey: Self.syncFolderPathKey)
+        let storedSyncFolderPath = defaults.string(forKey: Self.syncFolderPathKey)
+        self.syncFolderPath = storedSyncFolderPath
+        self.deviceFilter = storedSyncFolderPath?.isEmpty == false ? Self.allDevicesFilterId : device.id
         refresh()
     }
 
@@ -157,8 +159,9 @@ final class DashboardModel: ObservableObject {
         if !modelOptions.contains(modelFilter) {
             modelFilter = "All Models"
         }
-        if !deviceOptions.contains(where: { $0.id == deviceFilter }) {
-            deviceFilter = Self.allDevicesFilterId
+        let availableDeviceOptions = deviceOptions
+        if !availableDeviceOptions.contains(where: { $0.id == deviceFilter }) {
+            deviceFilter = availableDeviceOptions.first?.id ?? Self.allDevicesFilterId
         }
     }
 
@@ -171,6 +174,10 @@ final class DashboardModel: ObservableObject {
         return URL(fileURLWithPath: syncFolderPath)
     }
 
+    var isSyncConfigured: Bool {
+        syncFolderURL != nil
+    }
+
     var defaultICloudSyncFolderURL: URL? {
         let root = FileManager.default.homeDirectoryForCurrentUser
             .appendingPathComponent("Library", isDirectory: true)
@@ -185,10 +192,16 @@ final class DashboardModel: ObservableObject {
     }
 
     var selectedDeviceTitle: String {
-        deviceOptions.first { $0.id == deviceFilter }?.title ?? "All Devices"
+        deviceOptions.first { $0.id == deviceFilter }?.title ?? deviceOptions.first?.title ?? "This Mac"
     }
 
     var deviceOptions: [DashboardDeviceOption] {
+        guard isSyncConfigured else {
+            return [
+                DashboardDeviceOption(id: localDevice.id, title: "This Mac", deviceId: localDevice.id)
+            ]
+        }
+
         var options = [
             DashboardDeviceOption(id: Self.allDevicesFilterId, title: "All Devices", deviceId: nil),
             DashboardDeviceOption(id: localDevice.id, title: "This Mac", deviceId: localDevice.id)
@@ -231,7 +244,7 @@ final class DashboardModel: ObservableObject {
     func clearSyncFolder() {
         defaults.removeObject(forKey: Self.syncFolderPathKey)
         syncFolderPath = nil
-        deviceFilter = Self.allDevicesFilterId
+        deviceFilter = localDevice.id
         refresh(restartInProgress: true)
     }
 
diff --git a/Sources/TokenMeter/DashboardView.swift b/Sources/TokenMeter/DashboardView.swift
@@ -7,6 +7,7 @@ struct DashboardView: View {
     @EnvironmentObject private var updates: UpdateModel
     @State private var showingFilters = false
     @State private var showingDetails = false
+    @State private var showingSyncSettings = false
     @AppStorage("showFullTokenNumbers") private var showFullTokenNumbers = false
 
     private let refreshTimer = Timer.publish(every: 60, on: .main, in: .common).autoconnect()
@@ -43,7 +44,7 @@ struct DashboardView: View {
                             .padding(.top, 10)
                     }
 
-                    syncFolderBlock
+                    syncSettingsSection
 
                     dataSourcesBlock
 
@@ -142,15 +143,17 @@ struct DashboardView: View {
     }
 
     private var headerSubtitle: String {
-        let deviceScope = model.selectedDeviceTitle
+        let sectionTitle: String
         switch model.selectedSection {
         case .all:
-            return "Combined usage - \(deviceScope)"
+            sectionTitle = "Combined usage"
         case .codex:
-            return "Codex sessions - \(deviceScope)"
+            sectionTitle = "Codex sessions"
         case .claude:
-            return "Claude Code sessions - \(deviceScope)"
+            sectionTitle = "Claude Code sessions"
         }
+        guard model.isSyncConfigured else { return sectionTitle }
+        return "\(sectionTitle) - \(model.selectedDeviceTitle)"
     }
 
     private var numberFormat: TokenNumberFormat {
@@ -175,7 +178,9 @@ struct DashboardView: View {
                         .fill(TokenMeterTheme.control)
                 }
 
-                devicePicker
+                if model.isSyncConfigured {
+                    devicePicker
+                }
 
                 Spacer()
 
@@ -545,7 +550,9 @@ struct DashboardView: View {
             footerItem("Codex files", model.scanResult.codexFileCount)
             footerItem("Claude files", model.scanResult.claudeFileCount)
             footerItem("Events", model.scanResult.events.count)
-            footerItem("Devices", Set(model.scanResult.events.map(\.deviceId)).count)
+            if model.isSyncConfigured {
+                footerItem("Devices", Set(model.scanResult.events.map(\.deviceId)).count)
+            }
             footerItem("Errors", model.scanResult.parseErrorCount)
             Spacer()
             Text("Scanned \(model.scanResult.scannedAt.formatted(date: .omitted, time: .shortened))")
@@ -611,7 +618,7 @@ struct DashboardView: View {
             return DashboardNotice(
                 icon: "line.3.horizontal.decrease.circle",
                 title: "No matching events",
-                message: "The current section, time range, project, or model selection has no token events.",
+                message: noMatchingEventsMessage,
                 tint: TokenMeterTheme.secondaryText
             )
         }
@@ -646,6 +653,25 @@ struct DashboardView: View {
         return nil
     }
 
+    private var noMatchingEventsMessage: String {
+        if model.isSyncConfigured {
+            return "The current section, time range, project, model, or device selection has no token events."
+        }
+        return "The current section, time range, project, or model selection has no token events."
+    }
+
+    @ViewBuilder
+    private var syncSettingsSection: some View {
+        if model.isSyncConfigured {
+            syncFolderBlock
+        } else {
+            CollapsibleSection("Sync Folder", isExpanded: $showingSyncSettings) {
+                syncFolderBlock
+                    .padding(.top, 8)
+            }
+        }
+    }
+
     private func chooseSyncFolder() {
         let panel = NSOpenPanel()
         panel.title = "Choose TokenMeter Sync Folder"
diff --git a/Sources/TokenMeterCore/SyncLedger.swift b/Sources/TokenMeterCore/SyncLedger.swift
@@ -15,6 +15,7 @@ public final class TokenSyncLedgerStore: @unchecked Sendable {
     public func synchronize(
         localEvents: [TokenEvent],
         replaceLocalLedger: Bool = false,
+        importedAfter: Date? = nil,
         isCancelled: () -> Bool = { false }
     ) -> (events: [TokenEvent], status: SyncFolderStatus) {
         guard fileManager.fileExists(atPath: folder.path) else {
@@ -37,7 +38,7 @@ public final class TokenSyncLedgerStore: @unchecked Sendable {
             exportError = error.localizedDescription
         }
 
-        let readResult = readDeviceLedgers(isCancelled: isCancelled)
+        let readResult = readDeviceLedgers(importedAfter: importedAfter, isCancelled: isCancelled)
         let status = SyncFolderStatus(
             path: folder.path,
             exists: true,
@@ -63,35 +64,83 @@ public final class TokenSyncLedgerStore: @unchecked Sendable {
         let localLedgerURL = devicesURL.appendingPathComponent("\(safeFileName(localDevice.id)).jsonl")
 
         var recordsByKey: [String: SyncLedgerRecord] = [:]
-        if !replaceExisting, fileManager.fileExists(atPath: localLedgerURL.path) {
-            let existing = readRecords(from: localLedgerURL, isCancelled: isCancelled).records
-            for record in existing {
-                recordsByKey[record.identityKey] = record
-            }
-        }
-
         for event in events where !isCancelled() {
             let record = SyncLedgerRecord(event: event.withDevice(localDevice))
             recordsByKey[record.identityKey] = record
         }
 
-        let records = recordsByKey.values.sorted {
+        let records: [SyncLedgerRecord]
+        if replaceExisting {
+            records = sortedRecords(Array(recordsByKey.values))
+            try write(records: records, to: localLedgerURL)
+            return records.count
+        }
+
+        let existingKeys = readIdentityKeys(from: localLedgerURL, isCancelled: isCancelled)
+        records = sortedRecords(recordsByKey.values.filter { !existingKeys.contains($0.identityKey) })
+        guard !records.isEmpty else { return 0 }
+
+        try append(records: records, to: localLedgerURL)
+        return records.count
+    }
+
+    private func sortedRecords(_ records: [SyncLedgerRecord]) -> [SyncLedgerRecord] {
+        records.sorted {
             if $0.timestamp == $1.timestamp {
                 return $0.eventId < $1.eventId
             }
             return $0.timestamp < $1.timestamp
         }
+    }
+
+    private func write(records: [SyncLedgerRecord], to url: URL) throws {
         let encoder = Self.jsonEncoder
         let lines = try records.map { record in
             let data = try encoder.encode(record)
             return String(decoding: data, as: UTF8.self)
         }
         let data = (lines.joined(separator: "\n") + (lines.isEmpty ? "" : "\n")).data(using: .utf8) ?? Data()
-        try data.write(to: localLedgerURL, options: [.atomic])
-        return events.count
+        try data.write(to: url, options: [.atomic])
+    }
+
+    private func append(records: [SyncLedgerRecord], to url: URL) throws {
+        let encoder = Self.jsonEncoder
+        let lines = try records.map { record in
+            let data = try encoder.encode(record)
+            return String(decoding: data, as: UTF8.self)
+        }
+        guard !lines.isEmpty else { return }
+
+        if !fileManager.fileExists(atPath: url.path) {
+            let data = (lines.joined(separator: "\n") + "\n").data(using: .utf8) ?? Data()
+            try data.write(to: url, options: [.atomic])
+            return
+        }
+
+        let handle = try FileHandle(forUpdating: url)
+        defer {
+            try? handle.close()
+        }
+
+        let endOffset = try handle.seekToEnd()
+        if endOffset > 0 {
+            try handle.seek(toOffset: endOffset - 1)
+            let lastByte = handle.readDataToEndOfFile()
+            if lastByte != Data([0x0A]) {
+                try handle.seekToEnd()
+                try handle.write(contentsOf: Data([0x0A]))
+            }
+        }
+
+        try handle.seekToEnd()
+        let data = (lines.joined(separator: "\n") + "\n").data(using: .utf8) ?? Data()
+        try handle.write(contentsOf: data)
     }
 
-    private func readDeviceLedgers(isCancelled: () -> Bool) -> (events: [TokenEvent], deviceFileCount: Int, parseErrorCount: Int) {
+    private func readDeviceLedgers(
+        importedAfter: Date?,
+        isCancelled: () -> Bool
+    ) -> (events: [TokenEvent], deviceFileCount: Int, parseErrorCount: Int) {
         let devicesURL = folder.appendingPathComponent("devices", isDirectory: true)
         guard fileManager.fileExists(atPath: devicesURL.path) else {
             return ([], 0, 0)
@@ -111,15 +160,35 @@ public final class TokenSyncLedgerStore: @unchecked Sendable {
 
         for case let url as URL in enumerator where !isCancelled() && url.pathExtension == "jsonl" {
             fileCount += 1
-            let result = readRecords(from: url, isCancelled: isCancelled)
+            let result = readRecords(from: url, importedAfter: importedAfter, isCancelled: isCancelled)
             parseErrors += result.parseErrorCount
             events.append(contentsOf: result.records.map(\.tokenEvent))
         }
 
         return (events, fileCount, parseErrors)
     }
 
-    private func readRecords(from url: URL, isCancelled: () -> Bool) -> (records: [SyncLedgerRecord], parseErrorCount: Int) {
+    private func readIdentityKeys(from url: URL, isCancelled: () -> Bool) -> Set<String> {
+        guard fileManager.fileExists(atPath: url.path) else { return [] }
+        guard let content = try? String(contentsOf: url, encoding: .utf8) else { return [] }
+
+        let decoder = Self.jsonDecoder
+        var keys: Set<String> = []
+        for line in content.split(separator: "\n", omittingEmptySubsequences: true) where !isCancelled() {
+            guard let data = String(line).data(using: .utf8),
+                  let identity = try? decoder.decode(SyncLedgerIdentity.self, from: data) else {
+                continue
+            }
+            keys.insert(identity.identityKey)
+        }
+        return keys
+    }
+
+    private func readRecords(
+        from url: URL,
+        importedAfter: Date?,
+        isCancelled: () -> Bool
+    ) -> (records: [SyncLedgerRecord], parseErrorCount: Int) {
         guard let content = try? String(contentsOf: url, encoding: .utf8) else {
             return ([], 1)
         }
@@ -135,6 +204,10 @@ public final class TokenSyncLedgerStore: @unchecked Sendable {
             }
 
             do {
+                if let importedAfter {
+                    let header = try decoder.decode(SyncLedgerRecordHeader.self, from: data)
+                    guard header.timestamp >= importedAfter else { continue }
+                }
                 let record = try decoder.decode(SyncLedgerRecord.self, from: data)
                 records.append(record)
             } catch {
@@ -194,6 +267,24 @@ public final class TokenSyncLedgerStore: @unchecked Sendable {
     }
 }
 
+private struct SyncLedgerIdentity: Decodable {
+    var deviceId: String
+    var eventId: String
+
+    var identityKey: String {
+        "\(deviceId)|\(eventId)"
+    }
+
+    enum CodingKeys: String, CodingKey {
+        case deviceId = "device_id"
+        case eventId = "event_id"
+    }
+}
+
+private struct SyncLedgerRecordHeader: Decodable {
+    var timestamp: Date
+}
+
 private struct SyncLedgerRecord: Codable, Hashable {
     var schemaVersion: Int
     var deviceId: String
diff --git a/Sources/TokenMeterCore/TokenLogScanner.swift b/Sources/TokenMeterCore/TokenLogScanner.swift
@@ -74,6 +74,7 @@ public final class TokenLogScanner: @unchecked Sendable {
             localEvents: localEvents,
             syncFolder: syncFolder,
             replaceSyncLedger: replaceSyncLedger,
+            importedAfter: modifiedAfter,
             isCancelled: isCancelled
         )
         let events = deduplicated(events: localEvents + syncOutcome.events)
@@ -200,6 +201,7 @@ public final class TokenLogScanner: @unchecked Sendable {
         localEvents: [TokenEvent],
         syncFolder: URL?,
         replaceSyncLedger: Bool,
+        importedAfter: Date?,
         isCancelled: () -> Bool
     ) -> (events: [TokenEvent], status: SyncFolderStatus) {
         guard let syncFolder else {
@@ -214,6 +216,7 @@ public final class TokenLogScanner: @unchecked Sendable {
         return store.synchronize(
             localEvents: localEvents,
             replaceLocalLedger: replaceSyncLedger,
+            importedAfter: importedAfter,
             isCancelled: isCancelled
         )
     }
diff --git a/Sources/TokenMeterSelfTest/main.swift b/Sources/TokenMeterSelfTest/main.swift
