fix: add 2-day buffer to sync boundary and force_full_sync flag

ervwalter · ervwalter · commit e8f9bdb277dc · 2025-11-10T13:23:10.000-06:00
- Change sync splice from 90-day to 88-day cutoff (2-day buffer)
- Truncate both existing and new data at same boundary for clean splice
- Add force_full_sync boolean flag to source_data table
- Add GetForceFullSyncAsync() to check flag and trigger data clear
- Update MeasurementSyncService to check flag before sync
- Add test for force_full_sync functionality
- All 334 tests pass
diff --git a/apps/api/TrendWeight.Tests/Features/Measurements/Services/MeasurementSyncServiceTests.cs b/apps/api/TrendWeight.Tests/Features/Measurements/Services/MeasurementSyncServiceTests.cs
@@ -785,7 +785,8 @@ public async Task RefreshProviderAsync_WithExistingDataAndSyncWindow_MergesCorre
         var userId = Guid.NewGuid();
         var provider = "withings";
         var lastSyncTime = DateTime.UtcNow.AddDays(-1);
-        var syncStartDate = lastSyncTime.AddDays(-90); // 90 days before last sync
+        var syncStartDate = lastSyncTime.AddDays(-90); // Fetch starts 90 days before last sync
+        var cutoffDate = syncStartDate.AddDays(2); // Splice at 88 days before last sync (2-day buffer)
 
         // Existing data from previous sync (has both old and recent measurements)
         var existingSourceData = new List<SourceData>
@@ -796,23 +797,23 @@ public async Task RefreshProviderAsync_WithExistingDataAndSyncWindow_MergesCorre
                 LastUpdate = lastSyncTime,
                 Measurements = new List<RawMeasurement>
                 {
-                    // Old measurements (before sync window)
-                    CreateTestRawMeasurement(syncStartDate.AddDays(-10).ToString("yyyy-MM-dd"), 65.0m),
-                    CreateTestRawMeasurement(syncStartDate.AddDays(-5).ToString("yyyy-MM-dd"), 66.0m),
-                    // Recent measurements (within sync window)
-                    CreateTestRawMeasurement(syncStartDate.AddDays(5).ToString("yyyy-MM-dd"), 70.0m),
-                    CreateTestRawMeasurement(syncStartDate.AddDays(10).ToString("yyyy-MM-dd"), 71.0m),
-                    CreateTestRawMeasurement(syncStartDate.AddDays(15).ToString("yyyy-MM-dd"), 72.0m)
+                    // Old measurements (before cutoff at day -88, should be preserved)
+                    CreateTestRawMeasurement(syncStartDate.AddDays(-10).ToString("yyyy-MM-dd"), 65.0m), // day -100
+                    CreateTestRawMeasurement(syncStartDate.AddDays(-5).ToString("yyyy-MM-dd"), 66.0m),  // day -95
+                    // Recent measurements (after cutoff, should be replaced by provider data)
+                    CreateTestRawMeasurement(syncStartDate.AddDays(5).ToString("yyyy-MM-dd"), 70.0m),   // day -85
+                    CreateTestRawMeasurement(syncStartDate.AddDays(10).ToString("yyyy-MM-dd"), 71.0m),  // day -80
+                    CreateTestRawMeasurement(syncStartDate.AddDays(15).ToString("yyyy-MM-dd"), 72.0m)   // day -75
                 }
             }
         };
 
-        // New measurements from provider (only has data within sync window)
+        // New measurements from provider (fetched from day -90, truncated to day -88 onwards)
         var newMeasurements = new List<RawMeasurement>
         {
-            CreateTestRawMeasurement(syncStartDate.AddDays(5).ToString("yyyy-MM-dd"), 70.5m), // Updated weight
-            CreateTestRawMeasurement(syncStartDate.AddDays(12).ToString("yyyy-MM-dd"), 71.5m), // New measurement
-            // Note: measurements at day 10 and 15 are missing (user deleted them)
+            CreateTestRawMeasurement(syncStartDate.AddDays(5).ToString("yyyy-MM-dd"), 70.5m),  // day -85, updated weight
+            CreateTestRawMeasurement(syncStartDate.AddDays(12).ToString("yyyy-MM-dd"), 71.5m), // day -78, new measurement
+            // Note: measurements at days -80 and -75 are missing (user deleted them upstream)
         };
 
         var providerService = new Mock<IProviderService>();
@@ -939,6 +940,79 @@ public async Task RefreshProviderAsync_WithFullSync_ReplacesAllData()
             It.Is<List<SourceData>>(sd => VerifyFullSyncReplacement(sd, provider, newMeasurements))), Times.Once);
     }
 
+    [Fact]
+    public async Task GetMeasurementsForUserAsync_WithForceFullSyncFlag_ClearsDataAndPerformsFullSync()
+    {
+        // Arrange
+        var userId = Guid.NewGuid();
+        var provider = "fitbit";
+
+        // Existing data exists
+        var existingSourceData = new List<SourceData>
+        {
+            new SourceData
+            {
+                Source = provider,
+                LastUpdate = DateTime.UtcNow.AddDays(-30),
+                Measurements = new List<RawMeasurement>
+                {
+                    CreateTestRawMeasurement("2024-01-01", 70.0m),
+                    CreateTestRawMeasurement("2024-01-02", 71.0m)
+                }
+            }
+        };
+
+        // New measurements from full sync
+        var newMeasurements = new List<RawMeasurement>
+        {
+            CreateTestRawMeasurement("2024-01-01", 70.5m), // Updated
+            CreateTestRawMeasurement("2024-01-02", 71.5m), // Updated
+            CreateTestRawMeasurement("2024-01-03", 72.0m)  // New
+        };
+
+        var providerService = new Mock<IProviderService>();
+        // Verify that startDate is null (full sync) after clearing data
+        providerService.Setup(x => x.SyncMeasurementsAsync(userId, true, null))
+            .ReturnsAsync(new ProviderSyncResult
+            {
+                Provider = provider,
+                Success = true,
+                Measurements = newMeasurements
+            });
+
+        _providerIntegrationServiceMock.Setup(x => x.GetProviderService(provider))
+            .Returns(providerService.Object);
+
+        // Mock force_full_sync flag to return true
+        _sourceDataServiceMock.Setup(x => x.GetForceFullSyncAsync(userId, provider))
+            .ReturnsAsync(true);
+
+        // Mock GetLastSyncTimeAsync to return null (simulating cleared data after ClearProviderDataAsync is called)
+        _sourceDataServiceMock.Setup(x => x.GetLastSyncTimeAsync(userId, provider))
+            .ReturnsAsync((DateTime?)null);
+
+        _sourceDataServiceMock.Setup(x => x.GetSourceDataAsync(userId, new List<string> { provider }))
+            .ReturnsAsync(existingSourceData);
+
+        // Mock ClearSourceDataAsync to simulate data being cleared
+        _sourceDataServiceMock.Setup(x => x.ClearSourceDataAsync(userId, provider))
+            .Returns(Task.CompletedTask);
+
+        // Act
+        var result = await _sut.GetMeasurementsForUserAsync(userId, new List<string> { provider }, true);
+
+        // Assert
+        // Verify ClearProviderDataAsync was called (this is called on MeasurementSyncService, not the mock)
+        // We verify this indirectly by checking that a full sync was performed (startDate = null)
+        providerService.Verify(x => x.SyncMeasurementsAsync(userId, true, null), Times.Once,
+            "Should perform full sync (startDate = null) after force_full_sync flag is detected");
+
+        // Verify data was updated
+        _sourceDataServiceMock.Verify(x => x.UpdateSourceDataAsync(
+            userId,
+            It.Is<List<SourceData>>(sd => sd.Count == 1 && sd[0].Source == provider)), Times.Once);
+    }
+
     #endregion
 
     #region Private Helper Methods
@@ -981,14 +1055,17 @@ private static bool VerifyMergedData(List<SourceData> sd, string provider, DateT
             return false;
 
         var measurements = sd[0].Measurements!; // We already checked it's not null
-        return measurements.Count == 4 && // 2 old (preserved) + 2 new (from provider)
-                                          // Old measurements preserved
+        // Cutoff is at syncStartDate + 2 days (88 days before last sync)
+        // Old measurements (before cutoff): preserved
+        // New measurements (after cutoff): from provider
+        return measurements.Count == 4 && // 2 old (preserved before day -88) + 2 new (from provider after day -88)
+                                          // Old measurements preserved (days -100, -95 are before cutoff at day -88)
                measurements.Any(m => m.Date == syncStartDate.AddDays(-10).ToString("yyyy-MM-dd") && m.Weight == 65.0m) &&
                measurements.Any(m => m.Date == syncStartDate.AddDays(-5).ToString("yyyy-MM-dd") && m.Weight == 66.0m) &&
-               // New measurements from provider
+               // New measurements from provider (days -85, -78 are after cutoff at day -88)
                measurements.Any(m => m.Date == syncStartDate.AddDays(5).ToString("yyyy-MM-dd") && m.Weight == 70.5m) &&
                measurements.Any(m => m.Date == syncStartDate.AddDays(12).ToString("yyyy-MM-dd") && m.Weight == 71.5m) &&
-               // Deleted measurements are gone
+               // Deleted measurements are gone (days -80, -75 were in old data but not in provider data)
                !measurements.Any(m => m.Date == syncStartDate.AddDays(10).ToString("yyyy-MM-dd")) &&
                !measurements.Any(m => m.Date == syncStartDate.AddDays(15).ToString("yyyy-MM-dd"));
     }
diff --git a/apps/api/TrendWeight/Features/Measurements/ISourceDataService.cs b/apps/api/TrendWeight/Features/Measurements/ISourceDataService.cs
@@ -28,6 +28,14 @@ public interface ISourceDataService
     /// <returns>Last sync DateTime in UTC, or null if no data exists</returns>
     Task<DateTime?> GetLastSyncTimeAsync(Guid userId, string provider);
 
+    /// <summary>
+    /// Get the force_full_sync flag for a specific provider
+    /// </summary>
+    /// <param name="userId">User's Supabase UID</param>
+    /// <param name="provider">Provider name (e.g., "withings", "fitbit")</param>
+    /// <returns>True if force_full_sync is set, false otherwise</returns>
+    Task<bool> GetForceFullSyncAsync(Guid userId, string provider);
+
     /// <summary>
     /// Clears source data for a user
     /// </summary>
diff --git a/apps/api/TrendWeight/Features/Measurements/MeasurementSyncService.cs b/apps/api/TrendWeight/Features/Measurements/MeasurementSyncService.cs
@@ -57,6 +57,14 @@ public async Task<MeasurementsResult> GetMeasurementsForUserAsync(
             // Check each provider's last sync time and resync flag
             foreach (var provider in activeProviders)
             {
+                // Check force_full_sync flag - if set, clear data to trigger full resync
+                var forceFullSync = await _sourceDataService.GetForceFullSyncAsync(userId, provider);
+                if (forceFullSync)
+                {
+                    _logger.LogInformation("Force full sync set for user {UserId} provider {Provider}, clearing data", userId, provider);
+                    await ClearProviderDataAsync(userId, provider);
+                }
+
                 // Check last sync time for this provider
                 var lastSync = await _sourceDataService.GetLastSyncTimeAsync(userId, provider);
                 var needsRefresh = lastSync == null || (now - lastSync.Value).TotalSeconds > _cacheDurationSeconds;
@@ -155,12 +163,12 @@ private async Task<ProviderSyncResult> RefreshProviderAsync(Guid userId, string
 
             // Calculate start date for sync
             DateTime? startDate = null;
-            // For regular refresh, fetch from 90 days before last sync
+            // For regular refresh, fetch from 90 days before last sync (with 2-day buffer to avoid boundary issues)
             var lastSyncTime = await _sourceDataService.GetLastSyncTimeAsync(userId, provider);
             if (lastSyncTime.HasValue)
             {
                 startDate = lastSyncTime.Value.AddDays(-90);
-                _logger.LogDebug("Fetching {Provider} measurements from {StartDate} (90 days before last sync)",
+                _logger.LogDebug("Fetching {Provider} measurements from {StartDate} (90 days before last sync with 2-day buffer)",
                     provider, startDate.Value.ToString("o"));
             }
             else
@@ -194,15 +202,25 @@ await _progressReporter.ReportProviderProgressAsync(
                 if (existingProviderData?.Measurements != null && startDate.HasValue)
                 {
                     // We have existing data and a sync window - merge them
-                    var cutoffDate = startDate.Value.ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture);
+                    // Cutoff is 2 days after fetch start (88 days before last sync) to create a buffer zone
+                    // that handles timezone interpretation differences and API boundary quirks
+                    var cutoffDate = startDate.Value.AddDays(2).ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture);
+
+                    _logger.LogDebug("Merging {Provider} data with cutoff date {CutoffDate} (fetch started at {FetchStart})",
+                        provider, cutoffDate, startDate.Value.ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture));
 
                     // Keep only measurements before the cutoff date from existing data
                     var existingMeasurementsToKeep = existingProviderData.Measurements
                         .Where(m => string.Compare(m.Date, cutoffDate, StringComparison.Ordinal) < 0)
                         .ToList();
 
-                    // Combine: new measurements (truth for sync window) + older kept measurements
-                    mergedMeasurements = result.Measurements.Concat(existingMeasurementsToKeep).ToList();
+                    // Keep only measurements on or after the cutoff date from new data (discard buffer zone)
+                    var newMeasurementsToUse = result.Measurements
+                        .Where(m => string.Compare(m.Date, cutoffDate, StringComparison.Ordinal) >= 0)
+                        .ToList();
+
+                    // Combine: truncated new measurements (truth from cutoff forward) + older kept measurements
+                    mergedMeasurements = newMeasurementsToUse.Concat(existingMeasurementsToKeep).ToList();
                 }
                 else
                 {
diff --git a/apps/api/TrendWeight/Features/Measurements/SourceDataService.cs b/apps/api/TrendWeight/Features/Measurements/SourceDataService.cs
@@ -234,6 +234,21 @@ public async Task UpdateSourceDataAsync(Guid userId, List<SourceData> data)
         }
     }
 
+    /// <inheritdoc />
+    public async Task<bool> GetForceFullSyncAsync(Guid userId, string provider)
+    {
+        try
+        {
+            var dbSourceData = await GetOrFetchSourceDataAsync(userId, provider);
+            return dbSourceData?.ForceFullSync ?? false;
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "Error getting force_full_sync for user {UserId} provider {Provider}", userId, provider);
+            return false;
+        }
+    }
+
     /// <inheritdoc />
     public async Task ClearSourceDataAsync(Guid userId, string? provider = null)
     {
diff --git a/apps/api/TrendWeight/Infrastructure/DataAccess/Models/DbSourceData.cs b/apps/api/TrendWeight/Infrastructure/DataAccess/Models/DbSourceData.cs
@@ -21,6 +21,9 @@ public class DbSourceData : BaseModel
     [Column("last_sync")]
     public string? LastSync { get; set; }
 
+    [Column("force_full_sync")]
+    public bool ForceFullSync { get; set; } = false;
+
     [Column("updated_at")]
     public string UpdatedAt { get; set; } = string.Empty;
 }
diff --git a/apps/api/TrendWeight/supabase/migrations/20250210_add_force_full_sync_to_source_data.sql b/apps/api/TrendWeight/supabase/migrations/20250210_add_force_full_sync_to_source_data.sql
@@ -0,0 +1,10 @@
+-- Add force_full_sync flag to source_data table
+-- This flag allows triggering a full resync for specific provider data
+-- by clearing existing data before the next sync
+
+ALTER TABLE public.source_data
+ADD COLUMN IF NOT EXISTS force_full_sync BOOLEAN DEFAULT FALSE;
+
+-- Add comment explaining the purpose
+COMMENT ON COLUMN public.source_data.force_full_sync IS
+'When true, triggers deletion of this source_data row before next sync, causing a full resync. Auto-clears when new data is written.';
diff --git a/apps/api/TrendWeight/supabase/schema.sql b/apps/api/TrendWeight/supabase/schema.sql
@@ -50,9 +50,10 @@ CREATE TABLE IF NOT EXISTS public.source_data (
     provider VARCHAR NOT NULL,
     measurements JSONB NOT NULL DEFAULT '[]'::jsonb,
     last_sync TEXT,
+    force_full_sync BOOLEAN DEFAULT FALSE,
     updated_at TEXT DEFAULT now(),
     CONSTRAINT source_data_pkey PRIMARY KEY (uid, provider),
-    CONSTRAINT source_data_uid_fkey FOREIGN KEY (uid) 
+    CONSTRAINT source_data_uid_fkey FOREIGN KEY (uid)
         REFERENCES public.profiles(uid) ON DELETE CASCADE
 );
 

Original file line number	Diff line number	Diff line change
`@@ -21,6 +21,9 @@ public class DbSourceData : BaseModel`
`21`	`21`	`[Column("last_sync")]`
`22`	`22`	`public string? LastSync { get; set; }`
`23`	`23`
	`24`	`+ [Column("force_full_sync")]`
	`25`	`+ public bool ForceFullSync { get; set; } = false;`
	`26`	`+`
`24`	`27`	`[Column("updated_at")]`
`25`	`28`	`public string UpdatedAt { get; set; } = string.Empty;`
`26`	`29`	`}`