diff --git a/osu.Game/Database/BackgroundDataStoreProcessor.cs b/osu.Game/Database/BackgroundDataStoreProcessor.cs index a9abae72f94f..1315eb464518 100644 --- a/osu.Game/Database/BackgroundDataStoreProcessor.cs +++ b/osu.Game/Database/BackgroundDataStoreProcessor.cs @@ -90,6 +90,7 @@ protected override void LoadComplete() clearOutdatedStarRatings(); populateMissingStarRatings(); processOnlineBeatmapSetsWithNoUpdate(); + // Note that the previous method will also update these on a fresh run. processBeatmapsWithMissingObjectCounts(); processScoresWithMissingStatistics(); @@ -102,6 +103,7 @@ protected override void LoadComplete() if (t.Exception?.InnerException is ObjectDisposedException) { Logger.Log("Finished background aborted during shutdown"); + return; } @@ -234,7 +236,7 @@ void flushPendingUpdates() var beatmap = realmAccess.Run(r => r.Find(id)?.Detach()); if (beatmap == null) - return; + continue; try { @@ -277,6 +279,7 @@ private void processOnlineBeatmapSetsWithNoUpdate() // // We may eventually consider making the Process call more specific (or avoid this in any number // of other possible ways), but for now avoid queueing if the user isn't logged in at startup. + if (api.IsLoggedIn) { foreach (var b in r.All().Where(b => b.OnlineID > 0 && b.LastOnlineUpdate == null && b.BeatmapSet != null)) @@ -418,43 +421,89 @@ private void processScoresWithMissingStatistics() int processedCount = 0; int failedCount = 0; - foreach (var id in scoreIds) + foreach (var chunk in scoreIds.Chunk(100)) { if (notification?.State == ProgressNotificationState.Cancelled) break; updateNotificationProgress(notification, processedCount, scoreIds.Count); - sleepIfRequired(); - try + var updates = new List<(Guid id, string json)>(); + var failedIds = new List(); + int missingCount = 0; + + var detachedScores = realmAccess.Run(r => { - var score = scoreManager.Query(s => s.ID == id); + var scores = new List(); + + foreach (var id in chunk) + { + var s = r.Find(id); + + if (s != null) + scores.Add(s.Detach()); + else + missingCount++; + } - if (score != null) + return scores; + }); + + foreach (var score in detachedScores) + { + try { scoreManager.PopulateMaximumStatistics(score); + updates.Add((score.ID, JsonConvert.SerializeObject(score.MaximumStatistics))); + } + catch (Exception e) + { + Logger.Log(@$"Failed to populate maximum statistics for {score.ID}: {e}"); + failedIds.Add(score.ID); + } + } + if (updates.Count > 0 || failedIds.Count > 0) + { + try + { // Can't use async overload because we're not on the update thread. // ReSharper disable once MethodHasAsyncOverload realmAccess.Write(r => { - r.Find(id)!.MaximumStatisticsJson = JsonConvert.SerializeObject(score.MaximumStatistics); + foreach (var update in updates) + { + var s = r.Find(update.id); + + if (s != null) + s.MaximumStatisticsJson = update.json; + } + + foreach (var id in failedIds) + { + var s = r.Find(id); + + if (s != null) + s.BackgroundReprocessingFailed = true; + } }); - } - ++processedCount; - } - catch (ObjectDisposedException) - { - throw; - } - catch (Exception e) - { - Logger.Log(@$"Failed to populate maximum statistics for {id}: {e}"); - realmAccess.Write(r => r.Find(id)!.BackgroundReprocessingFailed = true); - ++failedCount; + processedCount += updates.Count; + failedCount += failedIds.Count; + } + catch (ObjectDisposedException) + { + throw; + } + catch (Exception e) + { + Logger.Log($"Fatal error writing batch in score statistics population: {e}"); + failedCount += updates.Count + failedIds.Count; + } } + + processedCount += missingCount; } completeNotification(notification, processedCount, scoreIds.Count, failedCount); @@ -468,16 +517,17 @@ private void convertLegacyTotalScoreToStandardised() r.All() .Filter($"{nameof(ScoreInfo.BackgroundReprocessingFailed)} == false && {nameof(ScoreInfo.BeatmapInfo)} != null && {nameof(ScoreInfo.IsLegacyScore)} == true && {nameof(ScoreInfo.TotalScoreVersion)} < $0", LegacyScoreEncoder.LATEST_VERSION) .AsEnumerable() + // must be done after materialisation, as realm doesn't want to support // nested property predicates .Where(s => s.Ruleset.IsLegacyRuleset()) .Select(s => s.ID))); - Logger.Log($"Found {scoreIds.Count} scores which require total score conversion."); - if (scoreIds.Count == 0) return; + Logger.Log($"Found {scoreIds.Count} scores which require total score conversion."); + var notification = showProgressNotification(scoreIds.Count, "Upgrading scores to new scoring algorithm", "scores have been upgraded to the new scoring algorithm"); int processedCount = 0; @@ -489,11 +539,11 @@ private void convertLegacyTotalScoreToStandardised() break; updateNotificationProgress(notification, processedCount, scoreIds.Count); - sleepIfRequired(); var updates = new List<(Guid id, long totalScore, long totalScoreWithoutMods, double accuracy, ScoreRank rank)>(); var failedIds = new List(); + int missingCount = 0; var detachedScores = realmAccess.Run(r => { @@ -505,6 +555,8 @@ private void convertLegacyTotalScoreToStandardised() if (score != null) scores.Add(score.Detach()); + else + missingCount++; } return scores; @@ -535,7 +587,9 @@ private void convertLegacyTotalScoreToStandardised() foreach (var update in updates) { var s = r.Find(update.id); - if (s == null) continue; + + if (s == null) + continue; s.TotalScore = update.totalScore; s.TotalScoreWithoutMods = update.totalScoreWithoutMods; @@ -547,6 +601,7 @@ private void convertLegacyTotalScoreToStandardised() foreach (var id in failedIds) { var s = r.Find(id); + if (s != null) s.BackgroundReprocessingFailed = true; } @@ -562,8 +617,11 @@ private void convertLegacyTotalScoreToStandardised() catch (Exception e) { Logger.Log($"Fatal error writing batch in score conversion: {e}"); + failedCount += updates.Count + failedIds.Count; } } + + processedCount += missingCount; } completeNotification(notification, processedCount, scoreIds.Count, failedCount); @@ -577,53 +635,107 @@ private void upgradeScoreRanks() r.All() .Where(s => s.TotalScoreVersion < 30000013 && !s.BackgroundReprocessingFailed) // last total score version with a significant change to ranks .AsEnumerable() + // must be done after materialisation, as realm doesn't support // filtering on nested property predicates or projection via `.Select()` .Where(s => s.Ruleset.IsLegacyRuleset()) .Select(s => s.ID))); - Logger.Log($"Found {scoreIds.Count} scores which require rank upgrades."); - if (scoreIds.Count == 0) return; + Logger.Log($"Found {scoreIds.Count} scores which require rank upgrades."); + var notification = showProgressNotification(scoreIds.Count, "Adjusting ranks of scores", "scores now have more correct ranks."); int processedCount = 0; int failedCount = 0; - foreach (var id in scoreIds) + foreach (var chunk in scoreIds.Chunk(100)) { if (notification?.State == ProgressNotificationState.Cancelled) break; updateNotificationProgress(notification, processedCount, scoreIds.Count); - sleepIfRequired(); - try + var updates = new List<(Guid id, ScoreRank rank)>(); + var failedIds = new List(); + int missingCount = 0; + + var detachedScores = realmAccess.Run(r => { - // Can't use async overload because we're not on the update thread. - // ReSharper disable once MethodHasAsyncOverload - realmAccess.Write(r => + var scores = new List(); + + foreach (var id in chunk) { - ScoreInfo s = r.Find(id)!; - s.Rank = StandardisedScoreMigrationTools.ComputeRank(s); - s.TotalScoreVersion = LegacyScoreEncoder.LATEST_VERSION; - }); + var s = r.Find(id); - ++processedCount; - } - catch (ObjectDisposedException) + if (s != null) + scores.Add(s.Detach()); + else + missingCount++; + } + + return scores; + }); + + foreach (var detachedScore in detachedScores) { - throw; + try + { + updates.Add((detachedScore.ID, StandardisedScoreMigrationTools.ComputeRank(detachedScore))); + } + catch (Exception e) + { + Logger.Log($"Failed to update rank score {detachedScore.ID}: {e}"); + failedIds.Add(detachedScore.ID); + } } - catch (Exception e) + + if (updates.Count > 0 || failedIds.Count > 0) { - Logger.Log($"Failed to update rank score {id}: {e}"); - realmAccess.Write(r => r.Find(id)!.BackgroundReprocessingFailed = true); - ++failedCount; + try + { + // Can't use async overload because we're not on the update thread. + // ReSharper disable once MethodHasAsyncOverload + realmAccess.Write(r => + { + foreach (var update in updates) + { + var s = r.Find(update.id); + + if (s != null) + { + s.Rank = update.rank; + s.TotalScoreVersion = LegacyScoreEncoder.LATEST_VERSION; + } + } + + foreach (var id in failedIds) + { + var s = r.Find(id); + + if (s != null) + s.BackgroundReprocessingFailed = true; + } + }); + + processedCount += updates.Count; + failedCount += failedIds.Count; + } + catch (ObjectDisposedException) + { + throw; + } + catch (Exception e) + { + Logger.Log($"Fatal error writing batch in score rank upgrade: {e}"); + failedCount += updates.Count + failedIds.Count; + } } + + processedCount += missingCount; } completeNotification(notification, processedCount, scoreIds.Count, failedCount); @@ -631,11 +743,13 @@ private void upgradeScoreRanks() private void backpopulateMissingSubmissionAndRankDates() { - if (DebugUtils.IsNUnitRunning) return; + if (DebugUtils.IsNUnitRunning) + return; if (!localMetadataSource.Available) { Logger.Log("Cannot backpopulate missing submission/rank dates because the local metadata cache is missing."); + return; } @@ -644,12 +758,14 @@ private void backpopulateMissingSubmissionAndRankDates() if (!localMetadataSource.IsAtLeastVersion(2)) { Logger.Log("Cannot backpopulate missing submission/rank dates because the local metadata cache is too old."); + return; } } catch (Exception ex) { Logger.Log($"Error when trying to query version of local metadata cache: {ex}"); + return; } @@ -703,10 +819,12 @@ private void backpopulateMissingSubmissionAndRankDates() Debug.Assert(result != null); beatmapSet.DateRanked = result.DateRanked; beatmapSet.DateSubmitted = result.DateSubmitted; + return true; } Logger.Log($"Could not find {beatmapSet.GetDisplayString()} in local cache while backpopulating missing submission/rank date"); + return false; }); @@ -738,7 +856,8 @@ protected virtual void BackpopulateUserTags() if (!localMetadataSource.Available || !localMetadataSource.IsAtLeastVersion(3)) { - if (DebugUtils.IsNUnitRunning) return; + if (DebugUtils.IsNUnitRunning) + return; Logger.Log(@"Local metadata cache has too low version to backpopulate user tags, attempting refetch..."); localMetadataSource.FetchCache().WaitSafely(); @@ -746,11 +865,13 @@ protected virtual void BackpopulateUserTags() if (!localMetadataSource.Available || !localMetadataSource.IsAtLeastVersion(3)) { Logger.Log(@"Local metadata cache refetch failed. Aborting user tags backpopulation."); + return; } } var lastPopulation = config.Get(OsuSetting.LastOnlineTagsPopulation); + // dropping time data here completely is intentional, because storing the date to config is a lossy operation // (truncates some ticks off of the date when it's being converted to string and back). // therefore, if precision isn't explicitly constrained, the condition below would always fail just because the date stored to config @@ -760,6 +881,7 @@ protected virtual void BackpopulateUserTags() if (metadataSourceFetchDate <= lastPopulation) { Logger.Log($@"Skipping user tag population because the local metadata source hasn't been updated since the last time user tags were checked ({lastPopulation.Value:d})"); + return; } @@ -813,6 +935,7 @@ protected virtual void BackpopulateUserTags() { beatmap.Metadata.UserTags.Clear(); beatmap.Metadata.UserTags.AddRange(userTags); + return true; } @@ -820,6 +943,7 @@ protected virtual void BackpopulateUserTags() } Logger.Log(@$"Could not find {beatmap.GetDisplayString()} in local cache while backpopulating missing user tags"); + return false; }); diff --git a/replace_method.py b/replace_method.py deleted file mode 100644 index f59b1702e96a..000000000000 --- a/replace_method.py +++ /dev/null @@ -1,51 +0,0 @@ -import sys - -filepath = "osu.Game/Screens/Edit/EditorBeatmap.cs" -with open(filepath, 'r') as f: - content = f.read() - -old_method = """ public int findInsertionIndex(IReadOnlyList list, double startTime) - { - for (int i = 0; i < list.Count; i++) - { - if (list[i].StartTime > startTime) - return i - 1; - } - - return list.Count - 1; - }""" - -new_method = """ public int findInsertionIndex(IReadOnlyList list, double startTime) - { - int min = 0; - int max = list.Count - 1; - - while (min <= max) - { - int mid = min + (max - min) / 2; - if (list[mid].StartTime <= startTime) - min = mid + 1; - else - max = mid - 1; - } - - return min - 1; - }""" - -if old_method not in content: - # Try normalizing line endings or whitespace if needed, but let's check exact match first - # Maybe try stripping whitespace - # Actually, I'll print a snippet to debug if it fails - print("Method not found!") - # Find approximate location - start_idx = content.find("public int findInsertionIndex") - if start_idx != -1: - print("Found start at:", start_idx) - print("Content snippet:") - print(content[start_idx:start_idx+300]) - sys.exit(1) - -new_content = content.replace(old_method, new_method) - -with open(filepath, 'w') as f: - f.write(new_content)