Skip to content

Commit eeb6e72

Browse files
michael-j-greenmichael-j-green
committed
Refactor report sending and metadata dump process to improve error handling and reduce sleep times
1 parent 7d61b60 commit eeb6e72

File tree

4 files changed

+134
-121
lines changed

4 files changed

+134
-121
lines changed

‎.vscode/launch.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
"program": "${workspaceFolder}/service-host/bin/Debug/net8.0/service-host.dll",
4646
"args": [
4747
"--service",
48-
"SignatureIngestor",
48+
"MetadataMapDump",
4949
"--reportingserver",
5050
"https://localhost:7023"
5151
],

‎hasheous-lib/Classes/Logging.cs‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -213,15 +213,22 @@ static public void Log(LogType EventType, string ServerProcess, string Message,
213213
/// <param name="performETACalculation">If true, performs ETA calculation for the progress item.</param>
214214
static public void SendReport(string progressItemKey, int? count, int? total, string description, bool performETACalculation = false)
215215
{
216-
if (report != null)
216+
try
217217
{
218-
_ = report.SendAsync(progressItemKey, count, total, description, performETACalculation).ContinueWith(task =>
218+
if (report != null)
219219
{
220-
if (task.IsFaulted)
220+
_ = report.SendAsync(progressItemKey, count, total, description, performETACalculation).ContinueWith(task =>
221221
{
222-
Console.WriteLine($"[Logging.SendReport] Error sending report: {task.Exception?.InnerException?.Message}");
223-
}
224-
}, TaskScheduler.Default);
222+
if (task.IsFaulted)
223+
{
224+
// swallow any exceptions from sending the report to avoid impacting the main process
225+
}
226+
}, TaskScheduler.Default);
227+
}
228+
}
229+
catch
230+
{
231+
// swallow any exceptions from sending the report to avoid impacting the main process
225232
}
226233
}
227234

‎hasheous-lib/Classes/ProcessQueue/Tasks/Dumps.cs‎

Lines changed: 118 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -101,121 +101,122 @@ public class Dumps : IQueueTask
101101
int totalGamesProcessed = 0;
102102
for (int pageNumber = 1; ; pageNumber++)
103103
{
104-
Logging.Log(Logging.LogType.Information, "Metadata Dump", $"Getting page {pageNumber} of game data objects for dump...");
105-
106-
var dataObjectsList = await dataObjects.GetDataObjects(DataObjects.DataObjectType.Game, pageNumber, 100, null, false, false);
107-
if (dataObjectsList == null || dataObjectsList.Objects.Count == 0)
108-
{
109-
Logging.Log(Logging.LogType.Information, "Metadata Dump", "No more game data objects to process.");
110-
break; // No more items to process
111-
}
112-
113-
// step 2: dump each game data object
114-
Logging.Log(Logging.LogType.Information, "Metadata Dump", $"Processing {dataObjectsList.Objects.Count} game data objects from page {pageNumber}...");
115-
foreach (var dataObjectItem in dataObjectsList.Objects)
104+
try
116105
{
117-
totalGamesProcessed++;
118-
Logging.Log(Logging.LogType.Information, "Metadata Dump", $"{totalGamesProcessed}/{dataObjectsList.Count}: Processing game (ID: {dataObjectItem.Id})...");
119-
Logging.SendReport(Config.LogName, totalGamesProcessed, dataObjectsList.Count, $"Processing {dataObjectItem.Name}...", true);
120-
121-
string platformName = "Unknown Platform";
122-
123-
DataObjectItem? dataObject;
124-
string cacheKey = RedisConnection.GenerateKey("Dumps", "Game_" + dataObjectItem.Id.ToString());
125-
if (await RedisConnection.CacheItemExists(cacheKey))
126-
{
127-
dataObject = await RedisConnection.GetCacheItem<DataObjectItem>(cacheKey);
128-
}
129-
else
130-
{
131-
dataObject = await dataObjects.GetDataObject(DataObjects.DataObjectType.Game, dataObjectItem.Id);
132-
if (dataObject != null)
133-
{
134-
RedisConnection.SetCacheItem<DataObjectItem>(cacheKey, dataObject, cacheDuration);
135-
}
136-
}
106+
Logging.Log(Logging.LogType.Information, "Metadata Dump", $"Getting page {pageNumber} of game data objects for dump...");
137107

138-
if (dataObject == null)
108+
var dataObjectsList = await dataObjects.GetDataObjects(DataObjects.DataObjectType.Game, pageNumber, 100, null, false, false);
109+
if (dataObjectsList == null || dataObjectsList.Objects.Count == 0)
139110
{
140-
Logging.Log(Logging.LogType.Information, "Metadata Dump", $"{totalGamesProcessed}/{dataObjectsList.Count}: Data object with ID {dataObjectItem.Id} not found. Skipping...");
141-
142-
continue; // Skip if the data object is not found
111+
Logging.Log(Logging.LogType.Information, "Metadata Dump", "No more game data objects to process.");
112+
break; // No more items to process
143113
}
144114

145-
// get the platform for the game
146-
DataObjectItem? platformItem = null;
147-
if (dataObject.Attributes != null && dataObject.Attributes.Any(attr => attr.attributeName == AttributeItem.AttributeName.Platform))
115+
// step 2: dump each game data object
116+
Logging.Log(Logging.LogType.Information, "Metadata Dump", $"Processing {dataObjectsList.Objects.Count} game data objects from page {pageNumber}...");
117+
foreach (var dataObjectItem in dataObjectsList.Objects)
148118
{
149-
platformItem = (DataObjectItem)dataObject.Attributes.First(attr => attr.attributeName == AttributeItem.AttributeName.Platform).Value;
150-
151-
if (platformItem != null)
119+
try
152120
{
153-
platformName = platformItem.Name;
154-
}
155-
}
121+
totalGamesProcessed++;
122+
Logging.Log(Logging.LogType.Information, "Metadata Dump", $"{totalGamesProcessed}/{dataObjectsList.Count}: Processing game (ID: {dataObjectItem.Id})...");
123+
Logging.SendReport(Config.LogName, totalGamesProcessed, dataObjectsList.Count, $"Processing {dataObjectItem.Name}...", true);
156124

157-
string platformPath = Path.Combine(outputPath, platformName);
158-
if (!Directory.Exists(platformPath))
159-
{
160-
Directory.CreateDirectory(platformPath);
125+
string platformName = "Unknown Platform";
161126

162-
// add the platform mapping file
163-
if (platformItem != null)
164-
{
165-
DataObjectItem? platformDataObject;
166-
string platformCacheKey = RedisConnection.GenerateKey("Dumps", "Platform_" + platformItem.Id.ToString());
167-
if (await RedisConnection.CacheItemExists(platformCacheKey))
127+
DataObjectItem? dataObject = await dataObjects.GetDataObject(DataObjects.DataObjectType.Game, dataObjectItem.Id);
128+
129+
if (dataObject == null)
168130
{
169-
platformDataObject = await RedisConnection.GetCacheItem<DataObjectItem>(platformCacheKey);
131+
Logging.Log(Logging.LogType.Information, "Metadata Dump", $"{totalGamesProcessed}/{dataObjectsList.Count}: Data object with ID {dataObjectItem.Id} not found. Skipping...");
132+
133+
continue; // Skip if the data object is not found
170134
}
171-
else
135+
136+
// get the platform for the game
137+
DataObjectItem? platformItem = null;
138+
if (dataObject.Attributes != null && dataObject.Attributes.Any(attr => attr.attributeName == AttributeItem.AttributeName.Platform))
172139
{
173-
platformDataObject = await dataObjects.GetDataObject(DataObjects.DataObjectType.Platform, platformItem.Id);
174-
if (platformDataObject != null)
140+
platformItem = (DataObjectItem)dataObject.Attributes.First(attr => attr.attributeName == AttributeItem.AttributeName.Platform).Value;
141+
142+
if (platformItem != null)
175143
{
176-
await RedisConnection.SetCacheItem<DataObjectItem>(platformCacheKey, platformDataObject, cacheDuration);
144+
platformName = platformItem.Name;
177145
}
178146
}
179-
if (platformDataObject != null)
147+
148+
string platformPath = Path.Combine(outputPath, platformName);
149+
if (!Directory.Exists(platformPath))
180150
{
181-
string platformFileName = "PlatformMapping.json";
182-
string platformFilePath = Path.Combine(platformPath, platformFileName);
151+
Directory.CreateDirectory(platformPath);
183152

184-
// serialize the dictionary to JSON and write to file
185-
string platformJsonContent = Newtonsoft.Json.JsonConvert.SerializeObject(platformDataObject, jsonSettings);
186-
await File.WriteAllTextAsync(platformFilePath, platformJsonContent);
153+
// add the platform mapping file
154+
if (platformItem != null)
155+
{
156+
DataObjectItem? platformDataObject;
157+
string platformCacheKey = RedisConnection.GenerateKey("Dumps", "Platform_" + platformItem.Id.ToString());
158+
if (await RedisConnection.CacheItemExists(platformCacheKey))
159+
{
160+
platformDataObject = await RedisConnection.GetCacheItem<DataObjectItem>(platformCacheKey);
161+
}
162+
else
163+
{
164+
platformDataObject = await dataObjects.GetDataObject(DataObjects.DataObjectType.Platform, platformItem.Id);
165+
if (platformDataObject != null)
166+
{
167+
await RedisConnection.SetCacheItem<DataObjectItem>(platformCacheKey, platformDataObject, cacheDuration);
168+
}
169+
}
170+
if (platformDataObject != null)
171+
{
172+
string platformFileName = "PlatformMapping.json";
173+
string platformFilePath = Path.Combine(platformPath, platformFileName);
174+
175+
// serialize the dictionary to JSON and write to file
176+
string platformJsonContent = Newtonsoft.Json.JsonConvert.SerializeObject(platformDataObject, jsonSettings);
177+
await File.WriteAllTextAsync(platformFilePath, platformJsonContent);
178+
}
179+
}
187180
}
188-
}
189-
}
190181

191-
// Add to the list of platforms if not already present
192-
if (!platforms.Contains(platformName))
193-
{
194-
platforms.Add(platformName);
195-
}
182+
// Add to the list of platforms if not already present
183+
if (!platforms.Contains(platformName))
184+
{
185+
platforms.Add(platformName);
186+
}
196187

197-
// Ensure the file name is safe for writing to disk
198-
string unsafeFileName = $"{dataObject.Name.Trim()} ({dataObject.Id}).json";
199-
foreach (char c in Path.GetInvalidFileNameChars())
200-
{
201-
unsafeFileName = unsafeFileName.Replace(c, '_');
202-
}
203-
string fileName = unsafeFileName;
204-
string filePath = Path.Combine(platformPath, fileName);
188+
// Ensure the file name is safe for writing to disk
189+
string unsafeFileName = $"{dataObject.Name.Trim()} ({dataObject.Id}).json";
190+
foreach (char c in Path.GetInvalidFileNameChars())
191+
{
192+
unsafeFileName = unsafeFileName.Replace(c, '_');
193+
}
194+
string fileName = unsafeFileName;
195+
string filePath = Path.Combine(platformPath, fileName);
205196

206-
// serialize the dictionary to JSON and write to file
207-
string jsonContent = Newtonsoft.Json.JsonConvert.SerializeObject(dataObject, jsonSettings);
208-
await File.WriteAllTextAsync(filePath, jsonContent);
197+
// serialize the dictionary to JSON and write to file
198+
string jsonContent = Newtonsoft.Json.JsonConvert.SerializeObject(dataObject, jsonSettings);
199+
await File.WriteAllTextAsync(filePath, jsonContent);
209200

210-
// if counter is a multiple of 10, introduce a short delay
211-
if (totalGamesProcessed % 10 == 0)
212-
{
213-
await Task.Delay(2000); // 2 seconds delay
201+
// if counter is a multiple of 10, introduce a short delay
202+
if (totalGamesProcessed % 10 == 0)
203+
{
204+
await Task.Delay(1000); // 1 second delay
205+
}
206+
}
207+
catch (Exception ex)
208+
{
209+
Logging.Log(Logging.LogType.Critical, "Metadata Dump", $"An error occurred while processing game with ID {dataObjectItem.Id}: {ex.Message}", ex);
210+
}
214211
}
215212
}
213+
catch (Exception ex)
214+
{
215+
Logging.Log(Logging.LogType.Critical, "Metadata Dump", $"An error occurred while dumping metadata: {ex.Message}", ex);
216+
}
216217

217-
// sleep for a 30 seconds to avoid overwhelming the system
218-
await Task.Delay(30000);
218+
// sleep for a 5 seconds to avoid overwhelming the system
219+
await Task.Delay(5000);
219220
}
220221

221222
Logging.SendReport(Config.LogName, null, null, "Compressing content.");
@@ -239,32 +240,39 @@ public class Dumps : IQueueTask
239240
int platformCounter = 0;
240241
foreach (string platform in platforms)
241242
{
242-
platformCounter++;
243-
Logging.SendReport(Config.LogName, platformCounter, platforms.Count, $"Creating zip for platform {platform}...", true);
244-
245-
// create a zip for the platform
246-
string platformSourcePath = Path.Combine(outputPath, platform);
247-
if (Directory.Exists(platformSourcePath))
243+
try
248244
{
249-
string safePlatformName = platform;
250-
foreach (char c in Path.GetInvalidFileNameChars())
251-
{
252-
safePlatformName = safePlatformName.Replace(c, '_');
253-
}
254-
string platformZipPath = Path.Combine(platformTempZipFilePath, $"{safePlatformName}.zip");
255-
System.IO.Compression.ZipFile.CreateFromDirectory(platformSourcePath, platformZipPath, System.IO.Compression.CompressionLevel.SmallestSize, false);
256-
// generate md5 checksum for the zip
257-
using (var md5 = System.Security.Cryptography.MD5.Create())
245+
platformCounter++;
246+
Logging.SendReport(Config.LogName, platformCounter, platforms.Count, $"Creating zip for platform {platform}...", true);
247+
248+
// create a zip for the platform
249+
string platformSourcePath = Path.Combine(outputPath, platform);
250+
if (Directory.Exists(platformSourcePath))
258251
{
259-
using (var stream = File.OpenRead(platformZipPath))
252+
string safePlatformName = platform;
253+
foreach (char c in Path.GetInvalidFileNameChars())
260254
{
261-
var hash = md5.ComputeHash(stream);
262-
string md5String = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
263-
string md5FilePath = platformZipPath + ".md5sum";
264-
await File.WriteAllTextAsync(md5FilePath, md5String);
255+
safePlatformName = safePlatformName.Replace(c, '_');
256+
}
257+
string platformZipPath = Path.Combine(platformTempZipFilePath, $"{safePlatformName}.zip");
258+
System.IO.Compression.ZipFile.CreateFromDirectory(platformSourcePath, platformZipPath, System.IO.Compression.CompressionLevel.SmallestSize, false);
259+
// generate md5 checksum for the zip
260+
using (var md5 = System.Security.Cryptography.MD5.Create())
261+
{
262+
using (var stream = File.OpenRead(platformZipPath))
263+
{
264+
var hash = md5.ComputeHash(stream);
265+
string md5String = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
266+
string md5FilePath = platformZipPath + ".md5sum";
267+
await File.WriteAllTextAsync(md5FilePath, md5String);
268+
}
265269
}
266270
}
267271
}
272+
catch (Exception ex)
273+
{
274+
Logging.Log(Logging.LogType.Critical, "Metadata Dump", $"An error occurred while creating zip for platform {platform}: {ex.Message}", ex);
275+
}
268276

269277
// sleep for 5 seconds to avoid overwhelming the system
270278
await Task.Delay(5000);

‎hasheous-lib/Classes/Report.cs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -82,14 +82,12 @@ public async System.Threading.Tasks.Task SendAsync(string progressItemKey, int?
8282
try
8383
{
8484
string url = $"{this.reportingServerUrl}/api/v1.0/BackgroundTasks/{this.processId}/{this.correlationId}/report";
85-
Console.WriteLine($"Sending report to {url}");
8685
var response = await httpClient.PostAsync(url, content);
8786
response.EnsureSuccessStatusCode();
8887
}
89-
catch (Exception ex)
88+
catch
9089
{
91-
// Log the error but do not throw
92-
Console.WriteLine($"Failed to send report to server: {ex.Message}");
90+
// swallow the error to prevent reporting issues to the console
9391
}
9492
}
9593
finally

0 commit comments

Comments
 (0)