diff --git a/.vscode/launch.json b/.vscode/launch.json
index ca380323..38f07daa 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -45,7 +45,7 @@
"program": "${workspaceFolder}/service-host/bin/Debug/net8.0/service-host.dll",
"args": [
"--service",
- "SignatureIngestor",
+ "MetadataMapDump",
"--reportingserver",
"https://localhost:7023"
],
diff --git a/hasheous-lib/Classes/DataObjects.cs b/hasheous-lib/Classes/DataObjects.cs
index b3afc2c9..2ec62da6 100644
--- a/hasheous-lib/Classes/DataObjects.cs
+++ b/hasheous-lib/Classes/DataObjects.cs
@@ -2523,7 +2523,8 @@ void AddArticleVariants(string value)
}
}
- Logging.Log(Logging.LogType.Information, "Import Game", "Search candidates: " + String.Join(", ", distinctCandidates));
+ // remove blank entries and any entries that are just punctuation or dashes
+ distinctCandidates = distinctCandidates.Where(c => !string.IsNullOrWhiteSpace(c) && c.Any(char.IsLetterOrDigit)).ToList();
return distinctCandidates;
}
diff --git a/hasheous-lib/Classes/Logging.cs b/hasheous-lib/Classes/Logging.cs
index b8bd15a3..1722e342 100644
--- a/hasheous-lib/Classes/Logging.cs
+++ b/hasheous-lib/Classes/Logging.cs
@@ -213,15 +213,22 @@ static public void Log(LogType EventType, string ServerProcess, string Message,
/// If true, performs ETA calculation for the progress item.
static public void SendReport(string progressItemKey, int? count, int? total, string description, bool performETACalculation = false)
{
- if (report != null)
+ try
{
- _ = report.SendAsync(progressItemKey, count, total, description, performETACalculation).ContinueWith(task =>
+ if (report != null)
{
- if (task.IsFaulted)
+ _ = report.SendAsync(progressItemKey, count, total, description, performETACalculation).ContinueWith(task =>
{
- Console.WriteLine($"[Logging.SendReport] Error sending report: {task.Exception?.InnerException?.Message}");
- }
- }, TaskScheduler.Default);
+ if (task.IsFaulted)
+ {
+ // swallow any exceptions from sending the report to avoid impacting the main process
+ }
+ }, TaskScheduler.Default);
+ }
+ }
+ catch
+ {
+ // swallow any exceptions from sending the report to avoid impacting the main process
}
}
diff --git a/hasheous-lib/Classes/ProcessQueue/Tasks/Dumps.cs b/hasheous-lib/Classes/ProcessQueue/Tasks/Dumps.cs
index 820a2e59..6d1e8d5d 100644
--- a/hasheous-lib/Classes/ProcessQueue/Tasks/Dumps.cs
+++ b/hasheous-lib/Classes/ProcessQueue/Tasks/Dumps.cs
@@ -67,6 +67,22 @@ public class Dumps : IQueueTask
}
};
+ private static string SanitizeFileName(string name)
+ {
+ if (string.IsNullOrWhiteSpace(name))
+ {
+ return "Unknown";
+ }
+
+ string sanitized = name.Trim();
+ foreach (char c in Path.GetInvalidFileNameChars())
+ {
+ sanitized = sanitized.Replace(c, '_');
+ }
+
+ return string.IsNullOrWhiteSpace(sanitized) ? "Unknown" : sanitized;
+ }
+
private async Task DumpMetadataAsync()
{
string outputPath = Path.Combine(Config.LibraryConfiguration.LibraryMetadataMapDumpsDirectory, "Content");
@@ -96,126 +112,137 @@ public class Dumps : IQueueTask
// initialize the DataObjects class
DataObjects dataObjects = new DataObjects();
+ Dictionary platformDataCache = new Dictionary();
+
// step 1: get all game data objects
// This will fetch all game data objects, which can be paginated.
int totalGamesProcessed = 0;
for (int pageNumber = 1; ; pageNumber++)
{
- Logging.Log(Logging.LogType.Information, "Metadata Dump", $"Getting page {pageNumber} of game data objects for dump...");
-
- var dataObjectsList = await dataObjects.GetDataObjects(DataObjects.DataObjectType.Game, pageNumber, 100, null, false, false);
- if (dataObjectsList == null || dataObjectsList.Objects.Count == 0)
+ try
{
- Logging.Log(Logging.LogType.Information, "Metadata Dump", "No more game data objects to process.");
- break; // No more items to process
- }
-
- // step 2: dump each game data object
- Logging.Log(Logging.LogType.Information, "Metadata Dump", $"Processing {dataObjectsList.Objects.Count} game data objects from page {pageNumber}...");
- foreach (var dataObjectItem in dataObjectsList.Objects)
- {
- totalGamesProcessed++;
- Logging.Log(Logging.LogType.Information, "Metadata Dump", $"{totalGamesProcessed}/{dataObjectsList.Count}: Processing game (ID: {dataObjectItem.Id})...");
- Logging.SendReport(Config.LogName, totalGamesProcessed, dataObjectsList.Count, $"Processing {dataObjectItem.Name}...", true);
-
- string platformName = "Unknown Platform";
+ Logging.Log(Logging.LogType.Information, "Metadata Dump", $"Getting page {pageNumber} of game data objects for dump...");
- DataObjectItem? dataObject;
- string cacheKey = RedisConnection.GenerateKey("Dumps", "Game_" + dataObjectItem.Id.ToString());
- if (await RedisConnection.CacheItemExists(cacheKey))
+ var dataObjectsList = await dataObjects.GetDataObjects(DataObjects.DataObjectType.Game, pageNumber, 100, null, false, false);
+ if (dataObjectsList == null || dataObjectsList.Objects.Count == 0)
{
- dataObject = await RedisConnection.GetCacheItem(cacheKey);
- }
- else
- {
- dataObject = await dataObjects.GetDataObject(DataObjects.DataObjectType.Game, dataObjectItem.Id);
- if (dataObject != null)
- {
- RedisConnection.SetCacheItem(cacheKey, dataObject, cacheDuration);
- }
- }
-
- if (dataObject == null)
- {
- Logging.Log(Logging.LogType.Information, "Metadata Dump", $"{totalGamesProcessed}/{dataObjectsList.Count}: Data object with ID {dataObjectItem.Id} not found. Skipping...");
-
- continue; // Skip if the data object is not found
+ Logging.Log(Logging.LogType.Information, "Metadata Dump", "No more game data objects to process.");
+ break; // No more items to process
}
- // get the platform for the game
- DataObjectItem? platformItem = null;
- if (dataObject.Attributes != null && dataObject.Attributes.Any(attr => attr.attributeName == AttributeItem.AttributeName.Platform))
+ // step 2: dump each game data object
+ Logging.Log(Logging.LogType.Information, "Metadata Dump", $"Processing {dataObjectsList.Objects.Count} game data objects from page {pageNumber}...");
+ foreach (var dataObjectItem in dataObjectsList.Objects)
{
- platformItem = (DataObjectItem)dataObject.Attributes.First(attr => attr.attributeName == AttributeItem.AttributeName.Platform).Value;
-
- if (platformItem != null)
+ try
{
- platformName = platformItem.Name;
- }
- }
+ totalGamesProcessed++;
+ Logging.Log(Logging.LogType.Information, "Metadata Dump", $"{totalGamesProcessed}/{dataObjectsList.Count}: Processing game (ID: {dataObjectItem.Id})...");
+ Logging.SendReport(Config.LogName, totalGamesProcessed, dataObjectsList.Count, $"Processing {dataObjectItem.Name}...", true);
- string platformPath = Path.Combine(outputPath, platformName);
- if (!Directory.Exists(platformPath))
- {
- Directory.CreateDirectory(platformPath);
+ string platformName = "Unknown Platform";
- // add the platform mapping file
- if (platformItem != null)
- {
- DataObjectItem? platformDataObject;
- string platformCacheKey = RedisConnection.GenerateKey("Dumps", "Platform_" + platformItem.Id.ToString());
- if (await RedisConnection.CacheItemExists(platformCacheKey))
+ DataObjectItem? dataObject = await dataObjects.GetDataObject(DataObjects.DataObjectType.Game, dataObjectItem.Id);
+
+ if (dataObject == null)
{
- platformDataObject = await RedisConnection.GetCacheItem(platformCacheKey);
+ Logging.Log(Logging.LogType.Information, "Metadata Dump", $"{totalGamesProcessed}/{dataObjectsList.Count}: Data object with ID {dataObjectItem.Id} not found. Skipping...");
+
+ continue; // Skip if the data object is not found
}
- else
+
+ // get the platform for the game
+ DataObjectItem? platformItem = null;
+ if (dataObject.Attributes != null && dataObject.Attributes.Any(attr => attr.attributeName == AttributeItem.AttributeName.Platform))
{
- platformDataObject = await dataObjects.GetDataObject(DataObjects.DataObjectType.Platform, platformItem.Id);
- if (platformDataObject != null)
+ platformItem = (DataObjectItem)dataObject.Attributes.First(attr => attr.attributeName == AttributeItem.AttributeName.Platform).Value;
+
+ if (platformItem != null)
{
- await RedisConnection.SetCacheItem(platformCacheKey, platformDataObject, cacheDuration);
+ platformName = platformItem.Name;
}
}
- if (platformDataObject != null)
+
+ string safePlatformName = SanitizeFileName(platformName);
+ string platformPath = Path.Combine(outputPath, safePlatformName);
+ if (!Directory.Exists(platformPath))
{
- string platformFileName = "PlatformMapping.json";
- string platformFilePath = Path.Combine(platformPath, platformFileName);
+ Directory.CreateDirectory(platformPath);
- // serialize the dictionary to JSON and write to file
- string platformJsonContent = Newtonsoft.Json.JsonConvert.SerializeObject(platformDataObject, jsonSettings);
- await File.WriteAllTextAsync(platformFilePath, platformJsonContent);
+ // add the platform mapping file
+ if (platformItem != null)
+ {
+ DataObjectItem? platformDataObject;
+ if (platformDataCache.TryGetValue(platformItem.Id, out DataObjectItem cachedPlatform))
+ {
+ platformDataObject = cachedPlatform;
+ }
+ else
+ {
+ string platformCacheKey = RedisConnection.GenerateKey("Dumps", "Platform_" + platformItem.Id.ToString());
+ if (await RedisConnection.CacheItemExists(platformCacheKey))
+ {
+ platformDataObject = await RedisConnection.GetCacheItem(platformCacheKey);
+ }
+ else
+ {
+ platformDataObject = await dataObjects.GetDataObject(DataObjects.DataObjectType.Platform, platformItem.Id);
+ if (platformDataObject != null)
+ {
+ await RedisConnection.SetCacheItem(platformCacheKey, platformDataObject, cacheDuration);
+ }
+ }
+ if (platformDataObject != null)
+ {
+ platformDataCache[platformItem.Id] = platformDataObject;
+ }
+ }
+ if (platformDataObject != null)
+ {
+ string platformFileName = "PlatformMapping.json";
+ string platformFilePath = Path.Combine(platformPath, platformFileName);
+
+ // serialize the dictionary to JSON and write to file
+ string platformJsonContent = Newtonsoft.Json.JsonConvert.SerializeObject(platformDataObject, jsonSettings);
+ await File.WriteAllTextAsync(platformFilePath, platformJsonContent);
+ }
+ }
}
- }
- }
- // Add to the list of platforms if not already present
- if (!platforms.Contains(platformName))
- {
- platforms.Add(platformName);
- }
+ // Add to the list of platforms if not already present
+ if (!platforms.Contains(safePlatformName))
+ {
+ platforms.Add(safePlatformName);
+ }
- // Ensure the file name is safe for writing to disk
- string unsafeFileName = $"{dataObject.Name.Trim()} ({dataObject.Id}).json";
- foreach (char c in Path.GetInvalidFileNameChars())
- {
- unsafeFileName = unsafeFileName.Replace(c, '_');
- }
- string fileName = unsafeFileName;
- string filePath = Path.Combine(platformPath, fileName);
+ // Ensure the file name is safe for writing to disk
+ string unsafeFileName = $"{dataObject.Name.Trim()} ({dataObject.Id}).json";
+ string fileName = SanitizeFileName(unsafeFileName);
+ string filePath = Path.Combine(platformPath, fileName);
- // serialize the dictionary to JSON and write to file
- string jsonContent = Newtonsoft.Json.JsonConvert.SerializeObject(dataObject, jsonSettings);
- await File.WriteAllTextAsync(filePath, jsonContent);
+ // serialize the dictionary to JSON and write to file
+ string jsonContent = Newtonsoft.Json.JsonConvert.SerializeObject(dataObject, jsonSettings);
+ await File.WriteAllTextAsync(filePath, jsonContent);
- // if counter is a multiple of 10, introduce a short delay
- if (totalGamesProcessed % 10 == 0)
- {
- await Task.Delay(2000); // 2 seconds delay
+ // if counter is a multiple of 10, introduce a short delay
+ if (totalGamesProcessed % 10 == 0)
+ {
+ await Task.Delay(500); // 0.5 second delay
+ }
+ }
+ catch (Exception ex)
+ {
+ Logging.Log(Logging.LogType.Critical, "Metadata Dump", $"An error occurred while processing game with ID {dataObjectItem.Id}: {ex.Message}", ex);
+ }
}
}
+ catch (Exception ex)
+ {
+ Logging.Log(Logging.LogType.Critical, "Metadata Dump", $"An error occurred while dumping metadata: {ex.Message}", ex);
+ }
- // sleep for a 30 seconds to avoid overwhelming the system
- await Task.Delay(30000);
+ // sleep for a 5 seconds to avoid overwhelming the system
+ await Task.Delay(5000);
}
Logging.SendReport(Config.LogName, null, null, "Compressing content.");
@@ -239,32 +266,35 @@ public class Dumps : IQueueTask
int platformCounter = 0;
foreach (string platform in platforms)
{
- platformCounter++;
- Logging.SendReport(Config.LogName, platformCounter, platforms.Count, $"Creating zip for platform {platform}...", true);
-
- // create a zip for the platform
- string platformSourcePath = Path.Combine(outputPath, platform);
- if (Directory.Exists(platformSourcePath))
+ try
{
- string safePlatformName = platform;
- foreach (char c in Path.GetInvalidFileNameChars())
- {
- safePlatformName = safePlatformName.Replace(c, '_');
- }
- string platformZipPath = Path.Combine(platformTempZipFilePath, $"{safePlatformName}.zip");
- System.IO.Compression.ZipFile.CreateFromDirectory(platformSourcePath, platformZipPath, System.IO.Compression.CompressionLevel.SmallestSize, false);
- // generate md5 checksum for the zip
- using (var md5 = System.Security.Cryptography.MD5.Create())
+ platformCounter++;
+ Logging.SendReport(Config.LogName, platformCounter, platforms.Count, $"Creating zip for platform {platform}...", true);
+
+ // create a zip for the platform
+ string platformSourcePath = Path.Combine(outputPath, platform);
+ if (Directory.Exists(platformSourcePath))
{
- using (var stream = File.OpenRead(platformZipPath))
+ string safePlatformName = SanitizeFileName(platform);
+ string platformZipPath = Path.Combine(platformTempZipFilePath, $"{safePlatformName}.zip");
+ System.IO.Compression.ZipFile.CreateFromDirectory(platformSourcePath, platformZipPath, System.IO.Compression.CompressionLevel.SmallestSize, false);
+ // generate md5 checksum for the zip
+ using (var md5 = System.Security.Cryptography.MD5.Create())
{
- var hash = md5.ComputeHash(stream);
- string md5String = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
- string md5FilePath = platformZipPath + ".md5sum";
- await File.WriteAllTextAsync(md5FilePath, md5String);
+ using (var stream = File.OpenRead(platformZipPath))
+ {
+ var hash = md5.ComputeHash(stream);
+ string md5String = BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
+ string md5FilePath = platformZipPath + ".md5sum";
+ await File.WriteAllTextAsync(md5FilePath, md5String);
+ }
}
}
}
+ catch (Exception ex)
+ {
+ Logging.Log(Logging.LogType.Critical, "Metadata Dump", $"An error occurred while creating zip for platform {platform}: {ex.Message}", ex);
+ }
// sleep for 5 seconds to avoid overwhelming the system
await Task.Delay(5000);
diff --git a/hasheous-lib/Classes/Report.cs b/hasheous-lib/Classes/Report.cs
index 56249b1f..cb0420fe 100644
--- a/hasheous-lib/Classes/Report.cs
+++ b/hasheous-lib/Classes/Report.cs
@@ -82,14 +82,12 @@ public async System.Threading.Tasks.Task SendAsync(string progressItemKey, int?
try
{
string url = $"{this.reportingServerUrl}/api/v1.0/BackgroundTasks/{this.processId}/{this.correlationId}/report";
- Console.WriteLine($"Sending report to {url}");
var response = await httpClient.PostAsync(url, content);
response.EnsureSuccessStatusCode();
}
- catch (Exception ex)
+ catch
{
- // Log the error but do not throw
- Console.WriteLine($"Failed to send report to server: {ex.Message}");
+ // swallow the error to prevent reporting issues to the console
}
}
finally
diff --git a/hasheous-lib/hasheous-lib.csproj b/hasheous-lib/hasheous-lib.csproj
index 6ecdda14..59bf2bb2 100644
--- a/hasheous-lib/hasheous-lib.csproj
+++ b/hasheous-lib/hasheous-lib.csproj
@@ -19,17 +19,17 @@
-
-
-
+
+
+
-
+
-
-
-
+
+
+
diff --git a/hasheous/Controllers/V1.0/TaskWorkerController.cs b/hasheous/Controllers/V1.0/TaskWorkerController.cs
index 3ca78ae6..9d101ffc 100644
--- a/hasheous/Controllers/V1.0/TaskWorkerController.cs
+++ b/hasheous/Controllers/V1.0/TaskWorkerController.cs
@@ -15,7 +15,7 @@ namespace hasheous_server.Controllers.v1_0
[ApiController]
[Route("api/v{version:apiVersion}/[controller]/")]
[ApiVersion("1.0")]
- [ApiExplorerSettings(IgnoreApi = false)]
+ [ApiExplorerSettings(IgnoreApi = true)]
[IgnoreAntiforgeryToken]
public class TaskWorkerController : ControllerBase
{