Skip to content

Commit 602ad61

Browse files
Toastbrot236Toastbrot236
authored
Implement helper method for asset reference validation (#1086)
This PR implements a helper method which applies various common validations to a given asset reference (common as in these kinds of validations are needed by various endpoints, like e.g. multiple endpoints needing to ensure a referenced asset is an image, or not a GUID, or not disallowed etc). The idea is that, instead of every endpoint implementing its own validation, they'd just call this method and handle its return value. This would deduplicate a relatively high amount of code, and it would potentially also prevent edge cases where some endpoints forget to validate some things (e.g. only root level hashes are ensured to be valid hashes right now, some API endpoints only ensure an asset is in the database and not the data store, the photo upload endpoint only ensures a photo's dependencies are in the data store and not the database etc). Since a complete refactor would've made this PR way larger than it already is (~850 lines just to implement and test the method), this PR only implements and tests the method. Modifying endpoint methods to actually use this method would have to be done in future PRs.
2 parents 2ec2e74 + cd6f58e commit 602ad61

7 files changed

Lines changed: 832 additions & 51 deletions

File tree

‎Refresh.Core/Helpers/ResourceValidationHelper.cs‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
using System.Diagnostics;
2+
using Bunkum.Core;
3+
using NotEnoughLogs;
4+
using Refresh.Common.Verification;
5+
using Refresh.Core.Types.Assets.Validation;
6+
using Refresh.Database.Models.Assets;
7+
using Refresh.Database.Models.Authentication;
8+
9+
namespace Refresh.Core.Helpers;
10+
11+
public abstract class ResourceValidationHelper
12+
{
13+
/// <summary>
14+
/// Validates the given asset reference (hash/guid/blank) using the given parameters, if necessary also by reading the referenced asset
15+
/// or getting data about it from database (see <see cref="GameAsset"/> and <see cref="DisallowedAsset"/>).
16+
/// </summary>
17+
public static ValidatedAssetResult ValidateReference(AssetValidationParameters parameters, Logger logger)
18+
{
19+
string assetTypeStr = parameters.AssetContextTypeStr ?? (parameters.MustBeTexture ? "image asset" : "asset");
20+
GameAsset? asset = null;
21+
bool existsInDataStore = false;
22+
bool isPSP = parameters.GameToUseIn == TokenGame.LittleBigPlanetPSP;
23+
Action<string>? onNewAssetRefCallback = parameters.OnNewAssetRefCallback;
24+
25+
if (parameters.AssetRef.IsBlankHash())
26+
{
27+
if (!parameters.MayBeBlank) return new(BadRequest, "0", $"The {assetTypeStr} must be set.", onNewAssetRefCallback);
28+
else return new(OK, "0", null, onNewAssetRefCallback);
29+
}
30+
31+
else if (parameters.AssetRef.StartsWith('g'))
32+
{
33+
if (!parameters.MayBeGuid) return new(BadRequest, null, $"The {assetTypeStr} may not be an in-game asset.", onNewAssetRefCallback);
34+
if (parameters.AssetRef.Length < 2) return new(BadRequest, null, $"The used in-game {assetTypeStr} is invalid (empty GUID).", onNewAssetRefCallback);
35+
36+
// This should only happen if the user is messing with mods/the API/beta builds, so give them a more detailed response
37+
bool canParseGuid = long.TryParse(parameters.AssetRef[1..], out long guid);
38+
if (!canParseGuid)
39+
return new(BadRequest, null, $"The used in-game {assetTypeStr} is invalid (badly formatted GUID).", onNewAssetRefCallback);
40+
41+
if (parameters.MustBeTexture && !parameters.GuidChecker.IsTextureGuid(parameters.GameToUseIn, guid))
42+
return new(BadRequest, null, $"The used in-game {assetTypeStr} was not a valid image (unknown GUID).", onNewAssetRefCallback);
43+
}
44+
45+
// At this point the reference is a hash
46+
else if (!parameters.MayBeHash)
47+
{
48+
return new(BadRequest, null, $"The {assetTypeStr} may not be a custom asset.", onNewAssetRefCallback);
49+
}
50+
51+
else if (!CommonPatterns.Sha1Regex().IsMatch(parameters.AssetRef))
52+
{
53+
// This should only happen if a player is messing with mods/the API, so give them a more detailed response
54+
return new(BadRequest, null, $"The used {assetTypeStr} had an invalid hash.", onNewAssetRefCallback);
55+
}
56+
57+
else
58+
{
59+
DisallowedAsset? disallowed = parameters.Database.GetDisallowedAssetInfo(parameters.AssetRef);
60+
if (disallowed != null)
61+
{
62+
logger.LogWarning(BunkumCategory.UserContent, $"{parameters.User} tried to use a manually disallowed {assetTypeStr}.");
63+
return new(Unauthorized, disallowanceInfo: disallowed, onNewAssetRefCallback: onNewAssetRefCallback);
64+
}
65+
66+
string filename = isPSP ? $"psp/{parameters.AssetRef}" : parameters.AssetRef;
67+
existsInDataStore = parameters.DataStore.ExistsInStore(filename);
68+
69+
if (!existsInDataStore)
70+
{
71+
logger.LogDebug(BunkumCategory.UserContent, $"Referenced asset '{filename}' could not be found in data store.");
72+
73+
if (parameters.MustBeInDataStoreIfHash)
74+
return new(NotFound, null, $"The used {assetTypeStr} did not exist on the server.", onNewAssetRefCallback);
75+
}
76+
77+
asset = parameters.Cache.GetAssetInfo(parameters.AssetRef, parameters.Database);
78+
79+
// Only try to import if the asset exists in the data store
80+
if (existsInDataStore && asset == null)
81+
{
82+
logger.LogInfo(BunkumCategory.UserContent, $"Referenced asset '{filename}' exists in data store but not in database, attempting to import automatically...");
83+
Stopwatch sw = new();
84+
sw.Start();
85+
86+
if (!parameters.DataStore.TryGetDataFromStore(filename, out byte[]? assetData) || assetData == null)
87+
{
88+
sw.Stop();
89+
logger.LogError(BunkumCategory.UserContent, $"Failed to read '{filename}' from data store!");
90+
logger.LogDebug(BunkumCategory.UserContent, $"Failed to get '{filename}' after {sw.ElapsedMilliseconds}ms.");
91+
return new(InternalServerError, null, $"Failed to read {assetTypeStr} internally. Please report this to the server owner.", onNewAssetRefCallback, existsInDataStore: existsInDataStore);
92+
}
93+
94+
asset = parameters.AssetImporter.ReadAndVerifyAsset(parameters.AssetRef, assetData, parameters.PlatformToUseIn, parameters.Database);
95+
if (asset == null)
96+
{
97+
sw.Stop();
98+
logger.LogDebug(BunkumCategory.UserContent, $"Failed to get '{filename}' after {sw.ElapsedMilliseconds}ms.");
99+
return new(BadRequest, null, $"The used {assetTypeStr} was invalid or corrupt.", onNewAssetRefCallback, existsInDataStore: existsInDataStore);
100+
}
101+
102+
sw.Stop();
103+
logger.LogInfo(BunkumCategory.UserContent, $"Successfully imported '{filename}' in {sw.ElapsedMilliseconds}ms.");
104+
}
105+
106+
// FIXME: for some reason, PSP texture detection/conversion broke so we can no longer tell if a PSP texture is actually a texture, so skip this for PSP
107+
if (asset != null && !isPSP)
108+
{
109+
bool isHashedTexture = (asset.AssetFlags & AssetFlags.Imagery) != 0;
110+
111+
if (parameters.MustBeTexture && !isHashedTexture)
112+
return new(BadRequest, null, $"The used {assetTypeStr} was not a valid custom image.", onNewAssetRefCallback, assetInfo: asset, existsInDataStore: existsInDataStore);
113+
114+
// TODO: actually use AIPI to scan image if not null
115+
}
116+
}
117+
118+
return new(OK, parameters.AssetRef, null, onNewAssetRefCallback, assetInfo: asset, existsInDataStore: existsInDataStore);
119+
}
120+
}

‎Refresh.Core/Importing/Importer.cs‎

Lines changed: 34 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -107,32 +107,40 @@ private static bool MatchesMagic(ReadOnlySpan<byte> data, ulong magic)
107107
/// <returns>Whether the file is likely of TGA format</returns>
108108
private static bool IsPspTga(ReadOnlySpan<byte> data)
109109
{
110-
byte imageIdLength = data[0];
111-
byte colorMapType = data[1];
112-
byte imageType = data[2];
113-
ReadOnlySpan<byte> colorMapSpecification = data[3..8];
114-
ReadOnlySpan<byte> imageSpecification = data[8..18];
115-
short xOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[..2]);
116-
short yOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[2..4]);
117-
ushort width = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[4..6]);
118-
ushort height = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[6..8]);
119-
byte depth = imageSpecification[8];
120-
byte descriptor = imageSpecification[9];
121-
122-
//PSP does not seem to fill out this information
123-
if (imageIdLength != 0) return false;
124-
if (xOrigin != 0) return false;
125-
if (yOrigin != 0) return false;
126-
//These are the fields set by PSP, that shouldn't change from image to image
127-
if (colorMapType != 1) return false;
128-
if (descriptor != 0) return false;
129-
if (imageType != 1) return false;
130-
if (depth != 8) return false;
131-
//Reasonable validation checks (PSP seems to only send images of max size 480x272)
132-
if (width > 500) return false;
133-
if (height > 300) return false;
134-
135-
return true;
110+
try
111+
{
112+
byte imageIdLength = data[0];
113+
byte colorMapType = data[1];
114+
byte imageType = data[2];
115+
ReadOnlySpan<byte> colorMapSpecification = data[3..8];
116+
ReadOnlySpan<byte> imageSpecification = data[8..18];
117+
short xOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[..2]);
118+
short yOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[2..4]);
119+
ushort width = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[4..6]);
120+
ushort height = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[6..8]);
121+
byte depth = imageSpecification[8];
122+
byte descriptor = imageSpecification[9];
123+
124+
//PSP does not seem to fill out this information
125+
if (imageIdLength != 0) return false;
126+
if (xOrigin != 0) return false;
127+
if (yOrigin != 0) return false;
128+
//These are the fields set by PSP, that shouldn't change from image to image
129+
if (colorMapType != 1) return false;
130+
if (descriptor != 0) return false;
131+
if (imageType != 1) return false;
132+
if (depth != 8) return false;
133+
//Reasonable validation checks (PSP seems to only send images of max size 480x272)
134+
if (width > 500) return false;
135+
if (height > 300) return false;
136+
137+
return true;
138+
}
139+
catch
140+
{
141+
//If the data couldn't be read, it's not a TGA
142+
return false;
143+
}
136144
}
137145

138146
private bool IsMip(Span<byte> rawData)

‎Refresh.Core/Types/Assets/Validation/AssetValidationParameters.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using Bunkum.Core.Storage;
2+
using Refresh.Core.Importing;
3+
using Refresh.Core.Services;
4+
using Refresh.Core.Types.Data;
5+
using Refresh.Database;
6+
using Refresh.Database.Models.Authentication;
7+
using Refresh.Database.Models.Users;
8+
9+
namespace Refresh.Core.Types.Assets.Validation;
10+
11+
public struct AssetValidationParameters
12+
{
13+
/// <summary>
14+
/// The reference (hash/guid/blank) to validate
15+
/// </summary>
16+
public string AssetRef { get; set; } = "0";
17+
public GameUser? User { get; set; }
18+
public TokenGame GameToUseIn { get; set; }
19+
public TokenPlatform PlatformToUseIn { get; set; }
20+
public GameDatabaseContext Database { get; set; } = null!;
21+
public IDataStore DataStore { get; set; } = null!;
22+
public CacheService Cache { get; set; } = null!;
23+
public GuidCheckerService GuidChecker { get; set; } = null!;
24+
public AssetImporter AssetImporter { get; set; } = null!;
25+
public AipiService? Aipi { get; set; }
26+
27+
public bool MayBeBlank { get; set; } = true;
28+
public bool MayBeGuid { get; set; } = true;
29+
public bool MayBeHash { get; set; } = true;
30+
public bool MustBeInDataStoreIfHash { get; set; } = true;
31+
public bool MustBeTexture { get; set; } = false;
32+
33+
/// <summary>
34+
/// What the asset should be referred as in user-faced error messages and in logs, e.g. "planet asset" or "icon".
35+
/// If null, we will default to calling it "asset" or "image" depending on MustBeTexture.
36+
/// </summary>
37+
public string? AssetContextTypeStr { get; set; }
38+
39+
/// <summary>
40+
/// Callback which is called with the new asset reference as parameter when constructing <see cref="ValidatedAssetResult"/>;
41+
/// useful to update asset references of entities during validation without requiring the caller to manually reassign them after validation;
42+
/// this way similar attributes like photo images or face icons can simply be iterated.
43+
/// If null, this will be skipped.
44+
/// </summary>
45+
public Action<string>? OnNewAssetRefCallback { get; set; }
46+
47+
public AssetValidationParameters(string assetKey, DataContext dataContext, AssetImporter assetImporter, AipiService? aipi = null)
48+
{
49+
this.AssetRef = assetKey;
50+
this.User = dataContext.User;
51+
this.GameToUseIn = dataContext.Game;
52+
this.PlatformToUseIn = dataContext.Platform;
53+
this.Database = dataContext.Database;
54+
this.DataStore = dataContext.DataStore;
55+
this.Cache = dataContext.Cache;
56+
this.GuidChecker = dataContext.GuidChecker;
57+
this.AssetImporter = assetImporter;
58+
this.Aipi = aipi;
59+
}
60+
61+
public AssetValidationParameters()
62+
{
63+
64+
}
65+
}

‎Refresh.Core/Types/Assets/Validation/ValidatedAssetResult.cs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
using System.Net;
2+
using Refresh.Database.Models.Assets;
3+
4+
namespace Refresh.Core.Types.Assets.Validation;
5+
6+
public struct ValidatedAssetResult
7+
{
8+
/// <summary>
9+
/// HTTP code to return if validation failed.
10+
/// OK: don't cancel request and proceed.
11+
/// </summary>
12+
public HttpStatusCode Status { get; set; }
13+
14+
/// <summary>
15+
/// new reference (hash/guid/blank) to use
16+
/// </summary>
17+
public string NewAssetRef { get; set; }
18+
19+
/// <summary>
20+
/// message to show to the user.
21+
/// null: don't show anything.
22+
/// </summary>
23+
public string? ErrorMessage { get; set; }
24+
25+
public GameAsset? AssetInfo { get; set; }
26+
public DisallowedAsset? DisallowanceInfo { get; set; }
27+
public bool ExistsInDataStore { get; set; }
28+
29+
public ValidatedAssetResult(HttpStatusCode status, string? newAssetRef = null, string? errorMessage = null, Action<string>? onNewAssetRefCallback = null,
30+
GameAsset? assetInfo = null, DisallowedAsset? disallowanceInfo = null, bool existsInDataStore = false)
31+
{
32+
this.Status = status;
33+
this.NewAssetRef = newAssetRef ?? "0";
34+
this.ErrorMessage = errorMessage;
35+
this.AssetInfo = assetInfo;
36+
this.DisallowanceInfo = disallowanceInfo;
37+
this.ExistsInDataStore = existsInDataStore;
38+
39+
onNewAssetRefCallback?.Invoke(this.NewAssetRef);
40+
}
41+
}

‎Refresh.Database/Models/Assets/AssetFlags.cs‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ public enum AssetFlags
2020
/// A planet is considered modded if it depends on this asset, or if the asset already has the Modded flag above.
2121
/// </summary>
2222
ModdedOnPlanets = 1 << 3,
23+
/// <summary>
24+
/// This asset is a texture or contains imagery, for now only given to asset types handled by image conversion.
25+
/// </summary>
26+
Imagery = 1 << 4,
2327
}
2428

2529
public static class AssetSafetyLevelExtensions
@@ -30,33 +34,33 @@ public static AssetFlags FromAssetType(GameAssetType type, GameAssetFormat? meth
3034
{
3135
// Common asset types created by the game
3236
GameAssetType.Level => AssetFlags.None,
33-
GameAssetType.StreamingLevelChunk => AssetFlags.None | AssetFlags.ModdedOnPlanets,
37+
GameAssetType.StreamingLevelChunk => AssetFlags.ModdedOnPlanets,
3438
GameAssetType.Plan => AssetFlags.None,
35-
GameAssetType.ThingRecording => AssetFlags.None | AssetFlags.ModdedOnPlanets,
36-
GameAssetType.SyncedProfile => AssetFlags.None | AssetFlags.ModdedOnPlanets,
37-
GameAssetType.GriefSongState => AssetFlags.None | AssetFlags.ModdedOnPlanets,
38-
GameAssetType.Quest => AssetFlags.None | AssetFlags.ModdedOnPlanets,
39-
GameAssetType.AdventureSharedData => AssetFlags.None | AssetFlags.ModdedOnPlanets,
40-
GameAssetType.AdventureCreateProfile => AssetFlags.None | AssetFlags.ModdedOnPlanets,
41-
GameAssetType.ChallengeGhost => AssetFlags.None | AssetFlags.ModdedOnPlanets,
39+
GameAssetType.ThingRecording => AssetFlags.ModdedOnPlanets,
40+
GameAssetType.SyncedProfile => AssetFlags.ModdedOnPlanets,
41+
GameAssetType.GriefSongState => AssetFlags.ModdedOnPlanets,
42+
GameAssetType.Quest => AssetFlags.ModdedOnPlanets,
43+
GameAssetType.AdventureSharedData => AssetFlags.ModdedOnPlanets,
44+
GameAssetType.AdventureCreateProfile => AssetFlags.ModdedOnPlanets,
45+
GameAssetType.ChallengeGhost => AssetFlags.ModdedOnPlanets,
4246

4347
// Common media types created by the game
4448
GameAssetType.VoiceRecording => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
4549
GameAssetType.Painting => AssetFlags.Media,
46-
GameAssetType.Texture => AssetFlags.Media,
47-
GameAssetType.Jpeg => AssetFlags.Media,
48-
GameAssetType.Png => AssetFlags.Media,
49-
GameAssetType.Tga => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
50-
GameAssetType.Mip => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
50+
GameAssetType.Texture => AssetFlags.Media | AssetFlags.Imagery,
51+
GameAssetType.Jpeg => AssetFlags.Media | AssetFlags.Imagery,
52+
GameAssetType.Png => AssetFlags.Media | AssetFlags.Imagery,
53+
GameAssetType.Tga => AssetFlags.Media | AssetFlags.Imagery | AssetFlags.ModdedOnPlanets,
54+
GameAssetType.Mip => AssetFlags.Media | AssetFlags.Imagery | AssetFlags.ModdedOnPlanets,
5155

5256
// Uncommon, but still vanilla assets created by the game in niche scenarios.
5357
// While not image/audio data like the other media types, GfxMaterial is marked as media because this file can contain full PS3 shaders.
5458
GameAssetType.GfxMaterial => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
55-
GameAssetType.Material => AssetFlags.None | AssetFlags.ModdedOnPlanets,
56-
GameAssetType.Bevel => AssetFlags.None | AssetFlags.ModdedOnPlanets,
59+
GameAssetType.Material => AssetFlags.ModdedOnPlanets,
60+
GameAssetType.Bevel => AssetFlags.ModdedOnPlanets,
5761

5862
// Modded media types
59-
GameAssetType.GameDataTexture => AssetFlags.Media | AssetFlags.Modded,
63+
GameAssetType.GameDataTexture => AssetFlags.Media | AssetFlags.Modded | AssetFlags.Imagery,
6064
GameAssetType.AnimatedTexture => AssetFlags.Media | AssetFlags.Modded,
6165

6266
// Normal modded assets

0 commit comments

Comments
 (0)