Skip to content

Commit 2aed7e2

Browse files
Toastbrot236Toastbrot236
authored
Merge branch 'main' into asset-include
2 parents 9fac398 + 602ad61 commit 2aed7e2

7 files changed

Lines changed: 832 additions & 51 deletions

File tree

‎Refresh.Core/Helpers/ResourceValidationHelper.cs‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
using System.Diagnostics;
2+
using Bunkum.Core;
3+
using NotEnoughLogs;
4+
using Refresh.Common.Verification;
5+
using Refresh.Core.Types.Assets.Validation;
6+
using Refresh.Database.Models.Assets;
7+
using Refresh.Database.Models.Authentication;
8+
9+
namespace Refresh.Core.Helpers;
10+
11+
public abstract class ResourceValidationHelper
12+
{
13+
/// <summary>
14+
/// Validates the given asset reference (hash/guid/blank) using the given parameters, if necessary also by reading the referenced asset
15+
/// or getting data about it from database (see <see cref="GameAsset"/> and <see cref="DisallowedAsset"/>).
16+
/// </summary>
17+
public static ValidatedAssetResult ValidateReference(AssetValidationParameters parameters, Logger logger)
18+
{
19+
string assetTypeStr = parameters.AssetContextTypeStr ?? (parameters.MustBeTexture ? "image asset" : "asset");
20+
GameAsset? asset = null;
21+
bool existsInDataStore = false;
22+
bool isPSP = parameters.GameToUseIn == TokenGame.LittleBigPlanetPSP;
23+
Action<string>? onNewAssetRefCallback = parameters.OnNewAssetRefCallback;
24+
25+
if (parameters.AssetRef.IsBlankHash())
26+
{
27+
if (!parameters.MayBeBlank) return new(BadRequest, "0", $"The {assetTypeStr} must be set.", onNewAssetRefCallback);
28+
else return new(OK, "0", null, onNewAssetRefCallback);
29+
}
30+
31+
else if (parameters.AssetRef.StartsWith('g'))
32+
{
33+
if (!parameters.MayBeGuid) return new(BadRequest, null, $"The {assetTypeStr} may not be an in-game asset.", onNewAssetRefCallback);
34+
if (parameters.AssetRef.Length < 2) return new(BadRequest, null, $"The used in-game {assetTypeStr} is invalid (empty GUID).", onNewAssetRefCallback);
35+
36+
// This should only happen if the user is messing with mods/the API/beta builds, so give them a more detailed response
37+
bool canParseGuid = long.TryParse(parameters.AssetRef[1..], out long guid);
38+
if (!canParseGuid)
39+
return new(BadRequest, null, $"The used in-game {assetTypeStr} is invalid (badly formatted GUID).", onNewAssetRefCallback);
40+
41+
if (parameters.MustBeTexture && !parameters.GuidChecker.IsTextureGuid(parameters.GameToUseIn, guid))
42+
return new(BadRequest, null, $"The used in-game {assetTypeStr} was not a valid image (unknown GUID).", onNewAssetRefCallback);
43+
}
44+
45+
// At this point the reference is a hash
46+
else if (!parameters.MayBeHash)
47+
{
48+
return new(BadRequest, null, $"The {assetTypeStr} may not be a custom asset.", onNewAssetRefCallback);
49+
}
50+
51+
else if (!CommonPatterns.Sha1Regex().IsMatch(parameters.AssetRef))
52+
{
53+
// This should only happen if a player is messing with mods/the API, so give them a more detailed response
54+
return new(BadRequest, null, $"The used {assetTypeStr} had an invalid hash.", onNewAssetRefCallback);
55+
}
56+
57+
else
58+
{
59+
DisallowedAsset? disallowed = parameters.Database.GetDisallowedAssetInfo(parameters.AssetRef);
60+
if (disallowed != null)
61+
{
62+
logger.LogWarning(BunkumCategory.UserContent, $"{parameters.User} tried to use a manually disallowed {assetTypeStr}.");
63+
return new(Unauthorized, disallowanceInfo: disallowed, onNewAssetRefCallback: onNewAssetRefCallback);
64+
}
65+
66+
string filename = isPSP ? $"psp/{parameters.AssetRef}" : parameters.AssetRef;
67+
existsInDataStore = parameters.DataStore.ExistsInStore(filename);
68+
69+
if (!existsInDataStore)
70+
{
71+
logger.LogDebug(BunkumCategory.UserContent, $"Referenced asset '{filename}' could not be found in data store.");
72+
73+
if (parameters.MustBeInDataStoreIfHash)
74+
return new(NotFound, null, $"The used {assetTypeStr} did not exist on the server.", onNewAssetRefCallback);
75+
}
76+
77+
asset = parameters.Cache.GetAssetInfo(parameters.AssetRef, parameters.Database);
78+
79+
// Only try to import if the asset exists in the data store
80+
if (existsInDataStore && asset == null)
81+
{
82+
logger.LogInfo(BunkumCategory.UserContent, $"Referenced asset '{filename}' exists in data store but not in database, attempting to import automatically...");
83+
Stopwatch sw = new();
84+
sw.Start();
85+
86+
if (!parameters.DataStore.TryGetDataFromStore(filename, out byte[]? assetData) || assetData == null)
87+
{
88+
sw.Stop();
89+
logger.LogError(BunkumCategory.UserContent, $"Failed to read '{filename}' from data store!");
90+
logger.LogDebug(BunkumCategory.UserContent, $"Failed to get '{filename}' after {sw.ElapsedMilliseconds}ms.");
91+
return new(InternalServerError, null, $"Failed to read {assetTypeStr} internally. Please report this to the server owner.", onNewAssetRefCallback, existsInDataStore: existsInDataStore);
92+
}
93+
94+
asset = parameters.AssetImporter.ReadAndVerifyAsset(parameters.AssetRef, assetData, parameters.PlatformToUseIn, parameters.Database);
95+
if (asset == null)
96+
{
97+
sw.Stop();
98+
logger.LogDebug(BunkumCategory.UserContent, $"Failed to get '{filename}' after {sw.ElapsedMilliseconds}ms.");
99+
return new(BadRequest, null, $"The used {assetTypeStr} was invalid or corrupt.", onNewAssetRefCallback, existsInDataStore: existsInDataStore);
100+
}
101+
102+
sw.Stop();
103+
logger.LogInfo(BunkumCategory.UserContent, $"Successfully imported '{filename}' in {sw.ElapsedMilliseconds}ms.");
104+
}
105+
106+
// FIXME: for some reason, PSP texture detection/conversion broke so we can no longer tell if a PSP texture is actually a texture, so skip this for PSP
107+
if (asset != null && !isPSP)
108+
{
109+
bool isHashedTexture = (asset.AssetFlags & AssetFlags.Imagery) != 0;
110+
111+
if (parameters.MustBeTexture && !isHashedTexture)
112+
return new(BadRequest, null, $"The used {assetTypeStr} was not a valid custom image.", onNewAssetRefCallback, assetInfo: asset, existsInDataStore: existsInDataStore);
113+
114+
// TODO: actually use AIPI to scan image if not null
115+
}
116+
}
117+
118+
return new(OK, parameters.AssetRef, null, onNewAssetRefCallback, assetInfo: asset, existsInDataStore: existsInDataStore);
119+
}
120+
}

‎Refresh.Core/Importing/Importer.cs‎

Lines changed: 34 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -107,32 +107,40 @@ private static bool MatchesMagic(ReadOnlySpan<byte> data, ulong magic)
107107
/// <returns>Whether the file is likely of TGA format</returns>
108108
private static bool IsPspTga(ReadOnlySpan<byte> data)
109109
{
110-
byte imageIdLength = data[0];
111-
byte colorMapType = data[1];
112-
byte imageType = data[2];
113-
ReadOnlySpan<byte> colorMapSpecification = data[3..8];
114-
ReadOnlySpan<byte> imageSpecification = data[8..18];
115-
short xOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[..2]);
116-
short yOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[2..4]);
117-
ushort width = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[4..6]);
118-
ushort height = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[6..8]);
119-
byte depth = imageSpecification[8];
120-
byte descriptor = imageSpecification[9];
121-
122-
//PSP does not seem to fill out this information
123-
if (imageIdLength != 0) return false;
124-
if (xOrigin != 0) return false;
125-
if (yOrigin != 0) return false;
126-
//These are the fields set by PSP, that shouldn't change from image to image
127-
if (colorMapType != 1) return false;
128-
if (descriptor != 0) return false;
129-
if (imageType != 1) return false;
130-
if (depth != 8) return false;
131-
//Reasonable validation checks (PSP seems to only send images of max size 480x272)
132-
if (width > 500) return false;
133-
if (height > 300) return false;
134-
135-
return true;
110+
try
111+
{
112+
byte imageIdLength = data[0];
113+
byte colorMapType = data[1];
114+
byte imageType = data[2];
115+
ReadOnlySpan<byte> colorMapSpecification = data[3..8];
116+
ReadOnlySpan<byte> imageSpecification = data[8..18];
117+
short xOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[..2]);
118+
short yOrigin = BinaryPrimitives.ReadInt16LittleEndian(imageSpecification[2..4]);
119+
ushort width = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[4..6]);
120+
ushort height = BinaryPrimitives.ReadUInt16LittleEndian(imageSpecification[6..8]);
121+
byte depth = imageSpecification[8];
122+
byte descriptor = imageSpecification[9];
123+
124+
//PSP does not seem to fill out this information
125+
if (imageIdLength != 0) return false;
126+
if (xOrigin != 0) return false;
127+
if (yOrigin != 0) return false;
128+
//These are the fields set by PSP, that shouldn't change from image to image
129+
if (colorMapType != 1) return false;
130+
if (descriptor != 0) return false;
131+
if (imageType != 1) return false;
132+
if (depth != 8) return false;
133+
//Reasonable validation checks (PSP seems to only send images of max size 480x272)
134+
if (width > 500) return false;
135+
if (height > 300) return false;
136+
137+
return true;
138+
}
139+
catch
140+
{
141+
//If the data couldn't be read, it's not a TGA
142+
return false;
143+
}
136144
}
137145

138146
private bool IsMip(Span<byte> rawData)

‎Refresh.Core/Types/Assets/Validation/AssetValidationParameters.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
using Bunkum.Core.Storage;
2+
using Refresh.Core.Importing;
3+
using Refresh.Core.Services;
4+
using Refresh.Core.Types.Data;
5+
using Refresh.Database;
6+
using Refresh.Database.Models.Authentication;
7+
using Refresh.Database.Models.Users;
8+
9+
namespace Refresh.Core.Types.Assets.Validation;
10+
11+
public struct AssetValidationParameters
12+
{
13+
/// <summary>
14+
/// The reference (hash/guid/blank) to validate
15+
/// </summary>
16+
public string AssetRef { get; set; } = "0";
17+
public GameUser? User { get; set; }
18+
public TokenGame GameToUseIn { get; set; }
19+
public TokenPlatform PlatformToUseIn { get; set; }
20+
public GameDatabaseContext Database { get; set; } = null!;
21+
public IDataStore DataStore { get; set; } = null!;
22+
public CacheService Cache { get; set; } = null!;
23+
public GuidCheckerService GuidChecker { get; set; } = null!;
24+
public AssetImporter AssetImporter { get; set; } = null!;
25+
public AipiService? Aipi { get; set; }
26+
27+
public bool MayBeBlank { get; set; } = true;
28+
public bool MayBeGuid { get; set; } = true;
29+
public bool MayBeHash { get; set; } = true;
30+
public bool MustBeInDataStoreIfHash { get; set; } = true;
31+
public bool MustBeTexture { get; set; } = false;
32+
33+
/// <summary>
34+
/// What the asset should be referred as in user-faced error messages and in logs, e.g. "planet asset" or "icon".
35+
/// If null, we will default to calling it "asset" or "image" depending on MustBeTexture.
36+
/// </summary>
37+
public string? AssetContextTypeStr { get; set; }
38+
39+
/// <summary>
40+
/// Callback which is called with the new asset reference as parameter when constructing <see cref="ValidatedAssetResult"/>;
41+
/// useful to update asset references of entities during validation without requiring the caller to manually reassign them after validation;
42+
/// this way similar attributes like photo images or face icons can simply be iterated.
43+
/// If null, this will be skipped.
44+
/// </summary>
45+
public Action<string>? OnNewAssetRefCallback { get; set; }
46+
47+
public AssetValidationParameters(string assetKey, DataContext dataContext, AssetImporter assetImporter, AipiService? aipi = null)
48+
{
49+
this.AssetRef = assetKey;
50+
this.User = dataContext.User;
51+
this.GameToUseIn = dataContext.Game;
52+
this.PlatformToUseIn = dataContext.Platform;
53+
this.Database = dataContext.Database;
54+
this.DataStore = dataContext.DataStore;
55+
this.Cache = dataContext.Cache;
56+
this.GuidChecker = dataContext.GuidChecker;
57+
this.AssetImporter = assetImporter;
58+
this.Aipi = aipi;
59+
}
60+
61+
public AssetValidationParameters()
62+
{
63+
64+
}
65+
}

‎Refresh.Core/Types/Assets/Validation/ValidatedAssetResult.cs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
using System.Net;
2+
using Refresh.Database.Models.Assets;
3+
4+
namespace Refresh.Core.Types.Assets.Validation;
5+
6+
public struct ValidatedAssetResult
7+
{
8+
/// <summary>
9+
/// HTTP code to return if validation failed.
10+
/// OK: don't cancel request and proceed.
11+
/// </summary>
12+
public HttpStatusCode Status { get; set; }
13+
14+
/// <summary>
15+
/// new reference (hash/guid/blank) to use
16+
/// </summary>
17+
public string NewAssetRef { get; set; }
18+
19+
/// <summary>
20+
/// message to show to the user.
21+
/// null: don't show anything.
22+
/// </summary>
23+
public string? ErrorMessage { get; set; }
24+
25+
public GameAsset? AssetInfo { get; set; }
26+
public DisallowedAsset? DisallowanceInfo { get; set; }
27+
public bool ExistsInDataStore { get; set; }
28+
29+
public ValidatedAssetResult(HttpStatusCode status, string? newAssetRef = null, string? errorMessage = null, Action<string>? onNewAssetRefCallback = null,
30+
GameAsset? assetInfo = null, DisallowedAsset? disallowanceInfo = null, bool existsInDataStore = false)
31+
{
32+
this.Status = status;
33+
this.NewAssetRef = newAssetRef ?? "0";
34+
this.ErrorMessage = errorMessage;
35+
this.AssetInfo = assetInfo;
36+
this.DisallowanceInfo = disallowanceInfo;
37+
this.ExistsInDataStore = existsInDataStore;
38+
39+
onNewAssetRefCallback?.Invoke(this.NewAssetRef);
40+
}
41+
}

‎Refresh.Database/Models/Assets/AssetFlags.cs‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ public enum AssetFlags
2020
/// A planet is considered modded if it depends on this asset, or if the asset already has the Modded flag above.
2121
/// </summary>
2222
ModdedOnPlanets = 1 << 3,
23+
/// <summary>
24+
/// This asset is a texture or contains imagery, for now only given to asset types handled by image conversion.
25+
/// </summary>
26+
Imagery = 1 << 4,
2327
}
2428

2529
public static class AssetSafetyLevelExtensions
@@ -30,33 +34,33 @@ public static AssetFlags FromAssetType(GameAssetType type, GameAssetFormat? meth
3034
{
3135
// Common asset types created by the game
3236
GameAssetType.Level => AssetFlags.None,
33-
GameAssetType.StreamingLevelChunk => AssetFlags.None | AssetFlags.ModdedOnPlanets,
37+
GameAssetType.StreamingLevelChunk => AssetFlags.ModdedOnPlanets,
3438
GameAssetType.Plan => AssetFlags.None,
35-
GameAssetType.ThingRecording => AssetFlags.None | AssetFlags.ModdedOnPlanets,
36-
GameAssetType.SyncedProfile => AssetFlags.None | AssetFlags.ModdedOnPlanets,
37-
GameAssetType.GriefSongState => AssetFlags.None | AssetFlags.ModdedOnPlanets,
38-
GameAssetType.Quest => AssetFlags.None | AssetFlags.ModdedOnPlanets,
39-
GameAssetType.AdventureSharedData => AssetFlags.None | AssetFlags.ModdedOnPlanets,
40-
GameAssetType.AdventureCreateProfile => AssetFlags.None | AssetFlags.ModdedOnPlanets,
41-
GameAssetType.ChallengeGhost => AssetFlags.None | AssetFlags.ModdedOnPlanets,
39+
GameAssetType.ThingRecording => AssetFlags.ModdedOnPlanets,
40+
GameAssetType.SyncedProfile => AssetFlags.ModdedOnPlanets,
41+
GameAssetType.GriefSongState => AssetFlags.ModdedOnPlanets,
42+
GameAssetType.Quest => AssetFlags.ModdedOnPlanets,
43+
GameAssetType.AdventureSharedData => AssetFlags.ModdedOnPlanets,
44+
GameAssetType.AdventureCreateProfile => AssetFlags.ModdedOnPlanets,
45+
GameAssetType.ChallengeGhost => AssetFlags.ModdedOnPlanets,
4246

4347
// Common media types created by the game
4448
GameAssetType.VoiceRecording => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
4549
GameAssetType.Painting => AssetFlags.Media,
46-
GameAssetType.Texture => AssetFlags.Media,
47-
GameAssetType.Jpeg => AssetFlags.Media,
48-
GameAssetType.Png => AssetFlags.Media,
49-
GameAssetType.Tga => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
50-
GameAssetType.Mip => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
50+
GameAssetType.Texture => AssetFlags.Media | AssetFlags.Imagery,
51+
GameAssetType.Jpeg => AssetFlags.Media | AssetFlags.Imagery,
52+
GameAssetType.Png => AssetFlags.Media | AssetFlags.Imagery,
53+
GameAssetType.Tga => AssetFlags.Media | AssetFlags.Imagery | AssetFlags.ModdedOnPlanets,
54+
GameAssetType.Mip => AssetFlags.Media | AssetFlags.Imagery | AssetFlags.ModdedOnPlanets,
5155

5256
// Uncommon, but still vanilla assets created by the game in niche scenarios.
5357
// While not image/audio data like the other media types, GfxMaterial is marked as media because this file can contain full PS3 shaders.
5458
GameAssetType.GfxMaterial => AssetFlags.Media | AssetFlags.ModdedOnPlanets,
55-
GameAssetType.Material => AssetFlags.None | AssetFlags.ModdedOnPlanets,
56-
GameAssetType.Bevel => AssetFlags.None | AssetFlags.ModdedOnPlanets,
59+
GameAssetType.Material => AssetFlags.ModdedOnPlanets,
60+
GameAssetType.Bevel => AssetFlags.ModdedOnPlanets,
5761

5862
// Modded media types
59-
GameAssetType.GameDataTexture => AssetFlags.Media | AssetFlags.Modded,
63+
GameAssetType.GameDataTexture => AssetFlags.Media | AssetFlags.Modded | AssetFlags.Imagery,
6064
GameAssetType.AnimatedTexture => AssetFlags.Media | AssetFlags.Modded,
6165

6266
// Normal modded assets

0 commit comments

Comments
 (0)