When VRAM is exhausted, flush gsKit draw commands before evicting an atlas that was used in the current frame to avoid graphics corruption

This also adds a "[THRASHING]" debug text when we tried evicting a texture atlas that was being used in the current frame

Author: MrPowerGamerBR

src/ps2/gs_renderer.c

@@ -296,7 +296,7 @@ static uint32_t countFreeChunks(GsRenderer* gs) {
 
 // Find the atlas with the oldest lastUsed time (LRU victim).
 // Returns the atlasId, or -1 if no loaded atlases.
-static int16_t findLRUVictim(GsRenderer* gs) {
+static int16_t findLRUVictim(GsRenderer* gs, bool* wasUsedOnThisFrame) {
     uint64_t oldest = UINT64_MAX;
     int16_t victimAtlas = -1;
     forEach(VRAMChunk, chunk, gs->chunks, gs->chunkCount) {
@@ -305,6 +305,8 @@ static int16_t findLRUVictim(GsRenderer* gs) {
             victimAtlas = chunk->atlasId;
         }
     }
+    if (victimAtlas != -1)
+        *wasUsedOnThisFrame = oldest == gs->frameCounter;
     return victimAtlas;
 }
 
@@ -351,10 +353,20 @@ static int32_t allocateChunks(GsRenderer* gs, int chunksNeeded) {
 
     // Attempt 2: evict LRU victims one at a time until space is found
     repeat(gs->chunkCount, attempts) {
-        int16_t victim = findLRUVictim(gs);
+        bool wasUsedOnThisFrame = false;
+
+        int16_t victim = findLRUVictim(gs, &wasUsedOnThisFrame);
         if (0 > victim)
             break;
 
+        // We only need to flush if the victim was used on this frame
+        // If it wasn't, then we can evict with no care in the world
+        if (wasUsedOnThisFrame) {
+            fprintf(stderr, "GsRenderer: Flushing draw queue before VRAM evicting because atlas was used on the current frame\n");
+            gs->evictedAtlasUsedInCurrentFrame = true;
+            gsKit_queue_exec(gs->gsGlobal);
+        }
+
         evictAtlas(gs, victim);
 
         idx = findConsecutiveFreeChunks(gs, chunksNeeded);
@@ -363,6 +375,11 @@ static int32_t allocateChunks(GsRenderer* gs, int chunksNeeded) {
             return idx;
     }
 
+    // At this point we are lost, just flush and hope for the best
+    gs->evictedAtlasUsedInCurrentFrame = true;
+    fprintf(stderr, "GsRenderer: Flushing draw queue before VRAM defrag\n");
+    gsKit_queue_exec(gs->gsGlobal);
+
     // Attempt 3: defrag - evict ALL and let them reload on demand
     // Handles fragmentation where enough free chunks exist but aren't consecutive
     if (countFreeChunks(gs) >= (uint32_t) chunksNeeded) {
@@ -901,6 +918,7 @@ static void gsBeginFrame(Renderer* renderer, MAYBE_UNUSED int32_t gameW, MAYBE_U
     GsRenderer* gs = (GsRenderer*) renderer;
     gs->zCounter = 1;
     gs->frameCounter++;
+    gs->evictedAtlasUsedInCurrentFrame = false;
 }
 
 static void gsEndFrame(MAYBE_UNUSED Renderer* renderer) {

src/ps2/gs_renderer.h

@@ -117,6 +117,7 @@ typedef struct {
     uint16_t atlasCount;       // Number of atlas IDs from ATLAS.BIN header
     uint8_t* atlasBpp;         // Bits per pixel per atlas (4 or 8), from ATLAS.BIN [atlasCount]
     uint64_t frameCounter;     // Incremented each frame for LRU tracking
+    bool evictedAtlasUsedInCurrentFrame; // Used for debugging, true if a atlas that was used on the current frame was evicted (VRAM thrashing)
 
     // EE RAM atlas cache (stores compressed atlas data to avoid repeated CDVD reads)
     uint8_t* eeCache;                  // 4 MiB contiguous buffer

src/ps2/main.c

@@ -674,7 +674,7 @@ int main(int argc, char* argv[]) {
                 if (gsRenderer->eeCacheEntries[ai].atlasId >= 0) eeramAtlasCount++;
             }
 
-            snprintf(debugText, sizeof(debugText), "Tick: %.2fms\nFree: %d bytes\nVRAM Free: %lu bytes\nRoom Speed: %u%s\nAtlas: (%u, %u, %u)", tickTime, freeBytes, (unsigned long) vramFreeBytes, roomSpeed, speedCapRemoved ? " [UNCAPPED]" : "", vramAtlasCount, eeramAtlasCount, gsRenderer->atlasCount);
+            snprintf(debugText, sizeof(debugText), "Tick: %.2fms\nFree: %d bytes\nVRAM Free: %lu bytes\nRoom Speed: %u%s\nAtlas: (%u, %u, %u)%s", tickTime, freeBytes, (unsigned long) vramFreeBytes, roomSpeed, speedCapRemoved ? " [UNCAPPED]" : "", vramAtlasCount, eeramAtlasCount, gsRenderer->atlasCount, gsRenderer->evictedAtlasUsedInCurrentFrame ? " [THRASHING]" : "");
             gsKit_fontm_print_scaled(gsGlobal, gsFontM, 10.0f, 10.0f, 10, 0.6f, debugColor, debugText);
         }