Skip to content

Fix a couple issues found by ASan/UBSan in the Metal backend #1844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
fabioarnold wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Fix a couple issues found by ASan/UBSan in the Metal backend #1844

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2ec9c00
Metal: convert triangle fan to triangle list
fabioarnold Mar 16, 2026
9870810
Metal: prevent OOB write in `MetalRenderer()`
fabioarnold Mar 18, 2026
9ad8b44
Metal: 0-init pointers in MetalPipelineCache
fabioarnold Mar 18, 2026
5554699
Metal: fix null pointer use in LatteTextureViewMtl destructor
fabioarnold Mar 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 44 additions & 25 deletions src/Cafe/HW/Latte/Core/LatteIndices.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,16 +117,22 @@ uint32 LatteIndices_calculateIndexOutputSize(LattePrimitiveMode primitiveMode, L
}
else if (primitiveMode == LattePrimitiveMode::TRIANGLE_FAN && g_renderer->GetType() == RendererAPI::Metal)
{
// Metal doesn't support triangle fan -> convert to triangle list
if (count <= 2)
{
return 0;
}
uint32 numTriangles = count - 2;
if (indexType == LatteIndexType::AUTO)
{
if (count <= 0xFFFF)
return count * sizeof(uint16);
return count * sizeof(uint32);
return numTriangles * 3 * sizeof(uint16);
return numTriangles * 3 * sizeof(uint32);
}
if (indexType == LatteIndexType::U16_BE || indexType == LatteIndexType::U16_LE)
return count * sizeof(uint16);
return numTriangles * 3 * sizeof(uint16);
if (indexType == LatteIndexType::U32_BE || indexType == LatteIndexType::U32_LE)
return count * sizeof(uint32);
return numTriangles * 3 * sizeof(uint32);
cemu_assert_suspicious();
return 0;
}
Expand Down Expand Up @@ -326,36 +332,46 @@ void LatteIndices_generateAutoLineLoopIndices(void* indexDataOutput, uint32 coun
template<typename T>
void LatteIndices_unpackTriangleFanAndConvert(const void* indexDataInput, void* indexDataOutput, uint32 count, uint32& indexMin, uint32& indexMax)
{
if (count <= 2)
return;
sint32 numTriangles = count - 2;
const betype<T>* src = (betype<T>*)indexDataInput;
T idx0 = src[0];
indexMin = std::min(indexMin, (uint32)idx0);
indexMax = std::max(indexMax, (uint32)idx0);
T* dst = (T*)indexDataOutput;
// TODO: check this
for (sint32 i = 0; i < count; i++)
{
uint32 i0;
if (i % 2 == 0)
i0 = i / 2;
else
i0 = count - 1 - i / 2;
T idx = src[i0];
indexMin = std::min(indexMin, (uint32)idx);
indexMax = std::max(indexMax, (uint32)idx);
dst[i] = idx;
for (sint32 i = 0; i < numTriangles; i++)
{
T idx1 = src[1];
T idx2 = src[2];
indexMin = std::min(indexMin, (uint32)idx1);
indexMax = std::max(indexMax, (uint32)idx1);
indexMin = std::min(indexMin, (uint32)idx2);
indexMax = std::max(indexMax, (uint32)idx2);
dst[0] = idx0;
dst[1] = idx1;
dst[2] = idx2;
src += 1;
dst += 3;
}
}

template<typename T>
void LatteIndices_generateAutoTriangleFanIndices(const void* indexDataInput, void* indexDataOutput, uint32 count, uint32& indexMin, uint32& indexMax)
{
const betype<T>* src = (betype<T>*)indexDataInput;
if (count <= 2)
return;
sint32 numTriangles = count - 2;
T* dst = (T*)indexDataOutput;
for (sint32 i = 0; i < count; i++)
for (sint32 i = 0; i < numTriangles; i++)
{
T idx = i;
if (idx % 2 == 0)
idx = idx / 2;
else
idx = count - 1 - idx / 2;
dst[i] = idx;
T idx0 = 0;
T idx1 = i + 1;
T idx2 = i + 2;
dst[0] = idx0;
dst[1] = idx1;
dst[2] = idx2;
dst += 3;
}
indexMin = 0;
indexMax = std::max(count, 1u) - 1;
Expand Down Expand Up @@ -868,7 +884,10 @@ void LatteIndices_decode(const void* indexData, LatteIndexType indexType, uint32
LatteIndices_unpackTriangleFanAndConvert<uint32>(indexData, indexOutputPtr, count, indexMin, indexMax);
else
cemu_assert_debug(false);
outputCount = count;
if (count >= 2)
outputCount = (count - 2) * 3;
else
outputCount = 0;
}
else
{
Expand Down
8 changes: 4 additions & 4 deletions src/Cafe/HW/Latte/Renderer/Metal/LatteTextureViewMtl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,17 +104,17 @@ MTL::Texture* LatteTextureViewMtl::GetSwizzledView(uint32 gpuSamplerSwizzle)
}

// Fallback cache
auto& fallbackEntry = m_fallbackViewCache[gpuSamplerSwizzle];
if (fallbackEntry)
auto it = m_fallbackViewCache.find(gpuSamplerSwizzle);
if (it != m_fallbackViewCache.end())
{
return fallbackEntry;
return it->second;
}

MTL::Texture* texture = CreateSwizzledView(gpuSamplerSwizzle);
if (freeIndex != -1)
m_viewCache[freeIndex] = {gpuSamplerSwizzle, texture};
else
fallbackEntry = texture;
m_fallbackViewCache[gpuSamplerSwizzle] = texture;

return texture;
}
Expand Down
2 changes: 1 addition & 1 deletion src/Cafe/HW/Latte/Renderer/Metal/LatteToMtl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ MTL::PrimitiveType GetMtlPrimitiveType(LattePrimitiveMode primitiveMode)
case Latte::LATTE_VGT_PRIMITIVE_TYPE::E_PRIMITIVE_TYPE::TRIANGLES:
return MTL::PrimitiveTypeTriangle;
case Latte::LATTE_VGT_PRIMITIVE_TYPE::E_PRIMITIVE_TYPE::TRIANGLE_FAN:
return MTL::PrimitiveTypeTriangleStrip;
return MTL::PrimitiveTypeTriangle; // triangle fans are emulated as a triangle list
case Latte::LATTE_VGT_PRIMITIVE_TYPE::E_PRIMITIVE_TYPE::TRIANGLE_STRIP:
return MTL::PrimitiveTypeTriangleStrip;
case Latte::LATTE_VGT_PRIMITIVE_TYPE::E_PRIMITIVE_TYPE::QUADS:
Expand Down
4 changes: 2 additions & 2 deletions src/Cafe/HW/Latte/Renderer/Metal/MetalPipelineCache.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ class MetalPipelineCache
std::map<uint64, PipelineObject*> m_pipelineCache;
FSpinlock m_pipelineCacheLock;

std::thread* m_pipelineCacheStoreThread;
std::thread* m_pipelineCacheStoreThread = nullptr;

class FileCache* s_cache;
class FileCache* s_cache = nullptr;

std::atomic_uint32_t m_numCompilationThreads{ 0 };
ConcurrentQueue<std::vector<uint8>> m_compilationQueue;
Expand Down
2 changes: 1 addition & 1 deletion src/Cafe/HW/Latte/Renderer/Metal/MetalRenderer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ MetalRenderer::MetalRenderer()
for (uint32 i = 0; i < MAX_MTL_VERTEX_BUFFERS; i++)
m_state.m_vertexBufferOffsets[i] = INVALID_OFFSET;

for (uint32 i = 0; i < METAL_SHADER_TYPE_TOTAL; i++)
for (uint32 i = 0; i < METAL_GENERAL_SHADER_TYPE_TOTAL; i++)
{
for (uint32 j = 0; j < MAX_MTL_BUFFERS; j++)
m_state.m_uniformBufferOffsets[i][j] = INVALID_OFFSET;
Expand Down