GS/SW: Rewrite vertex ST values if they might overflow the rasterizer fixed point format.

Author: TJnotJT

pcsx2/GS/Renderers/SW/GSRendererSW.cpp

@@ -287,6 +287,125 @@ void ConvertVertexBuffer(const GSDrawingContext* RESTRICT ctx, GSVertexSW* RESTR
 	}
 }
 
+// Fix ST coordinates that would overflow the rasterizer fixed point format by rewriting the vertices.
+template <u32 primclass>
+void GSRendererSW::RewriteVerticesIfSTOverflow()
+{
+	if (PRIM->TME && PRIM->FST == 0)
+	{
+		const GSVector4 tsize = GSVector4(
+			static_cast<float>(1 << m_context->TEX0.TW),
+			static_cast<float>(1 << m_context->TEX0.TH),
+			1.0f,
+			1.0f);
+
+		// SW rasterizer stores UV in 1.15.16 format so clamp to +/- (2^15 - 2) (-2 so bilinear doesn't overflow).
+		// Do the division by texture size here to avoid divisions for each vertex.
+		const GSVector4 OVERFLOW_VAL = GSVector4::cxpr(static_cast<float>((1 << 15) - 2)) / tsize;
+
+		// Only rewrite big/small S or T when the clamping mode is CLAMP or REGION_CLAMP.
+		const GSVector4i clamp_mode = GSVector4i(
+			(m_context->CLAMP.WMS == CLAMP_CLAMP || m_context->CLAMP.WMS == CLAMP_REGION_CLAMP) ? 0xFFFFFFFF : 0,
+			(m_context->CLAMP.WMT == CLAMP_CLAMP || m_context->CLAMP.WMT == CLAMP_REGION_CLAMP) ? 0xFFFFFFFF : 0,
+			0,
+			0);
+
+		const bool st_overflow =
+			((GSVector4i::cast(m_vt.m_min.t <= -OVERFLOW_VAL * tsize) & clamp_mode).mask() & 3) ||
+			((GSVector4i::cast(m_vt.m_max.t >= OVERFLOW_VAL * tsize) & clamp_mode).mask() & 3) ||
+			m_vt.nan.value;
+
+		if (st_overflow)
+		{
+			constexpr int n = GSUtil::GetClassVertexCount(primclass);
+
+			// Make sure the copy buffer is large enough.
+			while (m_vertex.maxcount < m_index.tail)
+				GrowVertexBuffer();
+
+			GSVertex* RESTRICT vertex = m_vertex.buff;
+			GSVertex* RESTRICT vertex_copy = m_vertex.buff_copy;
+			u16* RESTRICT index = m_index.buff;
+
+			for (int i = 0; i < static_cast<int>(m_index.tail); i += n)
+			{
+				GSVector4 stcq[n];
+
+				// Load STQ for this primitive.
+				for (int j = 0; j < n; j++)
+					stcq[j] = GSVector4::cast(GSVector4i(vertex[index[i + j]].m[0]));
+
+				// Perform Q division and see which values need to be rewritten.
+				GSVector4 uv[n];
+				GSVector4i small{}, big{}, nan{};
+				for (int j = 0; j < n; j++)
+				{
+					// For sprites always use Q of second vertex.
+					const GSVector4 q = primclass == GS_SPRITE_CLASS ? stcq[1].wwww() : stcq[j].wwww();
+					uv[j] = (stcq[j] / q).xyzw(GSVector4::zero());
+					small |= GSVector4i::cast(uv[j] <= -OVERFLOW_VAL);
+					big |= GSVector4i::cast(uv[j] >= OVERFLOW_VAL);
+					nan |= GSVector4i::cast(uv[j] != uv[j]);
+				}
+
+				// Get the new values for fields that will be rewritten.
+				// The follows rules are used:
+				// 1. If there are small values but not big or nans, make all vertices small.
+				// 2. If there are big values but not small or nans, make all vertices big.
+				// 3. If there are both big and small values, or nans, make all vertices zero.
+				GSVector4 uv_new = GSVector4::zero();
+				uv_new = uv_new.blend32(-OVERFLOW_VAL, GSVector4::cast(small));
+				uv_new = uv_new.blend32(OVERFLOW_VAL, GSVector4::cast(big));
+				uv_new = uv_new.blend32(GSVector4::zero(), GSVector4::cast((small & big) | nan));
+
+				const GSVector4i rewrite = (((small | big) & clamp_mode) | nan).upl64(GSVector4i::zero());
+
+				// If both S and T are rewritten, no point in keeping Q. Just set it to 1.0f;
+				if ((GSVector4::cast(rewrite).mask() & 3) == 3)
+				{
+					for (int j = 0; j < n; j++)
+						stcq[j] = stcq[j].template insert32<0, 3>(GSVector4::m_one);
+				}
+				
+				// Rewrite the fields that require it and write to the copy buffer.
+				for (int j = 0; j < n; j++)
+				{
+					// For sprites always use Q of second vertex.
+					const GSVector4 q = (primclass == GS_SPRITE_CLASS) ? stcq[1].wwww() : stcq[j].wwww();
+					stcq[j] = stcq[j].blend32(uv_new * q, GSVector4::cast(rewrite));
+
+					vertex_copy[i + j].m[0] = GSVector4i::cast(stcq[j]).m;
+					vertex_copy[i + j].m[1] = vertex[index[i + j]].m[1];
+					index[i + j] = i + j;
+				}
+			}
+
+			// Swap the buffers and fix the counts.
+			std::swap(m_vertex.buff, m_vertex.buff_copy);
+			m_vertex.head = m_vertex.next = m_vertex.tail = m_index.tail;
+			
+			// Recalculate ST min/max/eq in the vertex trace.
+			GSVector4 tmin = GSVector4::cxpr(FLT_MAX);
+			GSVector4 tmax = GSVector4::cxpr(-FLT_MAX);
+			for (int i = 0; i < static_cast<int>(m_index.tail); i += n)
+			{
+				for (int j = 0; j < n; j++)
+				{
+					GSVector4 stcq = GSVector4::cast(GSVector4i(m_vertex.buff[i + j].m[0]));
+					const float Q = (primclass == GS_SPRITE_CLASS) ? stcq.w : m_vertex.buff[i + 1].RGBAQ.Q;
+					stcq = (stcq / Q).xyzw(stcq);
+					
+					tmin = tmin.min(stcq);
+					tmax = tmax.max(stcq);
+				}
+			}
+			m_vt.m_min.t = tmin.xyww() * tsize;
+			m_vt.m_max.t = tmax.xyww() * tsize;
+			m_vt.m_eq.stq = (m_vt.m_min.t == m_vt.m_max.t).mask();
+		}
+	}
+}
+
 void GSVertexSWInitStatic()
 {
 #define InitCVB4(P, T, F, Q) GSVertexSW::s_cvb[P][T][F][Q] = ConvertVertexBuffer<P, T, F, Q>;
@@ -309,6 +428,25 @@ void GSRendererSW::Draw()
 {
 	const GSDrawingContext* context = m_context;
 
+	switch (m_vt.m_primclass)
+	{
+		case GS_POINT_CLASS:
+			RewriteVerticesIfSTOverflow<GS_POINT_CLASS>();
+			break;
+		case GS_LINE_CLASS:
+			RewriteVerticesIfSTOverflow<GS_LINE_CLASS>();
+			break;
+		case GS_TRIANGLE_CLASS:
+			RewriteVerticesIfSTOverflow<GS_TRIANGLE_CLASS>();
+			break;
+		case GS_SPRITE_CLASS:
+			RewriteVerticesIfSTOverflow<GS_SPRITE_CLASS>();
+			break;
+		default:
+			pxFailRel("Unknown primitive class.");
+			break;
+	}
+	
 	auto data = m_vertex_heap.make_shared<SharedData>().cast<GSRasterizerData>();
 	SharedData* sd = static_cast<SharedData*>(data.get());
 

pcsx2/GS/Renderers/SW/GSRendererSW.h

@@ -79,6 +79,9 @@ class GSRendererSW final : public GSRenderer
 
 	bool GetScanlineGlobalData(SharedData* data);
 
+	template <u32 primclass>
+	void RewriteVerticesIfSTOverflow();
+
 public:
 	GSRendererSW(int threads);
 	~GSRendererSW() override;