Add GPU-side texture format channel fixup via shader constants

D3D11 lacks native luminance texture formats and requires CPU-side
per-pixel conversion for some 32-bit channel orders. This commit adds
a constant-driven channel swizzle in both pixel shader paths (Fixed
Function and Programmatic/NV2A combiner) that handles format
differences entirely on the GPU.

New TEXFMTFIXUP per-texture-stage constant with five modes:
  0 = identity (no fixup)
  1 = .gbar (X_D3DFMT_B8G8R8A8 uploaded raw as R8G8B8A8_UNORM)
  2 = .abgr (X_D3DFMT_R8G8B8A8 uploaded raw as R8G8B8A8_UNORM)
  3 = luminance: (L,0,0,1) -> (L,L,L,1) for L8/L16 on D3D11
  4 = alpha-luminance: (L,A,0,1) -> (L,L,L,A) for A8L8 on D3D11

Changes:
- FixedFunctionPixelShader.hlsli: Add TEXFMTFIXUP field to
  PsTextureStageState and shared constant definitions
- FixedFunctionPixelShader.hlsl: Apply fixup after texture sampling
- CxbxPixelShaderTemplate.hlsl: Add TEXFMTFIXUP at c42, new
  ApplyTexFmtFixup() helper, apply in Sample2D/3D/6F
- XbPixelShaderCompiler.cpp: Add CxbxGetTexFmtFixup() that detects
  Xbox texture format per stage; set constants for both FF and
  programmatic PS paths; bump PSH_XBOX_CONSTANT_MAX to 43
- HostResourceCreate.cpp: On D3D11, skip CPU conversion for
  B8G8R8A8 and R8G8B8A8 — upload raw bytes as R8G8B8A8_UNORM
  (the shader swizzle handles the channel reorder)
- Backend_D3D11.h: Bump CXBX_D3D11_PS_CB_COUNT to 43

GPU-side benefits:
- Luminance formats (L8, A8L8, L16) now produce correct colors
  instead of red-only output (fixes Halo lightmaps, etc.)
- B8G8R8A8 and R8G8B8A8 textures skip CPU per-pixel conversion
  loop entirely — raw data stays on GPU with zero CPU touch
- [branch] on uniform ensures zero-cost for the common identity case

Author: PatrickvL

src/core/hle/D3D8/Rendering/Backend_D3D11.h

@@ -45,7 +45,7 @@ extern IDirect3DSurface            *g_pD3DCurrentHostRenderTarget;
 static const UINT CXBX_D3D11_VS_CB_SLOT = 0;
 static const UINT CXBX_D3D11_VS_CB_COUNT = 256;
 static const UINT CXBX_D3D11_PS_CB_SLOT = 0;
-static const UINT CXBX_D3D11_PS_CB_COUNT = 42;
+static const UINT CXBX_D3D11_PS_CB_COUNT = 43;
 
 // Constant buffers (created in RenderGlobals.cpp device init, used by Backend_D3D11.cpp)
 extern ID3D11Buffer *g_pD3D11VSConstantBuffer;

src/core/hle/D3D8/Rendering/CxbxPixelShaderTemplate.hlsl

@@ -76,6 +76,7 @@ uniform const float4 LUM[4] : register(c35); // Note : PSH_XBOX_CONSTANT_LUM for
 uniform const float  FRONTFACE_FACTOR : register(c39); // Note : PSH_XBOX_CONSTANT_FRONTFACE_FACTOR
 uniform const float4 FOGINFO : register(c40);
 uniform const float  FOGENABLE : register(c41);
+uniform const float4 TEXFMTFIXUP : register(c42); // D3D11: per-stage texture format channel fixup (0=identity, 1=.gbar, 2=.abgr, 3=luminance, 4=alpha-luminance)
 
 #define CM_LT(c) if(c < 0) clip(-1); // = PS_COMPAREMODE_[RSTQ]_LT
 #define CM_GE(c) if(c >= 0) clip(-1); // = PS_COMPAREMODE_[RSTQ]_GE
@@ -324,8 +325,20 @@ void PerformAlphaKill(const float AlphaKill, float4 t)
 }
 #endif
 
+float4 ApplyTexFmtFixup(float4 t, float fixup)
+{
+	[branch] if (fixup != 0) {
+		if (fixup == 1) return t.gbar;                       // B8G8R8A8 uploaded as R8G8B8A8
+		if (fixup == 2) return t.abgr;                       // R8G8B8A8 uploaded as R8G8B8A8
+		if (fixup == 3) return float4(t.r, t.r, t.r, t.a);   // Luminance: R→(R,R,R,A)
+		if (fixup == 4) return float4(t.r, t.r, t.r, t.g);   // Alpha-luminance: RG→(R,R,R,G)
+	}
+	return t;
+}
+
 float4 PostProcessTexel(const int ts, float4 t)
 {
+	t = ApplyTexFmtFixup(t, TEXFMTFIXUP[ts]);
 	// TODO : Figure out in which order the following operations should be performed :
 	t = PerformColorSign(COLORSIGN[ts], t);
 	t = PerformColorKeyOp(COLORKEYOP[ts].x, COLORKEYCOLOR[ts], t);

src/core/hle/D3D8/Rendering/FixedFunctionPixelShader.hlsl

@@ -243,6 +243,14 @@ TextureArgs ExecuteTextureStage(
 	else if (type == SAMPLE_CUBE)
 		t = texCUBE(samplers[i], TexCoord.xyz);
 
+	// Apply texture format channel fixup (D3D11: luminance replication, channel swizzle)
+	[branch] if (stage.TEXFMTFIXUP != TEXFMTFIXUP_IDENTITY) {
+		if (stage.TEXFMTFIXUP == TEXFMTFIXUP_GBAR) t = t.gbar;
+		else if (stage.TEXFMTFIXUP == TEXFMTFIXUP_ABGR) t = t.abgr;
+		else if (stage.TEXFMTFIXUP == TEXFMTFIXUP_LUM) t = float4(t.r, t.r, t.r, t.a);
+		else if (stage.TEXFMTFIXUP == TEXFMTFIXUP_ALUM) t = float4(t.r, t.r, t.r, t.g);
+	}
+
 	// Bump environment mapping with luminance special case
 	if (previousOp == X_D3DTOP_BUMPENVMAPLUMINANCE) {
 		// Multiply sampled texture rgb values by L'

src/core/hle/D3D8/Rendering/FixedFunctionPixelShader.hlsli

@@ -115,8 +115,16 @@ namespace FixedFunctionPixelShader {
 		// BORDERCOLOR // = 29; set on sampler
 		alignas(16) float4 COLORKEYCOLOR; // = 30; Xbox extension!
 		// UNSUPPORTED // = 31; // Note : Somehow, this one comes through D3DDevice_SetTextureStageStateNotInline sometimes
+		alignas(16) float TEXFMTFIXUP; // D3D11: texture format channel fixup (0=identity, 1=.gbar, 2=.abgr, 3=luminance .rrra, 4=alpha-luminance .rrrg)
 	};
 
+	// Texture format fixup constants (shared between C++ and HLSL)
+	const float TEXFMTFIXUP_IDENTITY = 0;
+	const float TEXFMTFIXUP_GBAR     = 1; // B8G8R8A8 uploaded as R8G8B8A8
+	const float TEXFMTFIXUP_ABGR     = 2; // R8G8B8A8 uploaded as R8G8B8A8
+	const float TEXFMTFIXUP_LUM      = 3; // Luminance: R8→(R,R,R,1)
+	const float TEXFMTFIXUP_ALUM     = 4; // Alpha-luminance: R8G8→(R,R,R,G);
+
 	// This state is compiled into the shader
 	// Values correspond to XD3D8 version of D3DTEXTURESTAGESTATETYPE
 	// https://docs.microsoft.com/en-us/windows/win32/direct3d9/d3dtexturestagestatetype

src/core/hle/D3D8/Rendering/HostResourceCreate.cpp

@@ -263,6 +263,17 @@ EMUFORMAT PCFormat;
 			// Unset D3DUSAGE_DEPTHSTENCIL: It's not possible for ARGB textures to be depth stencils
 			// Fixes CreateTexture error in Virtua Cop 3 (Chihiro)
 			D3DUsage &= ~D3DUSAGE_DEPTHSTENCIL;
+
+#ifdef CXBX_USE_D3D11
+			// D3D11: For 32-bit formats that differ only in channel order,
+			// skip CPU conversion and upload raw bytes as R8G8B8A8_UNORM.
+			// The pixel shader applies the correct channel swizzle via TEXFMTFIXUP.
+			if (X_Format == xbox::X_D3DFMT_B8G8R8A8 || X_Format == xbox::X_D3DFMT_LIN_B8G8R8A8 ||
+			    X_Format == xbox::X_D3DFMT_R8G8B8A8 || X_Format == xbox::X_D3DFMT_LIN_R8G8B8A8) {
+				bConvertTextureFormat = false;
+				PCFormat = EMUFMT_A8B8G8R8; // = DXGI_FORMAT_R8G8B8A8_UNORM (raw byte order)
+			}
+#endif
 		}
 		else {
 			// Does host CheckDeviceFormat() succeed on this format?

src/core/hle/D3D8/XbPixelShaderCompiler.cpp

@@ -371,8 +371,10 @@ constexpr int PSH_XBOX_CONSTANT_FRONTFACE_FACTOR = PSH_XBOX_CONSTANT_LUM + 4; //
 constexpr int CXBX_D3DPS_CONSTREG_FOGINFO = PSH_XBOX_CONSTANT_FRONTFACE_FACTOR + 1; // = 40
 //Fog enable flag
 constexpr int PSH_XBOX_CONSTANT_FOGENABLE = CXBX_D3DPS_CONSTREG_FOGINFO + 1; // = 41
+// D3D11: Per-stage texture format channel fixup (packed float4, one component per stage)
+constexpr int PSH_XBOX_CONSTANT_TEXFMTFIXUP = PSH_XBOX_CONSTANT_FOGENABLE + 1; // = 42
 // This concludes the set of constants that need to be set on host :
-constexpr int PSH_XBOX_CONSTANT_MAX = PSH_XBOX_CONSTANT_FOGENABLE + 1; // = 42
+constexpr int PSH_XBOX_CONSTANT_MAX = PSH_XBOX_CONSTANT_TEXFMTFIXUP + 1; // = 43
 
 std::string_view GetD3DTOPString(int d3dtop) {
 	static constexpr std::string_view opToString[] = {
@@ -671,6 +673,40 @@ float CxbxComponentColorSignFromXboxAndHost(bool XboxMarksComponentSigned, bool
 	return -1.0f; // Mark the component for scaling from signed_to_unsigned
 }
 
+float CxbxGetTexFmtFixup(int stage_nr)
+{
+	using namespace FixedFunctionPixelShader;
+
+#ifdef CXBX_USE_D3D11
+	auto pXboxTex = g_pXbox_SetTexture[stage_nr];
+	if (pXboxTex == xbox::zeroptr)
+		return TEXFMTFIXUP_IDENTITY;
+
+	xbox::X_D3DFORMAT xboxFmt = GetXboxPixelContainerFormat((xbox::X_D3DPixelContainer*)pXboxTex);
+	switch (xboxFmt) {
+	case xbox::X_D3DFMT_L8:
+	case xbox::X_D3DFMT_LIN_L8:
+	case xbox::X_D3DFMT_L16:
+	case xbox::X_D3DFMT_LIN_L16:
+		return TEXFMTFIXUP_LUM;
+	case xbox::X_D3DFMT_A8L8:
+	case xbox::X_D3DFMT_LIN_A8L8:
+		return TEXFMTFIXUP_ALUM;
+	// B8G8R8A8 and R8G8B8A8 use GBAR/ABGR swizzles when uploaded raw
+	// (requires corresponding skip of CPU conversion in HostResourceCreate.cpp)
+	case xbox::X_D3DFMT_B8G8R8A8:
+	case xbox::X_D3DFMT_LIN_B8G8R8A8:
+		return TEXFMTFIXUP_GBAR;
+	case xbox::X_D3DFMT_R8G8B8A8:
+	case xbox::X_D3DFMT_LIN_R8G8B8A8:
+		return TEXFMTFIXUP_ABGR;
+	default:
+		break;
+	}
+#endif
+	return TEXFMTFIXUP_IDENTITY;
+}
+
 D3DXCOLOR CxbxCalcColorSign(int stage_nr)
 {
 	// Initially use what the running executable put in COLORSIGN :
@@ -762,6 +798,7 @@ void UpdateFixedFunctionPixelShaderState()
 		stage->BUMPENVLSCALE = AsFloat(XboxTextureStates.Get(i, xbox::X_D3DTSS_BUMPENVLSCALE));
 		stage->BUMPENVLOFFSET = AsFloat(XboxTextureStates.Get(i, xbox::X_D3DTSS_BUMPENVLOFFSET));
 		{ D3DXCOLOR c(XboxTextureStates.Get(i, xbox::X_D3DTSS_COLORKEYCOLOR)); stage->COLORKEYCOLOR = D3DXVECTOR4(c.r, c.g, c.b, c.a); }
+		stage->TEXFMTFIXUP = CxbxGetTexFmtFixup(i);
 	}
 
 	const int size = (sizeof(FixedFunctionPixelShaderState) + 16 - 1) / 16;
@@ -989,6 +1026,12 @@ void CxbxUpdateActivePixelShader() // NOPATCH
   fColor[CXBX_D3DPS_CONSTREG_FOGINFO].a = fogEnd;
   fColor[PSH_XBOX_CONSTANT_FOGENABLE].r = fogEnable;
 
+  // D3D11: Compute per-stage texture format channel fixup
+  fColor[PSH_XBOX_CONSTANT_TEXFMTFIXUP].r = CxbxGetTexFmtFixup(0);
+  fColor[PSH_XBOX_CONSTANT_TEXFMTFIXUP].g = CxbxGetTexFmtFixup(1);
+  fColor[PSH_XBOX_CONSTANT_TEXFMTFIXUP].b = CxbxGetTexFmtFixup(2);
+  fColor[PSH_XBOX_CONSTANT_TEXFMTFIXUP].a = CxbxGetTexFmtFixup(3);
+
   // Assume all constants are in use (this is much easier than tracking them for no other purpose than to skip a few here)
   // Read the color from the corresponding render state slot :
   // Set all host constant values using a single call: