Usage of unused render states for special game scenarios

- use unused  RenderState 42 to assign a remix texture category for the next surface
- use unused RenderState 150 to set a custom material hash for the next surface
- add hidden remix option 'useUnusedRenderstates' (default: false) to enable usage of unused render states

Author: xoxor4d

src/d3d9/d3d9_rtx_utils.cpp

@@ -176,6 +176,14 @@ namespace dxvk {
 
     materialData.alphaBlendEnabled = d3d9State.renderStates[D3DRS_ALPHABLENDENABLE] != FALSE;
 
+    if (RtxOptions::Get()->useUnusedRenderstates()) {
+      materialData.remixTextureCategoryFlagsFromD3D = d3d9State.renderStates[42] != 0xfefefefe ? d3d9State.renderStates[42] : 0u; // D3DRENDERSTATETYPE index 42 is not in use - unused values are set to 0xfefefefe
+
+      if (d3d9State.renderStates[150] != 0 && d3d9State.renderStates[150] != 0xfefefefe) { // D3DRENDERSTATETYPE index 150 is not in use - unused values are set to 0xfefefefe
+        materialData.setHashOverride(d3d9State.renderStates[150]);
+      }
+    }
+
     if (materialData.alphaBlendEnabled) {
       D3D9BlendState color;
       color.Src = D3DBLEND(d3d9State.renderStates[D3DRS_SRCBLEND]);

src/dxvk/rtx_render/rtx_materials.h

@@ -1451,9 +1451,15 @@ struct LegacyMaterialData {
   uint32_t tFactor = 0xffffffff;  // Value for D3DRS_TEXTUREFACTOR, default value of is opaque white
   D3DMATERIAL9 d3dMaterial = {};
   bool isTextureFactorBlend = false;
+  uint32_t  remixTextureCategoryFlagsFromD3D = 0u;
 
   void setHashOverride(XXH64_hash_t hash) {
     m_cachedHash = hash;
+    m_isHashOverridden = true;
+  }
+
+  bool isHashOverridden() const {
+    return m_isHashOverridden;
   }
 
 private:
@@ -1468,7 +1474,9 @@ struct LegacyMaterialData {
     // incorporate more textures used to identify a material uniquely. Note this is not the same as the
     // plain data hash used by the RtSurfaceMaterial for storage in map-like data structures, but rather
     // one used to identify a material and compare to user-provided hashes.
-    m_cachedHash = colorTextures[0].getImageHash();
+    if (!m_isHashOverridden) {
+      m_cachedHash = colorTextures[0].getImageHash();
+    }
   }
 
   const static uint32_t kMaxSupportedTextures = 2;
@@ -1478,6 +1486,7 @@ struct LegacyMaterialData {
   uint32_t colorTextureSlot[kMaxSupportedTextures] = { kInvalidResourceSlot };
 
   XXH64_hash_t m_cachedHash = kEmptyHash;
+  bool m_isHashOverridden = false;
 };
 
 struct MaterialData {

src/dxvk/rtx_render/rtx_options.h

@@ -989,6 +989,9 @@ namespace dxvk {
     RTX_OPTION("rtx", float, effectLightRadius, 5.f, "");
     RTX_OPTION("rtx", bool, effectLightPlasmaBall, false, "");
 
+    RTX_OPTION("rtx", bool, useUnusedRenderstates, false, "Enable usage of unused D3D renderstates to aid with special game scenarios whilst having game code access.\n"
+               "(RS 42) Setting the remix texture category for the next surface. (RS 150) Setting a user defined material hash for the next surface.");
+
     RTX_OPTION("rtx", bool, useObsoleteHashOnTextureUpload, false,
                "Whether or not to use slower XXH64 hash on texture upload.\n"
                "New projects should not enable this option as this solely exists for compatibility with older hashing schemes.");

src/dxvk/rtx_render/rtx_types.cpp

@@ -133,6 +133,10 @@ namespace dxvk {
     //                   than doing N lookups per texture hash for each category.
     const XXH64_hash_t& textureHash = materialData.getColorTexture().getImageHash();
 
+    if (RtxOptions::Get()->useUnusedRenderstates()) {
+      categories = CategoryFlags(materialData.remixTextureCategoryFlagsFromD3D);
+    }
+
     setCategory(InstanceCategories::WorldUI, lookupHash(RtxOptions::worldSpaceUiTextures(), textureHash));
     setCategory(InstanceCategories::WorldMatte, lookupHash(RtxOptions::worldSpaceUiBackgroundTextures(), textureHash));
 
@@ -297,9 +301,12 @@ namespace dxvk {
       return true;
     }
 
-    // NOTE: we use color texture hash for sky detection, however the replacement is hashed with
-    // the whole legacy material hash (which, as of 12/9/2022, equals to color texture hash). Adding a check just in case.
-    assert(drawCallState.getMaterialData().getColorTexture().getImageHash() == drawCallState.getMaterialData().getHash() && "Texture or material hash method changed!");
+    // NOTE: disable assert if material is using a custom hash that was set via an unused D3D RenderState
+    if (!drawCallState.getMaterialData().isHashOverridden()) {
+      // NOTE: we use color texture hash for sky detection, however the replacement is hashed with
+      // the whole legacy material hash (which, as of 12/9/2022, equals to color texture hash). Adding a check just in case.
+      assert(drawCallState.getMaterialData().getColorTexture().getImageHash() == drawCallState.getMaterialData().getHash() && "Texture or material hash method changed!");
+    }
 
     if (drawCallState.getMaterialData().usesTexture()) {
       if (lookupHash(RtxOptions::skyBoxTextures(), drawCallState.getMaterialData().getHash())) {