Add hacks to fix PCH .data causing fake mismatches with decompctx (#2606)

Author: caseif

configure.py

@@ -220,6 +220,13 @@
 # Can be overridden in libraries or objects
 config.scratch_preset_id = 69 # Twilight Princess (DOL)
 
+# Globs to exclude from context files
+# *.mch excludes precompiled header output (which cannot be parsed)
+config.context_exclude_globs = ["*.mch"]
+
+# Macro definitions to inject into context files
+config.context_defines = ["DECOMPCTX"]
+
 # Base flags, common to most GC/Wii games.
 # Generally leave untouched, with overrides added below.
 cflags_base = [

include/JSystem/J3DGraphBase/J3DMatBlock.h

@@ -103,6 +103,12 @@ inline u16 calcColorChanID(u16 enable, u8 matSrc, u8 lightMask, u8 diffuseFn, u8
     return reg;
 }
 
+#ifdef DECOMPCTX
+// Hack to mitigate fake mismatches when building from decompctx output -
+// see comment in sqrtf in math.h
+static u8 AttnArr[] = {2, 0, 2, 1};
+#endif
+
 /**
  * @ingroup jsystem-j3d
  *
@@ -138,7 +144,9 @@ struct J3DColorChan {
     u8 getMatSrc() const { return (GXColorSrc)(mColorChanID & 1); }
     u8 getDiffuseFn() const { return ((u32)(mColorChanID & (3 << 7)) >> 7); }
     u8 getAttnFn() const {
+#ifndef DECOMPCTX
         u8 AttnArr[] = {2,0,2,1};
+#endif
         return AttnArr[(u32)(mColorChanID & (3 << 9)) >> 9];
     }
 

include/Z2AudioLib/Z2Calc.h

@@ -5,8 +5,13 @@
 #include "m_Do/m_Do_lib.h"
 
 namespace Z2Calc {
-// hack for f_op_actor, having this present breaks its weak func ordering
+#ifdef DECOMPCTX
+// Hack to mitigate fake mismatches when building from decompctx output -
+// see comment in sqrtf in math.h
+static Vec cNullVec = {0.0f, 0.0f, 0.0f};
+#else
 static const Vec cNullVec = {0.0f, 0.0f, 0.0f};
+#endif
 
 enum CurveSign {
     CURVE_SIGN_0 = 0,

include/d/dolzel.h

@@ -1,7 +1,7 @@
 #ifndef DOLZEL_H
 #define DOLZEL_H
 
-#if __MWERKS__
+#if __MWERKS__ && !defined(DECOMPCTX)
 #include "d/dolzel.mch"
 #else
 #include "d/dolzel.pch"

include/d/dolzel_rel.h

@@ -1,7 +1,7 @@
 #ifndef DOLZEL_REL_H
 #define DOLZEL_REL_H
 
-#if __MWERKS__
+#if __MWERKS__ && !defined(DECOMPCTX)
 #include "d/dolzel_rel.mch"
 #else
 #include "d/dolzel_rel.pch"

src/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Include/math.h

@@ -74,9 +74,24 @@ inline float sqrtf(float mag) {
     return sqrt(mag);
 }
 #else
+#ifdef DECOMPCTX
+// Hack to mitigate fake mismatches when building from decompctx output
+// (which doesn't support precompiled headers).
+//
+// When built without a PCH, these constants would end up .rodata instead of .data
+// which causes a variety of knock-on effects in individual functions' assembly.
+//
+// Making them into globals is a bit of a hack, but it generally fixes later
+// .data and .rodata offsets and allows individual functions to match.
+static double _half = 0.5;
+static double _three = 3.0;
+#endif
 inline float sqrtf(float mag) {
+#ifndef DECOMPCTX
+    // part of the same hack, these are defined outside of the function when using decompctx
     static const double _half = 0.5;
     static const double _three = 3.0;
+#endif
     if (mag > 0.0f) {
         double tmpd = __frsqrte(mag);
         tmpd = tmpd * _half * (_three - mag * (tmpd * tmpd));

tools/decompctx.py

@@ -11,6 +11,7 @@
 ###
 
 import argparse
+import fnmatch
 import os
 import re
 from typing import List
@@ -19,6 +20,7 @@
 root_dir = os.path.abspath(os.path.join(script_dir, ".."))
 src_dir = os.path.join(root_dir, "src")
 include_dirs: List[str] = []  # Set with -I flag
+exclude_globs: List[str] = []  # Set with -x flag
 
 include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]')
 guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$")
@@ -28,6 +30,23 @@
 deps = []
 
 
+def generate_prelude(defines) -> str:
+    if len(defines) == 0:
+        return ""
+
+    out_text = "/* decompctx prelude */\n"
+    for define in defines:
+        parts = define.split("=", 1)
+        if len(parts) == 2:
+            macro_name, macro_val = parts
+            out_text += f"#define {macro_name} {macro_val}\n"
+        else:
+            out_text += f"#define {parts[0]}\n"
+    out_text += "/* end decompctx prelude */\n\n"
+
+    return out_text
+
+
 def import_h_file(in_file: str, r_path: str) -> str:
     rel_path = os.path.join(root_dir, r_path, in_file)
     if os.path.exists(rel_path):
@@ -73,8 +92,17 @@ def process_file(in_file: str, lines: List[str]) -> str:
             print("Processing file", in_file)
         include_match = include_pattern.match(line.strip())
         if include_match and not include_match[1].endswith(".s"):
+            excluded = False
+            for glob in exclude_globs:
+                if fnmatch.fnmatch(include_match[1], glob):
+                    excluded = True
+                    break
+
             out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n'
-            out_text += import_h_file(include_match[1], os.path.dirname(in_file))
+            if excluded:
+                out_text += "/* Skipped excluded file */\n"
+            else:
+                out_text += import_h_file(include_match[1], os.path.dirname(in_file))
             out_text += f'/* end "{include_match[1]}" */\n'
         else:
             out_text += line
@@ -111,13 +139,29 @@ def main():
         help="""Include directory""",
         action="append",
     )
+    parser.add_argument(
+        "-x",
+        "--exclude",
+        help="""Excluded file name glob""",
+        action="append",
+    )
+    parser.add_argument(
+        "-D",
+        "--define",
+        help="""Macro definition""",
+        action="append",
+    )
     args = parser.parse_args()
 
     if args.include is None:
         exit("No include directories specified")
     global include_dirs
     include_dirs = args.include
-    output = import_c_file(args.c_file)
+    global exclude_globs
+    exclude_globs = args.exclude or []
+    prelude_defines = args.define or []
+    output = generate_prelude(prelude_defines)
+    output += import_c_file(args.c_file)
 
     with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f:
         f.write(output)

tools/project.py

@@ -198,6 +198,12 @@ def __init__(self) -> None:
         self.link_order_callback: Optional[Callable[[int, List[str]], List[str]]] = (
             None  # Callback to add/remove/reorder units within a module
         )
+        self.context_exclude_globs: List[str] = (
+            []  # Globs to exclude from context files
+        )
+        self.context_defines: List[str] = (
+            []  # Macros to define at the top of context files
+        )
 
         # Progress output and report.json config
         self.progress = True  # Enable report.json generation and CLI progress output
@@ -492,7 +498,7 @@ def generate_build_ninja(
     decompctx = config.tools_dir / "decompctx.py"
     n.rule(
         name="decompctx",
-        command=f"$python {decompctx} $in -o $out -d $out.d $includes",
+        command=f"$python {decompctx} $in -o $out -d $out.d $includes $excludes $defines",
         description="CTX $in",
         depfile="$out.d",
         deps="gcc",
@@ -1048,12 +1054,19 @@ def c_build(obj: Object, src_path: Path) -> Optional[Path]:
                     ):
                         include_dirs.append(flag[3:])
                 includes = " ".join([f"-I {d}" for d in include_dirs])
+                excludes = " ".join([f"-x {d}" for d in config.context_exclude_globs])
+                defines = " ".join([f"-D {d}" for d in config.context_defines])
+
                 n.build(
                     outputs=obj.ctx_path,
                     rule="decompctx",
                     inputs=src_path,
                     implicit=decompctx,
-                    variables={"includes": includes},
+                    variables={
+                        "includes": includes,
+                        "excludes": excludes,
+                        "defines": defines,
+                    },
                 )
             n.newline()